Merge tag 'drm-for-v4.8' of git://people.freedesktop.org/~airlied/linux

Merge drm updates from Dave Airlie:
 "This is the main drm pull request for 4.8.

  I'm down with a cold at the moment so hopefully this isn't in too bad
  a state, I finished pulling stuff last week mostly (nouveau fixes just
  went in today), so only this message should be influenced by illness.
  Apologies to anyone who's major feature I missed :-)

  Core:
        Lockless GEM BO freeing
        Non-blocking atomic work
        Documentation changes (rst/sphinx)
        Prep for new fencing changes
        Simple display helpers
        Master/auth changes
        Register/unregister rework
        Loads of trivial patches/fixes.

  New stuff:
        ARM Mali display driver (not the 3D chip)
        sii902x RGB->HDMI bridge

  Panel:
        Support for new panels
        Improved backlight support

  Bridge:
        Convert ADV7511 to bridge driver
        ADV7533 support
        TC358767 (DSI/DPI to eDP) encoder chip support

  i915:
        BXT support enabled by default
        GVT-g infrastructure
        GuC command submission and fixes
        BXT workarounds
        SKL/BKL workarounds
        Demidlayering device registration
        Thundering herd fixes
        Missing pci ids
        Atomic updates

  amdgpu/radeon:
        ATPX improvements for better dGPU power control on PX systems
        New power features for CZ/BR/ST
        Pipelined BO moves and evictions in TTM
        GPU scheduler improvements
        GPU reset improvements
        Overclocking on dGPUs with amdgpu
        Polaris powermanagement enabled

  nouveau:
        GK20A/GM20B volt and clock improvements.
        Initial support for GP100/GP104 GPUs, GP104 will not yet support
        acceleration due to NVIDIA having not released firmware for them as of yet.

  exynos:
        Exynos5433 SoC with IOMMU support.

  vc4:
        Shader validation for branching

  imx-drm:
        Atomic mode setting conversion
        Reworked DMFC FIFO allocation
        External bridge support

  analogix-dp:
        RK3399 eDP support
        Lots of fixes.

  rockchip:
        Lots of small fixes.

  msm:
        DT bindings cleanups
        Shrinker and madvise support
        ASoC HDMI codec support

  tegra:
        Host1x driver cleanups
        SOR reworking for DP support
        Runtime PM support

  omapdrm:
        PLL enhancements
        Header refactoring
        Gamma table support

  arcgpu:
        Simulator support

  virtio-gpu:
        Atomic modesetting fixes.

  rcar-du:
        Misc fixes.

  mediatek:
        MT8173 HDMI support

  sti:
        ASOC HDMI codec support
        Minor fixes

  fsl-dcu:
        Suspend/resume support
        Bridge support

  amdkfd:
        Minor fixes.

  etnaviv:
        Enable GPU clock gating

  hisilicon:
        Vblank and other fixes"

* tag 'drm-for-v4.8' of git://people.freedesktop.org/~airlied/linux: (1575 commits)
  drm/nouveau/gr/nv3x: fix instobj write offsets in gr setup
  drm/nouveau/acpi: fix lockup with PCIe runtime PM
  drm/nouveau/acpi: check for function 0x1B before using it
  drm/nouveau/acpi: return supported DSM functions
  drm/nouveau/acpi: ensure matching ACPI handle and supported functions
  drm/nouveau/fbcon: fix font width not divisible by 8
  drm/amd/powerplay: remove enable_clock_power_gatings_tasks from initialize and resume events
  drm/amd/powerplay: move clockgating to after ungating power in pp for uvd/vce
  drm/amdgpu: add query device id and revision id into system info entry at CGS
  drm/amdgpu: add new definition in bif header
  drm/amd/powerplay: rename smum header guards
  drm/amdgpu: enable UVD context buffer for older HW
  drm/amdgpu: fix default UVD context size
  drm/amdgpu: fix incorrect type of info_id
  drm/amdgpu: make amdgpu_cgs_call_acpi_method as static
  drm/amdgpu: comment out unused defaults_staturn_pro static const structure to fix the build
  drm/amdgpu: enable UVD VM only on polaris
  drm/amdgpu: increase timeout of IB test
  drm/amdgpu: add destroy session when generate VCE destroy msg.
  drm/amd: fix deadlock of job_list_lock V2
  ...

836 files changed, 42810 insertions, 20788 deletions

diff --git a/drivers/char/agp/intel-gtt.c b/drivers/char/agp/intel-gtt.c
index aef87fdbd187..44311296ec02 100644
--- a/drivers/char/agp/intel-gtt.c
+++ b/drivers/char/agp/intel-gtt.c
@@ -840,6 +840,14 @@ static bool i830_check_flags(unsigned int flags)
         return false;
 }
 
+void intel_gtt_insert_page(dma_addr_t addr,
+                           unsigned int pg,
+                           unsigned int flags)
+{
+        intel_private.driver->write_entry(addr, pg, flags);
+}
+EXPORT_SYMBOL(intel_gtt_insert_page);
+
 void intel_gtt_insert_sg_entries(struct sg_table *st,
                                  unsigned int pg_start,
                                  unsigned int flags)
diff --git a/drivers/dma-buf/Kconfig b/drivers/dma-buf/Kconfig
index 9824bc4addf8..25bcfa0b474f 100644
--- a/drivers/dma-buf/Kconfig
+++ b/drivers/dma-buf/Kconfig
@@ -1,11 +1,20 @@
 menu "DMABUF options"
 
 config SYNC_FILE
-        bool "sync_file support for fences"
+        bool "Explicit Synchronization Framework"
         default n
         select ANON_INODES
         select DMA_SHARED_BUFFER
         ---help---
-          This option enables the fence framework synchronization to export
-          sync_files to userspace that can represent one or more fences.
+          The Sync File Framework adds explicit syncronization via
+          userspace. It enables send/receive 'struct fence' objects to/from
+          userspace via Sync File fds for synchronization between drivers via
+          userspace components. It has been ported from Android.
+
+          The first and main user for this is graphics in which a fence is
+          associated with a buffer. When a job is submitted to the GPU a fence
+          is attached to the buffer and is transferred via userspace, using Sync
+          Files fds, to the DRM driver for example. More details at
+          Documentation/sync_file.txt.
+
 endmenu
diff --git a/drivers/dma-buf/Makefile b/drivers/dma-buf/Makefile
index 4a424eca75ed..f353db213a81 100644
--- a/drivers/dma-buf/Makefile
+++ b/drivers/dma-buf/Makefile
@@ -1,2 +1,2 @@
-obj-y := dma-buf.o fence.o reservation.o seqno-fence.o
+obj-y := dma-buf.o fence.o reservation.o seqno-fence.o fence-array.o
 obj-$(CONFIG_SYNC_FILE)         += sync_file.o
diff --git a/drivers/dma-buf/dma-buf.c b/drivers/dma-buf/dma-buf.c
index 6355ab38d630..ddaee60ae52a 100644
--- a/drivers/dma-buf/dma-buf.c
+++ b/drivers/dma-buf/dma-buf.c
@@ -334,6 +334,7 @@ struct dma_buf *dma_buf_export(const struct dma_buf_export_info *exp_info)
         struct reservation_object *resv = exp_info->resv;
         struct file *file;
         size_t alloc_size = sizeof(struct dma_buf);
+        int ret;
 
         if (!exp_info->resv)
                 alloc_size += sizeof(struct reservation_object);
@@ -357,8 +358,8 @@ struct dma_buf *dma_buf_export(const struct dma_buf_export_info *exp_info)
 
         dmabuf = kzalloc(alloc_size, GFP_KERNEL);
         if (!dmabuf) {
-                module_put(exp_info->owner);
-                return ERR_PTR(-ENOMEM);
+                ret = -ENOMEM;
+                goto err_module;
         }
 
         dmabuf->priv = exp_info->priv;
@@ -379,8 +380,8 @@ struct dma_buf *dma_buf_export(const struct dma_buf_export_info *exp_info)
         file = anon_inode_getfile("dmabuf", &dma_buf_fops, dmabuf,
                                         exp_info->flags);
         if (IS_ERR(file)) {
-                kfree(dmabuf);
-                return ERR_CAST(file);
+                ret = PTR_ERR(file);
+                goto err_dmabuf;
         }
 
         file->f_mode |= FMODE_LSEEK;
@@ -394,6 +395,12 @@ struct dma_buf *dma_buf_export(const struct dma_buf_export_info *exp_info)
         mutex_unlock(&db_list.lock);
 
         return dmabuf;
+
+err_dmabuf:
+        kfree(dmabuf);
+err_module:
+        module_put(exp_info->owner);
+        return ERR_PTR(ret);
 }
 EXPORT_SYMBOL_GPL(dma_buf_export);
 
@@ -824,7 +831,7 @@ void dma_buf_vunmap(struct dma_buf *dmabuf, void *vaddr)
 EXPORT_SYMBOL_GPL(dma_buf_vunmap);
 
 #ifdef CONFIG_DEBUG_FS
-static int dma_buf_describe(struct seq_file *s)
+static int dma_buf_debug_show(struct seq_file *s, void *unused)
 {
         int ret;
         struct dma_buf *buf_obj;
@@ -879,17 +886,9 @@ static int dma_buf_describe(struct seq_file *s)
         return 0;
 }
 
-static int dma_buf_show(struct seq_file *s, void *unused)
-{
-        void (*func)(struct seq_file *) = s->private;
-
-        func(s);
-        return 0;
-}
-
 static int dma_buf_debug_open(struct inode *inode, struct file *file)
 {
-        return single_open(file, dma_buf_show, inode->i_private);
+        return single_open(file, dma_buf_debug_show, NULL);
 }
 
 static const struct file_operations dma_buf_debug_fops = {
@@ -903,20 +902,23 @@ static struct dentry *dma_buf_debugfs_dir;
 
 static int dma_buf_init_debugfs(void)
 {
+        struct dentry *d;
         int err = 0;
 
-        dma_buf_debugfs_dir = debugfs_create_dir("dma_buf", NULL);
+        d = debugfs_create_dir("dma_buf", NULL);
+        if (IS_ERR(d))
+                return PTR_ERR(d);
 
-        if (IS_ERR(dma_buf_debugfs_dir)) {
-                err = PTR_ERR(dma_buf_debugfs_dir);
-                dma_buf_debugfs_dir = NULL;
-                return err;
-        }
-
-        err = dma_buf_debugfs_create_file("bufinfo", dma_buf_describe);
+        dma_buf_debugfs_dir = d;
 
-        if (err)
+        d = debugfs_create_file("bufinfo", S_IRUGO, dma_buf_debugfs_dir,
+                                NULL, &dma_buf_debug_fops);
+        if (IS_ERR(d)) {
                 pr_debug("dma_buf: debugfs: failed to create node bufinfo\n");
+                debugfs_remove_recursive(dma_buf_debugfs_dir);
+                dma_buf_debugfs_dir = NULL;
+                err = PTR_ERR(d);
+        }
 
         return err;
 }
@@ -926,17 +928,6 @@ static void dma_buf_uninit_debugfs(void)
         if (dma_buf_debugfs_dir)
                 debugfs_remove_recursive(dma_buf_debugfs_dir);
 }
-
-int dma_buf_debugfs_create_file(const char *name,
-                                int (*write)(struct seq_file *))
-{
-        struct dentry *d;
-
-        d = debugfs_create_file(name, S_IRUGO, dma_buf_debugfs_dir,
-                        write, &dma_buf_debug_fops);
-
-        return PTR_ERR_OR_ZERO(d);
-}
 #else
 static inline int dma_buf_init_debugfs(void)
 {
diff --git a/drivers/dma-buf/fence-array.c b/drivers/dma-buf/fence-array.c
new file mode 100644
index 000000000000..a8731c853da6
--- /dev/null
+++ b/drivers/dma-buf/fence-array.c
@@ -0,0 +1,144 @@
+/*
+ * fence-array: aggregate fences to be waited together
+ *
+ * Copyright (C) 2016 Collabora Ltd
+ * Copyright (C) 2016 Advanced Micro Devices, Inc.
+ * Authors:
+ *      Gustavo Padovan <gustavo@padovan.org>
+ *      Christian König <christian.koenig@amd.com>
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 as published by
+ * the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ */
+
+#include <linux/export.h>
+#include <linux/slab.h>
+#include <linux/fence-array.h>
+
+static void fence_array_cb_func(struct fence *f, struct fence_cb *cb);
+
+static const char *fence_array_get_driver_name(struct fence *fence)
+{
+        return "fence_array";
+}
+
+static const char *fence_array_get_timeline_name(struct fence *fence)
+{
+        return "unbound";
+}
+
+static void fence_array_cb_func(struct fence *f, struct fence_cb *cb)
+{
+        struct fence_array_cb *array_cb =
+                container_of(cb, struct fence_array_cb, cb);
+        struct fence_array *array = array_cb->array;
+
+        if (atomic_dec_and_test(&array->num_pending))
+                fence_signal(&array->base);
+        fence_put(&array->base);
+}
+
+static bool fence_array_enable_signaling(struct fence *fence)
+{
+        struct fence_array *array = to_fence_array(fence);
+        struct fence_array_cb *cb = (void *)(&array[1]);
+        unsigned i;
+
+        for (i = 0; i < array->num_fences; ++i) {
+                cb[i].array = array;
+                /*
+                 * As we may report that the fence is signaled before all
+                 * callbacks are complete, we need to take an additional
+                 * reference count on the array so that we do not free it too
+                 * early. The core fence handling will only hold the reference
+                 * until we signal the array as complete (but that is now
+                 * insufficient).
+                 */
+                fence_get(&array->base);
+                if (fence_add_callback(array->fences[i], &cb[i].cb,
+                                       fence_array_cb_func)) {
+                        fence_put(&array->base);
+                        if (atomic_dec_and_test(&array->num_pending))
+                                return false;
+                }
+        }
+
+        return true;
+}
+
+static bool fence_array_signaled(struct fence *fence)
+{
+        struct fence_array *array = to_fence_array(fence);
+
+        return atomic_read(&array->num_pending) <= 0;
+}
+
+static void fence_array_release(struct fence *fence)
+{
+        struct fence_array *array = to_fence_array(fence);
+        unsigned i;
+
+        for (i = 0; i < array->num_fences; ++i)
+                fence_put(array->fences[i]);
+
+        kfree(array->fences);
+        fence_free(fence);
+}
+
+const struct fence_ops fence_array_ops = {
+        .get_driver_name = fence_array_get_driver_name,
+        .get_timeline_name = fence_array_get_timeline_name,
+        .enable_signaling = fence_array_enable_signaling,
+        .signaled = fence_array_signaled,
+        .wait = fence_default_wait,
+        .release = fence_array_release,
+};
+
+/**
+ * fence_array_create - Create a custom fence array
+ * @num_fences:         [in]    number of fences to add in the array
+ * @fences:             [in]    array containing the fences
+ * @context:            [in]    fence context to use
+ * @seqno:              [in]    sequence number to use
+ * @signal_on_any       [in]    signal on any fence in the array
+ *
+ * Allocate a fence_array object and initialize the base fence with fence_init().
+ * In case of error it returns NULL.
+ *
+ * The caller should allocte the fences array with num_fences size
+ * and fill it with the fences it wants to add to the object. Ownership of this
+ * array is take and fence_put() is used on each fence on release.
+ *
+ * If @signal_on_any is true the fence array signals if any fence in the array
+ * signals, otherwise it signals when all fences in the array signal.
+ */
+struct fence_array *fence_array_create(int num_fences, struct fence **fences,
+                                       u64 context, unsigned seqno,
+                                       bool signal_on_any)
+{
+        struct fence_array *array;
+        size_t size = sizeof(*array);
+
+        /* Allocate the callback structures behind the array. */
+        size += num_fences * sizeof(struct fence_array_cb);
+        array = kzalloc(size, GFP_KERNEL);
+        if (!array)
+                return NULL;
+
+        spin_lock_init(&array->lock);
+        fence_init(&array->base, &fence_array_ops, &array->lock,
+                   context, seqno);
+
+        array->num_fences = num_fences;
+        atomic_set(&array->num_pending, signal_on_any ? 1 : num_fences);
+        array->fences = fences;
+
+        return array;
+}
+EXPORT_SYMBOL(fence_array_create);
diff --git a/drivers/dma-buf/fence.c b/drivers/dma-buf/fence.c
index 7b05dbe9b296..4d51f9e83fa8 100644
--- a/drivers/dma-buf/fence.c
+++ b/drivers/dma-buf/fence.c
@@ -35,7 +35,7 @@ EXPORT_TRACEPOINT_SYMBOL(fence_emit);
  * context or not. One device can have multiple separate contexts,
  * and they're used if some engine can run independently of another.
  */
-static atomic_t fence_context_counter = ATOMIC_INIT(0);
+static atomic64_t fence_context_counter = ATOMIC64_INIT(0);
 
 /**
  * fence_context_alloc - allocate an array of fence contexts
@@ -44,10 +44,10 @@ static atomic_t fence_context_counter = ATOMIC_INIT(0);
  * This function will return the first index of the number of fences allocated.
  * The fence context is used for setting fence->context to a unique number.
  */
-unsigned fence_context_alloc(unsigned num)
+u64 fence_context_alloc(unsigned num)
 {
         BUG_ON(!num);
-        return atomic_add_return(num, &fence_context_counter) - num;
+        return atomic64_add_return(num, &fence_context_counter) - num;
 }
 EXPORT_SYMBOL(fence_context_alloc);
 
@@ -513,7 +513,7 @@ EXPORT_SYMBOL(fence_wait_any_timeout);
  */
 void
 fence_init(struct fence *fence, const struct fence_ops *ops,
-             spinlock_t *lock, unsigned context, unsigned seqno)
+             spinlock_t *lock, u64 context, unsigned seqno)
 {
         BUG_ON(!lock);
         BUG_ON(!ops || !ops->wait || !ops->enable_signaling ||
diff --git a/drivers/dma-buf/sync_file.c b/drivers/dma-buf/sync_file.c
index f08cf2d8309e..9aaa608dfe01 100644
--- a/drivers/dma-buf/sync_file.c
+++ b/drivers/dma-buf/sync_file.c
@@ -82,7 +82,7 @@ struct sync_file *sync_file_create(struct fence *fence)
 
         sync_file->num_fences = 1;
         atomic_set(&sync_file->status, 1);
-        snprintf(sync_file->name, sizeof(sync_file->name), "%s-%s%d-%d",
+        snprintf(sync_file->name, sizeof(sync_file->name), "%s-%s%llu-%d",
                  fence->ops->get_driver_name(fence),
                  fence->ops->get_timeline_name(fence), fence->context,
                  fence->seqno);
diff --git a/drivers/gpu/drm/Makefile b/drivers/gpu/drm/Makefile
index be43afb08c69..e3dba6f44a79 100644
--- a/drivers/gpu/drm/Makefile
+++ b/drivers/gpu/drm/Makefile
@@ -8,7 +8,7 @@ drm-y       :=	drm_auth.o drm_bufs.o drm_cache.o \
                 drm_lock.o drm_memory.o drm_drv.o drm_vm.o \
                 drm_scatter.o drm_pci.o \
                 drm_platform.o drm_sysfs.o drm_hashtab.o drm_mm.o \
-                drm_crtc.o drm_modes.o drm_edid.o \
+                drm_crtc.o drm_fourcc.o drm_modes.o drm_edid.o \
                 drm_info.o drm_debugfs.o drm_encoder_slave.o \
                 drm_trace_points.o drm_global.o drm_prime.o \
                 drm_rect.o drm_vma_manager.o drm_flip_work.o \
@@ -23,7 +23,8 @@ drm-$(CONFIG_AGP) += drm_agpsupport.o
 
 drm_kms_helper-y := drm_crtc_helper.o drm_dp_helper.o drm_probe_helper.o \
                 drm_plane_helper.o drm_dp_mst_topology.o drm_atomic_helper.o \
-                drm_kms_helper_common.o drm_dp_dual_mode_helper.o
+                drm_kms_helper_common.o drm_dp_dual_mode_helper.o \
+                drm_simple_kms_helper.o
 
 drm_kms_helper-$(CONFIG_DRM_LOAD_EDID_FIRMWARE) += drm_edid_load.o
 drm_kms_helper-$(CONFIG_DRM_FBDEV_EMULATION) += drm_fb_helper.o
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu.h b/drivers/gpu/drm/amd/amdgpu/amdgpu.h
index e055d5be1c3c..8ebc5f1eb4c0 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu.h
@@ -85,8 +85,12 @@ extern int amdgpu_vm_debug;
 extern int amdgpu_sched_jobs;
 extern int amdgpu_sched_hw_submission;
 extern int amdgpu_powerplay;
+extern int amdgpu_powercontainment;
 extern unsigned amdgpu_pcie_gen_cap;
 extern unsigned amdgpu_pcie_lane_cap;
+extern unsigned amdgpu_cg_mask;
+extern unsigned amdgpu_pg_mask;
+extern char *amdgpu_disable_cu;
 
 #define AMDGPU_WAIT_IDLE_TIMEOUT_IN_MS          3000
 #define AMDGPU_MAX_USEC_TIMEOUT                 100000  /* 100 ms */
@@ -183,6 +187,10 @@ int amdgpu_set_clockgating_state(struct amdgpu_device *adev,
 int amdgpu_set_powergating_state(struct amdgpu_device *adev,
                                   enum amd_ip_block_type block_type,
                                   enum amd_powergating_state state);
+int amdgpu_wait_for_idle(struct amdgpu_device *adev,
+                         enum amd_ip_block_type block_type);
+bool amdgpu_is_idle(struct amdgpu_device *adev,
+                    enum amd_ip_block_type block_type);
 
 struct amdgpu_ip_block_version {
         enum amd_ip_block_type type;
@@ -298,13 +306,16 @@ struct amdgpu_ring_funcs {
                                 uint32_t oa_base, uint32_t oa_size);
         /* testing functions */
         int (*test_ring)(struct amdgpu_ring *ring);
-        int (*test_ib)(struct amdgpu_ring *ring);
+        int (*test_ib)(struct amdgpu_ring *ring, long timeout);
         /* insert NOP packets */
         void (*insert_nop)(struct amdgpu_ring *ring, uint32_t count);
         /* pad the indirect buffer to the necessary number of dw */
         void (*pad_ib)(struct amdgpu_ring *ring, struct amdgpu_ib *ib);
         unsigned (*init_cond_exec)(struct amdgpu_ring *ring);
         void (*patch_cond_exec)(struct amdgpu_ring *ring, unsigned offset);
+        /* note usage for clock and power gating */
+        void (*begin_use)(struct amdgpu_ring *ring);
+        void (*end_use)(struct amdgpu_ring *ring);
 };
 
 /*
@@ -594,11 +605,9 @@ int amdgpu_sync_resv(struct amdgpu_device *adev,
                      struct amdgpu_sync *sync,
                      struct reservation_object *resv,
                      void *owner);
-bool amdgpu_sync_is_idle(struct amdgpu_sync *sync);
-int amdgpu_sync_cycle_fences(struct amdgpu_sync *dst, struct amdgpu_sync *src,
-                             struct fence *fence);
+struct fence *amdgpu_sync_peek_fence(struct amdgpu_sync *sync,
+                                     struct amdgpu_ring *ring);
 struct fence *amdgpu_sync_get_fence(struct amdgpu_sync *sync);
-int amdgpu_sync_wait(struct amdgpu_sync *sync);
 void amdgpu_sync_free(struct amdgpu_sync *sync);
 int amdgpu_sync_init(void);
 void amdgpu_sync_fini(void);
@@ -754,12 +763,11 @@ int amdgpu_job_alloc(struct amdgpu_device *adev, unsigned num_ibs,
 int amdgpu_job_alloc_with_ib(struct amdgpu_device *adev, unsigned size,
                              struct amdgpu_job **job);
 
+void amdgpu_job_free_resources(struct amdgpu_job *job);
 void amdgpu_job_free(struct amdgpu_job *job);
-void amdgpu_job_free_func(struct kref *refcount);
 int amdgpu_job_submit(struct amdgpu_job *job, struct amdgpu_ring *ring,
                       struct amd_sched_entity *entity, void *owner,
                       struct fence **f);
-void amdgpu_job_timeout_func(struct work_struct *work);
 
 struct amdgpu_ring {
         struct amdgpu_device            *adev;
@@ -767,12 +775,9 @@ struct amdgpu_ring {
         struct amdgpu_fence_driver      fence_drv;
         struct amd_gpu_scheduler        sched;
 
-        spinlock_t              fence_lock;
         struct amdgpu_bo        *ring_obj;
         volatile uint32_t       *ring;
         unsigned                rptr_offs;
-        u64                     next_rptr_gpu_addr;
-        volatile u32            *next_rptr_cpu_addr;
         unsigned                wptr;
         unsigned                wptr_old;
         unsigned                ring_size;
@@ -791,14 +796,16 @@ struct amdgpu_ring {
         u32                     doorbell_index;
         bool                    use_doorbell;
         unsigned                wptr_offs;
-        unsigned                next_rptr_offs;
         unsigned                fence_offs;
         uint64_t                current_ctx;
         enum amdgpu_ring_type   type;
         char                    name[16];
         unsigned                cond_exe_offs;
-        u64                             cond_exe_gpu_addr;
-        volatile u32    *cond_exe_cpu_addr;
+        u64                     cond_exe_gpu_addr;
+        volatile u32            *cond_exe_cpu_addr;
+#if defined(CONFIG_DEBUG_FS)
+        struct dentry *ent;
+#endif
 };
 
 /*
@@ -861,6 +868,7 @@ struct amdgpu_vm {
         struct amdgpu_bo        *page_directory;
         unsigned                max_pde_used;
         struct fence            *page_directory_fence;
+        uint64_t                last_eviction_counter;
 
         /* array of page tables, one for each page directory entry */
         struct amdgpu_vm_pt     *page_tables;
@@ -883,13 +891,14 @@ struct amdgpu_vm_id {
         struct fence            *first;
         struct amdgpu_sync      active;
         struct fence            *last_flush;
-        struct amdgpu_ring      *last_user;
         atomic64_t              owner;
 
         uint64_t                pd_gpu_addr;
         /* last flushed PD/PT update */
         struct fence            *flushed_updates;
 
+        uint32_t                current_gpu_reset_count;
+
         uint32_t                gds_base;
         uint32_t                gds_size;
         uint32_t                gws_base;
@@ -905,6 +914,10 @@ struct amdgpu_vm_manager {
         struct list_head                        ids_lru;
         struct amdgpu_vm_id                     ids[AMDGPU_NUM_VM];
 
+        /* Handling of VM fences */
+        u64                                     fence_context;
+        unsigned                                seqno[AMDGPU_MAX_RINGS];
+
         uint32_t                                max_pfn;
         /* vram base address for page table entry  */
         u64                                     vram_base_offset;
@@ -926,17 +939,14 @@ void amdgpu_vm_fini(struct amdgpu_device *adev, struct amdgpu_vm *vm);
 void amdgpu_vm_get_pd_bo(struct amdgpu_vm *vm,
                          struct list_head *validated,
                          struct amdgpu_bo_list_entry *entry);
-void amdgpu_vm_get_pt_bos(struct amdgpu_vm *vm, struct list_head *duplicates);
+void amdgpu_vm_get_pt_bos(struct amdgpu_device *adev, struct amdgpu_vm *vm,
+                          struct list_head *duplicates);
 void amdgpu_vm_move_pt_bos_in_lru(struct amdgpu_device *adev,
                                   struct amdgpu_vm *vm);
 int amdgpu_vm_grab_id(struct amdgpu_vm *vm, struct amdgpu_ring *ring,
                       struct amdgpu_sync *sync, struct fence *fence,
-                      unsigned *vm_id, uint64_t *vm_pd_addr);
-int amdgpu_vm_flush(struct amdgpu_ring *ring,
-                    unsigned vm_id, uint64_t pd_addr,
-                    uint32_t gds_base, uint32_t gds_size,
-                    uint32_t gws_base, uint32_t gws_size,
-                    uint32_t oa_base, uint32_t oa_size);
+                      struct amdgpu_job *job);
+int amdgpu_vm_flush(struct amdgpu_ring *ring, struct amdgpu_job *job);
 void amdgpu_vm_reset_id(struct amdgpu_device *adev, unsigned vm_id);
 uint64_t amdgpu_vm_map_gart(const dma_addr_t *pages_addr, uint64_t addr);
 int amdgpu_vm_update_page_directory(struct amdgpu_device *adev,
@@ -1142,6 +1152,12 @@ struct amdgpu_cu_info {
         uint32_t bitmap[4][4];
 };
 
+struct amdgpu_gfx_funcs {
+        /* get the gpu clock counter */
+        uint64_t (*get_gpu_clock_counter)(struct amdgpu_device *adev);
+        void (*select_se_sh)(struct amdgpu_device *adev, u32 se_num, u32 sh_num, u32 instance);
+};
+
 struct amdgpu_gfx {
         struct mutex                    gpu_clock_mutex;
         struct amdgpu_gca_config        config;
@@ -1178,6 +1194,7 @@ struct amdgpu_gfx {
         /* ce ram size*/
         unsigned                        ce_ram_size;
         struct amdgpu_cu_info           cu_info;
+        const struct amdgpu_gfx_funcs   *funcs;
 };
 
 int amdgpu_ib_get(struct amdgpu_device *adev, struct amdgpu_vm *vm,
@@ -1195,10 +1212,6 @@ void amdgpu_ring_insert_nop(struct amdgpu_ring *ring, uint32_t count);
 void amdgpu_ring_generic_pad_ib(struct amdgpu_ring *ring, struct amdgpu_ib *ib);
 void amdgpu_ring_commit(struct amdgpu_ring *ring);
 void amdgpu_ring_undo(struct amdgpu_ring *ring);
-unsigned amdgpu_ring_backup(struct amdgpu_ring *ring,
-                            uint32_t **data);
-int amdgpu_ring_restore(struct amdgpu_ring *ring,
-                        unsigned size, uint32_t *data);
 int amdgpu_ring_init(struct amdgpu_device *adev, struct amdgpu_ring *ring,
                      unsigned ring_size, u32 nop, u32 align_mask,
                      struct amdgpu_irq_src *irq_src, unsigned irq_type,
@@ -1250,6 +1263,7 @@ struct amdgpu_job {
         uint32_t                num_ibs;
         void                    *owner;
         uint64_t                ctx;
+        bool                    vm_needs_flush;
         unsigned                vm_id;
         uint64_t                vm_pd_addr;
         uint32_t                gds_base, gds_size;
@@ -1257,8 +1271,7 @@ struct amdgpu_job {
         uint32_t                oa_base, oa_size;
 
         /* user fence handling */
-        struct amdgpu_bo        *uf_bo;
-        uint32_t                uf_offset;
+        uint64_t                uf_addr;
         uint64_t                uf_sequence;
 
 };
@@ -1560,6 +1573,12 @@ struct amdgpu_dpm_funcs {
         u32 (*get_fan_control_mode)(struct amdgpu_device *adev);
         int (*set_fan_speed_percent)(struct amdgpu_device *adev, u32 speed);
         int (*get_fan_speed_percent)(struct amdgpu_device *adev, u32 *speed);
+        int (*force_clock_level)(struct amdgpu_device *adev, enum pp_clock_type type, uint32_t mask);
+        int (*print_clock_levels)(struct amdgpu_device *adev, enum pp_clock_type type, char *buf);
+        int (*get_sclk_od)(struct amdgpu_device *adev);
+        int (*set_sclk_od)(struct amdgpu_device *adev, uint32_t value);
+        int (*get_mclk_od)(struct amdgpu_device *adev);
+        int (*set_mclk_od)(struct amdgpu_device *adev, uint32_t value);
 };
 
 struct amdgpu_dpm {
@@ -1662,6 +1681,7 @@ struct amdgpu_uvd {
         struct amdgpu_ring      ring;
         struct amdgpu_irq_src   irq;
         bool                    address_64_bit;
+        bool                    use_ctx_buf;
         struct amd_sched_entity entity;
 };
 
@@ -1683,6 +1703,7 @@ struct amdgpu_vce {
         struct drm_file         *filp[AMDGPU_MAX_VCE_HANDLES];
         uint32_t                img_size[AMDGPU_MAX_VCE_HANDLES];
         struct delayed_work     idle_work;
+        struct mutex            idle_mutex;
         const struct firmware   *fw;    /* VCE firmware */
         struct amdgpu_ring      ring[AMDGPU_MAX_VCE_RINGS];
         struct amdgpu_irq_src   irq;
@@ -1767,6 +1788,8 @@ int amdgpu_debugfs_init(struct drm_minor *minor);
 void amdgpu_debugfs_cleanup(struct drm_minor *minor);
 #endif
 
+int amdgpu_debugfs_firmware_init(struct amdgpu_device *adev);
+
 /*
  * amdgpu smumgr functions
  */
@@ -1811,12 +1834,8 @@ struct amdgpu_asic_funcs {
                              u32 sh_num, u32 reg_offset, u32 *value);
         void (*set_vga_state)(struct amdgpu_device *adev, bool state);
         int (*reset)(struct amdgpu_device *adev);
-        /* wait for mc_idle */
-        int (*wait_for_mc_idle)(struct amdgpu_device *adev);
         /* get the reference clock */
         u32 (*get_xclk)(struct amdgpu_device *adev);
-        /* get the gpu clock counter */
-        uint64_t (*get_gpu_clock_counter)(struct amdgpu_device *adev);
         /* MM block clocks */
         int (*set_uvd_clocks)(struct amdgpu_device *adev, u32 vclk, u32 dclk);
         int (*set_vce_clocks)(struct amdgpu_device *adev, u32 evclk, u32 ecclk);
@@ -2003,6 +2022,10 @@ struct amdgpu_device {
         spinlock_t didt_idx_lock;
         amdgpu_rreg_t                   didt_rreg;
         amdgpu_wreg_t                   didt_wreg;
+        /* protects concurrent gc_cac register access */
+        spinlock_t gc_cac_idx_lock;
+        amdgpu_rreg_t                   gc_cac_rreg;
+        amdgpu_wreg_t                   gc_cac_wreg;
         /* protects concurrent ENDPOINT (audio) register access */
         spinlock_t audio_endpt_idx_lock;
         amdgpu_block_rreg_t             audio_endpt_rreg;
@@ -2028,6 +2051,7 @@ struct amdgpu_device {
         atomic64_t                      vram_vis_usage;
         atomic64_t                      gtt_usage;
         atomic64_t                      num_bytes_moved;
+        atomic64_t                      num_evictions;
         atomic_t                        gpu_reset_counter;
 
         /* display */
@@ -2038,7 +2062,7 @@ struct amdgpu_device {
         struct amdgpu_irq_src           hpd_irq;
 
         /* rings */
-        unsigned                        fence_context;
+        u64                             fence_context;
         unsigned                        num_rings;
         struct amdgpu_ring              *rings[AMDGPU_MAX_RINGS];
         bool                            ib_pool_ready;
@@ -2131,6 +2155,8 @@ void amdgpu_mm_wdoorbell(struct amdgpu_device *adev, u32 index, u32 v);
 #define WREG32_UVD_CTX(reg, v) adev->uvd_ctx_wreg(adev, (reg), (v))
 #define RREG32_DIDT(reg) adev->didt_rreg(adev, (reg))
 #define WREG32_DIDT(reg, v) adev->didt_wreg(adev, (reg), (v))
+#define RREG32_GC_CAC(reg) adev->gc_cac_rreg(adev, (reg))
+#define WREG32_GC_CAC(reg, v) adev->gc_cac_wreg(adev, (reg), (v))
 #define RREG32_AUDIO_ENDPT(block, reg) adev->audio_endpt_rreg(adev, (block), (reg))
 #define WREG32_AUDIO_ENDPT(block, reg, v) adev->audio_endpt_wreg(adev, (block), (reg), (v))
 #define WREG32_P(reg, val, mask)                                \
@@ -2206,12 +2232,10 @@ amdgpu_get_sdma_instance(struct amdgpu_ring *ring)
  */
 #define amdgpu_asic_set_vga_state(adev, state) (adev)->asic_funcs->set_vga_state((adev), (state))
 #define amdgpu_asic_reset(adev) (adev)->asic_funcs->reset((adev))
-#define amdgpu_asic_wait_for_mc_idle(adev) (adev)->asic_funcs->wait_for_mc_idle((adev))
 #define amdgpu_asic_get_xclk(adev) (adev)->asic_funcs->get_xclk((adev))
 #define amdgpu_asic_set_uvd_clocks(adev, v, d) (adev)->asic_funcs->set_uvd_clocks((adev), (v), (d))
 #define amdgpu_asic_set_vce_clocks(adev, ev, ec) (adev)->asic_funcs->set_vce_clocks((adev), (ev), (ec))
 #define amdgpu_asic_get_virtual_caps(adev) ((adev)->asic_funcs->get_virtual_caps((adev)))
-#define amdgpu_asic_get_gpu_clock_counter(adev) (adev)->asic_funcs->get_gpu_clock_counter((adev))
 #define amdgpu_asic_read_disabled_bios(adev) (adev)->asic_funcs->read_disabled_bios((adev))
 #define amdgpu_asic_read_bios_from_rom(adev, b, l) (adev)->asic_funcs->read_bios_from_rom((adev), (b), (l))
 #define amdgpu_asic_read_register(adev, se, sh, offset, v)((adev)->asic_funcs->read_register((adev), (se), (sh), (offset), (v)))
@@ -2222,7 +2246,7 @@ amdgpu_get_sdma_instance(struct amdgpu_ring *ring)
 #define amdgpu_vm_set_pte_pde(adev, ib, pe, addr, count, incr, flags) ((adev)->vm_manager.vm_pte_funcs->set_pte_pde((ib), (pe), (addr), (count), (incr), (flags)))
 #define amdgpu_ring_parse_cs(r, p, ib) ((r)->funcs->parse_cs((p), (ib)))
 #define amdgpu_ring_test_ring(r) (r)->funcs->test_ring((r))
-#define amdgpu_ring_test_ib(r) (r)->funcs->test_ib((r))
+#define amdgpu_ring_test_ib(r, t) (r)->funcs->test_ib((r), (t))
 #define amdgpu_ring_get_rptr(r) (r)->funcs->get_rptr((r))
 #define amdgpu_ring_get_wptr(r) (r)->funcs->get_wptr((r))
 #define amdgpu_ring_set_wptr(r) (r)->funcs->set_wptr((r))
@@ -2264,6 +2288,8 @@ amdgpu_get_sdma_instance(struct amdgpu_ring *ring)
 #define amdgpu_dpm_print_power_state(adev, ps) (adev)->pm.funcs->print_power_state((adev), (ps))
 #define amdgpu_dpm_vblank_too_short(adev) (adev)->pm.funcs->vblank_too_short((adev))
 #define amdgpu_dpm_enable_bapm(adev, e) (adev)->pm.funcs->enable_bapm((adev), (e))
+#define amdgpu_gfx_get_gpu_clock_counter(adev) (adev)->gfx.funcs->get_gpu_clock_counter((adev))
+#define amdgpu_gfx_select_se_sh(adev, se, sh, instance) (adev)->gfx.funcs->select_se_sh((adev), (se), (sh), (instance))
 
 #define amdgpu_dpm_get_temperature(adev) \
         ((adev)->pp_enabled ?                                           \
@@ -2342,6 +2368,18 @@ amdgpu_get_sdma_instance(struct amdgpu_ring *ring)
 #define amdgpu_dpm_force_clock_level(adev, type, level) \
                 (adev)->powerplay.pp_funcs->force_clock_level((adev)->powerplay.pp_handle, type, level)
 
+#define amdgpu_dpm_get_sclk_od(adev) \
+        (adev)->powerplay.pp_funcs->get_sclk_od((adev)->powerplay.pp_handle)
+
+#define amdgpu_dpm_set_sclk_od(adev, value) \
+        (adev)->powerplay.pp_funcs->set_sclk_od((adev)->powerplay.pp_handle, value)
+
+#define amdgpu_dpm_get_mclk_od(adev) \
+        ((adev)->powerplay.pp_funcs->get_mclk_od((adev)->powerplay.pp_handle))
+
+#define amdgpu_dpm_set_mclk_od(adev, value) \
+        ((adev)->powerplay.pp_funcs->set_mclk_od((adev)->powerplay.pp_handle, value))
+
 #define amdgpu_dpm_dispatch_task(adev, event_id, input, output)         \
         (adev)->powerplay.pp_funcs->dispatch_tasks((adev)->powerplay.pp_handle, (event_id), (input), (output))
 
@@ -2383,9 +2421,13 @@ bool amdgpu_device_is_px(struct drm_device *dev);
 #if defined(CONFIG_VGA_SWITCHEROO)
 void amdgpu_register_atpx_handler(void);
 void amdgpu_unregister_atpx_handler(void);
+bool amdgpu_has_atpx_dgpu_power_cntl(void);
+bool amdgpu_is_atpx_hybrid(void);
 #else
 static inline void amdgpu_register_atpx_handler(void) {}
 static inline void amdgpu_unregister_atpx_handler(void) {}
+static inline bool amdgpu_has_atpx_dgpu_power_cntl(void) { return false; }
+static inline bool amdgpu_is_atpx_hybrid(void) { return false; }
 #endif
 
 /*
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c
index 32809f749903..d080d0807a5b 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd.c
@@ -240,8 +240,8 @@ uint64_t get_gpu_clock_counter(struct kgd_dev *kgd)
 {
         struct amdgpu_device *rdev = (struct amdgpu_device *)kgd;
 
-        if (rdev->asic_funcs->get_gpu_clock_counter)
-                return rdev->asic_funcs->get_gpu_clock_counter(rdev);
+        if (rdev->gfx.funcs->get_gpu_clock_counter)
+                return rdev->gfx.funcs->get_gpu_clock_counter(rdev);
         return 0;
 }
 
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_atombios.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_atombios.c
index 9df1bcb35bf0..983175363b06 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_atombios.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_atombios.c
@@ -551,28 +551,19 @@ int amdgpu_atombios_get_clock_info(struct amdgpu_device *adev)
                     le16_to_cpu(firmware_info->info.usReferenceClock);
                 ppll->reference_div = 0;
 
-                if (crev < 2)
-                        ppll->pll_out_min =
-                                le16_to_cpu(firmware_info->info.usMinPixelClockPLL_Output);
-                else
-                        ppll->pll_out_min =
-                                le32_to_cpu(firmware_info->info_12.ulMinPixelClockPLL_Output);
+                ppll->pll_out_min =
+                        le32_to_cpu(firmware_info->info_12.ulMinPixelClockPLL_Output);
                 ppll->pll_out_max =
                     le32_to_cpu(firmware_info->info.ulMaxPixelClockPLL_Output);
 
-                if (crev >= 4) {
-                        ppll->lcd_pll_out_min =
-                                le16_to_cpu(firmware_info->info_14.usLcdMinPixelClockPLL_Output) * 100;
-                        if (ppll->lcd_pll_out_min == 0)
-                                ppll->lcd_pll_out_min = ppll->pll_out_min;
-                        ppll->lcd_pll_out_max =
-                                le16_to_cpu(firmware_info->info_14.usLcdMaxPixelClockPLL_Output) * 100;
-                        if (ppll->lcd_pll_out_max == 0)
-                                ppll->lcd_pll_out_max = ppll->pll_out_max;
-                } else {
+                ppll->lcd_pll_out_min =
+                        le16_to_cpu(firmware_info->info_14.usLcdMinPixelClockPLL_Output) * 100;
+                if (ppll->lcd_pll_out_min == 0)
                         ppll->lcd_pll_out_min = ppll->pll_out_min;
+                ppll->lcd_pll_out_max =
+                        le16_to_cpu(firmware_info->info_14.usLcdMaxPixelClockPLL_Output) * 100;
+                if (ppll->lcd_pll_out_max == 0)
                         ppll->lcd_pll_out_max = ppll->pll_out_max;
-                }
 
                 if (ppll->pll_out_min == 0)
                         ppll->pll_out_min = 64800;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_atpx_handler.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_atpx_handler.c
index 35a1248aaa77..49de92600074 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_atpx_handler.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_atpx_handler.c
@@ -10,6 +10,7 @@
 #include <linux/slab.h>
 #include <linux/acpi.h>
 #include <linux/pci.h>
+#include <linux/delay.h>
 
 #include "amd_acpi.h"
 
@@ -27,6 +28,7 @@ struct amdgpu_atpx_functions {
 struct amdgpu_atpx {
         acpi_handle handle;
         struct amdgpu_atpx_functions functions;
+        bool is_hybrid;
 };
 
 static struct amdgpu_atpx_priv {
@@ -63,6 +65,14 @@ bool amdgpu_has_atpx(void) {
         return amdgpu_atpx_priv.atpx_detected;
 }
 
+bool amdgpu_has_atpx_dgpu_power_cntl(void) {
+        return amdgpu_atpx_priv.atpx.functions.power_cntl;
+}
+
+bool amdgpu_is_atpx_hybrid(void) {
+        return amdgpu_atpx_priv.atpx.is_hybrid;
+}
+
 /**
  * amdgpu_atpx_call - call an ATPX method
  *
@@ -142,18 +152,12 @@ static void amdgpu_atpx_parse_functions(struct amdgpu_atpx_functions *f, u32 mas
  */
 static int amdgpu_atpx_validate(struct amdgpu_atpx *atpx)
 {
-        /* make sure required functions are enabled */
-        /* dGPU power control is required */
-        if (atpx->functions.power_cntl == false) {
-                printk("ATPX dGPU power cntl not present, forcing\n");
-                atpx->functions.power_cntl = true;
-        }
+        u32 valid_bits = 0;
 
         if (atpx->functions.px_params) {
                 union acpi_object *info;
                 struct atpx_px_params output;
                 size_t size;
-                u32 valid_bits;
 
                 info = amdgpu_atpx_call(atpx->handle, ATPX_FUNCTION_GET_PX_PARAMETERS, NULL);
                 if (!info)
@@ -172,19 +176,43 @@ static int amdgpu_atpx_validate(struct amdgpu_atpx *atpx)
                 memcpy(&output, info->buffer.pointer, size);
 
                 valid_bits = output.flags & output.valid_flags;
-                /* if separate mux flag is set, mux controls are required */
-                if (valid_bits & ATPX_SEPARATE_MUX_FOR_I2C) {
-                        atpx->functions.i2c_mux_cntl = true;
-                        atpx->functions.disp_mux_cntl = true;
-                }
-                /* if any outputs are muxed, mux controls are required */
-                if (valid_bits & (ATPX_CRT1_RGB_SIGNAL_MUXED |
-                                  ATPX_TV_SIGNAL_MUXED |
-                                  ATPX_DFP_SIGNAL_MUXED))
-                        atpx->functions.disp_mux_cntl = true;
 
                 kfree(info);
         }
+
+        /* if separate mux flag is set, mux controls are required */
+        if (valid_bits & ATPX_SEPARATE_MUX_FOR_I2C) {
+                atpx->functions.i2c_mux_cntl = true;
+                atpx->functions.disp_mux_cntl = true;
+        }
+        /* if any outputs are muxed, mux controls are required */
+        if (valid_bits & (ATPX_CRT1_RGB_SIGNAL_MUXED |
+                          ATPX_TV_SIGNAL_MUXED |
+                          ATPX_DFP_SIGNAL_MUXED))
+                atpx->functions.disp_mux_cntl = true;
+
+
+        /* some bioses set these bits rather than flagging power_cntl as supported */
+        if (valid_bits & (ATPX_DYNAMIC_PX_SUPPORTED |
+                          ATPX_DYNAMIC_DGPU_POWER_OFF_SUPPORTED))
+                atpx->functions.power_cntl = true;
+
+        atpx->is_hybrid = false;
+        if (valid_bits & ATPX_MS_HYBRID_GFX_SUPPORTED) {
+                printk("ATPX Hybrid Graphics\n");
+#if 1
+                /* This is a temporary hack until the D3 cold support
+                 * makes it upstream.  The ATPX power_control method seems
+                 * to still work on even if the system should be using
+                 * the new standardized hybrid D3 cold ACPI interface.
+                 */
+                atpx->functions.power_cntl = true;
+#else
+                atpx->functions.power_cntl = false;
+#endif
+                atpx->is_hybrid = true;
+        }
+
         return 0;
 }
 
@@ -259,6 +287,10 @@ static int amdgpu_atpx_set_discrete_state(struct amdgpu_atpx *atpx, u8 state)
                 if (!info)
                         return -EIO;
                 kfree(info);
+
+                /* 200ms delay is required after off */
+                if (state == 0)
+                        msleep(200);
         }
         return 0;
 }
@@ -507,7 +539,6 @@ static int amdgpu_atpx_get_client_id(struct pci_dev *pdev)
 static const struct vga_switcheroo_handler amdgpu_atpx_handler = {
         .switchto = amdgpu_atpx_switchto,
         .power_state = amdgpu_atpx_power_state,
-        .init = amdgpu_atpx_init,
         .get_client_id = amdgpu_atpx_get_client_id,
 };
 
@@ -542,6 +573,7 @@ static bool amdgpu_atpx_detect(void)
                 printk(KERN_INFO "vga_switcheroo: detected switching method %s handle\n",
                        acpi_method_name);
                 amdgpu_atpx_priv.atpx_detected = true;
+                amdgpu_atpx_init();
                 return true;
         }
         return false;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_bios.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_bios.c
index 99ca75baa47d..2b6afe123f3d 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_bios.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_bios.c
@@ -352,22 +352,22 @@ bool amdgpu_get_bios(struct amdgpu_device *adev)
         uint16_t tmp, bios_header_start;
 
         r = amdgpu_atrm_get_bios(adev);
-        if (r == false)
+        if (!r)
                 r = amdgpu_acpi_vfct_bios(adev);
-        if (r == false)
+        if (!r)
                 r = igp_read_bios_from_vram(adev);
-        if (r == false)
+        if (!r)
                 r = amdgpu_read_bios(adev);
-        if (r == false) {
+        if (!r) {
                 r = amdgpu_read_bios_from_rom(adev);
         }
-        if (r == false) {
+        if (!r) {
                 r = amdgpu_read_disabled_bios(adev);
         }
-        if (r == false) {
+        if (!r) {
                 r = amdgpu_read_platform_bios(adev);
         }
-        if (r == false || adev->bios == NULL) {
+        if (!r || adev->bios == NULL) {
                 DRM_ERROR("Unable to locate a BIOS ROM\n");
                 adev->bios = NULL;
                 return false;
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_bo_list.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_bo_list.c
index 823bf5e0b0c8..651115dcce12 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_bo_list.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_bo_list.c
@@ -94,6 +94,7 @@ static int amdgpu_bo_list_set(struct amdgpu_device *adev,
         unsigned last_entry = 0, first_userptr = num_entries;
         unsigned i;
         int r;
+        unsigned long total_size = 0;
 
         array = drm_malloc_ab(num_entries, sizeof(struct amdgpu_bo_list_entry));
         if (!array)
@@ -140,6 +141,7 @@ static int amdgpu_bo_list_set(struct amdgpu_device *adev,
                 if (entry->robj->prefered_domains == AMDGPU_GEM_DOMAIN_OA)
                         oa_obj = entry->robj;
 
+                total_size += amdgpu_bo_size(entry->robj);
                 trace_amdgpu_bo_list_set(list, entry->robj);
         }
 
@@ -155,6 +157,7 @@ static int amdgpu_bo_list_set(struct amdgpu_device *adev,
         list->array = array;
         list->num_entries = num_entries;
 
+        trace_amdgpu_cs_bo_status(list->num_entries, total_size);
         return 0;
 
 error_free:
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_cgs.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_cgs.c
index cf6f49fc1c75..bc0440f7a31d 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_cgs.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_cgs.c
@@ -312,6 +312,8 @@ static uint32_t amdgpu_cgs_read_ind_register(struct cgs_device *cgs_device,
                 return RREG32_UVD_CTX(index);
         case CGS_IND_REG__DIDT:
                 return RREG32_DIDT(index);
+        case CGS_IND_REG_GC_CAC:
+                return RREG32_GC_CAC(index);
         case CGS_IND_REG__AUDIO_ENDPT:
                 DRM_ERROR("audio endpt register access not implemented.\n");
                 return 0;
@@ -336,6 +338,8 @@ static void amdgpu_cgs_write_ind_register(struct cgs_device *cgs_device,
                 return WREG32_UVD_CTX(index, value);
         case CGS_IND_REG__DIDT:
                 return WREG32_DIDT(index, value);
+        case CGS_IND_REG_GC_CAC:
+                return WREG32_GC_CAC(index, value);
         case CGS_IND_REG__AUDIO_ENDPT:
                 DRM_ERROR("audio endpt register access not implemented.\n");
                 return;
@@ -748,6 +752,9 @@ static int amdgpu_cgs_get_firmware_info(struct cgs_device *cgs_device,
 
                 if (!adev->pm.fw) {
                         switch (adev->asic_type) {
+                        case CHIP_TOPAZ:
+                                strcpy(fw_name, "amdgpu/topaz_smc.bin");
+                                break;
                         case CHIP_TONGA:
                                 strcpy(fw_name, "amdgpu/tonga_smc.bin");
                                 break;
@@ -787,6 +794,7 @@ static int amdgpu_cgs_get_firmware_info(struct cgs_device *cgs_device,
                 }
 
                 hdr = (const struct smc_firmware_header_v1_0 *) adev->pm.fw->data;
+                amdgpu_ucode_print_smc_hdr(&hdr->header);
                 adev->pm.fw_version = le32_to_cpu(hdr->header.ucode_version);
                 ucode_size = le32_to_cpu(hdr->header.ucode_size_bytes);
                 ucode_start_address = le32_to_cpu(hdr->ucode_start_addr);
@@ -795,13 +803,14 @@ static int amdgpu_cgs_get_firmware_info(struct cgs_device *cgs_device,
 
                 info->version = adev->pm.fw_version;
                 info->image_size = ucode_size;
+                info->ucode_start_address = ucode_start_address;
                 info->kptr = (void *)src;
         }
         return 0;
 }
 
 static int amdgpu_cgs_query_system_info(struct cgs_device *cgs_device,
-                                struct cgs_system_info *sys_info)
+                                        struct cgs_system_info *sys_info)
 {
         CGS_FUNC_ADEV;
 
@@ -821,6 +830,12 @@ static int amdgpu_cgs_query_system_info(struct cgs_device *cgs_device,
         case CGS_SYSTEM_INFO_PCIE_MLW:
                 sys_info->value = adev->pm.pcie_mlw_mask;
                 break;
+        case CGS_SYSTEM_INFO_PCIE_DEV:
+                sys_info->value = adev->pdev->device;
+                break;
+        case CGS_SYSTEM_INFO_PCIE_REV:
+                sys_info->value = adev->pdev->revision;
+                break;
         case CGS_SYSTEM_INFO_CG_FLAGS:
                 sys_info->value = adev->cg_flags;
                 break;
@@ -830,6 +845,9 @@ static int amdgpu_cgs_query_system_info(struct cgs_device *cgs_device,
         case CGS_SYSTEM_INFO_GFX_CU_INFO:
                 sys_info->value = adev->gfx.cu_info.number;
                 break;
+        case CGS_SYSTEM_INFO_GFX_SE_INFO:
+                sys_info->value = adev->gfx.config.max_shader_engines;
+                break;
         default:
                 return -ENODEV;
         }
@@ -903,14 +921,12 @@ static int amdgpu_cgs_acpi_eval_object(struct cgs_device *cgs_device,
         acpi_handle handle;
         struct acpi_object_list input;
         struct acpi_buffer output = { ACPI_ALLOCATE_BUFFER, NULL };
-        union acpi_object *params = NULL;
-        union acpi_object *obj = NULL;
+        union acpi_object *params, *obj;
         uint8_t name[5] = {'\0'};
-        struct cgs_acpi_method_argument *argument = NULL;
+        struct cgs_acpi_method_argument *argument;
         uint32_t i, count;
         acpi_status status;
-        int result = 0;
-        uint32_t func_no = 0xFFFFFFFF;
+        int result;
 
         handle = ACPI_HANDLE(&adev->pdev->dev);
         if (!handle)
@@ -927,7 +943,6 @@ static int amdgpu_cgs_acpi_eval_object(struct cgs_device *cgs_device,
                 if (info->pinput_argument == NULL)
                         return -EINVAL;
                 argument = info->pinput_argument;
-                func_no = argument->value;
                 for (i = 0; i < info->input_count; i++) {
                         if (((argument->type == ACPI_TYPE_STRING) ||
                              (argument->type == ACPI_TYPE_BUFFER)) &&
@@ -972,11 +987,11 @@ static int amdgpu_cgs_acpi_eval_object(struct cgs_device *cgs_device,
                                 params->integer.value = argument->value;
                                 break;
                         case ACPI_TYPE_STRING:
-                                params->string.length = argument->method_length;
+                                params->string.length = argument->data_length;
                                 params->string.pointer = argument->pointer;
                                 break;
                         case ACPI_TYPE_BUFFER:
-                                params->buffer.length = argument->method_length;
+                                params->buffer.length = argument->data_length;
                                 params->buffer.pointer = argument->pointer;
                                 break;
                         default:
@@ -996,7 +1011,7 @@ static int amdgpu_cgs_acpi_eval_object(struct cgs_device *cgs_device,
 
         if (ACPI_FAILURE(status)) {
                 result = -EIO;
-                goto error;
+                goto free_input;
         }
 
         /* return the output info */
@@ -1006,7 +1021,7 @@ static int amdgpu_cgs_acpi_eval_object(struct cgs_device *cgs_device,
                 if ((obj->type != ACPI_TYPE_PACKAGE) ||
                         (obj->package.count != count)) {
                         result = -EIO;
-                        goto error;
+                        goto free_obj;
                 }
                 params = obj->package.elements;
         } else
@@ -1014,13 +1029,13 @@ static int amdgpu_cgs_acpi_eval_object(struct cgs_device *cgs_device,
 
         if (params == NULL) {
                 result = -EIO;
-                goto error;
+                goto free_obj;
         }
 
         for (i = 0; i < count; i++) {
                 if (argument->type != params->type) {
                         result = -EIO;
-                        goto error;
+                        goto free_obj;
                 }
                 switch (params->type) {
                 case ACPI_TYPE_INTEGER:
@@ -1030,7 +1045,7 @@ static int amdgpu_cgs_acpi_eval_object(struct cgs_device *cgs_device,
                         if ((params->string.length != argument->data_length) ||
                                 (params->string.pointer == NULL)) {
                                 result = -EIO;
-                                goto error;
+                                goto free_obj;
                         }
                         strncpy(argument->pointer,
                                 params->string.pointer,
@@ -1039,7 +1054,7 @@ static int amdgpu_cgs_acpi_eval_object(struct cgs_device *cgs_device,
                 case ACPI_TYPE_BUFFER:
                         if (params->buffer.pointer == NULL) {
                                 result = -EIO;
-                                goto error;
+                                goto free_obj;
                         }
                         memcpy(argument->pointer,
                                 params->buffer.pointer,
@@ -1052,9 +1067,10 @@ static int amdgpu_cgs_acpi_eval_object(struct cgs_device *cgs_device,
                 params++;
         }
 
-error:
-        if (obj != NULL)
-                kfree(obj);
+        result = 0;
+free_obj:
+        kfree(obj);
+free_input:
         kfree((void *)input.pointer);
         return result;
 }
@@ -1066,7 +1082,7 @@ static int amdgpu_cgs_acpi_eval_object(struct cgs_device *cgs_device,
 }
 #endif
 
-int amdgpu_cgs_call_acpi_method(struct cgs_device *cgs_device,
+static int amdgpu_cgs_call_acpi_method(struct cgs_device *cgs_device,
                                         uint32_t acpi_method,
                                         uint32_t acpi_function,
                                         void *pinput, void *poutput,
@@ -1079,17 +1095,14 @@ int amdgpu_cgs_call_acpi_method(struct cgs_device *cgs_device,
         struct cgs_acpi_method_info info = {0};
 
         acpi_input[0].type = CGS_ACPI_TYPE_INTEGER;
-        acpi_input[0].method_length = sizeof(uint32_t);
         acpi_input[0].data_length = sizeof(uint32_t);
         acpi_input[0].value = acpi_function;
 
         acpi_input[1].type = CGS_ACPI_TYPE_BUFFER;
-        acpi_input[1].method_length = CGS_ACPI_MAX_BUFFER_SIZE;
         acpi_input[1].data_length = input_size;
         acpi_input[1].pointer = pinput;
 
         acpi_output.type = CGS_ACPI_TYPE_BUFFER;
-        acpi_output.method_length = CGS_ACPI_MAX_BUFFER_SIZE;
         acpi_output.data_length = output_size;
         acpi_output.pointer = poutput;
 
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_connectors.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_connectors.c
index cb07da41152b..ff0b55a65ca3 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_connectors.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_connectors.c
@@ -1690,7 +1690,6 @@ amdgpu_connector_add(struct amdgpu_device *adev,
                                                    DRM_MODE_SCALE_NONE);
                         /* no HPD on analog connectors */
                         amdgpu_connector->hpd.hpd = AMDGPU_HPD_NONE;
-                        connector->polled = DRM_CONNECTOR_POLL_CONNECT;
                         connector->interlace_allowed = true;
                         connector->doublescan_allowed = true;
                         break;
@@ -1893,8 +1892,10 @@ amdgpu_connector_add(struct amdgpu_device *adev,
         }
 
         if (amdgpu_connector->hpd.hpd == AMDGPU_HPD_NONE) {
-                if (i2c_bus->valid)
-                        connector->polled = DRM_CONNECTOR_POLL_CONNECT;
+                if (i2c_bus->valid) {
+                        connector->polled = DRM_CONNECTOR_POLL_CONNECT |
+                                            DRM_CONNECTOR_POLL_DISCONNECT;
+                }
         } else
                 connector->polled = DRM_CONNECTOR_POLL_HPD;
 
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c
index 9bc8f1d99733..0307ff5887c5 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_cs.c
@@ -216,11 +216,8 @@ int amdgpu_cs_parser_init(struct amdgpu_cs_parser *p, void *data)
         if (ret)
                 goto free_all_kdata;
 
-        if (p->uf_entry.robj) {
-                p->job->uf_bo = amdgpu_bo_ref(p->uf_entry.robj);
-                p->job->uf_offset = uf_offset;
-        }
-
+        if (p->uf_entry.robj)
+                p->job->uf_addr = uf_offset;
         kfree(chunk_array);
         return 0;
 
@@ -459,7 +456,7 @@ static int amdgpu_cs_parser_bos(struct amdgpu_cs_parser *p,
                 list_splice(&need_pages, &p->validated);
         }
 
-        amdgpu_vm_get_pt_bos(&fpriv->vm, &duplicates);
+        amdgpu_vm_get_pt_bos(p->adev, &fpriv->vm, &duplicates);
 
         p->bytes_moved_threshold = amdgpu_cs_get_threshold_for_moves(p->adev);
         p->bytes_moved = 0;
@@ -472,6 +469,9 @@ static int amdgpu_cs_parser_bos(struct amdgpu_cs_parser *p,
         if (r)
                 goto error_validate;
 
+        fpriv->vm.last_eviction_counter =
+                atomic64_read(&p->adev->num_evictions);
+
         if (p->bo_list) {
                 struct amdgpu_bo *gds = p->bo_list->gds_obj;
                 struct amdgpu_bo *gws = p->bo_list->gws_obj;
@@ -499,6 +499,9 @@ static int amdgpu_cs_parser_bos(struct amdgpu_cs_parser *p,
                 }
         }
 
+        if (p->uf_entry.robj)
+                p->job->uf_addr += amdgpu_bo_gpu_offset(p->uf_entry.robj);
+
 error_validate:
         if (r) {
                 amdgpu_vm_move_pt_bos_in_lru(p->adev, &fpriv->vm);
@@ -653,18 +656,21 @@ static int amdgpu_cs_ib_vm_chunk(struct amdgpu_device *adev,
 
         /* Only for UVD/VCE VM emulation */
         if (ring->funcs->parse_cs) {
+                p->job->vm = NULL;
                 for (i = 0; i < p->job->num_ibs; i++) {
                         r = amdgpu_ring_parse_cs(ring, p, i);
                         if (r)
                                 return r;
                 }
-        }
+        } else {
+                p->job->vm_pd_addr = amdgpu_bo_gpu_offset(vm->page_directory);
 
-        r = amdgpu_bo_vm_update_pte(p, vm);
-        if (!r)
-                amdgpu_cs_sync_rings(p);
+                r = amdgpu_bo_vm_update_pte(p, vm);
+                if (r)
+                        return r;
+        }
 
-        return r;
+        return amdgpu_cs_sync_rings(p);
 }
 
 static int amdgpu_cs_handle_lockup(struct amdgpu_device *adev, int r)
@@ -761,7 +767,7 @@ static int amdgpu_cs_ib_fill(struct amdgpu_device *adev,
         }
 
         /* UVD & VCE fw doesn't support user fences */
-        if (parser->job->uf_bo && (
+        if (parser->job->uf_addr && (
             parser->job->ring->type == AMDGPU_RING_TYPE_UVD ||
             parser->job->ring->type == AMDGPU_RING_TYPE_VCE))
                 return -EINVAL;
@@ -830,17 +836,13 @@ static int amdgpu_cs_submit(struct amdgpu_cs_parser *p,
 {
         struct amdgpu_ring *ring = p->job->ring;
         struct amd_sched_entity *entity = &p->ctx->rings[ring->idx].entity;
-        struct fence *fence;
         struct amdgpu_job *job;
         int r;
 
         job = p->job;
         p->job = NULL;
 
-        r = amd_sched_job_init(&job->base, &ring->sched,
-                               entity, amdgpu_job_timeout_func,
-                               amdgpu_job_free_func,
-                               p->filp, &fence);
+        r = amd_sched_job_init(&job->base, &ring->sched, entity, p->filp);
         if (r) {
                 amdgpu_job_free(job);
                 return r;
@@ -848,9 +850,10 @@ static int amdgpu_cs_submit(struct amdgpu_cs_parser *p,
 
         job->owner = p->filp;
         job->ctx = entity->fence_context;
-        p->fence = fence_get(fence);
-        cs->out.handle = amdgpu_ctx_add_fence(p->ctx, ring, fence);
+        p->fence = fence_get(&job->base.s_fence->finished);
+        cs->out.handle = amdgpu_ctx_add_fence(p->ctx, ring, p->fence);
         job->uf_sequence = cs->out.handle;
+        amdgpu_job_free_resources(job);
 
         trace_amdgpu_cs_ioctl(job);
         amd_sched_entity_push_job(&job->base);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
index 6e920086af46..df7ab2458e50 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_device.c
@@ -25,6 +25,7 @@
  *          Alex Deucher
  *          Jerome Glisse
  */
+#include <linux/kthread.h>
 #include <linux/console.h>
 #include <linux/slab.h>
 #include <linux/debugfs.h>
@@ -35,6 +36,7 @@
 #include <linux/vga_switcheroo.h>
 #include <linux/efi.h>
 #include "amdgpu.h"
+#include "amdgpu_trace.h"
 #include "amdgpu_i2c.h"
 #include "atom.h"
 #include "amdgpu_atombios.h"
@@ -79,24 +81,27 @@ bool amdgpu_device_is_px(struct drm_device *dev)
 uint32_t amdgpu_mm_rreg(struct amdgpu_device *adev, uint32_t reg,
                         bool always_indirect)
 {
+        uint32_t ret;
+
         if ((reg * 4) < adev->rmmio_size && !always_indirect)
-                return readl(((void __iomem *)adev->rmmio) + (reg * 4));
+                ret = readl(((void __iomem *)adev->rmmio) + (reg * 4));
         else {
                 unsigned long flags;
-                uint32_t ret;
 
                 spin_lock_irqsave(&adev->mmio_idx_lock, flags);
                 writel((reg * 4), ((void __iomem *)adev->rmmio) + (mmMM_INDEX * 4));
                 ret = readl(((void __iomem *)adev->rmmio) + (mmMM_DATA * 4));
                 spin_unlock_irqrestore(&adev->mmio_idx_lock, flags);
-
-                return ret;
         }
+        trace_amdgpu_mm_rreg(adev->pdev->device, reg, ret);
+        return ret;
 }
 
 void amdgpu_mm_wreg(struct amdgpu_device *adev, uint32_t reg, uint32_t v,
                     bool always_indirect)
 {
+        trace_amdgpu_mm_wreg(adev->pdev->device, reg, v);
+        
         if ((reg * 4) < adev->rmmio_size && !always_indirect)
                 writel(v, ((void __iomem *)adev->rmmio) + (reg * 4));
         else {
@@ -1070,11 +1075,14 @@ int amdgpu_set_clockgating_state(struct amdgpu_device *adev,
         int i, r = 0;
 
         for (i = 0; i < adev->num_ip_blocks; i++) {
+                if (!adev->ip_block_status[i].valid)
+                        continue;
                 if (adev->ip_blocks[i].type == block_type) {
                         r = adev->ip_blocks[i].funcs->set_clockgating_state((void *)adev,
                                                                             state);
                         if (r)
                                 return r;
+                        break;
                 }
         }
         return r;
@@ -1087,16 +1095,53 @@ int amdgpu_set_powergating_state(struct amdgpu_device *adev,
         int i, r = 0;
 
         for (i = 0; i < adev->num_ip_blocks; i++) {
+                if (!adev->ip_block_status[i].valid)
+                        continue;
                 if (adev->ip_blocks[i].type == block_type) {
                         r = adev->ip_blocks[i].funcs->set_powergating_state((void *)adev,
                                                                             state);
                         if (r)
                                 return r;
+                        break;
                 }
         }
         return r;
 }
 
+int amdgpu_wait_for_idle(struct amdgpu_device *adev,
+                         enum amd_ip_block_type block_type)
+{
+        int i, r;
+
+        for (i = 0; i < adev->num_ip_blocks; i++) {
+                if (!adev->ip_block_status[i].valid)
+                        continue;
+                if (adev->ip_blocks[i].type == block_type) {
+                        r = adev->ip_blocks[i].funcs->wait_for_idle((void *)adev);
+                        if (r)
+                                return r;
+                        break;
+                }
+        }
+        return 0;
+
+}
+
+bool amdgpu_is_idle(struct amdgpu_device *adev,
+                    enum amd_ip_block_type block_type)
+{
+        int i;
+
+        for (i = 0; i < adev->num_ip_blocks; i++) {
+                if (!adev->ip_block_status[i].valid)
+                        continue;
+                if (adev->ip_blocks[i].type == block_type)
+                        return adev->ip_blocks[i].funcs->is_idle((void *)adev);
+        }
+        return true;
+
+}
+
 const struct amdgpu_ip_block_version * amdgpu_get_ip_block(
                                         struct amdgpu_device *adev,
                                         enum amd_ip_block_type type)
@@ -1209,6 +1254,9 @@ static int amdgpu_early_init(struct amdgpu_device *adev)
                 }
         }
 
+        adev->cg_flags &= amdgpu_cg_mask;
+        adev->pg_flags &= amdgpu_pg_mask;
+
         return 0;
 }
 
@@ -1440,9 +1488,12 @@ int amdgpu_device_init(struct amdgpu_device *adev,
         adev->uvd_ctx_wreg = &amdgpu_invalid_wreg;
         adev->didt_rreg = &amdgpu_invalid_rreg;
         adev->didt_wreg = &amdgpu_invalid_wreg;
+        adev->gc_cac_rreg = &amdgpu_invalid_rreg;
+        adev->gc_cac_wreg = &amdgpu_invalid_wreg;
         adev->audio_endpt_rreg = &amdgpu_block_invalid_rreg;
         adev->audio_endpt_wreg = &amdgpu_block_invalid_wreg;
 
+
         DRM_INFO("initializing kernel modesetting (%s 0x%04X:0x%04X 0x%04X:0x%04X 0x%02X).\n",
                  amdgpu_asic_name[adev->asic_type], pdev->vendor, pdev->device,
                  pdev->subsystem_vendor, pdev->subsystem_device, pdev->revision);
@@ -1467,6 +1518,7 @@ int amdgpu_device_init(struct amdgpu_device *adev,
         spin_lock_init(&adev->pcie_idx_lock);
         spin_lock_init(&adev->uvd_ctx_idx_lock);
         spin_lock_init(&adev->didt_idx_lock);
+        spin_lock_init(&adev->gc_cac_idx_lock);
         spin_lock_init(&adev->audio_endpt_idx_lock);
 
         adev->rmmio_base = pci_resource_start(adev->pdev, 5);
@@ -1511,17 +1563,20 @@ int amdgpu_device_init(struct amdgpu_device *adev,
                 vga_switcheroo_init_domain_pm_ops(adev->dev, &adev->vga_pm_domain);
 
         /* Read BIOS */
-        if (!amdgpu_get_bios(adev))
-                return -EINVAL;
+        if (!amdgpu_get_bios(adev)) {
+                r = -EINVAL;
+                goto failed;
+        }
         /* Must be an ATOMBIOS */
         if (!adev->is_atom_bios) {
                 dev_err(adev->dev, "Expecting atombios for GPU\n");
-                return -EINVAL;
+                r = -EINVAL;
+                goto failed;
         }
         r = amdgpu_atombios_init(adev);
         if (r) {
                 dev_err(adev->dev, "amdgpu_atombios_init failed\n");
-                return r;
+                goto failed;
         }
 
         /* See if the asic supports SR-IOV */
@@ -1538,7 +1593,8 @@ int amdgpu_device_init(struct amdgpu_device *adev,
              !(adev->virtualization.caps & AMDGPU_VIRT_CAPS_SRIOV_EN))) {
                 if (!adev->bios) {
                         dev_err(adev->dev, "Card not posted and no BIOS - ignoring\n");
-                        return -EINVAL;
+                        r = -EINVAL;
+                        goto failed;
                 }
                 DRM_INFO("GPU not posted. posting now...\n");
                 amdgpu_atom_asic_init(adev->mode_info.atom_context);
@@ -1548,7 +1604,7 @@ int amdgpu_device_init(struct amdgpu_device *adev,
         r = amdgpu_atombios_get_clock_info(adev);
         if (r) {
                 dev_err(adev->dev, "amdgpu_atombios_get_clock_info failed\n");
-                return r;
+                goto failed;
         }
         /* init i2c buses */
         amdgpu_atombios_i2c_init(adev);
@@ -1557,7 +1613,7 @@ int amdgpu_device_init(struct amdgpu_device *adev,
         r = amdgpu_fence_driver_init(adev);
         if (r) {
                 dev_err(adev->dev, "amdgpu_fence_driver_init failed\n");
-                return r;
+                goto failed;
         }
 
         /* init the mode config */
@@ -1567,7 +1623,7 @@ int amdgpu_device_init(struct amdgpu_device *adev,
         if (r) {
                 dev_err(adev->dev, "amdgpu_init failed\n");
                 amdgpu_fini(adev);
-                return r;
+                goto failed;
         }
 
         adev->accel_working = true;
@@ -1577,7 +1633,7 @@ int amdgpu_device_init(struct amdgpu_device *adev,
         r = amdgpu_ib_pool_init(adev);
         if (r) {
                 dev_err(adev->dev, "IB initialization failed (%d).\n", r);
-                return r;
+                goto failed;
         }
 
         r = amdgpu_ib_ring_tests(adev);
@@ -1594,6 +1650,12 @@ int amdgpu_device_init(struct amdgpu_device *adev,
                 DRM_ERROR("registering register debugfs failed (%d).\n", r);
         }
 
+        r = amdgpu_debugfs_firmware_init(adev);
+        if (r) {
+                DRM_ERROR("registering firmware debugfs failed (%d).\n", r);
+                return r;
+        }
+
         if ((amdgpu_testing & 1)) {
                 if (adev->accel_working)
                         amdgpu_test_moves(adev);
@@ -1619,10 +1681,15 @@ int amdgpu_device_init(struct amdgpu_device *adev,
         r = amdgpu_late_init(adev);
         if (r) {
                 dev_err(adev->dev, "amdgpu_late_init failed\n");
-                return r;
+                goto failed;
         }
 
         return 0;
+
+failed:
+        if (runtime)
+                vga_switcheroo_fini_domain_pm_ops(adev->dev);
+        return r;
 }
 
 static void amdgpu_debugfs_remove_files(struct amdgpu_device *adev);
@@ -1645,6 +1712,7 @@ void amdgpu_device_fini(struct amdgpu_device *adev)
         amdgpu_bo_evict_vram(adev);
         amdgpu_ib_pool_fini(adev);
         amdgpu_fence_driver_fini(adev);
+        drm_crtc_force_disable_all(adev->ddev);
         amdgpu_fbdev_fini(adev);
         r = amdgpu_fini(adev);
         kfree(adev->ip_block_status);
@@ -1656,6 +1724,8 @@ void amdgpu_device_fini(struct amdgpu_device *adev)
         kfree(adev->bios);
         adev->bios = NULL;
         vga_switcheroo_unregister_client(adev->pdev);
+        if (adev->flags & AMD_IS_PX)
+                vga_switcheroo_fini_domain_pm_ops(adev->dev);
         vga_client_register(adev->pdev, NULL, NULL, NULL);
         if (adev->rio_mem)
                 pci_iounmap(adev->pdev, adev->rio_mem);
@@ -1841,7 +1911,23 @@ int amdgpu_resume_kms(struct drm_device *dev, bool resume, bool fbcon)
         }
 
         drm_kms_helper_poll_enable(dev);
+
+        /*
+         * Most of the connector probing functions try to acquire runtime pm
+         * refs to ensure that the GPU is powered on when connector polling is
+         * performed. Since we're calling this from a runtime PM callback,
+         * trying to acquire rpm refs will cause us to deadlock.
+         *
+         * Since we're guaranteed to be holding the rpm lock, it's safe to
+         * temporarily disable the rpm helpers so this doesn't deadlock us.
+         */
+#ifdef CONFIG_PM
+        dev->dev->power.disable_depth++;
+#endif
         drm_helper_hpd_irq_event(dev);
+#ifdef CONFIG_PM
+        dev->dev->power.disable_depth--;
+#endif
 
         if (fbcon) {
                 amdgpu_fbdev_set_suspend(adev, 0);
@@ -1861,11 +1947,6 @@ int amdgpu_resume_kms(struct drm_device *dev, bool resume, bool fbcon)
  */
 int amdgpu_gpu_reset(struct amdgpu_device *adev)
 {
-        unsigned ring_sizes[AMDGPU_MAX_RINGS];
-        uint32_t *ring_data[AMDGPU_MAX_RINGS];
-
-        bool saved = false;
-
         int i, r;
         int resched;
 
@@ -1874,22 +1955,30 @@ int amdgpu_gpu_reset(struct amdgpu_device *adev)
         /* block TTM */
         resched = ttm_bo_lock_delayed_workqueue(&adev->mman.bdev);
 
-        r = amdgpu_suspend(adev);
-
+        /* block scheduler */
         for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
                 struct amdgpu_ring *ring = adev->rings[i];
+
                 if (!ring)
                         continue;
-
-                ring_sizes[i] = amdgpu_ring_backup(ring, &ring_data[i]);
-                if (ring_sizes[i]) {
-                        saved = true;
-                        dev_info(adev->dev, "Saved %d dwords of commands "
-                                 "on ring %d.\n", ring_sizes[i], i);
-                }
+                kthread_park(ring->sched.thread);
+                amd_sched_hw_job_reset(&ring->sched);
         }
+        /* after all hw jobs are reset, hw fence is meaningless, so force_completion */
+        amdgpu_fence_driver_force_completion(adev);
+
+        /* save scratch */
+        amdgpu_atombios_scratch_regs_save(adev);
+        r = amdgpu_suspend(adev);
 
 retry:
+        /* Disable fb access */
+        if (adev->mode_info.num_crtc) {
+                struct amdgpu_mode_mc_save save;
+                amdgpu_display_stop_mc_access(adev, &save);
+                amdgpu_wait_for_idle(adev, AMD_IP_BLOCK_TYPE_GMC);
+        }
+
         r = amdgpu_asic_reset(adev);
         /* post card */
         amdgpu_atom_asic_init(adev->mode_info.atom_context);
@@ -1898,32 +1987,29 @@ retry:
                 dev_info(adev->dev, "GPU reset succeeded, trying to resume\n");
                 r = amdgpu_resume(adev);
         }
-
+        /* restore scratch */
+        amdgpu_atombios_scratch_regs_restore(adev);
         if (!r) {
+                r = amdgpu_ib_ring_tests(adev);
+                if (r) {
+                        dev_err(adev->dev, "ib ring test failed (%d).\n", r);
+                        r = amdgpu_suspend(adev);
+                        goto retry;
+                }
+
                 for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
                         struct amdgpu_ring *ring = adev->rings[i];
                         if (!ring)
                                 continue;
-
-                        amdgpu_ring_restore(ring, ring_sizes[i], ring_data[i]);
-                        ring_sizes[i] = 0;
-                        ring_data[i] = NULL;
-                }
-
-                r = amdgpu_ib_ring_tests(adev);
-                if (r) {
-                        dev_err(adev->dev, "ib ring test failed (%d).\n", r);
-                        if (saved) {
-                                saved = false;
-                                r = amdgpu_suspend(adev);
-                                goto retry;
-                        }
+                        amd_sched_job_recovery(&ring->sched);
+                        kthread_unpark(ring->sched.thread);
                 }
         } else {
-                amdgpu_fence_driver_force_completion(adev);
+                dev_err(adev->dev, "asic resume failed (%d).\n", r);
                 for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
-                        if (adev->rings[i])
-                                kfree(ring_data[i]);
+                        if (adev->rings[i]) {
+                                kthread_unpark(adev->rings[i]->sched.thread);
+                        }
                 }
         }
 
@@ -1934,13 +2020,11 @@ retry:
                 /* bad news, how to tell it to userspace ? */
                 dev_info(adev->dev, "GPU reset failed\n");
         }
+        amdgpu_irq_gpu_reset_resume_helper(adev);
 
         return r;
 }
 
-#define AMDGPU_DEFAULT_PCIE_GEN_MASK 0x30007  /* gen: chipset 1/2, asic 1/2/3 */
-#define AMDGPU_DEFAULT_PCIE_MLW_MASK 0x2f0000 /* 1/2/4/8/16 lanes */
-
 void amdgpu_get_pcie_info(struct amdgpu_device *adev)
 {
         u32 mask;
@@ -2094,20 +2178,43 @@ static ssize_t amdgpu_debugfs_regs_read(struct file *f, char __user *buf,
         struct amdgpu_device *adev = f->f_inode->i_private;
         ssize_t result = 0;
         int r;
+        bool use_bank;
+        unsigned instance_bank, sh_bank, se_bank;
 
         if (size & 0x3 || *pos & 0x3)
                 return -EINVAL;
 
+        if (*pos & (1ULL << 62)) {
+                se_bank = (*pos >> 24) & 0x3FF;
+                sh_bank = (*pos >> 34) & 0x3FF;
+                instance_bank = (*pos >> 44) & 0x3FF;
+                use_bank = 1;
+                *pos &= 0xFFFFFF;
+        } else {
+                use_bank = 0;
+        }
+
+        if (use_bank) {
+                if (sh_bank >= adev->gfx.config.max_sh_per_se ||
+                    se_bank >= adev->gfx.config.max_shader_engines)
+                        return -EINVAL;
+                mutex_lock(&adev->grbm_idx_mutex);
+                amdgpu_gfx_select_se_sh(adev, se_bank,
+                                        sh_bank, instance_bank);
+        }
+
         while (size) {
                 uint32_t value;
 
                 if (*pos > adev->rmmio_size)
-                        return result;
+                        goto end;
 
                 value = RREG32(*pos >> 2);
                 r = put_user(value, (uint32_t *)buf);
-                if (r)
-                        return r;
+                if (r) {
+                        result = r;
+                        goto end;
+                }
 
                 result += 4;
                 buf += 4;
@@ -2115,6 +2222,12 @@ static ssize_t amdgpu_debugfs_regs_read(struct file *f, char __user *buf,
                 size -= 4;
         }
 
+end:
+        if (use_bank) {
+                amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
+                mutex_unlock(&adev->grbm_idx_mutex);
+        }
+
         return result;
 }
 
@@ -2314,6 +2427,68 @@ static ssize_t amdgpu_debugfs_regs_smc_write(struct file *f, const char __user *
         return result;
 }
 
+static ssize_t amdgpu_debugfs_gca_config_read(struct file *f, char __user *buf,
+                                        size_t size, loff_t *pos)
+{
+        struct amdgpu_device *adev = f->f_inode->i_private;
+        ssize_t result = 0;
+        int r;
+        uint32_t *config, no_regs = 0;
+
+        if (size & 0x3 || *pos & 0x3)
+                return -EINVAL;
+
+        config = kmalloc(256 * sizeof(*config), GFP_KERNEL);
+        if (!config)
+                return -ENOMEM;
+
+        /* version, increment each time something is added */
+        config[no_regs++] = 0;
+        config[no_regs++] = adev->gfx.config.max_shader_engines;
+        config[no_regs++] = adev->gfx.config.max_tile_pipes;
+        config[no_regs++] = adev->gfx.config.max_cu_per_sh;
+        config[no_regs++] = adev->gfx.config.max_sh_per_se;
+        config[no_regs++] = adev->gfx.config.max_backends_per_se;
+        config[no_regs++] = adev->gfx.config.max_texture_channel_caches;
+        config[no_regs++] = adev->gfx.config.max_gprs;
+        config[no_regs++] = adev->gfx.config.max_gs_threads;
+        config[no_regs++] = adev->gfx.config.max_hw_contexts;
+        config[no_regs++] = adev->gfx.config.sc_prim_fifo_size_frontend;
+        config[no_regs++] = adev->gfx.config.sc_prim_fifo_size_backend;
+        config[no_regs++] = adev->gfx.config.sc_hiz_tile_fifo_size;
+        config[no_regs++] = adev->gfx.config.sc_earlyz_tile_fifo_size;
+        config[no_regs++] = adev->gfx.config.num_tile_pipes;
+        config[no_regs++] = adev->gfx.config.backend_enable_mask;
+        config[no_regs++] = adev->gfx.config.mem_max_burst_length_bytes;
+        config[no_regs++] = adev->gfx.config.mem_row_size_in_kb;
+        config[no_regs++] = adev->gfx.config.shader_engine_tile_size;
+        config[no_regs++] = adev->gfx.config.num_gpus;
+        config[no_regs++] = adev->gfx.config.multi_gpu_tile_size;
+        config[no_regs++] = adev->gfx.config.mc_arb_ramcfg;
+        config[no_regs++] = adev->gfx.config.gb_addr_config;
+        config[no_regs++] = adev->gfx.config.num_rbs;
+
+        while (size && (*pos < no_regs * 4)) {
+                uint32_t value;
+
+                value = config[*pos >> 2];
+                r = put_user(value, (uint32_t *)buf);
+                if (r) {
+                        kfree(config);
+                        return r;
+                }
+
+                result += 4;
+                buf += 4;
+                *pos += 4;
+                size -= 4;
+        }
+
+        kfree(config);
+        return result;
+}
+
+
 static const struct file_operations amdgpu_debugfs_regs_fops = {
         .owner = THIS_MODULE,
         .read = amdgpu_debugfs_regs_read,
@@ -2339,11 +2514,18 @@ static const struct file_operations amdgpu_debugfs_regs_smc_fops = {
         .llseek = default_llseek
 };
 
+static const struct file_operations amdgpu_debugfs_gca_config_fops = {
+        .owner = THIS_MODULE,
+        .read = amdgpu_debugfs_gca_config_read,
+        .llseek = default_llseek
+};
+
 static const struct file_operations *debugfs_regs[] = {
         &amdgpu_debugfs_regs_fops,
         &amdgpu_debugfs_regs_didt_fops,
         &amdgpu_debugfs_regs_pcie_fops,
         &amdgpu_debugfs_regs_smc_fops,
+        &amdgpu_debugfs_gca_config_fops,
 };
 
 static const char *debugfs_regs_names[] = {
@@ -2351,6 +2533,7 @@ static const char *debugfs_regs_names[] = {
         "amdgpu_regs_didt",
         "amdgpu_regs_pcie",
         "amdgpu_regs_smc",
+        "amdgpu_gca_config",
 };
 
 static int amdgpu_debugfs_regs_init(struct amdgpu_device *adev)
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_display.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_display.c
index b0832da2ef7e..76f96028313d 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_display.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_display.c
@@ -122,7 +122,7 @@ static void amdgpu_flip_work_func(struct work_struct *__work)
                 spin_unlock_irqrestore(&crtc->dev->event_lock, flags);
                 usleep_range(min_udelay, 2 * min_udelay);
                 spin_lock_irqsave(&crtc->dev->event_lock, flags);
-        };
+        }
 
         if (!repcnt)
                 DRM_DEBUG_DRIVER("Delay problem on crtc %d: min_udelay %d, "
@@ -220,19 +220,17 @@ int amdgpu_crtc_page_flip(struct drm_crtc *crtc,
 
         r = amdgpu_bo_pin_restricted(new_rbo, AMDGPU_GEM_DOMAIN_VRAM, 0, 0, &base);
         if (unlikely(r != 0)) {
-                amdgpu_bo_unreserve(new_rbo);
                 r = -EINVAL;
                 DRM_ERROR("failed to pin new rbo buffer before flip\n");
-                goto cleanup;
+                goto unreserve;
         }
 
         r = reservation_object_get_fences_rcu(new_rbo->tbo.resv, &work->excl,
                                               &work->shared_count,
                                               &work->shared);
         if (unlikely(r != 0)) {
-                amdgpu_bo_unreserve(new_rbo);
                 DRM_ERROR("failed to get fences for buffer\n");
-                goto cleanup;
+                goto unpin;
         }
 
         amdgpu_bo_get_tiling_flags(new_rbo, &tiling_flags);
@@ -240,7 +238,7 @@ int amdgpu_crtc_page_flip(struct drm_crtc *crtc,
 
         work->base = base;
 
-        r = drm_vblank_get(crtc->dev, amdgpu_crtc->crtc_id);
+        r = drm_crtc_vblank_get(crtc);
         if (r) {
                 DRM_ERROR("failed to get vblank before flip\n");
                 goto pflip_cleanup;
@@ -268,16 +266,18 @@ int amdgpu_crtc_page_flip(struct drm_crtc *crtc,
         return 0;
 
 vblank_cleanup:
-        drm_vblank_put(crtc->dev, amdgpu_crtc->crtc_id);
+        drm_crtc_vblank_put(crtc);
 
 pflip_cleanup:
         if (unlikely(amdgpu_bo_reserve(new_rbo, false) != 0)) {
                 DRM_ERROR("failed to reserve new rbo in error path\n");
                 goto cleanup;
         }
+unpin:
         if (unlikely(amdgpu_bo_unpin(new_rbo) != 0)) {
                 DRM_ERROR("failed to unpin new rbo in error path\n");
         }
+unreserve:
         amdgpu_bo_unreserve(new_rbo);
 
 cleanup:
@@ -516,9 +516,7 @@ static void amdgpu_user_framebuffer_destroy(struct drm_framebuffer *fb)
 {
         struct amdgpu_framebuffer *amdgpu_fb = to_amdgpu_framebuffer(fb);
 
-        if (amdgpu_fb->obj) {
-                drm_gem_object_unreference_unlocked(amdgpu_fb->obj);
-        }
+        drm_gem_object_unreference_unlocked(amdgpu_fb->obj);
         drm_framebuffer_cleanup(fb);
         kfree(amdgpu_fb);
 }
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
index f888c015f76c..9aa533cf4ad1 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_drv.c
@@ -52,9 +52,10 @@
  * - 3.1.0 - allow reading more status registers (GRBM, SRBM, SDMA, CP)
  * - 3.2.0 - GFX8: Uses EOP_TC_WB_ACTION_EN, so UMDs don't have to do the same
  *           at the end of IBs.
+ * - 3.3.0 - Add VM support for UVD on supported hardware.
  */
 #define KMS_DRIVER_MAJOR        3
-#define KMS_DRIVER_MINOR        2
+#define KMS_DRIVER_MINOR        3
 #define KMS_DRIVER_PATCHLEVEL   0
 
 int amdgpu_vram_limit = 0;
@@ -82,8 +83,12 @@ int amdgpu_exp_hw_support = 0;
 int amdgpu_sched_jobs = 32;
 int amdgpu_sched_hw_submission = 2;
 int amdgpu_powerplay = -1;
+int amdgpu_powercontainment = 1;
 unsigned amdgpu_pcie_gen_cap = 0;
 unsigned amdgpu_pcie_lane_cap = 0;
+unsigned amdgpu_cg_mask = 0xffffffff;
+unsigned amdgpu_pg_mask = 0xffffffff;
+char *amdgpu_disable_cu = NULL;
 
 MODULE_PARM_DESC(vramlimit, "Restrict VRAM for testing, in megabytes");
 module_param_named(vramlimit, amdgpu_vram_limit, int, 0600);
@@ -160,6 +165,9 @@ module_param_named(sched_hw_submission, amdgpu_sched_hw_submission, int, 0444);
 #ifdef CONFIG_DRM_AMD_POWERPLAY
 MODULE_PARM_DESC(powerplay, "Powerplay component (1 = enable, 0 = disable, -1 = auto (default))");
 module_param_named(powerplay, amdgpu_powerplay, int, 0444);
+
+MODULE_PARM_DESC(powercontainment, "Power Containment (1 = enable (default), 0 = disable)");
+module_param_named(powercontainment, amdgpu_powercontainment, int, 0444);
 #endif
 
 MODULE_PARM_DESC(pcie_gen_cap, "PCIE Gen Caps (0: autodetect (default))");
@@ -168,6 +176,15 @@ module_param_named(pcie_gen_cap, amdgpu_pcie_gen_cap, uint, 0444);
 MODULE_PARM_DESC(pcie_lane_cap, "PCIE Lane Caps (0: autodetect (default))");
 module_param_named(pcie_lane_cap, amdgpu_pcie_lane_cap, uint, 0444);
 
+MODULE_PARM_DESC(cg_mask, "Clockgating flags mask (0 = disable clock gating)");
+module_param_named(cg_mask, amdgpu_cg_mask, uint, 0444);
+
+MODULE_PARM_DESC(pg_mask, "Powergating flags mask (0 = disable power gating)");
+module_param_named(pg_mask, amdgpu_pg_mask, uint, 0444);
+
+MODULE_PARM_DESC(disable_cu, "Disable CUs (se.sh.cu,...)");
+module_param_named(disable_cu, amdgpu_disable_cu, charp, 0444);
+
 static const struct pci_device_id pciidlist[] = {
 #ifdef CONFIG_DRM_AMDGPU_CIK
         /* Kaveri */
@@ -413,7 +430,10 @@ static int amdgpu_pmops_runtime_suspend(struct device *dev)
         pci_save_state(pdev);
         pci_disable_device(pdev);
         pci_ignore_hotplug(pdev);
-        pci_set_power_state(pdev, PCI_D3cold);
+        if (amdgpu_is_atpx_hybrid())
+                pci_set_power_state(pdev, PCI_D3cold);
+        else if (!amdgpu_has_atpx_dgpu_power_cntl())
+                pci_set_power_state(pdev, PCI_D3hot);
         drm_dev->switch_power_state = DRM_SWITCH_POWER_DYNAMIC_OFF;
 
         return 0;
@@ -430,7 +450,9 @@ static int amdgpu_pmops_runtime_resume(struct device *dev)
 
         drm_dev->switch_power_state = DRM_SWITCH_POWER_CHANGING;
 
-        pci_set_power_state(pdev, PCI_D0);
+        if (amdgpu_is_atpx_hybrid() ||
+            !amdgpu_has_atpx_dgpu_power_cntl())
+                pci_set_power_state(pdev, PCI_D0);
         pci_restore_state(pdev);
         ret = pci_enable_device(pdev);
         if (ret)
@@ -515,7 +537,7 @@ static struct drm_driver kms_driver = {
         .driver_features =
             DRIVER_USE_AGP |
             DRIVER_HAVE_IRQ | DRIVER_IRQ_SHARED | DRIVER_GEM |
-            DRIVER_PRIME | DRIVER_RENDER,
+            DRIVER_PRIME | DRIVER_RENDER | DRIVER_MODESET,
         .dev_priv_size = 0,
         .load = amdgpu_driver_load_kms,
         .open = amdgpu_driver_open_kms,
@@ -590,7 +612,6 @@ static int __init amdgpu_init(void)
         DRM_INFO("amdgpu kernel modesetting enabled.\n");
         driver = &kms_driver;
         pdriver = &amdgpu_kms_pci_driver;
-        driver->driver_features |= DRIVER_MODESET;
         driver->num_ioctls = amdgpu_max_kms_ioctl;
         amdgpu_register_atpx_handler();
         /* let modprobe override vga console setting */
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c
index d1558768cfb7..0b109aebfec6 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_fence.c
@@ -204,16 +204,25 @@ void amdgpu_fence_process(struct amdgpu_ring *ring)
         if (seq != ring->fence_drv.sync_seq)
                 amdgpu_fence_schedule_fallback(ring);
 
-        while (last_seq != seq) {
+        if (unlikely(seq == last_seq))
+                return;
+
+        last_seq &= drv->num_fences_mask;
+        seq &= drv->num_fences_mask;
+
+        do {
                 struct fence *fence, **ptr;
 
-                ptr = &drv->fences[++last_seq & drv->num_fences_mask];
+                ++last_seq;
+                last_seq &= drv->num_fences_mask;
+                ptr = &drv->fences[last_seq];
 
                 /* There is always exactly one thread signaling this fence slot */
                 fence = rcu_dereference_protected(*ptr, 1);
                 RCU_INIT_POINTER(*ptr, NULL);
 
-                BUG_ON(!fence);
+                if (!fence)
+                        continue;
 
                 r = fence_signal(fence);
                 if (!r)
@@ -222,7 +231,7 @@ void amdgpu_fence_process(struct amdgpu_ring *ring)
                         BUG();
 
                 fence_put(fence);
-        }
+        } while (last_seq != seq);
 }
 
 /**
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c
index 8fab6486064f..88fbed2389c0 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gem.c
@@ -503,7 +503,7 @@ static void amdgpu_gem_va_update_vm(struct amdgpu_device *adev,
         if (r)
                 goto error_print;
 
-        amdgpu_vm_get_pt_bos(bo_va->vm, &duplicates);
+        amdgpu_vm_get_pt_bos(adev, bo_va->vm, &duplicates);
         list_for_each_entry(entry, &list, head) {
                 domain = amdgpu_mem_type_to_domain(entry->bo->mem.mem_type);
                 /* if anything is swapped out don't swap it in here,
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c
index 9f95da4f0536..a074edd95c70 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.c
@@ -70,3 +70,47 @@ void amdgpu_gfx_scratch_free(struct amdgpu_device *adev, uint32_t reg)
                 }
         }
 }
+
+/**
+ * amdgpu_gfx_parse_disable_cu - Parse the disable_cu module parameter
+ *
+ * @mask: array in which the per-shader array disable masks will be stored
+ * @max_se: number of SEs
+ * @max_sh: number of SHs
+ *
+ * The bitmask of CUs to be disabled in the shader array determined by se and
+ * sh is stored in mask[se * max_sh + sh].
+ */
+void amdgpu_gfx_parse_disable_cu(unsigned *mask, unsigned max_se, unsigned max_sh)
+{
+        unsigned se, sh, cu;
+        const char *p;
+
+        memset(mask, 0, sizeof(*mask) * max_se * max_sh);
+
+        if (!amdgpu_disable_cu || !*amdgpu_disable_cu)
+                return;
+
+        p = amdgpu_disable_cu;
+        for (;;) {
+                char *next;
+                int ret = sscanf(p, "%u.%u.%u", &se, &sh, &cu);
+                if (ret < 3) {
+                        DRM_ERROR("amdgpu: could not parse disable_cu\n");
+                        return;
+                }
+
+                if (se < max_se && sh < max_sh && cu < 16) {
+                        DRM_INFO("amdgpu: disabling CU %u.%u.%u\n", se, sh, cu);
+                        mask[se * max_sh + sh] |= 1u << cu;
+                } else {
+                        DRM_ERROR("amdgpu: disable_cu %u.%u.%u is out of range\n",
+                                  se, sh, cu);
+                }
+
+                next = strchr(p, ',');
+                if (!next)
+                        break;
+                p = next + 1;
+        }
+}
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.h
index dc06cbda7be6..51321e154c09 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_gfx.h
@@ -27,4 +27,6 @@
 int amdgpu_gfx_scratch_get(struct amdgpu_device *adev, uint32_t *reg);
 void amdgpu_gfx_scratch_free(struct amdgpu_device *adev, uint32_t reg);
 
+unsigned amdgpu_gfx_parse_disable_cu(unsigned *mask, unsigned max_se, unsigned max_sh);
+
 #endif
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c
index 34e35423b78e..a31d7ef3032c 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ib.c
@@ -33,6 +33,8 @@
 #include "amdgpu.h"
 #include "atom.h"
 
+#define AMDGPU_IB_TEST_TIMEOUT  msecs_to_jiffies(1000)
+
 /*
  * IB
  * IBs (Indirect Buffers) and areas of GPU accessible memory where
@@ -122,7 +124,6 @@ int amdgpu_ib_schedule(struct amdgpu_ring *ring, unsigned num_ibs,
         bool skip_preamble, need_ctx_switch;
         unsigned patch_offset = ~0;
         struct amdgpu_vm *vm;
-        struct fence *hwf;
         uint64_t ctx;
 
         unsigned i;
@@ -160,10 +161,7 @@ int amdgpu_ib_schedule(struct amdgpu_ring *ring, unsigned num_ibs,
                 patch_offset = amdgpu_ring_init_cond_exec(ring);
 
         if (vm) {
-                r = amdgpu_vm_flush(ring, job->vm_id, job->vm_pd_addr,
-                                    job->gds_base, job->gds_size,
-                                    job->gws_base, job->gws_size,
-                                    job->oa_base, job->oa_size);
+                r = amdgpu_vm_flush(ring, job);
                 if (r) {
                         amdgpu_ring_undo(ring);
                         return r;
@@ -193,7 +191,7 @@ int amdgpu_ib_schedule(struct amdgpu_ring *ring, unsigned num_ibs,
         if (ring->funcs->emit_hdp_invalidate)
                 amdgpu_ring_emit_hdp_invalidate(ring);
 
-        r = amdgpu_fence_emit(ring, &hwf);
+        r = amdgpu_fence_emit(ring, f);
         if (r) {
                 dev_err(adev->dev, "failed to emit fence (%d)\n", r);
                 if (job && job->vm_id)
@@ -203,17 +201,11 @@ int amdgpu_ib_schedule(struct amdgpu_ring *ring, unsigned num_ibs,
         }
 
         /* wrap the last IB with fence */
-        if (job && job->uf_bo) {
-                uint64_t addr = amdgpu_bo_gpu_offset(job->uf_bo);
-
-                addr += job->uf_offset;
-                amdgpu_ring_emit_fence(ring, addr, job->uf_sequence,
+        if (job && job->uf_addr) {
+                amdgpu_ring_emit_fence(ring, job->uf_addr, job->uf_sequence,
                                        AMDGPU_FENCE_FLAG_64BIT);
         }
 
-        if (f)
-                *f = fence_get(hwf);
-
         if (patch_offset != ~0 && ring->funcs->patch_cond_exec)
                 amdgpu_ring_patch_cond_exec(ring, patch_offset);
 
@@ -296,7 +288,7 @@ int amdgpu_ib_ring_tests(struct amdgpu_device *adev)
                 if (!ring || !ring->ready)
                         continue;
 
-                r = amdgpu_ring_test_ib(ring);
+                r = amdgpu_ring_test_ib(ring, AMDGPU_IB_TEST_TIMEOUT);
                 if (r) {
                         ring->ready = false;
 
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_irq.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_irq.c
index 835a3fa8d8df..278708f5a744 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_irq.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_irq.c
@@ -383,6 +383,18 @@ int amdgpu_irq_update(struct amdgpu_device *adev,
         return r;
 }
 
+void amdgpu_irq_gpu_reset_resume_helper(struct amdgpu_device *adev)
+{
+        int i, j;
+        for (i = 0; i < AMDGPU_MAX_IRQ_SRC_ID; i++) {
+                struct amdgpu_irq_src *src = adev->irq.sources[i];
+                if (!src)
+                        continue;
+                for (j = 0; j < src->num_types; j++)
+                        amdgpu_irq_update(adev, src, j);
+        }
+}
+
 /**
  * amdgpu_irq_get - enable interrupt
  *
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_irq.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_irq.h
index e124b59f39c1..7ef09352e534 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_irq.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_irq.h
@@ -94,6 +94,7 @@ int amdgpu_irq_put(struct amdgpu_device *adev, struct amdgpu_irq_src *src,
                    unsigned type);
 bool amdgpu_irq_enabled(struct amdgpu_device *adev, struct amdgpu_irq_src *src,
                         unsigned type);
+void amdgpu_irq_gpu_reset_resume_helper(struct amdgpu_device *adev);
 
 int amdgpu_irq_add_domain(struct amdgpu_device *adev);
 void amdgpu_irq_remove_domain(struct amdgpu_device *adev);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c
index f0dafa514fe4..6674d40eb3ab 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_job.c
@@ -28,21 +28,15 @@
 #include "amdgpu.h"
 #include "amdgpu_trace.h"
 
-static void amdgpu_job_free_handler(struct work_struct *ws)
+static void amdgpu_job_timedout(struct amd_sched_job *s_job)
 {
-        struct amdgpu_job *job = container_of(ws, struct amdgpu_job, base.work_free_job);
-        amd_sched_job_put(&job->base);
-}
+        struct amdgpu_job *job = container_of(s_job, struct amdgpu_job, base);
 
-void amdgpu_job_timeout_func(struct work_struct *work)
-{
-        struct amdgpu_job *job = container_of(work, struct amdgpu_job, base.work_tdr.work);
         DRM_ERROR("ring %s timeout, last signaled seq=%u, last emitted seq=%u\n",
-                                job->base.sched->name,
-                                (uint32_t)atomic_read(&job->ring->fence_drv.last_seq),
-                                job->ring->fence_drv.sync_seq);
-
-        amd_sched_job_put(&job->base);
+                  job->base.sched->name,
+                  atomic_read(&job->ring->fence_drv.last_seq),
+                  job->ring->fence_drv.sync_seq);
+        amdgpu_gpu_reset(job->adev);
 }
 
 int amdgpu_job_alloc(struct amdgpu_device *adev, unsigned num_ibs,
@@ -63,7 +57,6 @@ int amdgpu_job_alloc(struct amdgpu_device *adev, unsigned num_ibs,
         (*job)->vm = vm;
         (*job)->ibs = (void *)&(*job)[1];
         (*job)->num_ibs = num_ibs;
-        INIT_WORK(&(*job)->base.work_free_job, amdgpu_job_free_handler);
 
         amdgpu_sync_create(&(*job)->sync);
 
@@ -86,27 +79,33 @@ int amdgpu_job_alloc_with_ib(struct amdgpu_device *adev, unsigned size,
         return r;
 }
 
-void amdgpu_job_free(struct amdgpu_job *job)
+void amdgpu_job_free_resources(struct amdgpu_job *job)
 {
-        unsigned i;
         struct fence *f;
+        unsigned i;
+
         /* use sched fence if available */
-        f = (job->base.s_fence)? &job->base.s_fence->base : job->fence;
+        f = job->base.s_fence ? &job->base.s_fence->finished : job->fence;
 
         for (i = 0; i < job->num_ibs; ++i)
-                amdgpu_sa_bo_free(job->adev, &job->ibs[i].sa_bo, f);
-        fence_put(job->fence);
+                amdgpu_ib_free(job->adev, &job->ibs[i], f);
+}
 
-        amdgpu_bo_unref(&job->uf_bo);
-        amdgpu_sync_free(&job->sync);
+void amdgpu_job_free_cb(struct amd_sched_job *s_job)
+{
+        struct amdgpu_job *job = container_of(s_job, struct amdgpu_job, base);
 
-        if (!job->base.use_sched)
-                kfree(job);
+        fence_put(job->fence);
+        amdgpu_sync_free(&job->sync);
+        kfree(job);
 }
 
-void amdgpu_job_free_func(struct kref *refcount)
+void amdgpu_job_free(struct amdgpu_job *job)
 {
-        struct amdgpu_job *job = container_of(refcount, struct amdgpu_job, base.refcount);
+        amdgpu_job_free_resources(job);
+
+        fence_put(job->fence);
+        amdgpu_sync_free(&job->sync);
         kfree(job);
 }
 
@@ -114,22 +113,20 @@ int amdgpu_job_submit(struct amdgpu_job *job, struct amdgpu_ring *ring,
                       struct amd_sched_entity *entity, void *owner,
                       struct fence **f)
 {
-        struct fence *fence;
         int r;
         job->ring = ring;
 
         if (!f)
                 return -EINVAL;
 
-        r = amd_sched_job_init(&job->base, &ring->sched,
-                               entity, amdgpu_job_timeout_func,
-                               amdgpu_job_free_func, owner, &fence);
+        r = amd_sched_job_init(&job->base, &ring->sched, entity, owner);
         if (r)
                 return r;
 
         job->owner = owner;
         job->ctx = entity->fence_context;
-        *f = fence_get(fence);
+        *f = fence_get(&job->base.s_fence->finished);
+        amdgpu_job_free_resources(job);
         amd_sched_entity_push_job(&job->base);
 
         return 0;
@@ -147,8 +144,8 @@ static struct fence *amdgpu_job_dependency(struct amd_sched_job *sched_job)
                 int r;
 
                 r = amdgpu_vm_grab_id(vm, ring, &job->sync,
-                                      &job->base.s_fence->base,
-                                      &job->vm_id, &job->vm_pd_addr);
+                                      &job->base.s_fence->finished,
+                                      job);
                 if (r)
                         DRM_ERROR("Error getting VM ID (%d)\n", r);
 
@@ -170,29 +167,24 @@ static struct fence *amdgpu_job_run(struct amd_sched_job *sched_job)
         }
         job = to_amdgpu_job(sched_job);
 
-        r = amdgpu_sync_wait(&job->sync);
-        if (r) {
-                DRM_ERROR("failed to sync wait (%d)\n", r);
-                return NULL;
-        }
+        BUG_ON(amdgpu_sync_peek_fence(&job->sync, NULL));
 
         trace_amdgpu_sched_run_job(job);
         r = amdgpu_ib_schedule(job->ring, job->num_ibs, job->ibs,
                                job->sync.last_vm_update, job, &fence);
-        if (r) {
+        if (r)
                 DRM_ERROR("Error scheduling IBs (%d)\n", r);
-                goto err;
-        }
 
-err:
-        job->fence = fence;
-        amdgpu_job_free(job);
+        /* if gpu reset, hw fence will be replaced here */
+        fence_put(job->fence);
+        job->fence = fence_get(fence);
+        amdgpu_job_free_resources(job);
         return fence;
 }
 
 const struct amd_sched_backend_ops amdgpu_sched_ops = {
         .dependency = amdgpu_job_dependency,
         .run_job = amdgpu_job_run,
-        .begin_job = amd_sched_job_begin,
-        .finish_job = amd_sched_job_finish,
+        .timedout_job = amdgpu_job_timedout,
+        .free_job = amdgpu_job_free_cb
 };
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
index d851ea15059f..d942654a1de0 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_kms.c
@@ -60,7 +60,10 @@ int amdgpu_driver_unload_kms(struct drm_device *dev)
         if (adev->rmmio == NULL)
                 goto done_free;
 
-        pm_runtime_get_sync(dev->dev);
+        if (amdgpu_device_is_px(dev)) {
+                pm_runtime_get_sync(dev->dev);
+                pm_runtime_forbid(dev->dev);
+        }
 
         amdgpu_amdkfd_device_fini(adev);
 
@@ -135,13 +138,75 @@ int amdgpu_driver_load_kms(struct drm_device *dev, unsigned long flags)
         }
 
 out:
-        if (r)
+        if (r) {
+                /* balance pm_runtime_get_sync in amdgpu_driver_unload_kms */
+                if (adev->rmmio && amdgpu_device_is_px(dev))
+                        pm_runtime_put_noidle(dev->dev);
                 amdgpu_driver_unload_kms(dev);
-
+        }
 
         return r;
 }
 
+static int amdgpu_firmware_info(struct drm_amdgpu_info_firmware *fw_info,
+                                struct drm_amdgpu_query_fw *query_fw,
+                                struct amdgpu_device *adev)
+{
+        switch (query_fw->fw_type) {
+        case AMDGPU_INFO_FW_VCE:
+                fw_info->ver = adev->vce.fw_version;
+                fw_info->feature = adev->vce.fb_version;
+                break;
+        case AMDGPU_INFO_FW_UVD:
+                fw_info->ver = adev->uvd.fw_version;
+                fw_info->feature = 0;
+                break;
+        case AMDGPU_INFO_FW_GMC:
+                fw_info->ver = adev->mc.fw_version;
+                fw_info->feature = 0;
+                break;
+        case AMDGPU_INFO_FW_GFX_ME:
+                fw_info->ver = adev->gfx.me_fw_version;
+                fw_info->feature = adev->gfx.me_feature_version;
+                break;
+        case AMDGPU_INFO_FW_GFX_PFP:
+                fw_info->ver = adev->gfx.pfp_fw_version;
+                fw_info->feature = adev->gfx.pfp_feature_version;
+                break;
+        case AMDGPU_INFO_FW_GFX_CE:
+                fw_info->ver = adev->gfx.ce_fw_version;
+                fw_info->feature = adev->gfx.ce_feature_version;
+                break;
+        case AMDGPU_INFO_FW_GFX_RLC:
+                fw_info->ver = adev->gfx.rlc_fw_version;
+                fw_info->feature = adev->gfx.rlc_feature_version;
+                break;
+        case AMDGPU_INFO_FW_GFX_MEC:
+                if (query_fw->index == 0) {
+                        fw_info->ver = adev->gfx.mec_fw_version;
+                        fw_info->feature = adev->gfx.mec_feature_version;
+                } else if (query_fw->index == 1) {
+                        fw_info->ver = adev->gfx.mec2_fw_version;
+                        fw_info->feature = adev->gfx.mec2_feature_version;
+                } else
+                        return -EINVAL;
+                break;
+        case AMDGPU_INFO_FW_SMC:
+                fw_info->ver = adev->pm.fw_version;
+                fw_info->feature = 0;
+                break;
+        case AMDGPU_INFO_FW_SDMA:
+                if (query_fw->index >= adev->sdma.num_instances)
+                        return -EINVAL;
+                fw_info->ver = adev->sdma.instance[query_fw->index].fw_version;
+                fw_info->feature = adev->sdma.instance[query_fw->index].feature_version;
+                break;
+        default:
+                return -EINVAL;
+        }
+        return 0;
+}
+
 /*
  * Userspace get information ioctl
  */
@@ -288,67 +353,20 @@ static int amdgpu_info_ioctl(struct drm_device *dev, void *data, struct drm_file
                 return copy_to_user(out, &count, min(size, 4u)) ? -EFAULT : 0;
         }
         case AMDGPU_INFO_TIMESTAMP:
-                ui64 = amdgpu_asic_get_gpu_clock_counter(adev);
+                ui64 = amdgpu_gfx_get_gpu_clock_counter(adev);
                 return copy_to_user(out, &ui64, min(size, 8u)) ? -EFAULT : 0;
         case AMDGPU_INFO_FW_VERSION: {
                 struct drm_amdgpu_info_firmware fw_info;
+                int ret;
 
                 /* We only support one instance of each IP block right now. */
                 if (info->query_fw.ip_instance != 0)
                         return -EINVAL;
 
-                switch (info->query_fw.fw_type) {
-                case AMDGPU_INFO_FW_VCE:
-                        fw_info.ver = adev->vce.fw_version;
-                        fw_info.feature = adev->vce.fb_version;
-                        break;
-                case AMDGPU_INFO_FW_UVD:
-                        fw_info.ver = adev->uvd.fw_version;
-                        fw_info.feature = 0;
-                        break;
-                case AMDGPU_INFO_FW_GMC:
-                        fw_info.ver = adev->mc.fw_version;
-                        fw_info.feature = 0;
-                        break;
-                case AMDGPU_INFO_FW_GFX_ME:
-                        fw_info.ver = adev->gfx.me_fw_version;
-                        fw_info.feature = adev->gfx.me_feature_version;
-                        break;
-                case AMDGPU_INFO_FW_GFX_PFP:
-                        fw_info.ver = adev->gfx.pfp_fw_version;
-                        fw_info.feature = adev->gfx.pfp_feature_version;
-                        break;
-                case AMDGPU_INFO_FW_GFX_CE:
-                        fw_info.ver = adev->gfx.ce_fw_version;
-                        fw_info.feature = adev->gfx.ce_feature_version;
-                        break;
-                case AMDGPU_INFO_FW_GFX_RLC:
-                        fw_info.ver = adev->gfx.rlc_fw_version;
-                        fw_info.feature = adev->gfx.rlc_feature_version;
-                        break;
-                case AMDGPU_INFO_FW_GFX_MEC:
-                        if (info->query_fw.index == 0) {
-                                fw_info.ver = adev->gfx.mec_fw_version;
-                                fw_info.feature = adev->gfx.mec_feature_version;
-                        } else if (info->query_fw.index == 1) {
-                                fw_info.ver = adev->gfx.mec2_fw_version;
-                                fw_info.feature = adev->gfx.mec2_feature_version;
-                        } else
-                                return -EINVAL;
-                        break;
-                case AMDGPU_INFO_FW_SMC:
-                        fw_info.ver = adev->pm.fw_version;
-                        fw_info.feature = 0;
-                        break;
-                case AMDGPU_INFO_FW_SDMA:
-                        if (info->query_fw.index >= adev->sdma.num_instances)
-                                return -EINVAL;
-                        fw_info.ver = adev->sdma.instance[info->query_fw.index].fw_version;
-                        fw_info.feature = adev->sdma.instance[info->query_fw.index].feature_version;
-                        break;
-                default:
-                        return -EINVAL;
-                }
+                ret = amdgpu_firmware_info(&fw_info, &info->query_fw, adev);
+                if (ret)
+                        return ret;
+
                 return copy_to_user(out, &fw_info,
                                     min((size_t)size, sizeof(fw_info))) ? -EFAULT : 0;
         }
@@ -566,6 +584,9 @@ void amdgpu_driver_postclose_kms(struct drm_device *dev,
 
         amdgpu_ctx_mgr_fini(&fpriv->ctx_mgr);
 
+        amdgpu_uvd_free_handles(adev, file_priv);
+        amdgpu_vce_free_handles(adev, file_priv);
+
         amdgpu_vm_fini(adev, &fpriv->vm);
 
         idr_for_each_entry(&fpriv->bo_list_handles, list, handle)
@@ -590,10 +611,6 @@ void amdgpu_driver_postclose_kms(struct drm_device *dev,
 void amdgpu_driver_preclose_kms(struct drm_device *dev,
                                 struct drm_file *file_priv)
 {
-        struct amdgpu_device *adev = dev->dev_private;
-
-        amdgpu_uvd_free_handles(adev, file_priv);
-        amdgpu_vce_free_handles(adev, file_priv);
 }
 
 /*
@@ -756,3 +773,130 @@ const struct drm_ioctl_desc amdgpu_ioctls_kms[] = {
         DRM_IOCTL_DEF_DRV(AMDGPU_GEM_USERPTR, amdgpu_gem_userptr_ioctl, DRM_AUTH|DRM_RENDER_ALLOW),
 };
 const int amdgpu_max_kms_ioctl = ARRAY_SIZE(amdgpu_ioctls_kms);
+
+/*
+ * Debugfs info
+ */
+#if defined(CONFIG_DEBUG_FS)
+
+static int amdgpu_debugfs_firmware_info(struct seq_file *m, void *data)
+{
+        struct drm_info_node *node = (struct drm_info_node *) m->private;
+        struct drm_device *dev = node->minor->dev;
+        struct amdgpu_device *adev = dev->dev_private;
+        struct drm_amdgpu_info_firmware fw_info;
+        struct drm_amdgpu_query_fw query_fw;
+        int ret, i;
+
+        /* VCE */
+        query_fw.fw_type = AMDGPU_INFO_FW_VCE;
+        ret = amdgpu_firmware_info(&fw_info, &query_fw, adev);
+        if (ret)
+                return ret;
+        seq_printf(m, "VCE feature version: %u, firmware version: 0x%08x\n",
+                   fw_info.feature, fw_info.ver);
+
+        /* UVD */
+        query_fw.fw_type = AMDGPU_INFO_FW_UVD;
+        ret = amdgpu_firmware_info(&fw_info, &query_fw, adev);
+        if (ret)
+                return ret;
+        seq_printf(m, "UVD feature version: %u, firmware version: 0x%08x\n",
+                   fw_info.feature, fw_info.ver);
+
+        /* GMC */
+        query_fw.fw_type = AMDGPU_INFO_FW_GMC;
+        ret = amdgpu_firmware_info(&fw_info, &query_fw, adev);
+        if (ret)
+                return ret;
+        seq_printf(m, "MC feature version: %u, firmware version: 0x%08x\n",
+                   fw_info.feature, fw_info.ver);
+
+        /* ME */
+        query_fw.fw_type = AMDGPU_INFO_FW_GFX_ME;
+        ret = amdgpu_firmware_info(&fw_info, &query_fw, adev);
+        if (ret)
+                return ret;
+        seq_printf(m, "ME feature version: %u, firmware version: 0x%08x\n",
+                   fw_info.feature, fw_info.ver);
+
+        /* PFP */
+        query_fw.fw_type = AMDGPU_INFO_FW_GFX_PFP;
+        ret = amdgpu_firmware_info(&fw_info, &query_fw, adev);
+        if (ret)
+                return ret;
+        seq_printf(m, "PFP feature version: %u, firmware version: 0x%08x\n",
+                   fw_info.feature, fw_info.ver);
+
+        /* CE */
+        query_fw.fw_type = AMDGPU_INFO_FW_GFX_CE;
+        ret = amdgpu_firmware_info(&fw_info, &query_fw, adev);
+        if (ret)
+                return ret;
+        seq_printf(m, "CE feature version: %u, firmware version: 0x%08x\n",
+                   fw_info.feature, fw_info.ver);
+
+        /* RLC */
+        query_fw.fw_type = AMDGPU_INFO_FW_GFX_RLC;
+        ret = amdgpu_firmware_info(&fw_info, &query_fw, adev);
+        if (ret)
+                return ret;
+        seq_printf(m, "RLC feature version: %u, firmware version: 0x%08x\n",
+                   fw_info.feature, fw_info.ver);
+
+        /* MEC */
+        query_fw.fw_type = AMDGPU_INFO_FW_GFX_MEC;
+        query_fw.index = 0;
+        ret = amdgpu_firmware_info(&fw_info, &query_fw, adev);
+        if (ret)
+                return ret;
+        seq_printf(m, "MEC feature version: %u, firmware version: 0x%08x\n",
+                   fw_info.feature, fw_info.ver);
+
+        /* MEC2 */
+        if (adev->asic_type == CHIP_KAVERI ||
+            (adev->asic_type > CHIP_TOPAZ && adev->asic_type != CHIP_STONEY)) {
+                query_fw.index = 1;
+                ret = amdgpu_firmware_info(&fw_info, &query_fw, adev);
+                if (ret)
+                        return ret;
+                seq_printf(m, "MEC2 feature version: %u, firmware version: 0x%08x\n",
+                           fw_info.feature, fw_info.ver);
+        }
+
+        /* SMC */
+        query_fw.fw_type = AMDGPU_INFO_FW_SMC;
+        ret = amdgpu_firmware_info(&fw_info, &query_fw, adev);
+        if (ret)
+                return ret;
+        seq_printf(m, "SMC feature version: %u, firmware version: 0x%08x\n",
+                   fw_info.feature, fw_info.ver);
+
+        /* SDMA */
+        query_fw.fw_type = AMDGPU_INFO_FW_SDMA;
+        for (i = 0; i < adev->sdma.num_instances; i++) {
+                query_fw.index = i;
+                ret = amdgpu_firmware_info(&fw_info, &query_fw, adev);
+                if (ret)
+                        return ret;
+                seq_printf(m, "SDMA%d feature version: %u, firmware version: 0x%08x\n",
+                           i, fw_info.feature, fw_info.ver);
+        }
+
+        return 0;
+}
+
+static const struct drm_info_list amdgpu_firmware_info_list[] = {
+        {"amdgpu_firmware_info", amdgpu_debugfs_firmware_info, 0, NULL},
+};
+#endif
+
+int amdgpu_debugfs_firmware_init(struct amdgpu_device *adev)
+{
+#if defined(CONFIG_DEBUG_FS)
+        return amdgpu_debugfs_add_files(adev, amdgpu_firmware_info_list,
+                                        ARRAY_SIZE(amdgpu_firmware_info_list));
+#else
+        return 0;
+#endif
+}
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c
index 7ecea83ce453..6f0873c75a25 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_object.c
@@ -589,6 +589,7 @@ void amdgpu_bo_move_notify(struct ttm_buffer_object *bo,
                            struct ttm_mem_reg *new_mem)
 {
         struct amdgpu_bo *rbo;
+        struct ttm_mem_reg *old_mem = &bo->mem;
 
         if (!amdgpu_ttm_bo_is_amdgpu_bo(bo))
                 return;
@@ -602,6 +603,8 @@ void amdgpu_bo_move_notify(struct ttm_buffer_object *bo,
 
         /* move_notify is called before move happens */
         amdgpu_update_memory_usage(rbo->adev, &bo->mem, new_mem);
+
+        trace_amdgpu_ttm_bo_move(rbo, new_mem->mem_type, old_mem->mem_type);
 }
 
 int amdgpu_bo_fault_reserve_notify(struct ttm_buffer_object *bo)
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_pm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_pm.c
index 0e13d80d2a95..ff63b88b0ffa 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_pm.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_pm.c
@@ -347,6 +347,8 @@ static ssize_t amdgpu_get_pp_dpm_sclk(struct device *dev,
 
         if (adev->pp_enabled)
                 size = amdgpu_dpm_print_clock_levels(adev, PP_SCLK, buf);
+        else if (adev->pm.funcs->print_clock_levels)
+                size = adev->pm.funcs->print_clock_levels(adev, PP_SCLK, buf);
 
         return size;
 }
@@ -363,7 +365,9 @@ static ssize_t amdgpu_set_pp_dpm_sclk(struct device *dev,
         uint32_t i, mask = 0;
         char sub_str[2];
 
-        for (i = 0; i < strlen(buf) - 1; i++) {
+        for (i = 0; i < strlen(buf); i++) {
+                if (*(buf + i) == '\n')
+                        continue;
                 sub_str[0] = *(buf + i);
                 sub_str[1] = '\0';
                 ret = kstrtol(sub_str, 0, &level);
@@ -377,6 +381,8 @@ static ssize_t amdgpu_set_pp_dpm_sclk(struct device *dev,
 
         if (adev->pp_enabled)
                 amdgpu_dpm_force_clock_level(adev, PP_SCLK, mask);
+        else if (adev->pm.funcs->force_clock_level)
+                adev->pm.funcs->force_clock_level(adev, PP_SCLK, mask);
 fail:
         return count;
 }
@@ -391,6 +397,8 @@ static ssize_t amdgpu_get_pp_dpm_mclk(struct device *dev,
 
         if (adev->pp_enabled)
                 size = amdgpu_dpm_print_clock_levels(adev, PP_MCLK, buf);
+        else if (adev->pm.funcs->print_clock_levels)
+                size = adev->pm.funcs->print_clock_levels(adev, PP_MCLK, buf);
 
         return size;
 }
@@ -407,7 +415,9 @@ static ssize_t amdgpu_set_pp_dpm_mclk(struct device *dev,
         uint32_t i, mask = 0;
         char sub_str[2];
 
-        for (i = 0; i < strlen(buf) - 1; i++) {
+        for (i = 0; i < strlen(buf); i++) {
+                if (*(buf + i) == '\n')
+                        continue;
                 sub_str[0] = *(buf + i);
                 sub_str[1] = '\0';
                 ret = kstrtol(sub_str, 0, &level);
@@ -421,6 +431,8 @@ static ssize_t amdgpu_set_pp_dpm_mclk(struct device *dev,
 
         if (adev->pp_enabled)
                 amdgpu_dpm_force_clock_level(adev, PP_MCLK, mask);
+        else if (adev->pm.funcs->force_clock_level)
+                adev->pm.funcs->force_clock_level(adev, PP_MCLK, mask);
 fail:
         return count;
 }
@@ -435,6 +447,8 @@ static ssize_t amdgpu_get_pp_dpm_pcie(struct device *dev,
 
         if (adev->pp_enabled)
                 size = amdgpu_dpm_print_clock_levels(adev, PP_PCIE, buf);
+        else if (adev->pm.funcs->print_clock_levels)
+                size = adev->pm.funcs->print_clock_levels(adev, PP_PCIE, buf);
 
         return size;
 }
@@ -451,7 +465,9 @@ static ssize_t amdgpu_set_pp_dpm_pcie(struct device *dev,
         uint32_t i, mask = 0;
         char sub_str[2];
 
-        for (i = 0; i < strlen(buf) - 1; i++) {
+        for (i = 0; i < strlen(buf); i++) {
+                if (*(buf + i) == '\n')
+                        continue;
                 sub_str[0] = *(buf + i);
                 sub_str[1] = '\0';
                 ret = kstrtol(sub_str, 0, &level);
@@ -465,6 +481,100 @@ static ssize_t amdgpu_set_pp_dpm_pcie(struct device *dev,
 
         if (adev->pp_enabled)
                 amdgpu_dpm_force_clock_level(adev, PP_PCIE, mask);
+        else if (adev->pm.funcs->force_clock_level)
+                adev->pm.funcs->force_clock_level(adev, PP_PCIE, mask);
+fail:
+        return count;
+}
+
+static ssize_t amdgpu_get_pp_sclk_od(struct device *dev,
+                struct device_attribute *attr,
+                char *buf)
+{
+        struct drm_device *ddev = dev_get_drvdata(dev);
+        struct amdgpu_device *adev = ddev->dev_private;
+        uint32_t value = 0;
+
+        if (adev->pp_enabled)
+                value = amdgpu_dpm_get_sclk_od(adev);
+        else if (adev->pm.funcs->get_sclk_od)
+                value = adev->pm.funcs->get_sclk_od(adev);
+
+        return snprintf(buf, PAGE_SIZE, "%d\n", value);
+}
+
+static ssize_t amdgpu_set_pp_sclk_od(struct device *dev,
+                struct device_attribute *attr,
+                const char *buf,
+                size_t count)
+{
+        struct drm_device *ddev = dev_get_drvdata(dev);
+        struct amdgpu_device *adev = ddev->dev_private;
+        int ret;
+        long int value;
+
+        ret = kstrtol(buf, 0, &value);
+
+        if (ret) {
+                count = -EINVAL;
+                goto fail;
+        }
+
+        if (adev->pp_enabled) {
+                amdgpu_dpm_set_sclk_od(adev, (uint32_t)value);
+                amdgpu_dpm_dispatch_task(adev, AMD_PP_EVENT_READJUST_POWER_STATE, NULL, NULL);
+        } else if (adev->pm.funcs->set_sclk_od) {
+                adev->pm.funcs->set_sclk_od(adev, (uint32_t)value);
+                adev->pm.dpm.current_ps = adev->pm.dpm.boot_ps;
+                amdgpu_pm_compute_clocks(adev);
+        }
+
+fail:
+        return count;
+}
+
+static ssize_t amdgpu_get_pp_mclk_od(struct device *dev,
+                struct device_attribute *attr,
+                char *buf)
+{
+        struct drm_device *ddev = dev_get_drvdata(dev);
+        struct amdgpu_device *adev = ddev->dev_private;
+        uint32_t value = 0;
+
+        if (adev->pp_enabled)
+                value = amdgpu_dpm_get_mclk_od(adev);
+        else if (adev->pm.funcs->get_mclk_od)
+                value = adev->pm.funcs->get_mclk_od(adev);
+
+        return snprintf(buf, PAGE_SIZE, "%d\n", value);
+}
+
+static ssize_t amdgpu_set_pp_mclk_od(struct device *dev,
+                struct device_attribute *attr,
+                const char *buf,
+                size_t count)
+{
+        struct drm_device *ddev = dev_get_drvdata(dev);
+        struct amdgpu_device *adev = ddev->dev_private;
+        int ret;
+        long int value;
+
+        ret = kstrtol(buf, 0, &value);
+
+        if (ret) {
+                count = -EINVAL;
+                goto fail;
+        }
+
+        if (adev->pp_enabled) {
+                amdgpu_dpm_set_mclk_od(adev, (uint32_t)value);
+                amdgpu_dpm_dispatch_task(adev, AMD_PP_EVENT_READJUST_POWER_STATE, NULL, NULL);
+        } else if (adev->pm.funcs->set_mclk_od) {
+                adev->pm.funcs->set_mclk_od(adev, (uint32_t)value);
+                adev->pm.dpm.current_ps = adev->pm.dpm.boot_ps;
+                amdgpu_pm_compute_clocks(adev);
+        }
+
 fail:
         return count;
 }
@@ -490,6 +600,12 @@ static DEVICE_ATTR(pp_dpm_mclk, S_IRUGO | S_IWUSR,
 static DEVICE_ATTR(pp_dpm_pcie, S_IRUGO | S_IWUSR,
                 amdgpu_get_pp_dpm_pcie,
                 amdgpu_set_pp_dpm_pcie);
+static DEVICE_ATTR(pp_sclk_od, S_IRUGO | S_IWUSR,
+                amdgpu_get_pp_sclk_od,
+                amdgpu_set_pp_sclk_od);
+static DEVICE_ATTR(pp_mclk_od, S_IRUGO | S_IWUSR,
+                amdgpu_get_pp_mclk_od,
+                amdgpu_set_pp_mclk_od);
 
 static ssize_t amdgpu_hwmon_show_temp(struct device *dev,
                                       struct device_attribute *attr,
@@ -1108,22 +1224,34 @@ int amdgpu_pm_sysfs_init(struct amdgpu_device *adev)
                         DRM_ERROR("failed to create device file pp_table\n");
                         return ret;
                 }
-                ret = device_create_file(adev->dev, &dev_attr_pp_dpm_sclk);
-                if (ret) {
-                        DRM_ERROR("failed to create device file pp_dpm_sclk\n");
-                        return ret;
-                }
-                ret = device_create_file(adev->dev, &dev_attr_pp_dpm_mclk);
-                if (ret) {
-                        DRM_ERROR("failed to create device file pp_dpm_mclk\n");
-                        return ret;
-                }
-                ret = device_create_file(adev->dev, &dev_attr_pp_dpm_pcie);
-                if (ret) {
-                        DRM_ERROR("failed to create device file pp_dpm_pcie\n");
-                        return ret;
-                }
         }
+
+        ret = device_create_file(adev->dev, &dev_attr_pp_dpm_sclk);
+        if (ret) {
+                DRM_ERROR("failed to create device file pp_dpm_sclk\n");
+                return ret;
+        }
+        ret = device_create_file(adev->dev, &dev_attr_pp_dpm_mclk);
+        if (ret) {
+                DRM_ERROR("failed to create device file pp_dpm_mclk\n");
+                return ret;
+        }
+        ret = device_create_file(adev->dev, &dev_attr_pp_dpm_pcie);
+        if (ret) {
+                DRM_ERROR("failed to create device file pp_dpm_pcie\n");
+                return ret;
+        }
+        ret = device_create_file(adev->dev, &dev_attr_pp_sclk_od);
+        if (ret) {
+                DRM_ERROR("failed to create device file pp_sclk_od\n");
+                return ret;
+        }
+        ret = device_create_file(adev->dev, &dev_attr_pp_mclk_od);
+        if (ret) {
+                DRM_ERROR("failed to create device file pp_mclk_od\n");
+                return ret;
+        }
+
         ret = amdgpu_debugfs_pm_init(adev);
         if (ret) {
                 DRM_ERROR("Failed to register debugfs file for dpm!\n");
@@ -1146,10 +1274,12 @@ void amdgpu_pm_sysfs_fini(struct amdgpu_device *adev)
                 device_remove_file(adev->dev, &dev_attr_pp_cur_state);
                 device_remove_file(adev->dev, &dev_attr_pp_force_state);
                 device_remove_file(adev->dev, &dev_attr_pp_table);
-                device_remove_file(adev->dev, &dev_attr_pp_dpm_sclk);
-                device_remove_file(adev->dev, &dev_attr_pp_dpm_mclk);
-                device_remove_file(adev->dev, &dev_attr_pp_dpm_pcie);
         }
+        device_remove_file(adev->dev, &dev_attr_pp_dpm_sclk);
+        device_remove_file(adev->dev, &dev_attr_pp_dpm_mclk);
+        device_remove_file(adev->dev, &dev_attr_pp_dpm_pcie);
+        device_remove_file(adev->dev, &dev_attr_pp_sclk_od);
+        device_remove_file(adev->dev, &dev_attr_pp_mclk_od);
 }
 
 void amdgpu_pm_compute_clocks(struct amdgpu_device *adev)
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_powerplay.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_powerplay.c
index 82256558e0f5..c5738a22b690 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_powerplay.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_powerplay.c
@@ -52,6 +52,7 @@ static int amdgpu_powerplay_init(struct amdgpu_device *adev)
                 pp_init->chip_family = adev->family;
                 pp_init->chip_id = adev->asic_type;
                 pp_init->device = amdgpu_cgs_create_device(adev);
+                pp_init->powercontainment_enabled = amdgpu_powercontainment;
 
                 ret = amd_powerplay_init(pp_init, amd_pp);
                 kfree(pp_init);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.c
index 870f9494252c..85aeb0a804bb 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ring.c
@@ -28,6 +28,7 @@
  */
 #include <linux/seq_file.h>
 #include <linux/slab.h>
+#include <linux/debugfs.h>
 #include <drm/drmP.h>
 #include <drm/amdgpu_drm.h>
 #include "amdgpu.h"
@@ -48,6 +49,7 @@
  */
 static int amdgpu_debugfs_ring_init(struct amdgpu_device *adev,
                                     struct amdgpu_ring *ring);
+static void amdgpu_debugfs_ring_fini(struct amdgpu_ring *ring);
 
 /**
  * amdgpu_ring_alloc - allocate space on the ring buffer
@@ -73,6 +75,10 @@ int amdgpu_ring_alloc(struct amdgpu_ring *ring, unsigned ndw)
 
         ring->count_dw = ndw;
         ring->wptr_old = ring->wptr;
+
+        if (ring->funcs->begin_use)
+                ring->funcs->begin_use(ring);
+
         return 0;
 }
 
@@ -125,6 +131,9 @@ void amdgpu_ring_commit(struct amdgpu_ring *ring)
 
         mb();
         amdgpu_ring_set_wptr(ring);
+
+        if (ring->funcs->end_use)
+                ring->funcs->end_use(ring);
 }
 
 /**
@@ -137,78 +146,9 @@ void amdgpu_ring_commit(struct amdgpu_ring *ring)
 void amdgpu_ring_undo(struct amdgpu_ring *ring)
 {
         ring->wptr = ring->wptr_old;
-}
-
-/**
- * amdgpu_ring_backup - Back up the content of a ring
- *
- * @ring: the ring we want to back up
- *
- * Saves all unprocessed commits from a ring, returns the number of dwords saved.
- */
-unsigned amdgpu_ring_backup(struct amdgpu_ring *ring,
-                            uint32_t **data)
-{
-        unsigned size, ptr, i;
-
-        *data = NULL;
-
-        if (ring->ring_obj == NULL)
-                return 0;
-
-        /* it doesn't make sense to save anything if all fences are signaled */
-        if (!amdgpu_fence_count_emitted(ring))
-                return 0;
-
-        ptr = le32_to_cpu(*ring->next_rptr_cpu_addr);
-
-        size = ring->wptr + (ring->ring_size / 4);
-        size -= ptr;
-        size &= ring->ptr_mask;
-        if (size == 0)
-                return 0;
-
-        /* and then save the content of the ring */
-        *data = kmalloc_array(size, sizeof(uint32_t), GFP_KERNEL);
-        if (!*data)
-                return 0;
-        for (i = 0; i < size; ++i) {
-                (*data)[i] = ring->ring[ptr++];
-                ptr &= ring->ptr_mask;
-        }
-
-        return size;
-}
-
-/**
- * amdgpu_ring_restore - append saved commands to the ring again
- *
- * @ring: ring to append commands to
- * @size: number of dwords we want to write
- * @data: saved commands
- *
- * Allocates space on the ring and restore the previously saved commands.
- */
-int amdgpu_ring_restore(struct amdgpu_ring *ring,
-                        unsigned size, uint32_t *data)
-{
-        int i, r;
-
-        if (!size || !data)
-                return 0;
-
-        /* restore the saved ring content */
-        r = amdgpu_ring_alloc(ring, size);
-        if (r)
-                return r;
-
-        for (i = 0; i < size; ++i) {
-                amdgpu_ring_write(ring, data[i]);
-        }
 
-        amdgpu_ring_commit(ring);
-        kfree(data);
-        return 0;
+        if (ring->funcs->end_use)
+                ring->funcs->end_use(ring);
 }
 
 /**
@@ -260,14 +200,6 @@ int amdgpu_ring_init(struct amdgpu_device *adev, struct amdgpu_ring *ring,
                 return r;
         }
 
-        r = amdgpu_wb_get(adev, &ring->next_rptr_offs);
-        if (r) {
-                dev_err(adev->dev, "(%d) ring next_rptr wb alloc failed\n", r);
-                return r;
-        }
-        ring->next_rptr_gpu_addr = adev->wb.gpu_addr + ring->next_rptr_offs * 4;
-        ring->next_rptr_cpu_addr = &adev->wb.wb[ring->next_rptr_offs];
-
         r = amdgpu_wb_get(adev, &ring->cond_exe_offs);
         if (r) {
                 dev_err(adev->dev, "(%d) ring cond_exec_polling wb alloc failed\n", r);
@@ -276,7 +208,6 @@ int amdgpu_ring_init(struct amdgpu_device *adev, struct amdgpu_ring *ring,
         ring->cond_exe_gpu_addr = adev->wb.gpu_addr + (ring->cond_exe_offs * 4);
         ring->cond_exe_cpu_addr = &adev->wb.wb[ring->cond_exe_offs];
 
-        spin_lock_init(&ring->fence_lock);
         r = amdgpu_fence_driver_start_ring(ring, irq_src, irq_type);
         if (r) {
                 dev_err(adev->dev, "failed initializing fences (%d).\n", r);
@@ -310,6 +241,9 @@ int amdgpu_ring_init(struct amdgpu_device *adev, struct amdgpu_ring *ring,
                 }
                 r = amdgpu_bo_kmap(ring->ring_obj,
                                        (void **)&ring->ring);
+
+                memset((void *)ring->ring, 0, ring->ring_size);
+
                 amdgpu_bo_unreserve(ring->ring_obj);
                 if (r) {
                         dev_err(adev->dev, "(%d) ring map failed\n", r);
@@ -347,7 +281,6 @@ void amdgpu_ring_fini(struct amdgpu_ring *ring)
         amdgpu_wb_free(ring->adev, ring->fence_offs);
         amdgpu_wb_free(ring->adev, ring->rptr_offs);
         amdgpu_wb_free(ring->adev, ring->wptr_offs);
-        amdgpu_wb_free(ring->adev, ring->next_rptr_offs);
 
         if (ring_obj) {
                 r = amdgpu_bo_reserve(ring_obj, false);
@@ -358,6 +291,7 @@ void amdgpu_ring_fini(struct amdgpu_ring *ring)
                 }
                 amdgpu_bo_unref(&ring_obj);
         }
+        amdgpu_debugfs_ring_fini(ring);
 }
 
 /*
@@ -365,57 +299,62 @@ void amdgpu_ring_fini(struct amdgpu_ring *ring)
  */
 #if defined(CONFIG_DEBUG_FS)
 
-static int amdgpu_debugfs_ring_info(struct seq_file *m, void *data)
+/* Layout of file is 12 bytes consisting of
+ * - rptr
+ * - wptr
+ * - driver's copy of wptr
+ *
+ * followed by n-words of ring data
+ */
+static ssize_t amdgpu_debugfs_ring_read(struct file *f, char __user *buf,
+                                        size_t size, loff_t *pos)
 {
-        struct drm_info_node *node = (struct drm_info_node *) m->private;
-        struct drm_device *dev = node->minor->dev;
-        struct amdgpu_device *adev = dev->dev_private;
-        int roffset = (unsigned long)node->info_ent->data;
-        struct amdgpu_ring *ring = (void *)(((uint8_t*)adev) + roffset);
-        uint32_t rptr, wptr, rptr_next;
-        unsigned i;
-
-        wptr = amdgpu_ring_get_wptr(ring);
-        seq_printf(m, "wptr: 0x%08x [%5d]\n", wptr, wptr);
-
-        rptr = amdgpu_ring_get_rptr(ring);
-        rptr_next = le32_to_cpu(*ring->next_rptr_cpu_addr);
-
-        seq_printf(m, "rptr: 0x%08x [%5d]\n", rptr, rptr);
-
-        seq_printf(m, "driver's copy of the wptr: 0x%08x [%5d]\n",
-                   ring->wptr, ring->wptr);
-
-        if (!ring->ready)
-                return 0;
-
-        /* print 8 dw before current rptr as often it's the last executed
-         * packet that is the root issue
-         */
-        i = (rptr + ring->ptr_mask + 1 - 32) & ring->ptr_mask;
-        while (i != rptr) {
-                seq_printf(m, "r[%5d]=0x%08x", i, ring->ring[i]);
-                if (i == rptr)
-                        seq_puts(m, " *");
-                if (i == rptr_next)
-                        seq_puts(m, " #");
-                seq_puts(m, "\n");
-                i = (i + 1) & ring->ptr_mask;
+        struct amdgpu_ring *ring = (struct amdgpu_ring*)f->f_inode->i_private;
+        int r, i;
+        uint32_t value, result, early[3];
+
+        if (*pos & 3 || size & 3)
+                return -EINVAL;
+
+        result = 0;
+
+        if (*pos < 12) {
+                early[0] = amdgpu_ring_get_rptr(ring);
+                early[1] = amdgpu_ring_get_wptr(ring);
+                early[2] = ring->wptr;
+                for (i = *pos / 4; i < 3 && size; i++) {
+                        r = put_user(early[i], (uint32_t *)buf);
+                        if (r)
+                                return r;
+                        buf += 4;
+                        result += 4;
+                        size -= 4;
+                        *pos += 4;
+                }
         }
-        while (i != wptr) {
-                seq_printf(m, "r[%5d]=0x%08x", i, ring->ring[i]);
-                if (i == rptr)
-                        seq_puts(m, " *");
-                if (i == rptr_next)
-                        seq_puts(m, " #");
-                seq_puts(m, "\n");
-                i = (i + 1) & ring->ptr_mask;
+
+        while (size) {
+                if (*pos >= (ring->ring_size + 12))
+                        return result;
+                        
+                value = ring->ring[(*pos - 12)/4];
+                r = put_user(value, (uint32_t*)buf);
+                if (r)
+                        return r;
+                buf += 4;
+                result += 4;
+                size -= 4;
+                *pos += 4;
         }
-        return 0;
+
+        return result;
 }
 
-static struct drm_info_list amdgpu_debugfs_ring_info_list[AMDGPU_MAX_RINGS];
-static char amdgpu_debugfs_ring_names[AMDGPU_MAX_RINGS][32];
+static const struct file_operations amdgpu_debugfs_ring_fops = {
+        .owner = THIS_MODULE,
+        .read = amdgpu_debugfs_ring_read,
+        .llseek = default_llseek
+};
 
 #endif
 
@@ -423,28 +362,27 @@ static int amdgpu_debugfs_ring_init(struct amdgpu_device *adev,
                                     struct amdgpu_ring *ring)
 {
 #if defined(CONFIG_DEBUG_FS)
-        unsigned offset = (uint8_t*)ring - (uint8_t*)adev;
-        unsigned i;
-        struct drm_info_list *info;
-        char *name;
-
-        for (i = 0; i < ARRAY_SIZE(amdgpu_debugfs_ring_info_list); ++i) {
-                info = &amdgpu_debugfs_ring_info_list[i];
-                if (!info->data)
-                        break;
-        }
+        struct drm_minor *minor = adev->ddev->primary;
+        struct dentry *ent, *root = minor->debugfs_root;
+        char name[32];
 
-        if (i == ARRAY_SIZE(amdgpu_debugfs_ring_info_list))
-                return -ENOSPC;
-
-        name = &amdgpu_debugfs_ring_names[i][0];
         sprintf(name, "amdgpu_ring_%s", ring->name);
-        info->name = name;
-        info->show = amdgpu_debugfs_ring_info;
-        info->driver_features = 0;
-        info->data = (void*)(uintptr_t)offset;
 
-        return amdgpu_debugfs_add_files(adev, info, 1);
+        ent = debugfs_create_file(name,
+                                  S_IFREG | S_IRUGO, root,
+                                  ring, &amdgpu_debugfs_ring_fops);
+        if (IS_ERR(ent))
+                return PTR_ERR(ent);
+
+        i_size_write(ent->d_inode, ring->ring_size + 12);
+        ring->ent = ent;
 #endif
         return 0;
 }
+
+static void amdgpu_debugfs_ring_fini(struct amdgpu_ring *ring)
+{
+#if defined(CONFIG_DEBUG_FS)
+        debugfs_remove(ring->ent);
+#endif
+}
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_sa.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_sa.c
index 48618ee324eb..d8af37a845f4 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_sa.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_sa.c
@@ -428,7 +428,7 @@ void amdgpu_sa_bo_dump_debug_info(struct amdgpu_sa_manager *sa_manager,
                            soffset, eoffset, eoffset - soffset);
 
                 if (i->fence)
-                        seq_printf(m, " protected by 0x%08x on context %d",
+                        seq_printf(m, " protected by 0x%08x on context %llu",
                                    i->fence->seqno, i->fence->context);
 
                 seq_printf(m, "\n");
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_sync.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_sync.c
index 34a92808bbd4..5c8d3022fb87 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_sync.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_sync.c
@@ -223,13 +223,16 @@ int amdgpu_sync_resv(struct amdgpu_device *adev,
 }
 
 /**
- * amdgpu_sync_is_idle - test if all fences are signaled
+ * amdgpu_sync_peek_fence - get the next fence not signaled yet
  *
  * @sync: the sync object
+ * @ring: optional ring to use for test
  *
- * Returns true if all fences in the sync object are signaled.
+ * Returns the next fence not signaled yet without removing it from the sync
+ * object.
  */
-bool amdgpu_sync_is_idle(struct amdgpu_sync *sync)
+struct fence *amdgpu_sync_peek_fence(struct amdgpu_sync *sync,
+                                     struct amdgpu_ring *ring)
 {
         struct amdgpu_sync_entry *e;
         struct hlist_node *tmp;
@@ -237,6 +240,19 @@ bool amdgpu_sync_is_idle(struct amdgpu_sync *sync)
 
         hash_for_each_safe(sync->fences, i, tmp, e, node) {
                 struct fence *f = e->fence;
+                struct amd_sched_fence *s_fence = to_amd_sched_fence(f);
+
+                if (ring && s_fence) {
+                        /* For fences from the same ring it is sufficient
+                         * when they are scheduled.
+                         */
+                        if (s_fence->sched == &ring->sched) {
+                                if (fence_is_signaled(&s_fence->scheduled))
+                                        continue;
+
+                                return &s_fence->scheduled;
+                        }
+                }
 
                 if (fence_is_signaled(f)) {
                         hash_del(&e->node);
@@ -245,58 +261,19 @@ bool amdgpu_sync_is_idle(struct amdgpu_sync *sync)
                         continue;
                 }
 
-                return false;
+                return f;
         }
 
-        return true;
+        return NULL;
 }
 
 /**
- * amdgpu_sync_cycle_fences - move fences from one sync object into another
+ * amdgpu_sync_get_fence - get the next fence from the sync object
  *
- * @dst: the destination sync object
- * @src: the source sync object
- * @fence: fence to add to source
+ * @sync: sync object to use
  *
- * Remove all fences from source and put them into destination and add
- * fence as new one into source.
+ * Get and removes the next fence from the sync object not signaled yet.
  */
-int amdgpu_sync_cycle_fences(struct amdgpu_sync *dst, struct amdgpu_sync *src,
-                             struct fence *fence)
-{
-        struct amdgpu_sync_entry *e, *newone;
-        struct hlist_node *tmp;
-        int i;
-
-        /* Allocate the new entry before moving the old ones */
-        newone = kmem_cache_alloc(amdgpu_sync_slab, GFP_KERNEL);
-        if (!newone)
-                return -ENOMEM;
-
-        hash_for_each_safe(src->fences, i, tmp, e, node) {
-                struct fence *f = e->fence;
-
-                hash_del(&e->node);
-                if (fence_is_signaled(f)) {
-                        fence_put(f);
-                        kmem_cache_free(amdgpu_sync_slab, e);
-                        continue;
-                }
-
-                if (amdgpu_sync_add_later(dst, f)) {
-                        kmem_cache_free(amdgpu_sync_slab, e);
-                        continue;
-                }
-
-                hash_add(dst->fences, &e->node, f->context);
-        }
-
-        hash_add(src->fences, &newone->node, fence->context);
-        newone->fence = fence_get(fence);
-
-        return 0;
-}
-
 struct fence *amdgpu_sync_get_fence(struct amdgpu_sync *sync)
 {
         struct amdgpu_sync_entry *e;
@@ -319,25 +296,6 @@ struct fence *amdgpu_sync_get_fence(struct amdgpu_sync *sync)
         return NULL;
 }
 
-int amdgpu_sync_wait(struct amdgpu_sync *sync)
-{
-        struct amdgpu_sync_entry *e;
-        struct hlist_node *tmp;
-        int i, r;
-
-        hash_for_each_safe(sync->fences, i, tmp, e, node) {
-                r = fence_wait(e->fence, false);
-                if (r)
-                        return r;
-
-                hash_del(&e->node);
-                fence_put(e->fence);
-                kmem_cache_free(amdgpu_sync_slab, e);
-        }
-
-        return 0;
-}
-
 /**
  * amdgpu_sync_free - free the sync object
  *
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_trace.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_trace.h
index 26a5f4acf584..0d8d65eb46cd 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_trace.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_trace.h
@@ -11,19 +11,68 @@
 #define TRACE_SYSTEM amdgpu
 #define TRACE_INCLUDE_FILE amdgpu_trace
 
+TRACE_EVENT(amdgpu_mm_rreg,
+            TP_PROTO(unsigned did, uint32_t reg, uint32_t value),
+            TP_ARGS(did, reg, value),
+            TP_STRUCT__entry(
+                                __field(unsigned, did)
+                                __field(uint32_t, reg)
+                                __field(uint32_t, value)
+                            ),
+            TP_fast_assign(
+                           __entry->did = did;
+                           __entry->reg = reg;
+                           __entry->value = value;
+                           ),
+            TP_printk("0x%04lx, 0x%04lx, 0x%08lx",
+                      (unsigned long)__entry->did,
+                      (unsigned long)__entry->reg,
+                      (unsigned long)__entry->value)
+);
+
+TRACE_EVENT(amdgpu_mm_wreg,
+            TP_PROTO(unsigned did, uint32_t reg, uint32_t value),
+            TP_ARGS(did, reg, value),
+            TP_STRUCT__entry(
+                                __field(unsigned, did)
+                                __field(uint32_t, reg)
+                                __field(uint32_t, value)
+                            ),
+            TP_fast_assign(
+                           __entry->did = did;
+                           __entry->reg = reg;
+                           __entry->value = value;
+                           ),
+            TP_printk("0x%04lx, 0x%04lx, 0x%08lx",
+                      (unsigned long)__entry->did,
+                      (unsigned long)__entry->reg,
+                      (unsigned long)__entry->value)
+);
+
 TRACE_EVENT(amdgpu_bo_create,
             TP_PROTO(struct amdgpu_bo *bo),
             TP_ARGS(bo),
             TP_STRUCT__entry(
                              __field(struct amdgpu_bo *, bo)
                              __field(u32, pages)
+                             __field(u32, type)
+                             __field(u32, prefer)
+                             __field(u32, allow)
+                             __field(u32, visible)
                              ),
 
             TP_fast_assign(
                            __entry->bo = bo;
                            __entry->pages = bo->tbo.num_pages;
+                           __entry->type = bo->tbo.mem.mem_type;
+                           __entry->prefer = bo->prefered_domains;
+                           __entry->allow = bo->allowed_domains;
+                           __entry->visible = bo->flags;
                            ),
-            TP_printk("bo=%p, pages=%u", __entry->bo, __entry->pages)
+
+            TP_printk("bo=%p,pages=%u,type=%d,prefered=%d,allowed=%d,visible=%d",
+                       __entry->bo, __entry->pages, __entry->type,
+                       __entry->prefer, __entry->allow, __entry->visible)
 );
 
 TRACE_EVENT(amdgpu_cs,
@@ -64,7 +113,7 @@ TRACE_EVENT(amdgpu_cs_ioctl,
                            __entry->adev = job->adev;
                            __entry->sched_job = &job->base;
                            __entry->ib = job->ibs;
-                           __entry->fence = &job->base.s_fence->base;
+                           __entry->fence = &job->base.s_fence->finished;
                            __entry->ring_name = job->ring->name;
                            __entry->num_ibs = job->num_ibs;
                            ),
@@ -89,7 +138,7 @@ TRACE_EVENT(amdgpu_sched_run_job,
                            __entry->adev = job->adev;
                            __entry->sched_job = &job->base;
                            __entry->ib = job->ibs;
-                           __entry->fence = &job->base.s_fence->base;
+                           __entry->fence = &job->base.s_fence->finished;
                            __entry->ring_name = job->ring->name;
                            __entry->num_ibs = job->num_ibs;
                            ),
@@ -100,24 +149,26 @@ TRACE_EVENT(amdgpu_sched_run_job,
 
 
 TRACE_EVENT(amdgpu_vm_grab_id,
-            TP_PROTO(struct amdgpu_vm *vm, int ring, unsigned vmid,
-                     uint64_t pd_addr),
-            TP_ARGS(vm, ring, vmid, pd_addr),
+            TP_PROTO(struct amdgpu_vm *vm, int ring, struct amdgpu_job *job),
+            TP_ARGS(vm, ring, job),
             TP_STRUCT__entry(
                              __field(struct amdgpu_vm *, vm)
                              __field(u32, ring)
                              __field(u32, vmid)
                              __field(u64, pd_addr)
+                             __field(u32, needs_flush)
                              ),
 
             TP_fast_assign(
                            __entry->vm = vm;
                            __entry->ring = ring;
-                           __entry->vmid = vmid;
-                           __entry->pd_addr = pd_addr;
+                           __entry->vmid = job->vm_id;
+                           __entry->pd_addr = job->vm_pd_addr;
+                           __entry->needs_flush = job->vm_needs_flush;
                            ),
-            TP_printk("vm=%p, ring=%u, id=%u, pd_addr=%010Lx", __entry->vm,
-                      __entry->ring, __entry->vmid, __entry->pd_addr)
+            TP_printk("vm=%p, ring=%u, id=%u, pd_addr=%010Lx needs_flush=%u",
+                      __entry->vm, __entry->ring, __entry->vmid,
+                      __entry->pd_addr, __entry->needs_flush)
 );
 
 TRACE_EVENT(amdgpu_vm_bo_map,
@@ -244,13 +295,55 @@ TRACE_EVENT(amdgpu_bo_list_set,
             TP_STRUCT__entry(
                              __field(struct amdgpu_bo_list *, list)
                              __field(struct amdgpu_bo *, bo)
+                             __field(u64, bo_size)
                              ),
 
             TP_fast_assign(
                            __entry->list = list;
                            __entry->bo = bo;
+                           __entry->bo_size = amdgpu_bo_size(bo);
                            ),
-            TP_printk("list=%p, bo=%p", __entry->list, __entry->bo)
+            TP_printk("list=%p, bo=%p, bo_size = %Ld",
+                      __entry->list,
+                      __entry->bo,
+                      __entry->bo_size)
+);
+
+TRACE_EVENT(amdgpu_cs_bo_status,
+            TP_PROTO(uint64_t total_bo, uint64_t total_size),
+            TP_ARGS(total_bo, total_size),
+            TP_STRUCT__entry(
+                        __field(u64, total_bo)
+                        __field(u64, total_size)
+                        ),
+
+            TP_fast_assign(
+                        __entry->total_bo = total_bo;
+                        __entry->total_size = total_size;
+                        ),
+            TP_printk("total bo size = %Ld, total bo count = %Ld",
+                        __entry->total_bo, __entry->total_size)
+);
+
+TRACE_EVENT(amdgpu_ttm_bo_move,
+            TP_PROTO(struct amdgpu_bo* bo, uint32_t new_placement, uint32_t old_placement),
+            TP_ARGS(bo, new_placement, old_placement),
+            TP_STRUCT__entry(
+                        __field(struct amdgpu_bo *, bo)
+                        __field(u64, bo_size)
+                        __field(u32, new_placement)
+                        __field(u32, old_placement)
+                        ),
+
+            TP_fast_assign(
+                        __entry->bo      = bo;
+                        __entry->bo_size = amdgpu_bo_size(bo);
+                        __entry->new_placement = new_placement;
+                        __entry->old_placement = old_placement;
+                        ),
+            TP_printk("bo=%p from:%d to %d with size = %Ld",
+                        __entry->bo, __entry->old_placement,
+                        __entry->new_placement, __entry->bo_size)
 );
 
 #endif
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
index 3b9053af4762..b7742e62972a 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ttm.c
@@ -286,9 +286,10 @@ static int amdgpu_move_blit(struct ttm_buffer_object *bo,
         r = amdgpu_copy_buffer(ring, old_start, new_start,
                                new_mem->num_pages * PAGE_SIZE, /* bytes */
                                bo->resv, &fence);
-        /* FIXME: handle copy error */
-        r = ttm_bo_move_accel_cleanup(bo, fence,
-                                      evict, no_wait_gpu, new_mem);
+        if (r)
+                return r;
+
+        r = ttm_bo_pipeline_move(bo, fence, evict, new_mem);
         fence_put(fence);
         return r;
 }
@@ -396,6 +397,11 @@ static int amdgpu_bo_move(struct ttm_buffer_object *bo,
                 return -EINVAL;
 
         adev = amdgpu_get_adev(bo->bdev);
+
+        /* remember the eviction */
+        if (evict)
+                atomic64_inc(&adev->num_evictions);
+
         if (old_mem->mem_type == TTM_PL_SYSTEM && bo->ttm == NULL) {
                 amdgpu_move_null(bo, new_mem);
                 return 0;
@@ -429,7 +435,8 @@ static int amdgpu_bo_move(struct ttm_buffer_object *bo,
 
         if (r) {
 memcpy:
-                r = ttm_bo_move_memcpy(bo, evict, no_wait_gpu, new_mem);
+                r = ttm_bo_move_memcpy(bo, evict, interruptible,
+                                       no_wait_gpu, new_mem);
                 if (r) {
                         return r;
                 }
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_uvd.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_uvd.c
index d9c88d13f8db..b11f4e8868d7 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_uvd.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_uvd.c
@@ -40,9 +40,16 @@
 #include "uvd/uvd_4_2_d.h"
 
 /* 1 second timeout */
-#define UVD_IDLE_TIMEOUT_MS     1000
+#define UVD_IDLE_TIMEOUT        msecs_to_jiffies(1000)
+
+/* Firmware versions for VI */
+#define FW_1_65_10      ((1 << 24) | (65 << 16) | (10 << 8))
+#define FW_1_87_11      ((1 << 24) | (87 << 16) | (11 << 8))
+#define FW_1_87_12      ((1 << 24) | (87 << 16) | (12 << 8))
+#define FW_1_37_15      ((1 << 24) | (37 << 16) | (15 << 8))
+
 /* Polaris10/11 firmware version */
-#define FW_1_66_16 ((1 << 24) | (66 << 16) | (16 << 8))
+#define FW_1_66_16      ((1 << 24) | (66 << 16) | (16 << 8))
 
 /* Firmware Names */
 #ifdef CONFIG_DRM_AMDGPU_CIK
@@ -92,7 +99,6 @@ MODULE_FIRMWARE(FIRMWARE_STONEY);
 MODULE_FIRMWARE(FIRMWARE_POLARIS10);
 MODULE_FIRMWARE(FIRMWARE_POLARIS11);
 
-static void amdgpu_uvd_note_usage(struct amdgpu_device *adev);
 static void amdgpu_uvd_idle_work_handler(struct work_struct *work);
 
 int amdgpu_uvd_sw_init(struct amdgpu_device *adev)
@@ -246,6 +252,23 @@ int amdgpu_uvd_sw_init(struct amdgpu_device *adev)
         if (!amdgpu_ip_block_version_cmp(adev, AMD_IP_BLOCK_TYPE_UVD, 5, 0))
                 adev->uvd.address_64_bit = true;
 
+        switch (adev->asic_type) {
+        case CHIP_TONGA:
+                adev->uvd.use_ctx_buf = adev->uvd.fw_version >= FW_1_65_10;
+                break;
+        case CHIP_CARRIZO:
+                adev->uvd.use_ctx_buf = adev->uvd.fw_version >= FW_1_87_11;
+                break;
+        case CHIP_FIJI:
+                adev->uvd.use_ctx_buf = adev->uvd.fw_version >= FW_1_87_12;
+                break;
+        case CHIP_STONEY:
+                adev->uvd.use_ctx_buf = adev->uvd.fw_version >= FW_1_37_15;
+                break;
+        default:
+                adev->uvd.use_ctx_buf = adev->asic_type >= CHIP_POLARIS10;
+        }
+
         return 0;
 }
 
@@ -346,8 +369,6 @@ void amdgpu_uvd_free_handles(struct amdgpu_device *adev, struct drm_file *filp)
                 if (handle != 0 && adev->uvd.filp[i] == filp) {
                         struct fence *fence;
 
-                        amdgpu_uvd_note_usage(adev);
-
                         r = amdgpu_uvd_get_destroy_msg(ring, handle,
                                                        false, &fence);
                         if (r) {
@@ -438,7 +459,7 @@ static int amdgpu_uvd_cs_msg_decode(struct amdgpu_device *adev, uint32_t *msg,
         unsigned fs_in_mb = width_in_mb * height_in_mb;
 
         unsigned image_size, tmp, min_dpb_size, num_dpb_buffer;
-        unsigned min_ctx_size = 0;
+        unsigned min_ctx_size = ~0;
 
         image_size = width * height;
         image_size += image_size / 2;
@@ -557,7 +578,7 @@ static int amdgpu_uvd_cs_msg_decode(struct amdgpu_device *adev, uint32_t *msg,
                 /* reference picture buffer */
                 min_dpb_size = image_size * num_dpb_buffer;
 
-                if (adev->asic_type < CHIP_POLARIS10){
+                if (!adev->uvd.use_ctx_buf){
                         /* macroblock context buffer */
                         min_dpb_size +=
                                 width_in_mb * height_in_mb * num_dpb_buffer * 192;
@@ -662,7 +683,7 @@ static int amdgpu_uvd_cs_msg(struct amdgpu_uvd_cs_ctx *ctx,
                 }
 
                 DRM_ERROR("No more free UVD handles!\n");
-                return -EINVAL;
+                return -ENOSPC;
 
         case 1:
                 /* it's a decode msg, calc buffer sizes */
@@ -913,8 +934,6 @@ int amdgpu_uvd_ring_parse_cs(struct amdgpu_cs_parser *parser, uint32_t ib_idx)
                 return -EINVAL;
         }
 
-        amdgpu_uvd_note_usage(ctx.parser->adev);
-
         return 0;
 }
 
@@ -968,7 +987,7 @@ static int amdgpu_uvd_send_msg(struct amdgpu_ring *ring, struct amdgpu_bo *bo,
 
         if (direct) {
                 r = amdgpu_ib_schedule(ring, 1, ib, NULL, NULL, &f);
-                job->fence = f;
+                job->fence = fence_get(f);
                 if (r)
                         goto err_free;
 
@@ -1106,24 +1125,18 @@ static void amdgpu_uvd_idle_work_handler(struct work_struct *work)
         if (fences == 0 && handles == 0) {
                 if (adev->pm.dpm_enabled) {
                         amdgpu_dpm_enable_uvd(adev, false);
-                        /* just work around for uvd clock remain high even
-                         * when uvd dpm disabled on Polaris10 */
-                        if (adev->asic_type == CHIP_POLARIS10)
-                                amdgpu_asic_set_uvd_clocks(adev, 0, 0);
                 } else {
                         amdgpu_asic_set_uvd_clocks(adev, 0, 0);
                 }
         } else {
-                schedule_delayed_work(&adev->uvd.idle_work,
-                                      msecs_to_jiffies(UVD_IDLE_TIMEOUT_MS));
+                schedule_delayed_work(&adev->uvd.idle_work, UVD_IDLE_TIMEOUT);
         }
 }
 
-static void amdgpu_uvd_note_usage(struct amdgpu_device *adev)
+void amdgpu_uvd_ring_begin_use(struct amdgpu_ring *ring)
 {
+        struct amdgpu_device *adev = ring->adev;
         bool set_clocks = !cancel_delayed_work_sync(&adev->uvd.idle_work);
-        set_clocks &= schedule_delayed_work(&adev->uvd.idle_work,
-                                            msecs_to_jiffies(UVD_IDLE_TIMEOUT_MS));
 
         if (set_clocks) {
                 if (adev->pm.dpm_enabled) {
@@ -1133,3 +1146,48 @@ static void amdgpu_uvd_note_usage(struct amdgpu_device *adev)
                 }
         }
 }
+
+void amdgpu_uvd_ring_end_use(struct amdgpu_ring *ring)
+{
+        schedule_delayed_work(&ring->adev->uvd.idle_work, UVD_IDLE_TIMEOUT);
+}
+
+/**
+ * amdgpu_uvd_ring_test_ib - test ib execution
+ *
+ * @ring: amdgpu_ring pointer
+ *
+ * Test if we can successfully execute an IB
+ */
+int amdgpu_uvd_ring_test_ib(struct amdgpu_ring *ring, long timeout)
+{
+        struct fence *fence;
+        long r;
+
+        r = amdgpu_uvd_get_create_msg(ring, 1, NULL);
+        if (r) {
+                DRM_ERROR("amdgpu: failed to get create msg (%ld).\n", r);
+                goto error;
+        }
+
+        r = amdgpu_uvd_get_destroy_msg(ring, 1, true, &fence);
+        if (r) {
+                DRM_ERROR("amdgpu: failed to get destroy ib (%ld).\n", r);
+                goto error;
+        }
+
+        r = fence_wait_timeout(fence, false, timeout);
+        if (r == 0) {
+                DRM_ERROR("amdgpu: IB test timed out.\n");
+                r = -ETIMEDOUT;
+        } else if (r < 0) {
+                DRM_ERROR("amdgpu: fence wait failed (%ld).\n", r);
+        } else {
+                DRM_INFO("ib test on ring %d succeeded\n",  ring->idx);
+                r = 0;
+        }
+
+error:
+        fence_put(fence);
+        return r;
+}
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_uvd.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_uvd.h
index 9a3b449081a7..c850009602d1 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_uvd.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_uvd.h
@@ -35,5 +35,8 @@ int amdgpu_uvd_get_destroy_msg(struct amdgpu_ring *ring, uint32_t handle,
 void amdgpu_uvd_free_handles(struct amdgpu_device *adev,
                              struct drm_file *filp);
 int amdgpu_uvd_ring_parse_cs(struct amdgpu_cs_parser *parser, uint32_t ib_idx);
+void amdgpu_uvd_ring_begin_use(struct amdgpu_ring *ring);
+void amdgpu_uvd_ring_end_use(struct amdgpu_ring *ring);
+int amdgpu_uvd_ring_test_ib(struct amdgpu_ring *ring, long timeout);
 
 #endif
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vce.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vce.c
index 875626a2eccb..05865ce35351 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vce.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vce.c
@@ -36,7 +36,7 @@
 #include "cikd.h"
 
 /* 1 second timeout */
-#define VCE_IDLE_TIMEOUT_MS     1000
+#define VCE_IDLE_TIMEOUT        msecs_to_jiffies(1000)
 
 /* Firmware Names */
 #ifdef CONFIG_DRM_AMDGPU_CIK
@@ -85,8 +85,6 @@ int amdgpu_vce_sw_init(struct amdgpu_device *adev, unsigned long size)
         unsigned ucode_version, version_major, version_minor, binary_id;
         int i, r;
 
-        INIT_DELAYED_WORK(&adev->vce.idle_work, amdgpu_vce_idle_work_handler);
-
         switch (adev->asic_type) {
 #ifdef CONFIG_DRM_AMDGPU_CIK
         case CHIP_BONAIRE:
@@ -197,6 +195,9 @@ int amdgpu_vce_sw_init(struct amdgpu_device *adev, unsigned long size)
                 adev->vce.filp[i] = NULL;
         }
 
+        INIT_DELAYED_WORK(&adev->vce.idle_work, amdgpu_vce_idle_work_handler);
+        mutex_init(&adev->vce.idle_mutex);
+
         return 0;
 }
 
@@ -220,6 +221,7 @@ int amdgpu_vce_sw_fini(struct amdgpu_device *adev)
         amdgpu_ring_fini(&adev->vce.ring[1]);
 
         release_firmware(adev->vce.fw);
+        mutex_destroy(&adev->vce.idle_mutex);
 
         return 0;
 }
@@ -310,37 +312,44 @@ static void amdgpu_vce_idle_work_handler(struct work_struct *work)
                         amdgpu_asic_set_vce_clocks(adev, 0, 0);
                 }
         } else {
-                schedule_delayed_work(&adev->vce.idle_work,
-                                      msecs_to_jiffies(VCE_IDLE_TIMEOUT_MS));
+                schedule_delayed_work(&adev->vce.idle_work, VCE_IDLE_TIMEOUT);
         }
 }
 
 /**
- * amdgpu_vce_note_usage - power up VCE
+ * amdgpu_vce_ring_begin_use - power up VCE
  *
- * @adev: amdgpu_device pointer
+ * @ring: amdgpu ring
  *
  * Make sure VCE is powerd up when we want to use it
  */
-static void amdgpu_vce_note_usage(struct amdgpu_device *adev)
+void amdgpu_vce_ring_begin_use(struct amdgpu_ring *ring)
 {
-        bool streams_changed = false;
-        bool set_clocks = !cancel_delayed_work_sync(&adev->vce.idle_work);
-        set_clocks &= schedule_delayed_work(&adev->vce.idle_work,
-                                            msecs_to_jiffies(VCE_IDLE_TIMEOUT_MS));
-
-        if (adev->pm.dpm_enabled) {
-                /* XXX figure out if the streams changed */
-                streams_changed = false;
-        }
+        struct amdgpu_device *adev = ring->adev;
+        bool set_clocks;
 
-        if (set_clocks || streams_changed) {
+        mutex_lock(&adev->vce.idle_mutex);
+        set_clocks = !cancel_delayed_work_sync(&adev->vce.idle_work);
+        if (set_clocks) {
                 if (adev->pm.dpm_enabled) {
                         amdgpu_dpm_enable_vce(adev, true);
                 } else {
                         amdgpu_asic_set_vce_clocks(adev, 53300, 40000);
                 }
         }
+        mutex_unlock(&adev->vce.idle_mutex);
+}
+
+/**
+ * amdgpu_vce_ring_end_use - power VCE down
+ *
+ * @ring: amdgpu ring
+ *
+ * Schedule work to power VCE down again
+ */
+void amdgpu_vce_ring_end_use(struct amdgpu_ring *ring)
+{
+        schedule_delayed_work(&ring->adev->vce.idle_work, VCE_IDLE_TIMEOUT);
 }
 
 /**
@@ -357,11 +366,10 @@ void amdgpu_vce_free_handles(struct amdgpu_device *adev, struct drm_file *filp)
         int i, r;
         for (i = 0; i < AMDGPU_MAX_VCE_HANDLES; ++i) {
                 uint32_t handle = atomic_read(&adev->vce.handles[i]);
+
                 if (!handle || adev->vce.filp[i] != filp)
                         continue;
 
-                amdgpu_vce_note_usage(adev);
-
                 r = amdgpu_vce_get_destroy_msg(ring, handle, false, NULL);
                 if (r)
                         DRM_ERROR("Error destroying VCE handle (%d)!\n", r);
@@ -437,7 +445,7 @@ int amdgpu_vce_get_create_msg(struct amdgpu_ring *ring, uint32_t handle,
                 ib->ptr[i] = 0x0;
 
         r = amdgpu_ib_schedule(ring, 1, ib, NULL, NULL, &f);
-        job->fence = f;
+        job->fence = fence_get(f);
         if (r)
                 goto err;
 
@@ -469,7 +477,6 @@ int amdgpu_vce_get_destroy_msg(struct amdgpu_ring *ring, uint32_t handle,
         struct amdgpu_job *job;
         struct amdgpu_ib *ib;
         struct fence *f = NULL;
-        uint64_t dummy;
         int i, r;
 
         r = amdgpu_job_alloc_with_ib(ring->adev, ib_size_dw * 4, &job);
@@ -477,7 +484,6 @@ int amdgpu_vce_get_destroy_msg(struct amdgpu_ring *ring, uint32_t handle,
                 return r;
 
         ib = &job->ibs[0];
-        dummy = ib->gpu_addr + 1024;
 
         /* stitch together an VCE destroy msg */
         ib->length_dw = 0;
@@ -485,11 +491,14 @@ int amdgpu_vce_get_destroy_msg(struct amdgpu_ring *ring, uint32_t handle,
         ib->ptr[ib->length_dw++] = 0x00000001; /* session cmd */
         ib->ptr[ib->length_dw++] = handle;
 
-        ib->ptr[ib->length_dw++] = 0x00000014; /* len */
-        ib->ptr[ib->length_dw++] = 0x05000005; /* feedback buffer */
-        ib->ptr[ib->length_dw++] = upper_32_bits(dummy);
-        ib->ptr[ib->length_dw++] = dummy;
-        ib->ptr[ib->length_dw++] = 0x00000001;
+        ib->ptr[ib->length_dw++] = 0x00000020; /* len */
+        ib->ptr[ib->length_dw++] = 0x00000002; /* task info */
+        ib->ptr[ib->length_dw++] = 0xffffffff; /* next task info, set to 0xffffffff if no */
+        ib->ptr[ib->length_dw++] = 0x00000001; /* destroy session */
+        ib->ptr[ib->length_dw++] = 0x00000000;
+        ib->ptr[ib->length_dw++] = 0x00000000;
+        ib->ptr[ib->length_dw++] = 0xffffffff; /* feedback is not needed, set to 0xffffffff and firmware will not output feedback */
+        ib->ptr[ib->length_dw++] = 0x00000000;
 
         ib->ptr[ib->length_dw++] = 0x00000008; /* len */
         ib->ptr[ib->length_dw++] = 0x02000001; /* destroy cmd */
@@ -499,7 +508,7 @@ int amdgpu_vce_get_destroy_msg(struct amdgpu_ring *ring, uint32_t handle,
 
         if (direct) {
                 r = amdgpu_ib_schedule(ring, 1, ib, NULL, NULL, &f);
-                job->fence = f;
+                job->fence = fence_get(f);
                 if (r)
                         goto err;
 
@@ -580,12 +589,10 @@ static int amdgpu_vce_cs_reloc(struct amdgpu_cs_parser *p, uint32_t ib_idx,
  * we we don't have another free session index.
  */
 static int amdgpu_vce_validate_handle(struct amdgpu_cs_parser *p,
-                                      uint32_t handle, bool *allocated)
+                                      uint32_t handle, uint32_t *allocated)
 {
         unsigned i;
 
-        *allocated = false;
-
         /* validate the handle */
         for (i = 0; i < AMDGPU_MAX_VCE_HANDLES; ++i) {
                 if (atomic_read(&p->adev->vce.handles[i]) == handle) {
@@ -602,7 +609,7 @@ static int amdgpu_vce_validate_handle(struct amdgpu_cs_parser *p,
                 if (!atomic_cmpxchg(&p->adev->vce.handles[i], 0, handle)) {
                         p->adev->vce.filp[i] = p->filp;
                         p->adev->vce.img_size[i] = 0;
-                        *allocated = true;
+                        *allocated |= 1 << i;
                         return i;
                 }
         }
@@ -622,15 +629,13 @@ int amdgpu_vce_ring_parse_cs(struct amdgpu_cs_parser *p, uint32_t ib_idx)
         struct amdgpu_ib *ib = &p->job->ibs[ib_idx];
         unsigned fb_idx = 0, bs_idx = 0;
         int session_idx = -1;
-        bool destroyed = false;
-        bool created = false;
-        bool allocated = false;
+        uint32_t destroyed = 0;
+        uint32_t created = 0;
+        uint32_t allocated = 0;
         uint32_t tmp, handle = 0;
         uint32_t *size = &tmp;
         int i, r = 0, idx = 0;
 
-        amdgpu_vce_note_usage(p->adev);
-
         while (idx < ib->length_dw) {
                 uint32_t len = amdgpu_get_ib_value(p, ib_idx, idx);
                 uint32_t cmd = amdgpu_get_ib_value(p, ib_idx, idx + 1);
@@ -641,30 +646,30 @@ int amdgpu_vce_ring_parse_cs(struct amdgpu_cs_parser *p, uint32_t ib_idx)
                         goto out;
                 }
 
-                if (destroyed) {
-                        DRM_ERROR("No other command allowed after destroy!\n");
-                        r = -EINVAL;
-                        goto out;
-                }
-
                 switch (cmd) {
-                case 0x00000001: // session
+                case 0x00000001: /* session */
                         handle = amdgpu_get_ib_value(p, ib_idx, idx + 2);
                         session_idx = amdgpu_vce_validate_handle(p, handle,
                                                                  &allocated);
-                        if (session_idx < 0)
-                                return session_idx;
+                        if (session_idx < 0) {
+                                r = session_idx;
+                                goto out;
+                        }
                         size = &p->adev->vce.img_size[session_idx];
                         break;
 
-                case 0x00000002: // task info
+                case 0x00000002: /* task info */
                         fb_idx = amdgpu_get_ib_value(p, ib_idx, idx + 6);
                         bs_idx = amdgpu_get_ib_value(p, ib_idx, idx + 7);
                         break;
 
-                case 0x01000001: // create
-                        created = true;
-                        if (!allocated) {
+                case 0x01000001: /* create */
+                        created |= 1 << session_idx;
+                        if (destroyed & (1 << session_idx)) {
+                                destroyed &= ~(1 << session_idx);
+                                allocated |= 1 << session_idx;
+
+                        } else if (!(allocated & (1 << session_idx))) {
                                 DRM_ERROR("Handle already in use!\n");
                                 r = -EINVAL;
                                 goto out;
@@ -675,16 +680,16 @@ int amdgpu_vce_ring_parse_cs(struct amdgpu_cs_parser *p, uint32_t ib_idx)
                                 8 * 3 / 2;
                         break;
 
-                case 0x04000001: // config extension
-                case 0x04000002: // pic control
-                case 0x04000005: // rate control
-                case 0x04000007: // motion estimation
-                case 0x04000008: // rdo
-                case 0x04000009: // vui
-                case 0x05000002: // auxiliary buffer
+                case 0x04000001: /* config extension */
+                case 0x04000002: /* pic control */
+                case 0x04000005: /* rate control */
+                case 0x04000007: /* motion estimation */
+                case 0x04000008: /* rdo */
+                case 0x04000009: /* vui */
+                case 0x05000002: /* auxiliary buffer */
                         break;
 
-                case 0x03000001: // encode
+                case 0x03000001: /* encode */
                         r = amdgpu_vce_cs_reloc(p, ib_idx, idx + 10, idx + 9,
                                                 *size, 0);
                         if (r)
@@ -696,18 +701,18 @@ int amdgpu_vce_ring_parse_cs(struct amdgpu_cs_parser *p, uint32_t ib_idx)
                                 goto out;
                         break;
 
-                case 0x02000001: // destroy
-                        destroyed = true;
+                case 0x02000001: /* destroy */
+                        destroyed |= 1 << session_idx;
                         break;
 
-                case 0x05000001: // context buffer
+                case 0x05000001: /* context buffer */
                         r = amdgpu_vce_cs_reloc(p, ib_idx, idx + 3, idx + 2,
                                                 *size * 2, 0);
                         if (r)
                                 goto out;
                         break;
 
-                case 0x05000004: // video bitstream buffer
+                case 0x05000004: /* video bitstream buffer */
                         tmp = amdgpu_get_ib_value(p, ib_idx, idx + 4);
                         r = amdgpu_vce_cs_reloc(p, ib_idx, idx + 3, idx + 2,
                                                 tmp, bs_idx);
@@ -715,7 +720,7 @@ int amdgpu_vce_ring_parse_cs(struct amdgpu_cs_parser *p, uint32_t ib_idx)
                                 goto out;
                         break;
 
-                case 0x05000005: // feedback buffer
+                case 0x05000005: /* feedback buffer */
                         r = amdgpu_vce_cs_reloc(p, ib_idx, idx + 3, idx + 2,
                                                 4096, fb_idx);
                         if (r)
@@ -737,21 +742,24 @@ int amdgpu_vce_ring_parse_cs(struct amdgpu_cs_parser *p, uint32_t ib_idx)
                 idx += len / 4;
         }
 
-        if (allocated && !created) {
+        if (allocated & ~created) {
                 DRM_ERROR("New session without create command!\n");
                 r = -ENOENT;
         }
 
 out:
-        if ((!r && destroyed) || (r && allocated)) {
-                /*
-                 * IB contains a destroy msg or we have allocated an
-                 * handle and got an error, anyway free the handle
-                 */
-                for (i = 0; i < AMDGPU_MAX_VCE_HANDLES; ++i)
-                        atomic_cmpxchg(&p->adev->vce.handles[i], handle, 0);
+        if (!r) {
+                /* No error, free all destroyed handle slots */
+                tmp = destroyed;
+        } else {
+                /* Error during parsing, free all allocated handle slots */
+                tmp = allocated;
         }
 
+        for (i = 0; i < AMDGPU_MAX_VCE_HANDLES; ++i)
+                if (tmp & (1 << i))
+                        atomic_set(&p->adev->vce.handles[i], 0);
+
         return r;
 }
 
@@ -837,10 +845,10 @@ int amdgpu_vce_ring_test_ring(struct amdgpu_ring *ring)
  * @ring: the engine to test on
  *
  */
-int amdgpu_vce_ring_test_ib(struct amdgpu_ring *ring)
+int amdgpu_vce_ring_test_ib(struct amdgpu_ring *ring, long timeout)
 {
         struct fence *fence = NULL;
-        int r;
+        long r;
 
         /* skip vce ring1 ib test for now, since it's not reliable */
         if (ring == &ring->adev->vce.ring[1])
@@ -848,21 +856,25 @@ int amdgpu_vce_ring_test_ib(struct amdgpu_ring *ring)
 
         r = amdgpu_vce_get_create_msg(ring, 1, NULL);
         if (r) {
-                DRM_ERROR("amdgpu: failed to get create msg (%d).\n", r);
+                DRM_ERROR("amdgpu: failed to get create msg (%ld).\n", r);
                 goto error;
         }
 
         r = amdgpu_vce_get_destroy_msg(ring, 1, true, &fence);
         if (r) {
-                DRM_ERROR("amdgpu: failed to get destroy ib (%d).\n", r);
+                DRM_ERROR("amdgpu: failed to get destroy ib (%ld).\n", r);
                 goto error;
         }
 
-        r = fence_wait(fence, false);
-        if (r) {
-                DRM_ERROR("amdgpu: fence wait failed (%d).\n", r);
+        r = fence_wait_timeout(fence, false, timeout);
+        if (r == 0) {
+                DRM_ERROR("amdgpu: IB test timed out.\n");
+                r = -ETIMEDOUT;
+        } else if (r < 0) {
+                DRM_ERROR("amdgpu: fence wait failed (%ld).\n", r);
         } else {
                 DRM_INFO("ib test on ring %d succeeded\n", ring->idx);
+                r = 0;
         }
 error:
         fence_put(fence);
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vce.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_vce.h
index f40cf761c66f..63f83d0d985c 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vce.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vce.h
@@ -39,6 +39,8 @@ void amdgpu_vce_ring_emit_ib(struct amdgpu_ring *ring, struct amdgpu_ib *ib,
 void amdgpu_vce_ring_emit_fence(struct amdgpu_ring *ring, u64 addr, u64 seq,
                                 unsigned flags);
 int amdgpu_vce_ring_test_ring(struct amdgpu_ring *ring);
-int amdgpu_vce_ring_test_ib(struct amdgpu_ring *ring);
+int amdgpu_vce_ring_test_ib(struct amdgpu_ring *ring, long timeout);
+void amdgpu_vce_ring_begin_use(struct amdgpu_ring *ring);
+void amdgpu_vce_ring_end_use(struct amdgpu_ring *ring);
 
 #endif
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
index 9f36ed30ba11..8e642fc48df4 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
@@ -25,6 +25,7 @@
  *          Alex Deucher
  *          Jerome Glisse
  */
+#include <linux/fence-array.h>
 #include <drm/drmP.h>
 #include <drm/amdgpu_drm.h>
 #include "amdgpu.h"
@@ -114,16 +115,26 @@ void amdgpu_vm_get_pd_bo(struct amdgpu_vm *vm,
 /**
  * amdgpu_vm_get_bos - add the vm BOs to a duplicates list
  *
+ * @adev: amdgpu device pointer
  * @vm: vm providing the BOs
  * @duplicates: head of duplicates list
  *
  * Add the page directory to the BO duplicates list
  * for command submission.
  */
-void amdgpu_vm_get_pt_bos(struct amdgpu_vm *vm, struct list_head *duplicates)
+void amdgpu_vm_get_pt_bos(struct amdgpu_device *adev, struct amdgpu_vm *vm,
+                          struct list_head *duplicates)
 {
+        uint64_t num_evictions;
         unsigned i;
 
+        /* We only need to validate the page tables
+         * if they aren't already valid.
+         */
+        num_evictions = atomic64_read(&adev->num_evictions);
+        if (num_evictions == vm->last_eviction_counter)
+                return;
+
         /* add the vm page table to the list */
         for (i = 0; i <= vm->max_pde_used; ++i) {
                 struct amdgpu_bo_list_entry *entry = &vm->page_tables[i].entry;
@@ -162,6 +173,13 @@ void amdgpu_vm_move_pt_bos_in_lru(struct amdgpu_device *adev,
         spin_unlock(&glob->lru_lock);
 }
 
+static bool amdgpu_vm_is_gpu_reset(struct amdgpu_device *adev,
+                              struct amdgpu_vm_id *id)
+{
+        return id->current_gpu_reset_count !=
+                atomic_read(&adev->gpu_reset_counter) ? true : false;
+}
+
 /**
  * amdgpu_vm_grab_id - allocate the next free VMID
  *
@@ -174,18 +192,67 @@ void amdgpu_vm_move_pt_bos_in_lru(struct amdgpu_device *adev,
  */
 int amdgpu_vm_grab_id(struct amdgpu_vm *vm, struct amdgpu_ring *ring,
                       struct amdgpu_sync *sync, struct fence *fence,
-                      unsigned *vm_id, uint64_t *vm_pd_addr)
+                      struct amdgpu_job *job)
 {
-        uint64_t pd_addr = amdgpu_bo_gpu_offset(vm->page_directory);
         struct amdgpu_device *adev = ring->adev;
+        uint64_t fence_context = adev->fence_context + ring->idx;
         struct fence *updates = sync->last_vm_update;
-        struct amdgpu_vm_id *id;
-        unsigned i = ring->idx;
-        int r;
+        struct amdgpu_vm_id *id, *idle;
+        struct fence **fences;
+        unsigned i;
+        int r = 0;
+
+        fences = kmalloc_array(sizeof(void *), adev->vm_manager.num_ids,
+                               GFP_KERNEL);
+        if (!fences)
+                return -ENOMEM;
 
         mutex_lock(&adev->vm_manager.lock);
 
+        /* Check if we have an idle VMID */
+        i = 0;
+        list_for_each_entry(idle, &adev->vm_manager.ids_lru, list) {
+                fences[i] = amdgpu_sync_peek_fence(&idle->active, ring);
+                if (!fences[i])
+                        break;
+                ++i;
+        }
+
+        /* If we can't find a idle VMID to use, wait till one becomes available */
+        if (&idle->list == &adev->vm_manager.ids_lru) {
+                u64 fence_context = adev->vm_manager.fence_context + ring->idx;
+                unsigned seqno = ++adev->vm_manager.seqno[ring->idx];
+                struct fence_array *array;
+                unsigned j;
+
+                for (j = 0; j < i; ++j)
+                        fence_get(fences[j]);
+
+                array = fence_array_create(i, fences, fence_context,
+                                           seqno, true);
+                if (!array) {
+                        for (j = 0; j < i; ++j)
+                                fence_put(fences[j]);
+                        kfree(fences);
+                        r = -ENOMEM;
+                        goto error;
+                }
+
+
+                r = amdgpu_sync_fence(ring->adev, sync, &array->base);
+                fence_put(&array->base);
+                if (r)
+                        goto error;
+
+                mutex_unlock(&adev->vm_manager.lock);
+                return 0;
+
+        }
+        kfree(fences);
+
+        job->vm_needs_flush = true;
         /* Check if we can use a VMID already assigned to this VM */
+        i = ring->idx;
         do {
                 struct fence *flushed;
 
@@ -196,67 +263,52 @@ int amdgpu_vm_grab_id(struct amdgpu_vm *vm, struct amdgpu_ring *ring,
                 /* Check all the prerequisites to using this VMID */
                 if (!id)
                         continue;
+                if (amdgpu_vm_is_gpu_reset(adev, id))
+                        continue;
 
                 if (atomic64_read(&id->owner) != vm->client_id)
                         continue;
 
-                if (pd_addr != id->pd_gpu_addr)
+                if (job->vm_pd_addr != id->pd_gpu_addr)
                         continue;
 
-                if (id->last_user != ring &&
-                    (!id->last_flush || !fence_is_signaled(id->last_flush)))
+                if (!id->last_flush)
                         continue;
 
-                flushed  = id->flushed_updates;
-                if (updates && (!flushed || fence_is_later(updates, flushed)))
+                if (id->last_flush->context != fence_context &&
+                    !fence_is_signaled(id->last_flush))
                         continue;
 
-                /* Good we can use this VMID */
-                if (id->last_user == ring) {
-                        r = amdgpu_sync_fence(ring->adev, sync,
-                                              id->first);
-                        if (r)
-                                goto error;
-                }
+                flushed  = id->flushed_updates;
+                if (updates &&
+                    (!flushed || fence_is_later(updates, flushed)))
+                        continue;
 
-                /* And remember this submission as user of the VMID */
+                /* Good we can use this VMID. Remember this submission as
+                 * user of the VMID.
+                 */
                 r = amdgpu_sync_fence(ring->adev, &id->active, fence);
                 if (r)
                         goto error;
 
+                id->current_gpu_reset_count = atomic_read(&adev->gpu_reset_counter);
                 list_move_tail(&id->list, &adev->vm_manager.ids_lru);
                 vm->ids[ring->idx] = id;
 
-                *vm_id = id - adev->vm_manager.ids;
-                *vm_pd_addr = AMDGPU_VM_NO_FLUSH;
-                trace_amdgpu_vm_grab_id(vm, ring->idx, *vm_id, *vm_pd_addr);
+                job->vm_id = id - adev->vm_manager.ids;
+                job->vm_needs_flush = false;
+                trace_amdgpu_vm_grab_id(vm, ring->idx, job);
 
                 mutex_unlock(&adev->vm_manager.lock);
                 return 0;
 
         } while (i != ring->idx);
 
-        id = list_first_entry(&adev->vm_manager.ids_lru,
-                              struct amdgpu_vm_id,
-                              list);
-
-        if (!amdgpu_sync_is_idle(&id->active)) {
-                struct list_head *head = &adev->vm_manager.ids_lru;
-                struct amdgpu_vm_id *tmp;
+        /* Still no ID to use? Then use the idle one found earlier */
+        id = idle;
 
-                list_for_each_entry_safe(id, tmp, &adev->vm_manager.ids_lru,
-                                         list) {
-                        if (amdgpu_sync_is_idle(&id->active)) {
-                                list_move(&id->list, head);
-                                head = &id->list;
-                        }
-                }
-                id = list_first_entry(&adev->vm_manager.ids_lru,
-                                      struct amdgpu_vm_id,
-                                      list);
-        }
-
-        r = amdgpu_sync_cycle_fences(sync, &id->active, fence);
+        /* Remember this submission as user of the VMID */
+        r = amdgpu_sync_fence(ring->adev, &id->active, fence);
         if (r)
                 goto error;
 
@@ -269,22 +321,46 @@ int amdgpu_vm_grab_id(struct amdgpu_vm *vm, struct amdgpu_ring *ring,
         fence_put(id->flushed_updates);
         id->flushed_updates = fence_get(updates);
 
-        id->pd_gpu_addr = pd_addr;
-
+        id->pd_gpu_addr = job->vm_pd_addr;
+        id->current_gpu_reset_count = atomic_read(&adev->gpu_reset_counter);
         list_move_tail(&id->list, &adev->vm_manager.ids_lru);
-        id->last_user = ring;
         atomic64_set(&id->owner, vm->client_id);
         vm->ids[ring->idx] = id;
 
-        *vm_id = id - adev->vm_manager.ids;
-        *vm_pd_addr = pd_addr;
-        trace_amdgpu_vm_grab_id(vm, ring->idx, *vm_id, *vm_pd_addr);
+        job->vm_id = id - adev->vm_manager.ids;
+        trace_amdgpu_vm_grab_id(vm, ring->idx, job);
 
 error:
         mutex_unlock(&adev->vm_manager.lock);
         return r;
 }
 
+static bool amdgpu_vm_ring_has_compute_vm_bug(struct amdgpu_ring *ring)
+{
+        struct amdgpu_device *adev = ring->adev;
+        const struct amdgpu_ip_block_version *ip_block;
+
+        if (ring->type != AMDGPU_RING_TYPE_COMPUTE)
+                /* only compute rings */
+                return false;
+
+        ip_block = amdgpu_get_ip_block(adev, AMD_IP_BLOCK_TYPE_GFX);
+        if (!ip_block)
+                return false;
+
+        if (ip_block->major <= 7) {
+                /* gfx7 has no workaround */
+                return true;
+        } else if (ip_block->major == 8) {
+                if (adev->gfx.mec_fw_version >= 673)
+                        /* gfx8 is fixed in MEC firmware 673 */
+                        return false;
+                else
+                        return true;
+        }
+        return false;
+}
+
 /**
  * amdgpu_vm_flush - hardware flush the vm
  *
@@ -294,59 +370,52 @@ error:
  *
  * Emit a VM flush when it is necessary.
  */
-int amdgpu_vm_flush(struct amdgpu_ring *ring,
-                    unsigned vm_id, uint64_t pd_addr,
-                    uint32_t gds_base, uint32_t gds_size,
-                    uint32_t gws_base, uint32_t gws_size,
-                    uint32_t oa_base, uint32_t oa_size)
+int amdgpu_vm_flush(struct amdgpu_ring *ring, struct amdgpu_job *job)
 {
         struct amdgpu_device *adev = ring->adev;
-        struct amdgpu_vm_id *id = &adev->vm_manager.ids[vm_id];
+        struct amdgpu_vm_id *id = &adev->vm_manager.ids[job->vm_id];
         bool gds_switch_needed = ring->funcs->emit_gds_switch && (
-                id->gds_base != gds_base ||
-                id->gds_size != gds_size ||
-                id->gws_base != gws_base ||
-                id->gws_size != gws_size ||
-                id->oa_base != oa_base ||
-                id->oa_size != oa_size);
+                id->gds_base != job->gds_base ||
+                id->gds_size != job->gds_size ||
+                id->gws_base != job->gws_base ||
+                id->gws_size != job->gws_size ||
+                id->oa_base != job->oa_base ||
+                id->oa_size != job->oa_size);
         int r;
 
         if (ring->funcs->emit_pipeline_sync && (
-            pd_addr != AMDGPU_VM_NO_FLUSH || gds_switch_needed ||
-                    ring->type == AMDGPU_RING_TYPE_COMPUTE))
+            job->vm_needs_flush || gds_switch_needed ||
+            amdgpu_vm_ring_has_compute_vm_bug(ring)))
                 amdgpu_ring_emit_pipeline_sync(ring);
 
-        if (ring->funcs->emit_vm_flush &&
-            pd_addr != AMDGPU_VM_NO_FLUSH) {
+        if (ring->funcs->emit_vm_flush && (job->vm_needs_flush ||
+            amdgpu_vm_is_gpu_reset(adev, id))) {
                 struct fence *fence;
 
-                trace_amdgpu_vm_flush(pd_addr, ring->idx, vm_id);
-                amdgpu_ring_emit_vm_flush(ring, vm_id, pd_addr);
+                trace_amdgpu_vm_flush(job->vm_pd_addr, ring->idx, job->vm_id);
+                amdgpu_ring_emit_vm_flush(ring, job->vm_id, job->vm_pd_addr);
+
+                r = amdgpu_fence_emit(ring, &fence);
+                if (r)
+                        return r;
 
                 mutex_lock(&adev->vm_manager.lock);
-                if ((id->pd_gpu_addr == pd_addr) && (id->last_user == ring)) {
-                        r = amdgpu_fence_emit(ring, &fence);
-                        if (r) {
-                                mutex_unlock(&adev->vm_manager.lock);
-                                return r;
-                        }
-                        fence_put(id->last_flush);
-                        id->last_flush = fence;
-                }
+                fence_put(id->last_flush);
+                id->last_flush = fence;
                 mutex_unlock(&adev->vm_manager.lock);
         }
 
         if (gds_switch_needed) {
-                id->gds_base = gds_base;
-                id->gds_size = gds_size;
-                id->gws_base = gws_base;
-                id->gws_size = gws_size;
-                id->oa_base = oa_base;
-                id->oa_size = oa_size;
-                amdgpu_ring_emit_gds_switch(ring, vm_id,
-                                            gds_base, gds_size,
-                                            gws_base, gws_size,
-                                            oa_base, oa_size);
+                id->gds_base = job->gds_base;
+                id->gds_size = job->gds_size;
+                id->gws_base = job->gws_base;
+                id->gws_size = job->gws_size;
+                id->oa_base = job->oa_base;
+                id->oa_size = job->oa_size;
+                amdgpu_ring_emit_gds_switch(ring, job->vm_id,
+                                            job->gds_base, job->gds_size,
+                                            job->gws_base, job->gws_size,
+                                            job->oa_base, job->oa_size);
         }
 
         return 0;
@@ -723,7 +792,7 @@ static void amdgpu_vm_frag_ptes(struct amdgpu_device *adev,
  * @vm: requested vm
  * @start: start of GPU address range
  * @end: end of GPU address range
- * @dst: destination address to map to
+ * @dst: destination address to map to, the next dst inside the function
  * @flags: mapping flags
  *
  * Update the page tables in the range @start - @end.
@@ -737,49 +806,75 @@ static void amdgpu_vm_update_ptes(struct amdgpu_device *adev,
 {
         const uint64_t mask = AMDGPU_VM_PTE_COUNT - 1;
 
-        uint64_t last_pe_start = ~0, last_pe_end = ~0, last_dst = ~0;
-        uint64_t addr;
+        uint64_t cur_pe_start, cur_pe_end, cur_dst;
+        uint64_t addr; /* next GPU address to be updated */
+        uint64_t pt_idx;
+        struct amdgpu_bo *pt;
+        unsigned nptes; /* next number of ptes to be updated */
+        uint64_t next_pe_start;
+
+        /* initialize the variables */
+        addr = start;
+        pt_idx = addr >> amdgpu_vm_block_size;
+        pt = vm->page_tables[pt_idx].entry.robj;
+
+        if ((addr & ~mask) == (end & ~mask))
+                nptes = end - addr;
+        else
+                nptes = AMDGPU_VM_PTE_COUNT - (addr & mask);
+
+        cur_pe_start = amdgpu_bo_gpu_offset(pt);
+        cur_pe_start += (addr & mask) * 8;
+        cur_pe_end = cur_pe_start + 8 * nptes;
+        cur_dst = dst;
+
+        /* for next ptb*/
+        addr += nptes;
+        dst += nptes * AMDGPU_GPU_PAGE_SIZE;
 
         /* walk over the address space and update the page tables */
-        for (addr = start; addr < end; ) {
-                uint64_t pt_idx = addr >> amdgpu_vm_block_size;
-                struct amdgpu_bo *pt = vm->page_tables[pt_idx].entry.robj;
-                unsigned nptes;
-                uint64_t pe_start;
+        while (addr < end) {
+                pt_idx = addr >> amdgpu_vm_block_size;
+                pt = vm->page_tables[pt_idx].entry.robj;
 
                 if ((addr & ~mask) == (end & ~mask))
                         nptes = end - addr;
                 else
                         nptes = AMDGPU_VM_PTE_COUNT - (addr & mask);
 
-                pe_start = amdgpu_bo_gpu_offset(pt);
-                pe_start += (addr & mask) * 8;
-
-                if (last_pe_end != pe_start) {
+                next_pe_start = amdgpu_bo_gpu_offset(pt);
+                next_pe_start += (addr & mask) * 8;
 
+                if (cur_pe_end == next_pe_start) {
+                        /* The next ptb is consecutive to current ptb.
+                         * Don't call amdgpu_vm_frag_ptes now.
+                         * Will update two ptbs together in future.
+                        */
+                        cur_pe_end += 8 * nptes;
+                } else {
                         amdgpu_vm_frag_ptes(adev, vm_update_params,
-                                            last_pe_start, last_pe_end,
-                                            last_dst, flags);
+                                            cur_pe_start, cur_pe_end,
+                                            cur_dst, flags);
 
-                        last_pe_start = pe_start;
-                        last_pe_end = pe_start + 8 * nptes;
-                        last_dst = dst;
-                } else {
-                        last_pe_end += 8 * nptes;
+                        cur_pe_start = next_pe_start;
+                        cur_pe_end = next_pe_start + 8 * nptes;
+                        cur_dst = dst;
                 }
 
+                /* for next ptb*/
                 addr += nptes;
                 dst += nptes * AMDGPU_GPU_PAGE_SIZE;
         }
 
-        amdgpu_vm_frag_ptes(adev, vm_update_params, last_pe_start,
-                            last_pe_end, last_dst, flags);
+        amdgpu_vm_frag_ptes(adev, vm_update_params, cur_pe_start,
+                            cur_pe_end, cur_dst, flags);
 }
 
 /**
  * amdgpu_vm_bo_update_mapping - update a mapping in the vm page table
  *
  * @adev: amdgpu_device pointer
+ * @exclusive: fence we need to sync to
  * @src: address where to copy page table entries from
  * @pages_addr: DMA addresses to use for mapping
  * @vm: requested vm
@@ -793,6 +888,7 @@ static void amdgpu_vm_update_ptes(struct amdgpu_device *adev,
  * Returns 0 for success, -EINVAL for failure.
  */
 static int amdgpu_vm_bo_update_mapping(struct amdgpu_device *adev,
+                                       struct fence *exclusive,
                                        uint64_t src,
                                        dma_addr_t *pages_addr,
                                        struct amdgpu_vm *vm,
@@ -853,6 +949,10 @@ static int amdgpu_vm_bo_update_mapping(struct amdgpu_device *adev,
 
         vm_update_params.ib = &job->ibs[0];
 
+        r = amdgpu_sync_fence(adev, &job->sync, exclusive);
+        if (r)
+                goto error_free;
+
         r = amdgpu_sync_resv(adev, &job->sync, vm->page_directory->tbo.resv,
                              owner);
         if (r)
@@ -889,6 +989,7 @@ error_free:
  * amdgpu_vm_bo_split_mapping - split a mapping into smaller chunks
  *
  * @adev: amdgpu_device pointer
+ * @exclusive: fence we need to sync to
  * @gtt_flags: flags as they are used for GTT
  * @pages_addr: DMA addresses to use for mapping
  * @vm: requested vm
@@ -902,6 +1003,7 @@ error_free:
  * Returns 0 for success, -EINVAL for failure.
  */
 static int amdgpu_vm_bo_split_mapping(struct amdgpu_device *adev,
+                                      struct fence *exclusive,
                                       uint32_t gtt_flags,
                                       dma_addr_t *pages_addr,
                                       struct amdgpu_vm *vm,
@@ -932,7 +1034,8 @@ static int amdgpu_vm_bo_split_mapping(struct amdgpu_device *adev,
         addr += mapping->offset;
 
         if (!pages_addr || src)
-                return amdgpu_vm_bo_update_mapping(adev, src, pages_addr, vm,
+                return amdgpu_vm_bo_update_mapping(adev, exclusive,
+                                                   src, pages_addr, vm,
                                                    start, mapping->it.last,
                                                    flags, addr, fence);
 
@@ -940,7 +1043,8 @@ static int amdgpu_vm_bo_split_mapping(struct amdgpu_device *adev,
                 uint64_t last;
 
                 last = min((uint64_t)mapping->it.last, start + max_size - 1);
-                r = amdgpu_vm_bo_update_mapping(adev, src, pages_addr, vm,
+                r = amdgpu_vm_bo_update_mapping(adev, exclusive,
+                                                src, pages_addr, vm,
                                                 start, last, flags, addr,
                                                 fence);
                 if (r)
@@ -973,6 +1077,7 @@ int amdgpu_vm_bo_update(struct amdgpu_device *adev,
         struct amdgpu_bo_va_mapping *mapping;
         dma_addr_t *pages_addr = NULL;
         uint32_t gtt_flags, flags;
+        struct fence *exclusive;
         uint64_t addr;
         int r;
 
@@ -994,8 +1099,11 @@ int amdgpu_vm_bo_update(struct amdgpu_device *adev,
                 default:
                         break;
                 }
+
+                exclusive = reservation_object_get_excl(bo_va->bo->tbo.resv);
         } else {
                 addr = 0;
+                exclusive = NULL;
         }
 
         flags = amdgpu_ttm_tt_pte_flags(adev, bo_va->bo->tbo.ttm, mem);
@@ -1007,7 +1115,8 @@ int amdgpu_vm_bo_update(struct amdgpu_device *adev,
         spin_unlock(&vm->status_lock);
 
         list_for_each_entry(mapping, &bo_va->invalids, list) {
-                r = amdgpu_vm_bo_split_mapping(adev, gtt_flags, pages_addr, vm,
+                r = amdgpu_vm_bo_split_mapping(adev, exclusive,
+                                               gtt_flags, pages_addr, vm,
                                                mapping, flags, addr,
                                                &bo_va->last_pt_update);
                 if (r)
@@ -1054,7 +1163,7 @@ int amdgpu_vm_clear_freed(struct amdgpu_device *adev,
                         struct amdgpu_bo_va_mapping, list);
                 list_del(&mapping->list);
 
-                r = amdgpu_vm_bo_split_mapping(adev, 0, NULL, vm, mapping,
+                r = amdgpu_vm_bo_split_mapping(adev, NULL, 0, NULL, vm, mapping,
                                                0, 0, NULL);
                 kfree(mapping);
                 if (r)
@@ -1445,6 +1554,7 @@ int amdgpu_vm_init(struct amdgpu_device *adev, struct amdgpu_vm *vm)
         amdgpu_bo_unreserve(vm->page_directory);
         if (r)
                 goto error_free_page_directory;
+        vm->last_eviction_counter = atomic64_read(&adev->num_evictions);
 
         return 0;
 
@@ -1516,6 +1626,10 @@ void amdgpu_vm_manager_init(struct amdgpu_device *adev)
                               &adev->vm_manager.ids_lru);
         }
 
+        adev->vm_manager.fence_context = fence_context_alloc(AMDGPU_MAX_RINGS);
+        for (i = 0; i < AMDGPU_MAX_RINGS; ++i)
+                adev->vm_manager.seqno[i] = 0;
+
         atomic_set(&adev->vm_manager.vm_pte_next_ring, 0);
         atomic64_set(&adev->vm_manager.client_counter, 0);
 }
diff --git a/drivers/gpu/drm/amd/amdgpu/atombios_encoders.c b/drivers/gpu/drm/amd/amdgpu/atombios_encoders.c
index 48b6bd671cda..c32eca26155c 100644
--- a/drivers/gpu/drm/amd/amdgpu/atombios_encoders.c
+++ b/drivers/gpu/drm/amd/amdgpu/atombios_encoders.c
@@ -98,6 +98,7 @@ amdgpu_atombios_encoder_set_backlight_level(struct amdgpu_encoder *amdgpu_encode
                 case ENCODER_OBJECT_ID_INTERNAL_KLDSCP_LVTMA:
                 case ENCODER_OBJECT_ID_INTERNAL_UNIPHY1:
                 case ENCODER_OBJECT_ID_INTERNAL_UNIPHY2:
+                case ENCODER_OBJECT_ID_INTERNAL_UNIPHY3:
                         if (dig->backlight_level == 0)
                                 amdgpu_atombios_encoder_setup_dig_transmitter(encoder,
                                                                        ATOM_TRANSMITTER_ACTION_LCD_BLOFF, 0, 0);
diff --git a/drivers/gpu/drm/amd/amdgpu/ci_dpm.c b/drivers/gpu/drm/amd/amdgpu/ci_dpm.c
index 5ec1f1e9c983..e2f0e5d58d5c 100644
--- a/drivers/gpu/drm/amd/amdgpu/ci_dpm.c
+++ b/drivers/gpu/drm/amd/amdgpu/ci_dpm.c
@@ -50,7 +50,9 @@
 #include "gmc/gmc_7_1_sh_mask.h"
 
 MODULE_FIRMWARE("radeon/bonaire_smc.bin");
+MODULE_FIRMWARE("radeon/bonaire_k_smc.bin");
 MODULE_FIRMWARE("radeon/hawaii_smc.bin");
+MODULE_FIRMWARE("radeon/hawaii_k_smc.bin");
 
 #define MC_CG_ARB_FREQ_F0           0x0a
 #define MC_CG_ARB_FREQ_F1           0x0b
@@ -84,12 +86,14 @@ static const struct ci_pt_defaults defaults_bonaire_xt =
         { 0x17C, 0x172, 0x180, 0x1BC, 0x1B3, 0x1BD, 0x206, 0x200, 0x203, 0x25D, 0x25A, 0x255, 0x2C3, 0x2C5, 0x2B4 }
 };
 
+#if 0
 static const struct ci_pt_defaults defaults_bonaire_pro =
 {
         1, 0xF, 0xFD, 0x19, 5, 45, 0, 0x65062,
         { 0x8C,  0x23F, 0x244, 0xA6,  0x83,  0x85,  0x86,  0x86,  0x83,  0xDB,  0xDB,  0xDA,  0x67,  0x60,  0x5F  },
         { 0x187, 0x193, 0x193, 0x1C7, 0x1D1, 0x1D1, 0x210, 0x219, 0x219, 0x266, 0x26C, 0x26C, 0x2C9, 0x2CB, 0x2CB }
 };
+#endif
 
 static const struct ci_pt_defaults defaults_saturn_xt =
 {
@@ -98,12 +102,14 @@ static const struct ci_pt_defaults defaults_saturn_xt =
         { 0x187, 0x187, 0x187, 0x1C7, 0x1C7, 0x1C7, 0x210, 0x210, 0x210, 0x266, 0x266, 0x266, 0x2C9, 0x2C9, 0x2C9 }
 };
 
+#if 0
 static const struct ci_pt_defaults defaults_saturn_pro =
 {
         1, 0xF, 0xFD, 0x19, 5, 55, 0, 0x30000,
         { 0x96,  0x21D, 0x23B, 0xA1,  0x85,  0x87,  0x83,  0x84,  0x81,  0xE6,  0xE6,  0xE6,  0x71,  0x6A,  0x6A  },
         { 0x193, 0x19E, 0x19E, 0x1D2, 0x1DC, 0x1DC, 0x21A, 0x223, 0x223, 0x26E, 0x27E, 0x274, 0x2CF, 0x2D2, 0x2D2 }
 };
+#endif
 
 static const struct ci_pt_config_reg didt_config_ci[] =
 {
@@ -736,19 +742,19 @@ static int ci_enable_didt(struct amdgpu_device *adev, bool enable)
 
         if (pi->caps_sq_ramping || pi->caps_db_ramping ||
             pi->caps_td_ramping || pi->caps_tcp_ramping) {
-                gfx_v7_0_enter_rlc_safe_mode(adev);
+                adev->gfx.rlc.funcs->enter_safe_mode(adev);
 
                 if (enable) {
                         ret = ci_program_pt_config_registers(adev, didt_config_ci);
                         if (ret) {
-                                gfx_v7_0_exit_rlc_safe_mode(adev);
+                                adev->gfx.rlc.funcs->exit_safe_mode(adev);
                                 return ret;
                         }
                 }
 
                 ci_do_enable_didt(adev, enable);
 
-                gfx_v7_0_exit_rlc_safe_mode(adev);
+                adev->gfx.rlc.funcs->exit_safe_mode(adev);
         }
 
         return 0;
@@ -3030,7 +3036,7 @@ static int ci_populate_single_memory_level(struct amdgpu_device *adev,
 
         if (pi->mclk_stutter_mode_threshold &&
             (memory_clock <= pi->mclk_stutter_mode_threshold) &&
-            (pi->uvd_enabled == false) &&
+            (!pi->uvd_enabled) &&
             (RREG32(mmDPG_PIPE_STUTTER_CONTROL) & DPG_PIPE_STUTTER_CONTROL__STUTTER_ENABLE_MASK) &&
             (adev->pm.dpm.new_active_crtc_count <= 2))
                 memory_level->StutterEnable = true;
@@ -3636,6 +3642,10 @@ static int ci_setup_default_dpm_tables(struct amdgpu_device *adev)
 
         ci_setup_default_pcie_tables(adev);
 
+        /* save a copy of the default DPM table */
+        memcpy(&(pi->golden_dpm_table), &(pi->dpm_table),
+                        sizeof(struct ci_dpm_table));
+
         return 0;
 }
 
@@ -5754,10 +5764,18 @@ static int ci_dpm_init_microcode(struct amdgpu_device *adev)
 
         switch (adev->asic_type) {
         case CHIP_BONAIRE:
-                chip_name = "bonaire";
+                if ((adev->pdev->revision == 0x80) ||
+                    (adev->pdev->revision == 0x81) ||
+                    (adev->pdev->device == 0x665f))
+                        chip_name = "bonaire_k";
+                else
+                        chip_name = "bonaire";
                 break;
         case CHIP_HAWAII:
-                chip_name = "hawaii";
+                if (adev->pdev->revision == 0x80)
+                        chip_name = "hawaii_k";
+                else
+                        chip_name = "hawaii";
                 break;
         case CHIP_KAVERI:
         case CHIP_KABINI:
@@ -6404,6 +6422,186 @@ static int ci_dpm_set_powergating_state(void *handle,
         return 0;
 }
 
+static int ci_dpm_print_clock_levels(struct amdgpu_device *adev,
+                enum pp_clock_type type, char *buf)
+{
+        struct ci_power_info *pi = ci_get_pi(adev);
+        struct ci_single_dpm_table *sclk_table = &pi->dpm_table.sclk_table;
+        struct ci_single_dpm_table *mclk_table = &pi->dpm_table.mclk_table;
+        struct ci_single_dpm_table *pcie_table = &pi->dpm_table.pcie_speed_table;
+
+        int i, now, size = 0;
+        uint32_t clock, pcie_speed;
+
+        switch (type) {
+        case PP_SCLK:
+                amdgpu_ci_send_msg_to_smc(adev, PPSMC_MSG_API_GetSclkFrequency);
+                clock = RREG32(mmSMC_MSG_ARG_0);
+
+                for (i = 0; i < sclk_table->count; i++) {
+                        if (clock > sclk_table->dpm_levels[i].value)
+                                continue;
+                        break;
+                }
+                now = i;
+
+                for (i = 0; i < sclk_table->count; i++)
+                        size += sprintf(buf + size, "%d: %uMhz %s\n",
+                                        i, sclk_table->dpm_levels[i].value / 100,
+                                        (i == now) ? "*" : "");
+                break;
+        case PP_MCLK:
+                amdgpu_ci_send_msg_to_smc(adev, PPSMC_MSG_API_GetMclkFrequency);
+                clock = RREG32(mmSMC_MSG_ARG_0);
+
+                for (i = 0; i < mclk_table->count; i++) {
+                        if (clock > mclk_table->dpm_levels[i].value)
+                                continue;
+                        break;
+                }
+                now = i;
+
+                for (i = 0; i < mclk_table->count; i++)
+                        size += sprintf(buf + size, "%d: %uMhz %s\n",
+                                        i, mclk_table->dpm_levels[i].value / 100,
+                                        (i == now) ? "*" : "");
+                break;
+        case PP_PCIE:
+                pcie_speed = ci_get_current_pcie_speed(adev);
+                for (i = 0; i < pcie_table->count; i++) {
+                        if (pcie_speed != pcie_table->dpm_levels[i].value)
+                                continue;
+                        break;
+                }
+                now = i;
+
+                for (i = 0; i < pcie_table->count; i++)
+                        size += sprintf(buf + size, "%d: %s %s\n", i,
+                                        (pcie_table->dpm_levels[i].value == 0) ? "2.5GB, x1" :
+                                        (pcie_table->dpm_levels[i].value == 1) ? "5.0GB, x16" :
+                                        (pcie_table->dpm_levels[i].value == 2) ? "8.0GB, x16" : "",
+                                        (i == now) ? "*" : "");
+                break;
+        default:
+                break;
+        }
+
+        return size;
+}
+
+static int ci_dpm_force_clock_level(struct amdgpu_device *adev,
+                enum pp_clock_type type, uint32_t mask)
+{
+        struct ci_power_info *pi = ci_get_pi(adev);
+
+        if (adev->pm.dpm.forced_level
+                        != AMDGPU_DPM_FORCED_LEVEL_MANUAL)
+                return -EINVAL;
+
+        switch (type) {
+        case PP_SCLK:
+                if (!pi->sclk_dpm_key_disabled)
+                        amdgpu_ci_send_msg_to_smc_with_parameter(adev,
+                                        PPSMC_MSG_SCLKDPM_SetEnabledMask,
+                                        pi->dpm_level_enable_mask.sclk_dpm_enable_mask & mask);
+                break;
+
+        case PP_MCLK:
+                if (!pi->mclk_dpm_key_disabled)
+                        amdgpu_ci_send_msg_to_smc_with_parameter(adev,
+                                        PPSMC_MSG_MCLKDPM_SetEnabledMask,
+                                        pi->dpm_level_enable_mask.mclk_dpm_enable_mask & mask);
+                break;
+
+        case PP_PCIE:
+        {
+                uint32_t tmp = mask & pi->dpm_level_enable_mask.pcie_dpm_enable_mask;
+                uint32_t level = 0;
+
+                while (tmp >>= 1)
+                        level++;
+
+                if (!pi->pcie_dpm_key_disabled)
+                        amdgpu_ci_send_msg_to_smc_with_parameter(adev,
+                                        PPSMC_MSG_PCIeDPM_ForceLevel,
+                                        level);
+                break;
+        }
+        default:
+                break;
+        }
+
+        return 0;
+}
+
+static int ci_dpm_get_sclk_od(struct amdgpu_device *adev)
+{
+        struct ci_power_info *pi = ci_get_pi(adev);
+        struct ci_single_dpm_table *sclk_table = &(pi->dpm_table.sclk_table);
+        struct ci_single_dpm_table *golden_sclk_table =
+                        &(pi->golden_dpm_table.sclk_table);
+        int value;
+
+        value = (sclk_table->dpm_levels[sclk_table->count - 1].value -
+                        golden_sclk_table->dpm_levels[golden_sclk_table->count - 1].value) *
+                        100 /
+                        golden_sclk_table->dpm_levels[golden_sclk_table->count - 1].value;
+
+        return value;
+}
+
+static int ci_dpm_set_sclk_od(struct amdgpu_device *adev, uint32_t value)
+{
+        struct ci_power_info *pi = ci_get_pi(adev);
+        struct ci_ps *ps = ci_get_ps(adev->pm.dpm.requested_ps);
+        struct ci_single_dpm_table *golden_sclk_table =
+                        &(pi->golden_dpm_table.sclk_table);
+
+        if (value > 20)
+                value = 20;
+
+        ps->performance_levels[ps->performance_level_count - 1].sclk =
+                        golden_sclk_table->dpm_levels[golden_sclk_table->count - 1].value *
+                        value / 100 +
+                        golden_sclk_table->dpm_levels[golden_sclk_table->count - 1].value;
+
+        return 0;
+}
+
+static int ci_dpm_get_mclk_od(struct amdgpu_device *adev)
+{
+        struct ci_power_info *pi = ci_get_pi(adev);
+        struct ci_single_dpm_table *mclk_table = &(pi->dpm_table.mclk_table);
+        struct ci_single_dpm_table *golden_mclk_table =
+                        &(pi->golden_dpm_table.mclk_table);
+        int value;
+
+        value = (mclk_table->dpm_levels[mclk_table->count - 1].value -
+                        golden_mclk_table->dpm_levels[golden_mclk_table->count - 1].value) *
+                        100 /
+                        golden_mclk_table->dpm_levels[golden_mclk_table->count - 1].value;
+
+        return value;
+}
+
+static int ci_dpm_set_mclk_od(struct amdgpu_device *adev, uint32_t value)
+{
+        struct ci_power_info *pi = ci_get_pi(adev);
+        struct ci_ps *ps = ci_get_ps(adev->pm.dpm.requested_ps);
+        struct ci_single_dpm_table *golden_mclk_table =
+                        &(pi->golden_dpm_table.mclk_table);
+
+        if (value > 20)
+                value = 20;
+
+        ps->performance_levels[ps->performance_level_count - 1].mclk =
+                        golden_mclk_table->dpm_levels[golden_mclk_table->count - 1].value *
+                        value / 100 +
+                        golden_mclk_table->dpm_levels[golden_mclk_table->count - 1].value;
+
+        return 0;
+}
+
 const struct amd_ip_funcs ci_dpm_ip_funcs = {
         .name = "ci_dpm",
         .early_init = ci_dpm_early_init,
@@ -6438,6 +6636,12 @@ static const struct amdgpu_dpm_funcs ci_dpm_funcs = {
         .get_fan_control_mode = &ci_dpm_get_fan_control_mode,
         .set_fan_speed_percent = &ci_dpm_set_fan_speed_percent,
         .get_fan_speed_percent = &ci_dpm_get_fan_speed_percent,
+        .print_clock_levels = ci_dpm_print_clock_levels,
+        .force_clock_level = ci_dpm_force_clock_level,
+        .get_sclk_od = ci_dpm_get_sclk_od,
+        .set_sclk_od = ci_dpm_set_sclk_od,
+        .get_mclk_od = ci_dpm_get_mclk_od,
+        .set_mclk_od = ci_dpm_set_mclk_od,
 };
 
 static void ci_dpm_set_dpm_funcs(struct amdgpu_device *adev)
diff --git a/drivers/gpu/drm/amd/amdgpu/ci_dpm.h b/drivers/gpu/drm/amd/amdgpu/ci_dpm.h
index faccc30c93bf..91be2996ae7c 100644
--- a/drivers/gpu/drm/amd/amdgpu/ci_dpm.h
+++ b/drivers/gpu/drm/amd/amdgpu/ci_dpm.h
@@ -193,6 +193,7 @@ struct ci_pt_defaults {
 
 struct ci_power_info {
         struct ci_dpm_table dpm_table;
+        struct ci_dpm_table golden_dpm_table;
         u32 voltage_control;
         u32 mvdd_control;
         u32 vddci_control;
diff --git a/drivers/gpu/drm/amd/amdgpu/cik.c b/drivers/gpu/drm/amd/amdgpu/cik.c
index 910431808542..4efc901f658c 100644
--- a/drivers/gpu/drm/amd/amdgpu/cik.c
+++ b/drivers/gpu/drm/amd/amdgpu/cik.c
@@ -879,7 +879,7 @@ static void cik_vga_set_state(struct amdgpu_device *adev, bool state)
         uint32_t tmp;
 
         tmp = RREG32(mmCONFIG_CNTL);
-        if (state == false)
+        if (!state)
                 tmp |= CONFIG_CNTL__VGA_DIS_MASK;
         else
                 tmp &= ~CONFIG_CNTL__VGA_DIS_MASK;
@@ -1035,12 +1035,12 @@ static uint32_t cik_read_indexed_register(struct amdgpu_device *adev,
 
         mutex_lock(&adev->grbm_idx_mutex);
         if (se_num != 0xffffffff || sh_num != 0xffffffff)
-                gfx_v7_0_select_se_sh(adev, se_num, sh_num);
+                amdgpu_gfx_select_se_sh(adev, se_num, sh_num, 0xffffffff);
 
         val = RREG32(reg_offset);
 
         if (se_num != 0xffffffff || sh_num != 0xffffffff)
-                gfx_v7_0_select_se_sh(adev, 0xffffffff, 0xffffffff);
+                amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
         mutex_unlock(&adev->grbm_idx_mutex);
         return val;
 }
@@ -1158,10 +1158,11 @@ static void kv_restore_regs_for_reset(struct amdgpu_device *adev,
         WREG32(mmGMCON_RENG_EXECUTE, save->gmcon_reng_execute);
 }
 
-static void cik_gpu_pci_config_reset(struct amdgpu_device *adev)
+static int cik_gpu_pci_config_reset(struct amdgpu_device *adev)
 {
         struct kv_reset_save_regs kv_save = { 0 };
         u32 i;
+        int r = -EINVAL;
 
         dev_info(adev->dev, "GPU pci config reset\n");
 
@@ -1177,14 +1178,20 @@ static void cik_gpu_pci_config_reset(struct amdgpu_device *adev)
 
         /* wait for asic to come out of reset */
         for (i = 0; i < adev->usec_timeout; i++) {
-                if (RREG32(mmCONFIG_MEMSIZE) != 0xffffffff)
+                if (RREG32(mmCONFIG_MEMSIZE) != 0xffffffff) {
+                        /* enable BM */
+                        pci_set_master(adev->pdev);
+                        r = 0;
                         break;
+                }
                 udelay(1);
         }
 
         /* does asic init need to be run first??? */
         if (adev->flags & AMD_IS_APU)
                 kv_restore_regs_for_reset(adev, &kv_save);
+
+        return r;
 }
 
 static void cik_set_bios_scratch_engine_hung(struct amdgpu_device *adev, bool hung)
@@ -1210,13 +1217,14 @@ static void cik_set_bios_scratch_engine_hung(struct amdgpu_device *adev, bool hu
  */
 static int cik_asic_reset(struct amdgpu_device *adev)
 {
+        int r;
         cik_set_bios_scratch_engine_hung(adev, true);
 
-        cik_gpu_pci_config_reset(adev);
+        r = cik_gpu_pci_config_reset(adev);
 
         cik_set_bios_scratch_engine_hung(adev, false);
 
-        return 0;
+        return r;
 }
 
 static int cik_set_uvd_clock(struct amdgpu_device *adev, u32 clock,
@@ -2014,9 +2022,6 @@ static const struct amdgpu_asic_funcs cik_asic_funcs =
         .set_uvd_clocks = &cik_set_uvd_clocks,
         .set_vce_clocks = &cik_set_vce_clocks,
         .get_virtual_caps = &cik_get_virtual_caps,
-        /* these should be moved to their own ip modules */
-        .get_gpu_clock_counter = &gfx_v7_0_get_gpu_clock_counter,
-        .wait_for_mc_idle = &gmc_v7_0_mc_wait_for_idle,
 };
 
 static int cik_common_early_init(void *handle)
diff --git a/drivers/gpu/drm/amd/amdgpu/cik_sdma.c b/drivers/gpu/drm/amd/amdgpu/cik_sdma.c
index 9dc4e24e31e7..ee6466912497 100644
--- a/drivers/gpu/drm/amd/amdgpu/cik_sdma.c
+++ b/drivers/gpu/drm/amd/amdgpu/cik_sdma.c
@@ -224,17 +224,6 @@ static void cik_sdma_ring_emit_ib(struct amdgpu_ring *ring,
                                   unsigned vm_id, bool ctx_switch)
 {
         u32 extra_bits = vm_id & 0xf;
-        u32 next_rptr = ring->wptr + 5;
-
-        while ((next_rptr & 7) != 4)
-                next_rptr++;
-
-        next_rptr += 4;
-        amdgpu_ring_write(ring, SDMA_PACKET(SDMA_OPCODE_WRITE, SDMA_WRITE_SUB_OPCODE_LINEAR, 0));
-        amdgpu_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc);
-        amdgpu_ring_write(ring, upper_32_bits(ring->next_rptr_gpu_addr) & 0xffffffff);
-        amdgpu_ring_write(ring, 1); /* number of DWs to follow */
-        amdgpu_ring_write(ring, next_rptr);
 
         /* IB packet must end on a 8 DW boundary */
         cik_sdma_ring_insert_nop(ring, (12 - (ring->wptr & 7)) % 8);
@@ -365,7 +354,7 @@ static void cik_sdma_enable(struct amdgpu_device *adev, bool enable)
         u32 me_cntl;
         int i;
 
-        if (enable == false) {
+        if (!enable) {
                 cik_sdma_gfx_stop(adev);
                 cik_sdma_rlc_stop(adev);
         }
@@ -628,20 +617,19 @@ static int cik_sdma_ring_test_ring(struct amdgpu_ring *ring)
  * Test a simple IB in the DMA ring (CIK).
  * Returns 0 on success, error on failure.
  */
-static int cik_sdma_ring_test_ib(struct amdgpu_ring *ring)
+static int cik_sdma_ring_test_ib(struct amdgpu_ring *ring, long timeout)
 {
         struct amdgpu_device *adev = ring->adev;
         struct amdgpu_ib ib;
         struct fence *f = NULL;
-        unsigned i;
         unsigned index;
-        int r;
         u32 tmp = 0;
         u64 gpu_addr;
+        long r;
 
         r = amdgpu_wb_get(adev, &index);
         if (r) {
-                dev_err(adev->dev, "(%d) failed to allocate wb slot\n", r);
+                dev_err(adev->dev, "(%ld) failed to allocate wb slot\n", r);
                 return r;
         }
 
@@ -651,11 +639,12 @@ static int cik_sdma_ring_test_ib(struct amdgpu_ring *ring)
         memset(&ib, 0, sizeof(ib));
         r = amdgpu_ib_get(adev, NULL, 256, &ib);
         if (r) {
-                DRM_ERROR("amdgpu: failed to get ib (%d).\n", r);
+                DRM_ERROR("amdgpu: failed to get ib (%ld).\n", r);
                 goto err0;
         }
 
-        ib.ptr[0] = SDMA_PACKET(SDMA_OPCODE_WRITE, SDMA_WRITE_SUB_OPCODE_LINEAR, 0);
+        ib.ptr[0] = SDMA_PACKET(SDMA_OPCODE_WRITE,
+                                SDMA_WRITE_SUB_OPCODE_LINEAR, 0);
         ib.ptr[1] = lower_32_bits(gpu_addr);
         ib.ptr[2] = upper_32_bits(gpu_addr);
         ib.ptr[3] = 1;
@@ -665,28 +654,25 @@ static int cik_sdma_ring_test_ib(struct amdgpu_ring *ring)
         if (r)
                 goto err1;
 
-        r = fence_wait(f, false);
-        if (r) {
-                DRM_ERROR("amdgpu: fence wait failed (%d).\n", r);
+        r = fence_wait_timeout(f, false, timeout);
+        if (r == 0) {
+                DRM_ERROR("amdgpu: IB test timed out\n");
+                r = -ETIMEDOUT;
                 goto err1;
-        }
-        for (i = 0; i < adev->usec_timeout; i++) {
-                tmp = le32_to_cpu(adev->wb.wb[index]);
-                if (tmp == 0xDEADBEEF)
-                        break;
-                DRM_UDELAY(1);
-        }
-        if (i < adev->usec_timeout) {
-                DRM_INFO("ib test on ring %d succeeded in %u usecs\n",
-                         ring->idx, i);
+        } else if (r < 0) {
+                DRM_ERROR("amdgpu: fence wait failed (%ld).\n", r);
                 goto err1;
+        }
+        tmp = le32_to_cpu(adev->wb.wb[index]);
+        if (tmp == 0xDEADBEEF) {
+                DRM_INFO("ib test on ring %d succeeded\n", ring->idx);
+                r = 0;
         } else {
                 DRM_ERROR("amdgpu: ib test failed (0x%08X)\n", tmp);
                 r = -EINVAL;
         }
 
 err1:
-        fence_put(f);
         amdgpu_ib_free(adev, &ib, NULL);
         fence_put(f);
 err0:
diff --git a/drivers/gpu/drm/amd/amdgpu/cz_dpm.c b/drivers/gpu/drm/amd/amdgpu/cz_dpm.c
index 933e425a8154..2a11413ed54a 100644
--- a/drivers/gpu/drm/amd/amdgpu/cz_dpm.c
+++ b/drivers/gpu/drm/amd/amdgpu/cz_dpm.c
@@ -425,7 +425,7 @@ static int cz_dpm_init(struct amdgpu_device *adev)
         pi->mgcg_cgtt_local1 = 0x0;
         pi->clock_slow_down_step = 25000;
         pi->skip_clock_slow_down = 1;
-        pi->enable_nb_ps_policy = 0;
+        pi->enable_nb_ps_policy = false;
         pi->caps_power_containment = true;
         pi->caps_cac = true;
         pi->didt_enabled = false;
@@ -2219,6 +2219,7 @@ static void cz_dpm_powergate_vce(struct amdgpu_device *adev, bool gate)
                         }
                 }
         } else { /*pi->caps_vce_pg*/
+                pi->vce_power_gated = gate;
                 cz_update_vce_dpm(adev);
                 cz_enable_vce_dpm(adev, !gate);
         }
diff --git a/drivers/gpu/drm/amd/amdgpu/dce_v10_0.c b/drivers/gpu/drm/amd/amdgpu/dce_v10_0.c
index 8227344d2ff6..c1b04e9aab57 100644
--- a/drivers/gpu/drm/amd/amdgpu/dce_v10_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/dce_v10_0.c
@@ -2667,19 +2667,21 @@ static void dce_v10_0_cursor_reset(struct drm_crtc *crtc)
         }
 }
 
-static void dce_v10_0_crtc_gamma_set(struct drm_crtc *crtc, u16 *red, u16 *green,
-                                    u16 *blue, uint32_t start, uint32_t size)
+static int dce_v10_0_crtc_gamma_set(struct drm_crtc *crtc, u16 *red, u16 *green,
+                                    u16 *blue, uint32_t size)
 {
         struct amdgpu_crtc *amdgpu_crtc = to_amdgpu_crtc(crtc);
-        int end = (start + size > 256) ? 256 : start + size, i;
+        int i;
 
         /* userspace palettes are always correct as is */
-        for (i = start; i < end; i++) {
+        for (i = 0; i < size; i++) {
                 amdgpu_crtc->lut_r[i] = red[i] >> 6;
                 amdgpu_crtc->lut_g[i] = green[i] >> 6;
                 amdgpu_crtc->lut_b[i] = blue[i] >> 6;
         }
         dce_v10_0_crtc_load_lut(crtc);
+
+        return 0;
 }
 
 static void dce_v10_0_crtc_destroy(struct drm_crtc *crtc)
@@ -2717,13 +2719,13 @@ static void dce_v10_0_crtc_dpms(struct drm_crtc *crtc, int mode)
                 type = amdgpu_crtc_idx_to_irq_type(adev, amdgpu_crtc->crtc_id);
                 amdgpu_irq_update(adev, &adev->crtc_irq, type);
                 amdgpu_irq_update(adev, &adev->pageflip_irq, type);
-                drm_vblank_on(dev, amdgpu_crtc->crtc_id);
+                drm_crtc_vblank_on(crtc);
                 dce_v10_0_crtc_load_lut(crtc);
                 break;
         case DRM_MODE_DPMS_STANDBY:
         case DRM_MODE_DPMS_SUSPEND:
         case DRM_MODE_DPMS_OFF:
-                drm_vblank_off(dev, amdgpu_crtc->crtc_id);
+                drm_crtc_vblank_off(crtc);
                 if (amdgpu_crtc->enabled) {
                         dce_v10_0_vga_enable(crtc, true);
                         amdgpu_atombios_crtc_blank(crtc, ATOM_ENABLE);
@@ -3372,7 +3374,7 @@ static int dce_v10_0_pageflip_irq(struct amdgpu_device *adev,
 
         spin_unlock_irqrestore(&adev->ddev->event_lock, flags);
 
-        drm_vblank_put(adev->ddev, amdgpu_crtc->crtc_id);
+        drm_crtc_vblank_put(&amdgpu_crtc->base);
         schedule_work(&works->unpin_work);
 
         return 0;
diff --git a/drivers/gpu/drm/amd/amdgpu/dce_v11_0.c b/drivers/gpu/drm/amd/amdgpu/dce_v11_0.c
index af26ec0bc59d..d4bf133908b1 100644
--- a/drivers/gpu/drm/amd/amdgpu/dce_v11_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/dce_v11_0.c
@@ -307,11 +307,10 @@ static void dce_v11_0_page_flip(struct amdgpu_device *adev,
         struct amdgpu_crtc *amdgpu_crtc = adev->mode_info.crtcs[crtc_id];
         u32 tmp;
 
-        /* flip at hsync for async, default is vsync */
-        /* use UPDATE_IMMEDIATE_EN instead for async? */
+        /* flip immediate for async, default is vsync */
         tmp = RREG32(mmGRPH_FLIP_CONTROL + amdgpu_crtc->crtc_offset);
         tmp = REG_SET_FIELD(tmp, GRPH_FLIP_CONTROL,
-                            GRPH_SURFACE_UPDATE_H_RETRACE_EN, async ? 1 : 0);
+                            GRPH_SURFACE_UPDATE_IMMEDIATE_EN, async ? 1 : 0);
         WREG32(mmGRPH_FLIP_CONTROL + amdgpu_crtc->crtc_offset, tmp);
         /* update the scanout addresses */
         WREG32(mmGRPH_PRIMARY_SURFACE_ADDRESS_HIGH + amdgpu_crtc->crtc_offset,
@@ -2678,19 +2677,21 @@ static void dce_v11_0_cursor_reset(struct drm_crtc *crtc)
         }
 }
 
-static void dce_v11_0_crtc_gamma_set(struct drm_crtc *crtc, u16 *red, u16 *green,
-                                    u16 *blue, uint32_t start, uint32_t size)
+static int dce_v11_0_crtc_gamma_set(struct drm_crtc *crtc, u16 *red, u16 *green,
+                                    u16 *blue, uint32_t size)
 {
         struct amdgpu_crtc *amdgpu_crtc = to_amdgpu_crtc(crtc);
-        int end = (start + size > 256) ? 256 : start + size, i;
+        int i;
 
         /* userspace palettes are always correct as is */
-        for (i = start; i < end; i++) {
+        for (i = 0; i < size; i++) {
                 amdgpu_crtc->lut_r[i] = red[i] >> 6;
                 amdgpu_crtc->lut_g[i] = green[i] >> 6;
                 amdgpu_crtc->lut_b[i] = blue[i] >> 6;
         }
         dce_v11_0_crtc_load_lut(crtc);
+
+        return 0;
 }
 
 static void dce_v11_0_crtc_destroy(struct drm_crtc *crtc)
@@ -2728,13 +2729,13 @@ static void dce_v11_0_crtc_dpms(struct drm_crtc *crtc, int mode)
                 type = amdgpu_crtc_idx_to_irq_type(adev, amdgpu_crtc->crtc_id);
                 amdgpu_irq_update(adev, &adev->crtc_irq, type);
                 amdgpu_irq_update(adev, &adev->pageflip_irq, type);
-                drm_vblank_on(dev, amdgpu_crtc->crtc_id);
+                drm_crtc_vblank_on(crtc);
                 dce_v11_0_crtc_load_lut(crtc);
                 break;
         case DRM_MODE_DPMS_STANDBY:
         case DRM_MODE_DPMS_SUSPEND:
         case DRM_MODE_DPMS_OFF:
-                drm_vblank_off(dev, amdgpu_crtc->crtc_id);
+                drm_crtc_vblank_off(crtc);
                 if (amdgpu_crtc->enabled) {
                         dce_v11_0_vga_enable(crtc, true);
                         amdgpu_atombios_crtc_blank(crtc, ATOM_ENABLE);
@@ -3433,7 +3434,7 @@ static int dce_v11_0_pageflip_irq(struct amdgpu_device *adev,
 
         spin_unlock_irqrestore(&adev->ddev->event_lock, flags);
 
-        drm_vblank_put(adev->ddev, amdgpu_crtc->crtc_id);
+        drm_crtc_vblank_put(&amdgpu_crtc->base);
         schedule_work(&works->unpin_work);
 
         return 0;
diff --git a/drivers/gpu/drm/amd/amdgpu/dce_v8_0.c b/drivers/gpu/drm/amd/amdgpu/dce_v8_0.c
index 3fb65e41a6ef..4fdfab1e9200 100644
--- a/drivers/gpu/drm/amd/amdgpu/dce_v8_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/dce_v8_0.c
@@ -526,36 +526,16 @@ static void dce_v8_0_stop_mc_access(struct amdgpu_device *adev,
                 crtc_enabled = REG_GET_FIELD(RREG32(mmCRTC_CONTROL + crtc_offsets[i]),
                                              CRTC_CONTROL, CRTC_MASTER_EN);
                 if (crtc_enabled) {
-#if 0
-                        u32 frame_count;
-                        int j;
-
+#if 1
                         save->crtc_enabled[i] = true;
                         tmp = RREG32(mmCRTC_BLANK_CONTROL + crtc_offsets[i]);
                         if (REG_GET_FIELD(tmp, CRTC_BLANK_CONTROL, CRTC_BLANK_DATA_EN) == 0) {
-                                amdgpu_display_vblank_wait(adev, i);
-                                WREG32(mmCRTC_UPDATE_LOCK + crtc_offsets[i], 1);
+                                /*it is correct only for RGB ; black is 0*/
+                                WREG32(mmCRTC_BLANK_DATA_COLOR + crtc_offsets[i], 0);
                                 tmp = REG_SET_FIELD(tmp, CRTC_BLANK_CONTROL, CRTC_BLANK_DATA_EN, 1);
                                 WREG32(mmCRTC_BLANK_CONTROL + crtc_offsets[i], tmp);
-                                WREG32(mmCRTC_UPDATE_LOCK + crtc_offsets[i], 0);
-                        }
-                        /* wait for the next frame */
-                        frame_count = amdgpu_display_vblank_get_counter(adev, i);
-                        for (j = 0; j < adev->usec_timeout; j++) {
-                                if (amdgpu_display_vblank_get_counter(adev, i) != frame_count)
-                                        break;
-                                udelay(1);
-                        }
-                        tmp = RREG32(mmGRPH_UPDATE + crtc_offsets[i]);
-                        if (REG_GET_FIELD(tmp, GRPH_UPDATE, GRPH_UPDATE_LOCK) == 0) {
-                                tmp = REG_SET_FIELD(tmp, GRPH_UPDATE, GRPH_UPDATE_LOCK, 1);
-                                WREG32(mmGRPH_UPDATE + crtc_offsets[i], tmp);
-                        }
-                        tmp = RREG32(mmMASTER_UPDATE_LOCK + crtc_offsets[i]);
-                        if (REG_GET_FIELD(tmp, MASTER_UPDATE_LOCK, MASTER_UPDATE_LOCK) == 0) {
-                                tmp = REG_SET_FIELD(tmp, MASTER_UPDATE_LOCK, MASTER_UPDATE_LOCK, 1);
-                                WREG32(mmMASTER_UPDATE_LOCK + crtc_offsets[i], tmp);
                         }
+                        mdelay(20);
 #else
                         /* XXX this is a hack to avoid strange behavior with EFI on certain systems */
                         WREG32(mmCRTC_UPDATE_LOCK + crtc_offsets[i], 1);
@@ -575,55 +555,22 @@ static void dce_v8_0_stop_mc_access(struct amdgpu_device *adev,
 static void dce_v8_0_resume_mc_access(struct amdgpu_device *adev,
                                       struct amdgpu_mode_mc_save *save)
 {
-        u32 tmp, frame_count;
-        int i, j;
+        u32 tmp;
+        int i;
 
         /* update crtc base addresses */
         for (i = 0; i < adev->mode_info.num_crtc; i++) {
                 WREG32(mmGRPH_PRIMARY_SURFACE_ADDRESS_HIGH + crtc_offsets[i],
                        upper_32_bits(adev->mc.vram_start));
-                WREG32(mmGRPH_SECONDARY_SURFACE_ADDRESS_HIGH + crtc_offsets[i],
-                       upper_32_bits(adev->mc.vram_start));
                 WREG32(mmGRPH_PRIMARY_SURFACE_ADDRESS + crtc_offsets[i],
                        (u32)adev->mc.vram_start);
-                WREG32(mmGRPH_SECONDARY_SURFACE_ADDRESS + crtc_offsets[i],
-                       (u32)adev->mc.vram_start);
 
                 if (save->crtc_enabled[i]) {
-                        tmp = RREG32(mmMASTER_UPDATE_MODE + crtc_offsets[i]);
-                        if (REG_GET_FIELD(tmp, MASTER_UPDATE_MODE, MASTER_UPDATE_MODE) != 3) {
-                                tmp = REG_SET_FIELD(tmp, MASTER_UPDATE_MODE, MASTER_UPDATE_MODE, 3);
-                                WREG32(mmMASTER_UPDATE_MODE + crtc_offsets[i], tmp);
-                        }
-                        tmp = RREG32(mmGRPH_UPDATE + crtc_offsets[i]);
-                        if (REG_GET_FIELD(tmp, GRPH_UPDATE, GRPH_UPDATE_LOCK)) {
-                                tmp = REG_SET_FIELD(tmp, GRPH_UPDATE, GRPH_UPDATE_LOCK, 0);
-                                WREG32(mmGRPH_UPDATE + crtc_offsets[i], tmp);
-                        }
-                        tmp = RREG32(mmMASTER_UPDATE_LOCK + crtc_offsets[i]);
-                        if (REG_GET_FIELD(tmp, MASTER_UPDATE_LOCK, MASTER_UPDATE_LOCK)) {
-                                tmp = REG_SET_FIELD(tmp, MASTER_UPDATE_LOCK, MASTER_UPDATE_LOCK, 0);
-                                WREG32(mmMASTER_UPDATE_LOCK + crtc_offsets[i], tmp);
-                        }
-                        for (j = 0; j < adev->usec_timeout; j++) {
-                                tmp = RREG32(mmGRPH_UPDATE + crtc_offsets[i]);
-                                if (REG_GET_FIELD(tmp, GRPH_UPDATE, GRPH_SURFACE_UPDATE_PENDING) == 0)
-                                        break;
-                                udelay(1);
-                        }
                         tmp = RREG32(mmCRTC_BLANK_CONTROL + crtc_offsets[i]);
                         tmp = REG_SET_FIELD(tmp, CRTC_BLANK_CONTROL, CRTC_BLANK_DATA_EN, 0);
-                        WREG32(mmCRTC_UPDATE_LOCK + crtc_offsets[i], 1);
                         WREG32(mmCRTC_BLANK_CONTROL + crtc_offsets[i], tmp);
-                        WREG32(mmCRTC_UPDATE_LOCK + crtc_offsets[i], 0);
-                        /* wait for the next frame */
-                        frame_count = amdgpu_display_vblank_get_counter(adev, i);
-                        for (j = 0; j < adev->usec_timeout; j++) {
-                                if (amdgpu_display_vblank_get_counter(adev, i) != frame_count)
-                                        break;
-                                udelay(1);
-                        }
                 }
+                mdelay(20);
         }
 
         WREG32(mmVGA_MEMORY_BASE_ADDRESS_HIGH, upper_32_bits(adev->mc.vram_start));
@@ -2574,19 +2521,21 @@ static void dce_v8_0_cursor_reset(struct drm_crtc *crtc)
         }
 }
 
-static void dce_v8_0_crtc_gamma_set(struct drm_crtc *crtc, u16 *red, u16 *green,
-                                    u16 *blue, uint32_t start, uint32_t size)
+static int dce_v8_0_crtc_gamma_set(struct drm_crtc *crtc, u16 *red, u16 *green,
+                                   u16 *blue, uint32_t size)
 {
         struct amdgpu_crtc *amdgpu_crtc = to_amdgpu_crtc(crtc);
-        int end = (start + size > 256) ? 256 : start + size, i;
+        int i;
 
         /* userspace palettes are always correct as is */
-        for (i = start; i < end; i++) {
+        for (i = 0; i < size; i++) {
                 amdgpu_crtc->lut_r[i] = red[i] >> 6;
                 amdgpu_crtc->lut_g[i] = green[i] >> 6;
                 amdgpu_crtc->lut_b[i] = blue[i] >> 6;
         }
         dce_v8_0_crtc_load_lut(crtc);
+
+        return 0;
 }
 
 static void dce_v8_0_crtc_destroy(struct drm_crtc *crtc)
@@ -2624,13 +2573,13 @@ static void dce_v8_0_crtc_dpms(struct drm_crtc *crtc, int mode)
                 type = amdgpu_crtc_idx_to_irq_type(adev, amdgpu_crtc->crtc_id);
                 amdgpu_irq_update(adev, &adev->crtc_irq, type);
                 amdgpu_irq_update(adev, &adev->pageflip_irq, type);
-                drm_vblank_on(dev, amdgpu_crtc->crtc_id);
+                drm_crtc_vblank_on(crtc);
                 dce_v8_0_crtc_load_lut(crtc);
                 break;
         case DRM_MODE_DPMS_STANDBY:
         case DRM_MODE_DPMS_SUSPEND:
         case DRM_MODE_DPMS_OFF:
-                drm_vblank_off(dev, amdgpu_crtc->crtc_id);
+                drm_crtc_vblank_off(crtc);
                 if (amdgpu_crtc->enabled) {
                         dce_v8_0_vga_enable(crtc, true);
                         amdgpu_atombios_crtc_blank(crtc, ATOM_ENABLE);
@@ -3376,7 +3325,7 @@ static int dce_v8_0_pageflip_irq(struct amdgpu_device *adev,
 
         spin_unlock_irqrestore(&adev->ddev->event_lock, flags);
 
-        drm_vblank_put(adev->ddev, amdgpu_crtc->crtc_id);
+        drm_crtc_vblank_put(&amdgpu_crtc->base);
         schedule_work(&works->unpin_work);
 
         return 0;
diff --git a/drivers/gpu/drm/amd/amdgpu/fiji_smc.c b/drivers/gpu/drm/amd/amdgpu/fiji_smc.c
index b336c918d6a7..b3e19ba4c57f 100644
--- a/drivers/gpu/drm/amd/amdgpu/fiji_smc.c
+++ b/drivers/gpu/drm/amd/amdgpu/fiji_smc.c
@@ -173,7 +173,7 @@ static int fiji_send_msg_to_smc(struct amdgpu_device *adev, PPSMC_Msg msg)
 {
         if (!fiji_is_smc_ram_running(adev))
         {
-                return -EINVAL;;
+                return -EINVAL;
         }
 
         if (wait_smu_response(adev)) {
diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v7_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v7_0.c
index fc8ff4d3ccf8..d869d058ef24 100644
--- a/drivers/gpu/drm/amd/amdgpu/gfx_v7_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gfx_v7_0.c
@@ -1583,9 +1583,15 @@ static void gfx_v7_0_tiling_mode_table_init(struct amdgpu_device *adev)
  * registers are instanced per SE or SH.  0xffffffff means
  * broadcast to all SEs or SHs (CIK).
  */
-void gfx_v7_0_select_se_sh(struct amdgpu_device *adev, u32 se_num, u32 sh_num)
+static void gfx_v7_0_select_se_sh(struct amdgpu_device *adev,
+                                  u32 se_num, u32 sh_num, u32 instance)
 {
-        u32 data = GRBM_GFX_INDEX__INSTANCE_BROADCAST_WRITES_MASK;
+        u32 data;
+
+        if (instance == 0xffffffff)
+                data = REG_SET_FIELD(0, GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES, 1);
+        else
+                data = REG_SET_FIELD(0, GRBM_GFX_INDEX, INSTANCE_INDEX, instance);
 
         if ((se_num == 0xffffffff) && (sh_num == 0xffffffff))
                 data |= GRBM_GFX_INDEX__SH_BROADCAST_WRITES_MASK |
@@ -1659,13 +1665,13 @@ static void gfx_v7_0_setup_rb(struct amdgpu_device *adev)
         mutex_lock(&adev->grbm_idx_mutex);
         for (i = 0; i < adev->gfx.config.max_shader_engines; i++) {
                 for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) {
-                        gfx_v7_0_select_se_sh(adev, i, j);
+                        gfx_v7_0_select_se_sh(adev, i, j, 0xffffffff);
                         data = gfx_v7_0_get_rb_active_bitmap(adev);
                         active_rbs |= data << ((i * adev->gfx.config.max_sh_per_se + j) *
                                                rb_bitmap_width_per_sh);
                 }
         }
-        gfx_v7_0_select_se_sh(adev, 0xffffffff, 0xffffffff);
+        gfx_v7_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
         mutex_unlock(&adev->grbm_idx_mutex);
 
         adev->gfx.config.backend_enable_mask = active_rbs;
@@ -1746,7 +1752,7 @@ static void gfx_v7_0_gpu_init(struct amdgpu_device *adev)
          * making sure that the following register writes will be broadcasted
          * to all the shaders
          */
-        gfx_v7_0_select_se_sh(adev, 0xffffffff, 0xffffffff);
+        gfx_v7_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
 
         /* XXX SH_MEM regs */
         /* where to put LDS, scratch, GPUVM in FSA64 space */
@@ -2050,17 +2056,6 @@ static void gfx_v7_0_ring_emit_ib_gfx(struct amdgpu_ring *ring,
                                       unsigned vm_id, bool ctx_switch)
 {
         u32 header, control = 0;
-        u32 next_rptr = ring->wptr + 5;
-
-        if (ctx_switch)
-                next_rptr += 2;
-
-        next_rptr += 4;
-        amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
-        amdgpu_ring_write(ring, WRITE_DATA_DST_SEL(5) | WR_CONFIRM);
-        amdgpu_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc);
-        amdgpu_ring_write(ring, upper_32_bits(ring->next_rptr_gpu_addr) & 0xffffffff);
-        amdgpu_ring_write(ring, next_rptr);
 
         /* insert SWITCH_BUFFER packet before first IB in the ring frame */
         if (ctx_switch) {
@@ -2089,22 +2084,9 @@ static void gfx_v7_0_ring_emit_ib_compute(struct amdgpu_ring *ring,
                                           struct amdgpu_ib *ib,
                                           unsigned vm_id, bool ctx_switch)
 {
-        u32 header, control = 0;
-        u32 next_rptr = ring->wptr + 5;
-
-        control |= INDIRECT_BUFFER_VALID;
-        next_rptr += 4;
-        amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
-        amdgpu_ring_write(ring, WRITE_DATA_DST_SEL(5) | WR_CONFIRM);
-        amdgpu_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc);
-        amdgpu_ring_write(ring, upper_32_bits(ring->next_rptr_gpu_addr) & 0xffffffff);
-        amdgpu_ring_write(ring, next_rptr);
-
-        header = PACKET3(PACKET3_INDIRECT_BUFFER, 2);
+        u32 control = INDIRECT_BUFFER_VALID | ib->length_dw | (vm_id << 24);
 
-        control |= ib->length_dw | (vm_id << 24);
-
-        amdgpu_ring_write(ring, header);
+        amdgpu_ring_write(ring, PACKET3(PACKET3_INDIRECT_BUFFER, 2));
         amdgpu_ring_write(ring,
 #ifdef __BIG_ENDIAN
                                           (2 << 0) |
@@ -2123,26 +2105,25 @@ static void gfx_v7_0_ring_emit_ib_compute(struct amdgpu_ring *ring,
  * Provides a basic gfx ring test to verify that IBs are working.
  * Returns 0 on success, error on failure.
  */
-static int gfx_v7_0_ring_test_ib(struct amdgpu_ring *ring)
+static int gfx_v7_0_ring_test_ib(struct amdgpu_ring *ring, long timeout)
 {
         struct amdgpu_device *adev = ring->adev;
         struct amdgpu_ib ib;
         struct fence *f = NULL;
         uint32_t scratch;
         uint32_t tmp = 0;
-        unsigned i;
-        int r;
+        long r;
 
         r = amdgpu_gfx_scratch_get(adev, &scratch);
         if (r) {
-                DRM_ERROR("amdgpu: failed to get scratch reg (%d).\n", r);
+                DRM_ERROR("amdgpu: failed to get scratch reg (%ld).\n", r);
                 return r;
         }
         WREG32(scratch, 0xCAFEDEAD);
         memset(&ib, 0, sizeof(ib));
         r = amdgpu_ib_get(adev, NULL, 256, &ib);
         if (r) {
-                DRM_ERROR("amdgpu: failed to get ib (%d).\n", r);
+                DRM_ERROR("amdgpu: failed to get ib (%ld).\n", r);
                 goto err1;
         }
         ib.ptr[0] = PACKET3(PACKET3_SET_UCONFIG_REG, 1);
@@ -2154,21 +2135,19 @@ static int gfx_v7_0_ring_test_ib(struct amdgpu_ring *ring)
         if (r)
                 goto err2;
 
-        r = fence_wait(f, false);
-        if (r) {
-                DRM_ERROR("amdgpu: fence wait failed (%d).\n", r);
+        r = fence_wait_timeout(f, false, timeout);
+        if (r == 0) {
+                DRM_ERROR("amdgpu: IB test timed out\n");
+                r = -ETIMEDOUT;
                 goto err2;
-        }
-        for (i = 0; i < adev->usec_timeout; i++) {
-                tmp = RREG32(scratch);
-                if (tmp == 0xDEADBEEF)
-                        break;
-                DRM_UDELAY(1);
-        }
-        if (i < adev->usec_timeout) {
-                DRM_INFO("ib test on ring %d succeeded in %u usecs\n",
-                         ring->idx, i);
+        } else if (r < 0) {
+                DRM_ERROR("amdgpu: fence wait failed (%ld).\n", r);
                 goto err2;
+        }
+        tmp = RREG32(scratch);
+        if (tmp == 0xDEADBEEF) {
+                DRM_INFO("ib test on ring %d succeeded\n", ring->idx);
+                r = 0;
         } else {
                 DRM_ERROR("amdgpu: ib test failed (scratch(0x%04X)=0x%08X)\n",
                           scratch, tmp);
@@ -2176,7 +2155,6 @@ static int gfx_v7_0_ring_test_ib(struct amdgpu_ring *ring)
         }
 
 err2:
-        fence_put(f);
         amdgpu_ib_free(adev, &ib, NULL);
         fence_put(f);
 err1:
@@ -3221,7 +3199,8 @@ static int gfx_v7_0_rlc_init(struct amdgpu_device *adev)
                 }
         }
         adev->gfx.rlc.cs_data = ci_cs_data;
-        adev->gfx.rlc.cp_table_size = CP_ME_TABLE_SIZE * 5 * 4;
+        adev->gfx.rlc.cp_table_size = ALIGN(CP_ME_TABLE_SIZE * 5 * 4, 2048); /* CP JT */
+        adev->gfx.rlc.cp_table_size += 64 * 1024; /* GDS */
 
         src_ptr = adev->gfx.rlc.reg_list;
         dws = adev->gfx.rlc.reg_list_size;
@@ -3379,7 +3358,7 @@ static void gfx_v7_0_wait_for_rlc_serdes(struct amdgpu_device *adev)
         mutex_lock(&adev->grbm_idx_mutex);
         for (i = 0; i < adev->gfx.config.max_shader_engines; i++) {
                 for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) {
-                        gfx_v7_0_select_se_sh(adev, i, j);
+                        gfx_v7_0_select_se_sh(adev, i, j, 0xffffffff);
                         for (k = 0; k < adev->usec_timeout; k++) {
                                 if (RREG32(mmRLC_SERDES_CU_MASTER_BUSY) == 0)
                                         break;
@@ -3387,7 +3366,7 @@ static void gfx_v7_0_wait_for_rlc_serdes(struct amdgpu_device *adev)
                         }
                 }
         }
-        gfx_v7_0_select_se_sh(adev, 0xffffffff, 0xffffffff);
+        gfx_v7_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
         mutex_unlock(&adev->grbm_idx_mutex);
 
         mask = RLC_SERDES_NONCU_MASTER_BUSY__SE_MASTER_BUSY_MASK |
@@ -3434,7 +3413,7 @@ static u32 gfx_v7_0_halt_rlc(struct amdgpu_device *adev)
         return orig;
 }
 
-void gfx_v7_0_enter_rlc_safe_mode(struct amdgpu_device *adev)
+static void gfx_v7_0_enter_rlc_safe_mode(struct amdgpu_device *adev)
 {
         u32 tmp, i, mask;
 
@@ -3456,7 +3435,7 @@ void gfx_v7_0_enter_rlc_safe_mode(struct amdgpu_device *adev)
         }
 }
 
-void gfx_v7_0_exit_rlc_safe_mode(struct amdgpu_device *adev)
+static void gfx_v7_0_exit_rlc_safe_mode(struct amdgpu_device *adev)
 {
         u32 tmp;
 
@@ -3471,7 +3450,7 @@ void gfx_v7_0_exit_rlc_safe_mode(struct amdgpu_device *adev)
  *
  * Halt the RLC ME (MicroEngine) (CIK).
  */
-void gfx_v7_0_rlc_stop(struct amdgpu_device *adev)
+static void gfx_v7_0_rlc_stop(struct amdgpu_device *adev)
 {
         WREG32(mmRLC_CNTL, 0);
 
@@ -3547,7 +3526,7 @@ static int gfx_v7_0_rlc_resume(struct amdgpu_device *adev)
         WREG32(mmRLC_LB_CNTR_MAX, 0x00008000);
 
         mutex_lock(&adev->grbm_idx_mutex);
-        gfx_v7_0_select_se_sh(adev, 0xffffffff, 0xffffffff);
+        gfx_v7_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
         WREG32(mmRLC_LB_INIT_CU_MASK, 0xffffffff);
         WREG32(mmRLC_LB_PARAMS, 0x00600408);
         WREG32(mmRLC_LB_CNTL, 0x80000004);
@@ -3587,7 +3566,7 @@ static void gfx_v7_0_enable_cgcg(struct amdgpu_device *adev, bool enable)
                 tmp = gfx_v7_0_halt_rlc(adev);
 
                 mutex_lock(&adev->grbm_idx_mutex);
-                gfx_v7_0_select_se_sh(adev, 0xffffffff, 0xffffffff);
+                gfx_v7_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
                 WREG32(mmRLC_SERDES_WR_CU_MASTER_MASK, 0xffffffff);
                 WREG32(mmRLC_SERDES_WR_NONCU_MASTER_MASK, 0xffffffff);
                 tmp2 = RLC_SERDES_WR_CTRL__BPM_ADDR_MASK |
@@ -3638,7 +3617,7 @@ static void gfx_v7_0_enable_mgcg(struct amdgpu_device *adev, bool enable)
                 tmp = gfx_v7_0_halt_rlc(adev);
 
                 mutex_lock(&adev->grbm_idx_mutex);
-                gfx_v7_0_select_se_sh(adev, 0xffffffff, 0xffffffff);
+                gfx_v7_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
                 WREG32(mmRLC_SERDES_WR_CU_MASTER_MASK, 0xffffffff);
                 WREG32(mmRLC_SERDES_WR_NONCU_MASTER_MASK, 0xffffffff);
                 data = RLC_SERDES_WR_CTRL__BPM_ADDR_MASK |
@@ -3689,7 +3668,7 @@ static void gfx_v7_0_enable_mgcg(struct amdgpu_device *adev, bool enable)
                 tmp = gfx_v7_0_halt_rlc(adev);
 
                 mutex_lock(&adev->grbm_idx_mutex);
-                gfx_v7_0_select_se_sh(adev, 0xffffffff, 0xffffffff);
+                gfx_v7_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
                 WREG32(mmRLC_SERDES_WR_CU_MASTER_MASK, 0xffffffff);
                 WREG32(mmRLC_SERDES_WR_NONCU_MASTER_MASK, 0xffffffff);
                 data = RLC_SERDES_WR_CTRL__BPM_ADDR_MASK | RLC_SERDES_WR_CTRL__MGCG_OVERRIDE_1_MASK;
@@ -3867,6 +3846,20 @@ static void gfx_v7_0_enable_gfx_cgpg(struct amdgpu_device *adev,
         }
 }
 
+static void gfx_v7_0_set_user_cu_inactive_bitmap(struct amdgpu_device *adev,
+                                                 u32 bitmap)
+{
+        u32 data;
+
+        if (!bitmap)
+                return;
+
+        data = bitmap << GC_USER_SHADER_ARRAY_CONFIG__INACTIVE_CUS__SHIFT;
+        data &= GC_USER_SHADER_ARRAY_CONFIG__INACTIVE_CUS_MASK;
+
+        WREG32(mmGC_USER_SHADER_ARRAY_CONFIG, data);
+}
+
 static u32 gfx_v7_0_get_cu_active_bitmap(struct amdgpu_device *adev)
 {
         u32 data, mask;
@@ -4123,7 +4116,7 @@ static void gfx_v7_0_fini_pg(struct amdgpu_device *adev)
  * Fetches a GPU clock counter snapshot (SI).
  * Returns the 64 bit clock counter snapshot.
  */
-uint64_t gfx_v7_0_get_gpu_clock_counter(struct amdgpu_device *adev)
+static uint64_t gfx_v7_0_get_gpu_clock_counter(struct amdgpu_device *adev)
 {
         uint64_t clock;
 
@@ -4183,12 +4176,24 @@ static void gfx_v7_0_ring_emit_gds_switch(struct amdgpu_ring *ring,
         amdgpu_ring_write(ring, (1 << (oa_size + oa_base)) - (1 << oa_base));
 }
 
+static const struct amdgpu_gfx_funcs gfx_v7_0_gfx_funcs = {
+        .get_gpu_clock_counter = &gfx_v7_0_get_gpu_clock_counter,
+        .select_se_sh = &gfx_v7_0_select_se_sh,
+};
+
+static const struct amdgpu_rlc_funcs gfx_v7_0_rlc_funcs = {
+        .enter_safe_mode = gfx_v7_0_enter_rlc_safe_mode,
+        .exit_safe_mode = gfx_v7_0_exit_rlc_safe_mode
+};
+
 static int gfx_v7_0_early_init(void *handle)
 {
         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 
         adev->gfx.num_gfx_rings = GFX7_NUM_GFX_RINGS;
         adev->gfx.num_compute_rings = GFX7_NUM_COMPUTE_RINGS;
+        adev->gfx.funcs = &gfx_v7_0_gfx_funcs;
+        adev->gfx.rlc.funcs = &gfx_v7_0_rlc_funcs;
         gfx_v7_0_set_ring_funcs(adev);
         gfx_v7_0_set_irq_funcs(adev);
         gfx_v7_0_set_gds_init(adev);
@@ -5032,16 +5037,22 @@ static void gfx_v7_0_get_cu_info(struct amdgpu_device *adev)
         int i, j, k, counter, active_cu_number = 0;
         u32 mask, bitmap, ao_bitmap, ao_cu_mask = 0;
         struct amdgpu_cu_info *cu_info = &adev->gfx.cu_info;
+        unsigned disable_masks[4 * 2];
 
         memset(cu_info, 0, sizeof(*cu_info));
 
+        amdgpu_gfx_parse_disable_cu(disable_masks, 4, 2);
+
         mutex_lock(&adev->grbm_idx_mutex);
         for (i = 0; i < adev->gfx.config.max_shader_engines; i++) {
                 for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) {
                         mask = 1;
                         ao_bitmap = 0;
                         counter = 0;
-                        gfx_v7_0_select_se_sh(adev, i, j);
+                        gfx_v7_0_select_se_sh(adev, i, j, 0xffffffff);
+                        if (i < 4 && j < 2)
+                                gfx_v7_0_set_user_cu_inactive_bitmap(
+                                        adev, disable_masks[i * 2 + j]);
                         bitmap = gfx_v7_0_get_cu_active_bitmap(adev);
                         cu_info->bitmap[i][j] = bitmap;
 
@@ -5057,7 +5068,7 @@ static void gfx_v7_0_get_cu_info(struct amdgpu_device *adev)
                         ao_cu_mask |= (ao_bitmap << (i * 16 + j * 8));
                 }
         }
-        gfx_v7_0_select_se_sh(adev, 0xffffffff, 0xffffffff);
+        gfx_v7_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
         mutex_unlock(&adev->grbm_idx_mutex);
 
         cu_info->number = active_cu_number;
diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v7_0.h b/drivers/gpu/drm/amd/amdgpu/gfx_v7_0.h
index e747aa935c88..94e3ea147c26 100644
--- a/drivers/gpu/drm/amd/amdgpu/gfx_v7_0.h
+++ b/drivers/gpu/drm/amd/amdgpu/gfx_v7_0.h
@@ -26,11 +26,4 @@
 
 extern const struct amd_ip_funcs gfx_v7_0_ip_funcs;
 
-/* XXX these shouldn't be exported */
-void gfx_v7_0_enter_rlc_safe_mode(struct amdgpu_device *adev);
-void gfx_v7_0_exit_rlc_safe_mode(struct amdgpu_device *adev);
-void gfx_v7_0_rlc_stop(struct amdgpu_device *adev);
-uint64_t gfx_v7_0_get_gpu_clock_counter(struct amdgpu_device *adev);
-void gfx_v7_0_select_se_sh(struct amdgpu_device *adev, u32 se_num, u32 sh_num);
-
 #endif
diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c b/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c
index c2ef94511f70..bff8668e9e6d 100644
--- a/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.c
@@ -787,26 +787,25 @@ static int gfx_v8_0_ring_test_ring(struct amdgpu_ring *ring)
         return r;
 }
 
-static int gfx_v8_0_ring_test_ib(struct amdgpu_ring *ring)
+static int gfx_v8_0_ring_test_ib(struct amdgpu_ring *ring, long timeout)
 {
         struct amdgpu_device *adev = ring->adev;
         struct amdgpu_ib ib;
         struct fence *f = NULL;
         uint32_t scratch;
         uint32_t tmp = 0;
-        unsigned i;
-        int r;
+        long r;
 
         r = amdgpu_gfx_scratch_get(adev, &scratch);
         if (r) {
-                DRM_ERROR("amdgpu: failed to get scratch reg (%d).\n", r);
+                DRM_ERROR("amdgpu: failed to get scratch reg (%ld).\n", r);
                 return r;
         }
         WREG32(scratch, 0xCAFEDEAD);
         memset(&ib, 0, sizeof(ib));
         r = amdgpu_ib_get(adev, NULL, 256, &ib);
         if (r) {
-                DRM_ERROR("amdgpu: failed to get ib (%d).\n", r);
+                DRM_ERROR("amdgpu: failed to get ib (%ld).\n", r);
                 goto err1;
         }
         ib.ptr[0] = PACKET3(PACKET3_SET_UCONFIG_REG, 1);
@@ -818,28 +817,25 @@ static int gfx_v8_0_ring_test_ib(struct amdgpu_ring *ring)
         if (r)
                 goto err2;
 
-        r = fence_wait(f, false);
-        if (r) {
-                DRM_ERROR("amdgpu: fence wait failed (%d).\n", r);
+        r = fence_wait_timeout(f, false, timeout);
+        if (r == 0) {
+                DRM_ERROR("amdgpu: IB test timed out.\n");
+                r = -ETIMEDOUT;
                 goto err2;
-        }
-        for (i = 0; i < adev->usec_timeout; i++) {
-                tmp = RREG32(scratch);
-                if (tmp == 0xDEADBEEF)
-                        break;
-                DRM_UDELAY(1);
-        }
-        if (i < adev->usec_timeout) {
-                DRM_INFO("ib test on ring %d succeeded in %u usecs\n",
-                         ring->idx, i);
+        } else if (r < 0) {
+                DRM_ERROR("amdgpu: fence wait failed (%ld).\n", r);
                 goto err2;
+        }
+        tmp = RREG32(scratch);
+        if (tmp == 0xDEADBEEF) {
+                DRM_INFO("ib test on ring %d succeeded\n", ring->idx);
+                r = 0;
         } else {
                 DRM_ERROR("amdgpu: ib test failed (scratch(0x%04X)=0x%08X)\n",
                           scratch, tmp);
                 r = -EINVAL;
         }
 err2:
-        fence_put(f);
         amdgpu_ib_free(adev, &ib, NULL);
         fence_put(f);
 err1:
@@ -1160,6 +1156,71 @@ static void gfx_v8_0_get_csb_buffer(struct amdgpu_device *adev,
         buffer[count++] = cpu_to_le32(0);
 }
 
+static void cz_init_cp_jump_table(struct amdgpu_device *adev)
+{
+        const __le32 *fw_data;
+        volatile u32 *dst_ptr;
+        int me, i, max_me = 4;
+        u32 bo_offset = 0;
+        u32 table_offset, table_size;
+
+        if (adev->asic_type == CHIP_CARRIZO)
+                max_me = 5;
+
+        /* write the cp table buffer */
+        dst_ptr = adev->gfx.rlc.cp_table_ptr;
+        for (me = 0; me < max_me; me++) {
+                if (me == 0) {
+                        const struct gfx_firmware_header_v1_0 *hdr =
+                                (const struct gfx_firmware_header_v1_0 *)adev->gfx.ce_fw->data;
+                        fw_data = (const __le32 *)
+                                (adev->gfx.ce_fw->data +
+                                 le32_to_cpu(hdr->header.ucode_array_offset_bytes));
+                        table_offset = le32_to_cpu(hdr->jt_offset);
+                        table_size = le32_to_cpu(hdr->jt_size);
+                } else if (me == 1) {
+                        const struct gfx_firmware_header_v1_0 *hdr =
+                                (const struct gfx_firmware_header_v1_0 *)adev->gfx.pfp_fw->data;
+                        fw_data = (const __le32 *)
+                                (adev->gfx.pfp_fw->data +
+                                 le32_to_cpu(hdr->header.ucode_array_offset_bytes));
+                        table_offset = le32_to_cpu(hdr->jt_offset);
+                        table_size = le32_to_cpu(hdr->jt_size);
+                } else if (me == 2) {
+                        const struct gfx_firmware_header_v1_0 *hdr =
+                                (const struct gfx_firmware_header_v1_0 *)adev->gfx.me_fw->data;
+                        fw_data = (const __le32 *)
+                                (adev->gfx.me_fw->data +
+                                 le32_to_cpu(hdr->header.ucode_array_offset_bytes));
+                        table_offset = le32_to_cpu(hdr->jt_offset);
+                        table_size = le32_to_cpu(hdr->jt_size);
+                } else if (me == 3) {
+                        const struct gfx_firmware_header_v1_0 *hdr =
+                                (const struct gfx_firmware_header_v1_0 *)adev->gfx.mec_fw->data;
+                        fw_data = (const __le32 *)
+                                (adev->gfx.mec_fw->data +
+                                 le32_to_cpu(hdr->header.ucode_array_offset_bytes));
+                        table_offset = le32_to_cpu(hdr->jt_offset);
+                        table_size = le32_to_cpu(hdr->jt_size);
+                } else  if (me == 4) {
+                        const struct gfx_firmware_header_v1_0 *hdr =
+                                (const struct gfx_firmware_header_v1_0 *)adev->gfx.mec2_fw->data;
+                        fw_data = (const __le32 *)
+                                (adev->gfx.mec2_fw->data +
+                                 le32_to_cpu(hdr->header.ucode_array_offset_bytes));
+                        table_offset = le32_to_cpu(hdr->jt_offset);
+                        table_size = le32_to_cpu(hdr->jt_size);
+                }
+
+                for (i = 0; i < table_size; i ++) {
+                        dst_ptr[bo_offset + i] =
+                                cpu_to_le32(le32_to_cpu(fw_data[table_offset + i]));
+                }
+
+                bo_offset += table_size;
+        }
+}
+
 static void gfx_v8_0_rlc_fini(struct amdgpu_device *adev)
 {
         int r;
@@ -1175,6 +1236,18 @@ static void gfx_v8_0_rlc_fini(struct amdgpu_device *adev)
                 amdgpu_bo_unref(&adev->gfx.rlc.clear_state_obj);
                 adev->gfx.rlc.clear_state_obj = NULL;
         }
+
+        /* jump table block */
+        if (adev->gfx.rlc.cp_table_obj) {
+                r = amdgpu_bo_reserve(adev->gfx.rlc.cp_table_obj, false);
+                if (unlikely(r != 0))
+                        dev_warn(adev->dev, "(%d) reserve RLC cp table bo failed\n", r);
+                amdgpu_bo_unpin(adev->gfx.rlc.cp_table_obj);
+                amdgpu_bo_unreserve(adev->gfx.rlc.cp_table_obj);
+
+                amdgpu_bo_unref(&adev->gfx.rlc.cp_table_obj);
+                adev->gfx.rlc.cp_table_obj = NULL;
+        }
 }
 
 static int gfx_v8_0_rlc_init(struct amdgpu_device *adev)
@@ -1231,6 +1304,46 @@ static int gfx_v8_0_rlc_init(struct amdgpu_device *adev)
                 amdgpu_bo_unreserve(adev->gfx.rlc.clear_state_obj);
         }
 
+        if ((adev->asic_type == CHIP_CARRIZO) ||
+            (adev->asic_type == CHIP_STONEY)) {
+                adev->gfx.rlc.cp_table_size = ALIGN(96 * 5 * 4, 2048) + (64 * 1024); /* JT + GDS */
+                if (adev->gfx.rlc.cp_table_obj == NULL) {
+                        r = amdgpu_bo_create(adev, adev->gfx.rlc.cp_table_size, PAGE_SIZE, true,
+                                             AMDGPU_GEM_DOMAIN_VRAM,
+                                             AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED,
+                                             NULL, NULL,
+                                             &adev->gfx.rlc.cp_table_obj);
+                        if (r) {
+                                dev_warn(adev->dev, "(%d) create RLC cp table bo failed\n", r);
+                                return r;
+                        }
+                }
+
+                r = amdgpu_bo_reserve(adev->gfx.rlc.cp_table_obj, false);
+                if (unlikely(r != 0)) {
+                        dev_warn(adev->dev, "(%d) reserve RLC cp table bo failed\n", r);
+                        return r;
+                }
+                r = amdgpu_bo_pin(adev->gfx.rlc.cp_table_obj, AMDGPU_GEM_DOMAIN_VRAM,
+                                  &adev->gfx.rlc.cp_table_gpu_addr);
+                if (r) {
+                        amdgpu_bo_unreserve(adev->gfx.rlc.cp_table_obj);
+                        dev_warn(adev->dev, "(%d) pin RLC cp_table bo failed\n", r);
+                        return r;
+                }
+                r = amdgpu_bo_kmap(adev->gfx.rlc.cp_table_obj, (void **)&adev->gfx.rlc.cp_table_ptr);
+                if (r) {
+                        dev_warn(adev->dev, "(%d) map RLC cp table bo failed\n", r);
+                        return r;
+                }
+
+                cz_init_cp_jump_table(adev);
+
+                amdgpu_bo_kunmap(adev->gfx.rlc.cp_table_obj);
+                amdgpu_bo_unreserve(adev->gfx.rlc.cp_table_obj);
+
+        }
+
         return 0;
 }
 
@@ -1612,7 +1725,6 @@ static int gfx_v8_0_do_edc_gpr_workarounds(struct amdgpu_device *adev)
                 RREG32(sec_ded_counter_registers[i]);
 
 fail:
-        fence_put(f);
         amdgpu_ib_free(adev, &ib, NULL);
         fence_put(f);
 
@@ -3339,9 +3451,15 @@ static void gfx_v8_0_tiling_mode_table_init(struct amdgpu_device *adev)
         }
 }
 
-void gfx_v8_0_select_se_sh(struct amdgpu_device *adev, u32 se_num, u32 sh_num)
+static void gfx_v8_0_select_se_sh(struct amdgpu_device *adev,
+                                  u32 se_num, u32 sh_num, u32 instance)
 {
-        u32 data = REG_SET_FIELD(0, GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES, 1);
+        u32 data;
+
+        if (instance == 0xffffffff)
+                data = REG_SET_FIELD(0, GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES, 1);
+        else
+                data = REG_SET_FIELD(0, GRBM_GFX_INDEX, INSTANCE_INDEX, instance);
 
         if ((se_num == 0xffffffff) && (sh_num == 0xffffffff)) {
                 data = REG_SET_FIELD(data, GRBM_GFX_INDEX, SH_BROADCAST_WRITES, 1);
@@ -3391,13 +3509,13 @@ static void gfx_v8_0_setup_rb(struct amdgpu_device *adev)
         mutex_lock(&adev->grbm_idx_mutex);
         for (i = 0; i < adev->gfx.config.max_shader_engines; i++) {
                 for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) {
-                        gfx_v8_0_select_se_sh(adev, i, j);
+                        gfx_v8_0_select_se_sh(adev, i, j, 0xffffffff);
                         data = gfx_v8_0_get_rb_active_bitmap(adev);
                         active_rbs |= data << ((i * adev->gfx.config.max_sh_per_se + j) *
                                                rb_bitmap_width_per_sh);
                 }
         }
-        gfx_v8_0_select_se_sh(adev, 0xffffffff, 0xffffffff);
+        gfx_v8_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
         mutex_unlock(&adev->grbm_idx_mutex);
 
         adev->gfx.config.backend_enable_mask = active_rbs;
@@ -3501,7 +3619,7 @@ static void gfx_v8_0_gpu_init(struct amdgpu_device *adev)
          * making sure that the following register writes will be broadcasted
          * to all the shaders
          */
-        gfx_v8_0_select_se_sh(adev, 0xffffffff, 0xffffffff);
+        gfx_v8_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
 
         WREG32(mmPA_SC_FIFO_SIZE,
                    (adev->gfx.config.sc_prim_fifo_size_frontend <<
@@ -3524,7 +3642,7 @@ static void gfx_v8_0_wait_for_rlc_serdes(struct amdgpu_device *adev)
         mutex_lock(&adev->grbm_idx_mutex);
         for (i = 0; i < adev->gfx.config.max_shader_engines; i++) {
                 for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) {
-                        gfx_v8_0_select_se_sh(adev, i, j);
+                        gfx_v8_0_select_se_sh(adev, i, j, 0xffffffff);
                         for (k = 0; k < adev->usec_timeout; k++) {
                                 if (RREG32(mmRLC_SERDES_CU_MASTER_BUSY) == 0)
                                         break;
@@ -3532,7 +3650,7 @@ static void gfx_v8_0_wait_for_rlc_serdes(struct amdgpu_device *adev)
                         }
                 }
         }
-        gfx_v8_0_select_se_sh(adev, 0xffffffff, 0xffffffff);
+        gfx_v8_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
         mutex_unlock(&adev->grbm_idx_mutex);
 
         mask = RLC_SERDES_NONCU_MASTER_BUSY__SE_MASTER_BUSY_MASK |
@@ -3693,13 +3811,13 @@ static void gfx_v8_0_enable_save_restore_machine(struct amdgpu_device *adev)
         WREG32(mmRLC_SRM_CNTL, data);
 }
 
-static void polaris11_init_power_gating(struct amdgpu_device *adev)
+static void gfx_v8_0_init_power_gating(struct amdgpu_device *adev)
 {
         uint32_t data;
 
         if (adev->pg_flags & (AMD_PG_SUPPORT_GFX_PG |
-                        AMD_PG_SUPPORT_GFX_SMG |
-                        AMD_PG_SUPPORT_GFX_DMG)) {
+                              AMD_PG_SUPPORT_GFX_SMG |
+                              AMD_PG_SUPPORT_GFX_DMG)) {
                 data = RREG32(mmCP_RB_WPTR_POLL_CNTL);
                 data &= ~CP_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT_MASK;
                 data |= (0x60 << CP_RB_WPTR_POLL_CNTL__IDLE_POLL_COUNT__SHIFT);
@@ -3724,6 +3842,53 @@ static void polaris11_init_power_gating(struct amdgpu_device *adev)
         }
 }
 
+static void cz_enable_sck_slow_down_on_power_up(struct amdgpu_device *adev,
+                                                bool enable)
+{
+        u32 data, orig;
+
+        orig = data = RREG32(mmRLC_PG_CNTL);
+
+        if (enable)
+                data |= RLC_PG_CNTL__SMU_CLK_SLOWDOWN_ON_PU_ENABLE_MASK;
+        else
+                data &= ~RLC_PG_CNTL__SMU_CLK_SLOWDOWN_ON_PU_ENABLE_MASK;
+
+        if (orig != data)
+                WREG32(mmRLC_PG_CNTL, data);
+}
+
+static void cz_enable_sck_slow_down_on_power_down(struct amdgpu_device *adev,
+                                                  bool enable)
+{
+        u32 data, orig;
+
+        orig = data = RREG32(mmRLC_PG_CNTL);
+
+        if (enable)
+                data |= RLC_PG_CNTL__SMU_CLK_SLOWDOWN_ON_PD_ENABLE_MASK;
+        else
+                data &= ~RLC_PG_CNTL__SMU_CLK_SLOWDOWN_ON_PD_ENABLE_MASK;
+
+        if (orig != data)
+                WREG32(mmRLC_PG_CNTL, data);
+}
+
+static void cz_enable_cp_power_gating(struct amdgpu_device *adev, bool enable)
+{
+        u32 data, orig;
+
+        orig = data = RREG32(mmRLC_PG_CNTL);
+
+        if (enable)
+                data &= ~RLC_PG_CNTL__CP_PG_DISABLE_MASK;
+        else
+                data |= RLC_PG_CNTL__CP_PG_DISABLE_MASK;
+
+        if (orig != data)
+                WREG32(mmRLC_PG_CNTL, data);
+}
+
 static void gfx_v8_0_init_pg(struct amdgpu_device *adev)
 {
         if (adev->pg_flags & (AMD_PG_SUPPORT_GFX_PG |
@@ -3736,8 +3901,25 @@ static void gfx_v8_0_init_pg(struct amdgpu_device *adev)
                 gfx_v8_0_init_save_restore_list(adev);
                 gfx_v8_0_enable_save_restore_machine(adev);
 
-                if (adev->asic_type == CHIP_POLARIS11)
-                        polaris11_init_power_gating(adev);
+                if ((adev->asic_type == CHIP_CARRIZO) ||
+                    (adev->asic_type == CHIP_STONEY)) {
+                        WREG32(mmRLC_JUMP_TABLE_RESTORE, adev->gfx.rlc.cp_table_gpu_addr >> 8);
+                        gfx_v8_0_init_power_gating(adev);
+                        WREG32(mmRLC_PG_ALWAYS_ON_CU_MASK, adev->gfx.cu_info.ao_cu_mask);
+                        if (adev->pg_flags & AMD_PG_SUPPORT_RLC_SMU_HS) {
+                                cz_enable_sck_slow_down_on_power_up(adev, true);
+                                cz_enable_sck_slow_down_on_power_down(adev, true);
+                        } else {
+                                cz_enable_sck_slow_down_on_power_up(adev, false);
+                                cz_enable_sck_slow_down_on_power_down(adev, false);
+                        }
+                        if (adev->pg_flags & AMD_PG_SUPPORT_CP)
+                                cz_enable_cp_power_gating(adev, true);
+                        else
+                                cz_enable_cp_power_gating(adev, false);
+                } else if (adev->asic_type == CHIP_POLARIS11) {
+                        gfx_v8_0_init_power_gating(adev);
+                }
         }
 }
 
@@ -4976,7 +5158,7 @@ static int gfx_v8_0_soft_reset(void *handle)
  * Fetches a GPU clock counter snapshot.
  * Returns the 64 bit clock counter snapshot.
  */
-uint64_t gfx_v8_0_get_gpu_clock_counter(struct amdgpu_device *adev)
+static uint64_t gfx_v8_0_get_gpu_clock_counter(struct amdgpu_device *adev)
 {
         uint64_t clock;
 
@@ -5036,12 +5218,18 @@ static void gfx_v8_0_ring_emit_gds_switch(struct amdgpu_ring *ring,
         amdgpu_ring_write(ring, (1 << (oa_size + oa_base)) - (1 << oa_base));
 }
 
+static const struct amdgpu_gfx_funcs gfx_v8_0_gfx_funcs = {
+        .get_gpu_clock_counter = &gfx_v8_0_get_gpu_clock_counter,
+        .select_se_sh = &gfx_v8_0_select_se_sh,
+};
+
 static int gfx_v8_0_early_init(void *handle)
 {
         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
 
         adev->gfx.num_gfx_rings = GFX8_NUM_GFX_RINGS;
         adev->gfx.num_compute_rings = GFX8_NUM_COMPUTE_RINGS;
+        adev->gfx.funcs = &gfx_v8_0_gfx_funcs;
         gfx_v8_0_set_ring_funcs(adev);
         gfx_v8_0_set_irq_funcs(adev);
         gfx_v8_0_set_gds_init(adev);
@@ -5074,51 +5262,43 @@ static int gfx_v8_0_late_init(void *handle)
         return 0;
 }
 
-static void polaris11_enable_gfx_static_mg_power_gating(struct amdgpu_device *adev,
-                bool enable)
+static void gfx_v8_0_enable_gfx_static_mg_power_gating(struct amdgpu_device *adev,
+                                                       bool enable)
 {
         uint32_t data, temp;
 
-        /* Send msg to SMU via Powerplay */
-        amdgpu_set_powergating_state(adev,
-                        AMD_IP_BLOCK_TYPE_SMC,
-                        enable ? AMD_PG_STATE_GATE : AMD_PG_STATE_UNGATE);
+        if (adev->asic_type == CHIP_POLARIS11)
+                /* Send msg to SMU via Powerplay */
+                amdgpu_set_powergating_state(adev,
+                                             AMD_IP_BLOCK_TYPE_SMC,
+                                             enable ?
+                                             AMD_PG_STATE_GATE : AMD_PG_STATE_UNGATE);
 
-        if (enable) {
-                /* Enable static MGPG */
-                temp = data = RREG32(mmRLC_PG_CNTL);
+        temp = data = RREG32(mmRLC_PG_CNTL);
+        /* Enable static MGPG */
+        if (enable)
                 data |= RLC_PG_CNTL__STATIC_PER_CU_PG_ENABLE_MASK;
-
-                if (temp != data)
-                        WREG32(mmRLC_PG_CNTL, data);
-        } else {
-                temp = data = RREG32(mmRLC_PG_CNTL);
+        else
                 data &= ~RLC_PG_CNTL__STATIC_PER_CU_PG_ENABLE_MASK;
 
-                if (temp != data)
-                        WREG32(mmRLC_PG_CNTL, data);
-        }
+        if (temp != data)
+                WREG32(mmRLC_PG_CNTL, data);
 }
 
-static void polaris11_enable_gfx_dynamic_mg_power_gating(struct amdgpu_device *adev,
-                bool enable)
+static void gfx_v8_0_enable_gfx_dynamic_mg_power_gating(struct amdgpu_device *adev,
+                                                        bool enable)
 {
         uint32_t data, temp;
 
-        if (enable) {
-                /* Enable dynamic MGPG */
-                temp = data = RREG32(mmRLC_PG_CNTL);
+        temp = data = RREG32(mmRLC_PG_CNTL);
+        /* Enable dynamic MGPG */
+        if (enable)
                 data |= RLC_PG_CNTL__DYN_PER_CU_PG_ENABLE_MASK;
-
-                if (temp != data)
-                        WREG32(mmRLC_PG_CNTL, data);
-        } else {
-                temp = data = RREG32(mmRLC_PG_CNTL);
+        else
                 data &= ~RLC_PG_CNTL__DYN_PER_CU_PG_ENABLE_MASK;
 
-                if (temp != data)
-                        WREG32(mmRLC_PG_CNTL, data);
-        }
+        if (temp != data)
+                WREG32(mmRLC_PG_CNTL, data);
 }
 
 static void polaris11_enable_gfx_quick_mg_power_gating(struct amdgpu_device *adev,
@@ -5126,19 +5306,63 @@ static void polaris11_enable_gfx_quick_mg_power_gating(struct amdgpu_device *ade
 {
         uint32_t data, temp;
 
-        if (enable) {
-                /* Enable quick PG */
-                temp = data = RREG32(mmRLC_PG_CNTL);
-                data |= 0x100000;
+        temp = data = RREG32(mmRLC_PG_CNTL);
+        /* Enable quick PG */
+        if (enable)
+                data |= RLC_PG_CNTL__QUICK_PG_ENABLE_MASK;
+        else
+                data &= ~RLC_PG_CNTL__QUICK_PG_ENABLE_MASK;
 
-                if (temp != data)
-                        WREG32(mmRLC_PG_CNTL, data);
-        } else {
-                temp = data = RREG32(mmRLC_PG_CNTL);
-                data &= ~0x100000;
+        if (temp != data)
+                WREG32(mmRLC_PG_CNTL, data);
+}
 
-                if (temp != data)
-                        WREG32(mmRLC_PG_CNTL, data);
+static void cz_enable_gfx_cg_power_gating(struct amdgpu_device *adev,
+                                          bool enable)
+{
+        u32 data, orig;
+
+        orig = data = RREG32(mmRLC_PG_CNTL);
+
+        if (enable)
+                data |= RLC_PG_CNTL__GFX_POWER_GATING_ENABLE_MASK;
+        else
+                data &= ~RLC_PG_CNTL__GFX_POWER_GATING_ENABLE_MASK;
+
+        if (orig != data)
+                WREG32(mmRLC_PG_CNTL, data);
+}
+
+static void cz_enable_gfx_pipeline_power_gating(struct amdgpu_device *adev,
+                                                bool enable)
+{
+        u32 data, orig;
+
+        orig = data = RREG32(mmRLC_PG_CNTL);
+
+        if (enable)
+                data |= RLC_PG_CNTL__GFX_PIPELINE_PG_ENABLE_MASK;
+        else
+                data &= ~RLC_PG_CNTL__GFX_PIPELINE_PG_ENABLE_MASK;
+
+        if (orig != data)
+                WREG32(mmRLC_PG_CNTL, data);
+
+        /* Read any GFX register to wake up GFX. */
+        if (!enable)
+                data = RREG32(mmDB_RENDER_CONTROL);
+}
+
+static void cz_update_gfx_cg_power_gating(struct amdgpu_device *adev,
+                                          bool enable)
+{
+        if ((adev->pg_flags & AMD_PG_SUPPORT_GFX_PG) && enable) {
+                cz_enable_gfx_cg_power_gating(adev, true);
+                if (adev->pg_flags & AMD_PG_SUPPORT_GFX_PIPELINE)
+                        cz_enable_gfx_pipeline_power_gating(adev, true);
+        } else {
+                cz_enable_gfx_cg_power_gating(adev, false);
+                cz_enable_gfx_pipeline_power_gating(adev, false);
         }
 }
 
@@ -5146,21 +5370,42 @@ static int gfx_v8_0_set_powergating_state(void *handle,
                                           enum amd_powergating_state state)
 {
         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+        bool enable = (state == AMD_PG_STATE_GATE) ? true : false;
 
         if (!(adev->pg_flags & AMD_PG_SUPPORT_GFX_PG))
                 return 0;
 
         switch (adev->asic_type) {
+        case CHIP_CARRIZO:
+        case CHIP_STONEY:
+                if (adev->pg_flags & AMD_PG_SUPPORT_GFX_PG)
+                        cz_update_gfx_cg_power_gating(adev, enable);
+
+                if ((adev->pg_flags & AMD_PG_SUPPORT_GFX_SMG) && enable)
+                        gfx_v8_0_enable_gfx_static_mg_power_gating(adev, true);
+                else
+                        gfx_v8_0_enable_gfx_static_mg_power_gating(adev, false);
+
+                if ((adev->pg_flags & AMD_PG_SUPPORT_GFX_DMG) && enable)
+                        gfx_v8_0_enable_gfx_dynamic_mg_power_gating(adev, true);
+                else
+                        gfx_v8_0_enable_gfx_dynamic_mg_power_gating(adev, false);
+                break;
         case CHIP_POLARIS11:
-                if (adev->pg_flags & AMD_PG_SUPPORT_GFX_SMG)
-                        polaris11_enable_gfx_static_mg_power_gating(adev,
-                                        state == AMD_PG_STATE_GATE ? true : false);
-                else if (adev->pg_flags & AMD_PG_SUPPORT_GFX_DMG)
-                        polaris11_enable_gfx_dynamic_mg_power_gating(adev,
-                                        state == AMD_PG_STATE_GATE ? true : false);
+                if ((adev->pg_flags & AMD_PG_SUPPORT_GFX_SMG) && enable)
+                        gfx_v8_0_enable_gfx_static_mg_power_gating(adev, true);
+                else
+                        gfx_v8_0_enable_gfx_static_mg_power_gating(adev, false);
+
+                if ((adev->pg_flags & AMD_PG_SUPPORT_GFX_DMG) && enable)
+                        gfx_v8_0_enable_gfx_dynamic_mg_power_gating(adev, true);
+                else
+                        gfx_v8_0_enable_gfx_dynamic_mg_power_gating(adev, false);
+
+                if ((adev->pg_flags & AMD_PG_SUPPORT_GFX_QUICK_MG) && enable)
+                        polaris11_enable_gfx_quick_mg_power_gating(adev, true);
                 else
-                        polaris11_enable_gfx_quick_mg_power_gating(adev,
-                                        state == AMD_PG_STATE_GATE ? true : false);
+                        polaris11_enable_gfx_quick_mg_power_gating(adev, false);
                 break;
         default:
                 break;
@@ -5174,7 +5419,7 @@ static void gfx_v8_0_send_serdes_cmd(struct amdgpu_device *adev,
 {
         uint32_t data;
 
-        gfx_v8_0_select_se_sh(adev, 0xffffffff, 0xffffffff);
+        gfx_v8_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
 
         WREG32(mmRLC_SERDES_WR_CU_MASTER_MASK, 0xffffffff);
         WREG32(mmRLC_SERDES_WR_NONCU_MASTER_MASK, 0xffffffff);
@@ -5562,6 +5807,8 @@ static void gfx_v8_0_update_coarse_grain_clock_gating(struct amdgpu_device *adev
                         WREG32(mmRLC_CGCG_CGLS_CTRL, data);
         }
 
+        gfx_v8_0_wait_for_rlc_serdes(adev);
+
         adev->gfx.rlc.funcs->exit_safe_mode(adev);
 }
 static int gfx_v8_0_update_gfx_clock_gating(struct amdgpu_device *adev,
@@ -5687,17 +5934,6 @@ static void gfx_v8_0_ring_emit_ib_gfx(struct amdgpu_ring *ring,
                                       unsigned vm_id, bool ctx_switch)
 {
         u32 header, control = 0;
-        u32 next_rptr = ring->wptr + 5;
-
-        if (ctx_switch)
-                next_rptr += 2;
-
-        next_rptr += 4;
-        amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
-        amdgpu_ring_write(ring, WRITE_DATA_DST_SEL(5) | WR_CONFIRM);
-        amdgpu_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc);
-        amdgpu_ring_write(ring, upper_32_bits(ring->next_rptr_gpu_addr) & 0xffffffff);
-        amdgpu_ring_write(ring, next_rptr);
 
         /* insert SWITCH_BUFFER packet before first IB in the ring frame */
         if (ctx_switch) {
@@ -5726,23 +5962,9 @@ static void gfx_v8_0_ring_emit_ib_compute(struct amdgpu_ring *ring,
                                           struct amdgpu_ib *ib,
                                           unsigned vm_id, bool ctx_switch)
 {
-        u32 header, control = 0;
-        u32 next_rptr = ring->wptr + 5;
-
-        control |= INDIRECT_BUFFER_VALID;
-
-        next_rptr += 4;
-        amdgpu_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
-        amdgpu_ring_write(ring, WRITE_DATA_DST_SEL(5) | WR_CONFIRM);
-        amdgpu_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc);
-        amdgpu_ring_write(ring, upper_32_bits(ring->next_rptr_gpu_addr) & 0xffffffff);
-        amdgpu_ring_write(ring, next_rptr);
+        u32 control = INDIRECT_BUFFER_VALID | ib->length_dw | (vm_id << 24);
 
-        header = PACKET3(PACKET3_INDIRECT_BUFFER, 2);
-
-        control |= ib->length_dw | (vm_id << 24);
-
-        amdgpu_ring_write(ring, header);
+        amdgpu_ring_write(ring, PACKET3(PACKET3_INDIRECT_BUFFER, 2));
         amdgpu_ring_write(ring,
 #ifdef __BIG_ENDIAN
                                           (2 << 0) |
@@ -6195,9 +6417,9 @@ static void gfx_v8_0_set_rlc_funcs(struct amdgpu_device *adev)
 {
         switch (adev->asic_type) {
         case CHIP_TOPAZ:
-        case CHIP_STONEY:
                 adev->gfx.rlc.funcs = &iceland_rlc_funcs;
                 break;
+        case CHIP_STONEY:
         case CHIP_CARRIZO:
                 adev->gfx.rlc.funcs = &cz_rlc_funcs;
                 break;
@@ -6235,6 +6457,20 @@ static void gfx_v8_0_set_gds_init(struct amdgpu_device *adev)
         }
 }
 
+static void gfx_v8_0_set_user_cu_inactive_bitmap(struct amdgpu_device *adev,
+                                                 u32 bitmap)
+{
+        u32 data;
+
+        if (!bitmap)
+                return;
+
+        data = bitmap << GC_USER_SHADER_ARRAY_CONFIG__INACTIVE_CUS__SHIFT;
+        data &= GC_USER_SHADER_ARRAY_CONFIG__INACTIVE_CUS_MASK;
+
+        WREG32(mmGC_USER_SHADER_ARRAY_CONFIG, data);
+}
+
 static u32 gfx_v8_0_get_cu_active_bitmap(struct amdgpu_device *adev)
 {
         u32 data, mask;
@@ -6255,16 +6491,22 @@ static void gfx_v8_0_get_cu_info(struct amdgpu_device *adev)
         int i, j, k, counter, active_cu_number = 0;
         u32 mask, bitmap, ao_bitmap, ao_cu_mask = 0;
         struct amdgpu_cu_info *cu_info = &adev->gfx.cu_info;
+        unsigned disable_masks[4 * 2];
 
         memset(cu_info, 0, sizeof(*cu_info));
 
+        amdgpu_gfx_parse_disable_cu(disable_masks, 4, 2);
+
         mutex_lock(&adev->grbm_idx_mutex);
         for (i = 0; i < adev->gfx.config.max_shader_engines; i++) {
                 for (j = 0; j < adev->gfx.config.max_sh_per_se; j++) {
                         mask = 1;
                         ao_bitmap = 0;
                         counter = 0;
-                        gfx_v8_0_select_se_sh(adev, i, j);
+                        gfx_v8_0_select_se_sh(adev, i, j, 0xffffffff);
+                        if (i < 4 && j < 2)
+                                gfx_v8_0_set_user_cu_inactive_bitmap(
+                                        adev, disable_masks[i * 2 + j]);
                         bitmap = gfx_v8_0_get_cu_active_bitmap(adev);
                         cu_info->bitmap[i][j] = bitmap;
 
@@ -6280,7 +6522,7 @@ static void gfx_v8_0_get_cu_info(struct amdgpu_device *adev)
                         ao_cu_mask |= (ao_bitmap << (i * 16 + j * 8));
                 }
         }
-        gfx_v8_0_select_se_sh(adev, 0xffffffff, 0xffffffff);
+        gfx_v8_0_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
         mutex_unlock(&adev->grbm_idx_mutex);
 
         cu_info->number = active_cu_number;
diff --git a/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.h b/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.h
index 16a49f53a2fa..bc82c794312c 100644
--- a/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.h
+++ b/drivers/gpu/drm/amd/amdgpu/gfx_v8_0.h
@@ -26,7 +26,6 @@
 
 extern const struct amd_ip_funcs gfx_v8_0_ip_funcs;
 
-uint64_t gfx_v8_0_get_gpu_clock_counter(struct amdgpu_device *adev);
 void gfx_v8_0_select_se_sh(struct amdgpu_device *adev, u32 se_num, u32 sh_num);
 
 #endif
diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v7_0.c b/drivers/gpu/drm/amd/amdgpu/gmc_v7_0.c
index 1feb6439cb0b..d24a82bd0c7a 100644
--- a/drivers/gpu/drm/amd/amdgpu/gmc_v7_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gmc_v7_0.c
@@ -39,6 +39,7 @@
 
 static void gmc_v7_0_set_gart_funcs(struct amdgpu_device *adev);
 static void gmc_v7_0_set_irq_funcs(struct amdgpu_device *adev);
+static int gmc_v7_0_wait_for_idle(void *handle);
 
 MODULE_FIRMWARE("radeon/bonaire_mc.bin");
 MODULE_FIRMWARE("radeon/hawaii_mc.bin");
@@ -73,39 +74,15 @@ static void gmc_v7_0_init_golden_registers(struct amdgpu_device *adev)
         }
 }
 
-/**
- * gmc7_mc_wait_for_idle - wait for MC idle callback.
- *
- * @adev: amdgpu_device pointer
- *
- * Wait for the MC (memory controller) to be idle.
- * (evergreen+).
- * Returns 0 if the MC is idle, -1 if not.
- */
-int gmc_v7_0_mc_wait_for_idle(struct amdgpu_device *adev)
-{
-        unsigned i;
-        u32 tmp;
-
-        for (i = 0; i < adev->usec_timeout; i++) {
-                /* read MC_STATUS */
-                tmp = RREG32(mmSRBM_STATUS) & 0x1F00;
-                if (!tmp)
-                        return 0;
-                udelay(1);
-        }
-        return -1;
-}
-
-void gmc_v7_0_mc_stop(struct amdgpu_device *adev,
-                      struct amdgpu_mode_mc_save *save)
+static void gmc_v7_0_mc_stop(struct amdgpu_device *adev,
+                             struct amdgpu_mode_mc_save *save)
 {
         u32 blackout;
 
         if (adev->mode_info.num_crtc)
                 amdgpu_display_stop_mc_access(adev, save);
 
-        amdgpu_asic_wait_for_mc_idle(adev);
+        gmc_v7_0_wait_for_idle((void *)adev);
 
         blackout = RREG32(mmMC_SHARED_BLACKOUT_CNTL);
         if (REG_GET_FIELD(blackout, MC_SHARED_BLACKOUT_CNTL, BLACKOUT_MODE) != 1) {
@@ -120,8 +97,8 @@ void gmc_v7_0_mc_stop(struct amdgpu_device *adev,
         udelay(100);
 }
 
-void gmc_v7_0_mc_resume(struct amdgpu_device *adev,
-                        struct amdgpu_mode_mc_save *save)
+static void gmc_v7_0_mc_resume(struct amdgpu_device *adev,
+                               struct amdgpu_mode_mc_save *save)
 {
         u32 tmp;
 
@@ -311,7 +288,7 @@ static void gmc_v7_0_mc_program(struct amdgpu_device *adev)
                 amdgpu_display_set_vga_render_state(adev, false);
 
         gmc_v7_0_mc_stop(adev, &save);
-        if (amdgpu_asic_wait_for_mc_idle(adev)) {
+        if (gmc_v7_0_wait_for_idle((void *)adev)) {
                 dev_warn(adev->dev, "Wait for MC idle timedout !\n");
         }
         /* Update configuration */
@@ -331,7 +308,7 @@ static void gmc_v7_0_mc_program(struct amdgpu_device *adev)
         WREG32(mmMC_VM_AGP_BASE, 0);
         WREG32(mmMC_VM_AGP_TOP, 0x0FFFFFFF);
         WREG32(mmMC_VM_AGP_BOT, 0x0FFFFFFF);
-        if (amdgpu_asic_wait_for_mc_idle(adev)) {
+        if (gmc_v7_0_wait_for_idle((void *)adev)) {
                 dev_warn(adev->dev, "Wait for MC idle timedout !\n");
         }
         gmc_v7_0_mc_resume(adev, &save);
@@ -1137,7 +1114,7 @@ static int gmc_v7_0_soft_reset(void *handle)
 
         if (srbm_soft_reset) {
                 gmc_v7_0_mc_stop(adev, &save);
-                if (gmc_v7_0_wait_for_idle(adev)) {
+                if (gmc_v7_0_wait_for_idle((void *)adev)) {
                         dev_warn(adev->dev, "Wait for GMC idle timed out !\n");
                 }
 
diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v7_0.h b/drivers/gpu/drm/amd/amdgpu/gmc_v7_0.h
index 36fcbbc46ada..0b386b5d2f7a 100644
--- a/drivers/gpu/drm/amd/amdgpu/gmc_v7_0.h
+++ b/drivers/gpu/drm/amd/amdgpu/gmc_v7_0.h
@@ -26,11 +26,4 @@
 
 extern const struct amd_ip_funcs gmc_v7_0_ip_funcs;
 
-/* XXX these shouldn't be exported */
-void gmc_v7_0_mc_stop(struct amdgpu_device *adev,
-                      struct amdgpu_mode_mc_save *save);
-void gmc_v7_0_mc_resume(struct amdgpu_device *adev,
-                        struct amdgpu_mode_mc_save *save);
-int gmc_v7_0_mc_wait_for_idle(struct amdgpu_device *adev);
-
 #endif
diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v8_0.c b/drivers/gpu/drm/amd/amdgpu/gmc_v8_0.c
index 9945d5bbf1fe..717359d3ba8c 100644
--- a/drivers/gpu/drm/amd/amdgpu/gmc_v8_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/gmc_v8_0.c
@@ -41,6 +41,7 @@
 
 static void gmc_v8_0_set_gart_funcs(struct amdgpu_device *adev);
 static void gmc_v8_0_set_irq_funcs(struct amdgpu_device *adev);
+static int gmc_v8_0_wait_for_idle(void *handle);
 
 MODULE_FIRMWARE("amdgpu/tonga_mc.bin");
 MODULE_FIRMWARE("amdgpu/polaris11_mc.bin");
@@ -147,44 +148,15 @@ static void gmc_v8_0_init_golden_registers(struct amdgpu_device *adev)
         }
 }
 
-/**
- * gmc8_mc_wait_for_idle - wait for MC idle callback.
- *
- * @adev: amdgpu_device pointer
- *
- * Wait for the MC (memory controller) to be idle.
- * (evergreen+).
- * Returns 0 if the MC is idle, -1 if not.
- */
-int gmc_v8_0_mc_wait_for_idle(struct amdgpu_device *adev)
-{
-        unsigned i;
-        u32 tmp;
-
-        for (i = 0; i < adev->usec_timeout; i++) {
-                /* read MC_STATUS */
-                tmp = RREG32(mmSRBM_STATUS) & (SRBM_STATUS__VMC_BUSY_MASK |
-                                               SRBM_STATUS__MCB_BUSY_MASK |
-                                               SRBM_STATUS__MCB_NON_DISPLAY_BUSY_MASK |
-                                               SRBM_STATUS__MCC_BUSY_MASK |
-                                               SRBM_STATUS__MCD_BUSY_MASK |
-                                               SRBM_STATUS__VMC1_BUSY_MASK);
-                if (!tmp)
-                        return 0;
-                udelay(1);
-        }
-        return -1;
-}
-
-void gmc_v8_0_mc_stop(struct amdgpu_device *adev,
-                      struct amdgpu_mode_mc_save *save)
+static void gmc_v8_0_mc_stop(struct amdgpu_device *adev,
+                             struct amdgpu_mode_mc_save *save)
 {
         u32 blackout;
 
         if (adev->mode_info.num_crtc)
                 amdgpu_display_stop_mc_access(adev, save);
 
-        amdgpu_asic_wait_for_mc_idle(adev);
+        gmc_v8_0_wait_for_idle(adev);
 
         blackout = RREG32(mmMC_SHARED_BLACKOUT_CNTL);
         if (REG_GET_FIELD(blackout, MC_SHARED_BLACKOUT_CNTL, BLACKOUT_MODE) != 1) {
@@ -199,8 +171,8 @@ void gmc_v8_0_mc_stop(struct amdgpu_device *adev,
         udelay(100);
 }
 
-void gmc_v8_0_mc_resume(struct amdgpu_device *adev,
-                        struct amdgpu_mode_mc_save *save)
+static void gmc_v8_0_mc_resume(struct amdgpu_device *adev,
+                               struct amdgpu_mode_mc_save *save)
 {
         u32 tmp;
 
@@ -393,7 +365,7 @@ static void gmc_v8_0_mc_program(struct amdgpu_device *adev)
                 amdgpu_display_set_vga_render_state(adev, false);
 
         gmc_v8_0_mc_stop(adev, &save);
-        if (amdgpu_asic_wait_for_mc_idle(adev)) {
+        if (gmc_v8_0_wait_for_idle((void *)adev)) {
                 dev_warn(adev->dev, "Wait for MC idle timedout !\n");
         }
         /* Update configuration */
@@ -413,7 +385,7 @@ static void gmc_v8_0_mc_program(struct amdgpu_device *adev)
         WREG32(mmMC_VM_AGP_BASE, 0);
         WREG32(mmMC_VM_AGP_TOP, 0x0FFFFFFF);
         WREG32(mmMC_VM_AGP_BOT, 0x0FFFFFFF);
-        if (amdgpu_asic_wait_for_mc_idle(adev)) {
+        if (gmc_v8_0_wait_for_idle((void *)adev)) {
                 dev_warn(adev->dev, "Wait for MC idle timedout !\n");
         }
         gmc_v8_0_mc_resume(adev, &save);
@@ -1140,7 +1112,7 @@ static int gmc_v8_0_soft_reset(void *handle)
 
         if (srbm_soft_reset) {
                 gmc_v8_0_mc_stop(adev, &save);
-                if (gmc_v8_0_wait_for_idle(adev)) {
+                if (gmc_v8_0_wait_for_idle((void *)adev)) {
                         dev_warn(adev->dev, "Wait for GMC idle timed out !\n");
                 }
 
diff --git a/drivers/gpu/drm/amd/amdgpu/gmc_v8_0.h b/drivers/gpu/drm/amd/amdgpu/gmc_v8_0.h
index 973436086b38..fc5001a8119d 100644
--- a/drivers/gpu/drm/amd/amdgpu/gmc_v8_0.h
+++ b/drivers/gpu/drm/amd/amdgpu/gmc_v8_0.h
@@ -26,11 +26,4 @@
 
 extern const struct amd_ip_funcs gmc_v8_0_ip_funcs;
 
-/* XXX these shouldn't be exported */
-void gmc_v8_0_mc_stop(struct amdgpu_device *adev,
-                      struct amdgpu_mode_mc_save *save);
-void gmc_v8_0_mc_resume(struct amdgpu_device *adev,
-                        struct amdgpu_mode_mc_save *save);
-int gmc_v8_0_mc_wait_for_idle(struct amdgpu_device *adev);
-
 #endif
diff --git a/drivers/gpu/drm/amd/amdgpu/iceland_dpm.c b/drivers/gpu/drm/amd/amdgpu/iceland_dpm.c
index 825ccd63f2dc..2f078ad6095c 100644
--- a/drivers/gpu/drm/amd/amdgpu/iceland_dpm.c
+++ b/drivers/gpu/drm/amd/amdgpu/iceland_dpm.c
@@ -24,7 +24,7 @@
 #include <linux/firmware.h>
 #include "drmP.h"
 #include "amdgpu.h"
-#include "iceland_smumgr.h"
+#include "iceland_smum.h"
 
 MODULE_FIRMWARE("amdgpu/topaz_smc.bin");
 
diff --git a/drivers/gpu/drm/amd/amdgpu/iceland_smc.c b/drivers/gpu/drm/amd/amdgpu/iceland_smc.c
index 52ee08193295..211839913728 100644
--- a/drivers/gpu/drm/amd/amdgpu/iceland_smc.c
+++ b/drivers/gpu/drm/amd/amdgpu/iceland_smc.c
@@ -25,7 +25,7 @@
 #include "drmP.h"
 #include "amdgpu.h"
 #include "ppsmc.h"
-#include "iceland_smumgr.h"
+#include "iceland_smum.h"
 #include "smu_ucode_xfer_vi.h"
 #include "amdgpu_ucode.h"
 
@@ -211,7 +211,7 @@ static int iceland_send_msg_to_smc_without_waiting(struct amdgpu_device *adev,
                                                    PPSMC_Msg msg)
 {
         if (!iceland_is_smc_ram_running(adev))
-                return -EINVAL;;
+                return -EINVAL;
 
         if (wait_smu_response(adev)) {
                 DRM_ERROR("Failed to send previous message\n");
diff --git a/drivers/gpu/drm/amd/amdgpu/iceland_smumgr.h b/drivers/gpu/drm/amd/amdgpu/iceland_smum.h
index 1e0769e110fa..5983e3150cc5 100644
--- a/drivers/gpu/drm/amd/amdgpu/iceland_smumgr.h
+++ b/drivers/gpu/drm/amd/amdgpu/iceland_smum.h
@@ -21,8 +21,8 @@
  *
  */
 
-#ifndef ICELAND_SMUMGR_H
-#define ICELAND_SMUMGR_H
+#ifndef ICELAND_SMUM_H
+#define ICELAND_SMUM_H
 
 #include "ppsmc.h"
 
diff --git a/drivers/gpu/drm/amd/amdgpu/kv_dpm.c b/drivers/gpu/drm/amd/amdgpu/kv_dpm.c
index a789a863d677..a845e883f5fa 100644
--- a/drivers/gpu/drm/amd/amdgpu/kv_dpm.c
+++ b/drivers/gpu/drm/amd/amdgpu/kv_dpm.c
@@ -191,6 +191,7 @@ static void sumo_construct_vid_mapping_table(struct amdgpu_device *adev,
         vid_mapping_table->num_entries = i;
 }
 
+#if 0
 static const struct kv_lcac_config_values sx_local_cac_cfg_kv[] =
 {
         {  0,       4,        1    },
@@ -289,6 +290,7 @@ static const struct kv_lcac_config_reg cpl_cac_config_reg[] =
 {
         { 0xc0400d80, 0x003e0000, 17, 0x3fc00000, 22, 0x0001fffe, 1, 0x00000001, 0 }
 };
+#endif
 
 static const struct kv_pt_config_reg didt_config_kv[] =
 {
@@ -507,19 +509,19 @@ static int kv_enable_didt(struct amdgpu_device *adev, bool enable)
             pi->caps_db_ramping ||
             pi->caps_td_ramping ||
             pi->caps_tcp_ramping) {
-                gfx_v7_0_enter_rlc_safe_mode(adev);
+                adev->gfx.rlc.funcs->enter_safe_mode(adev);
 
                 if (enable) {
                         ret = kv_program_pt_config_registers(adev, didt_config_kv);
                         if (ret) {
-                                gfx_v7_0_exit_rlc_safe_mode(adev);
+                                adev->gfx.rlc.funcs->exit_safe_mode(adev);
                                 return ret;
                         }
                 }
 
                 kv_do_enable_didt(adev, enable);
 
-                gfx_v7_0_exit_rlc_safe_mode(adev);
+                adev->gfx.rlc.funcs->exit_safe_mode(adev);
         }
 
         return 0;
diff --git a/drivers/gpu/drm/amd/amdgpu/ppsmc.h b/drivers/gpu/drm/amd/amdgpu/ppsmc.h
index 7837f2ecc357..8463245f424f 100644
--- a/drivers/gpu/drm/amd/amdgpu/ppsmc.h
+++ b/drivers/gpu/drm/amd/amdgpu/ppsmc.h
@@ -90,7 +90,9 @@ typedef uint8_t PPSMC_Result;
 #define PPSMC_StartFanControl               ((uint8_t)0x5B)
 #define PPSMC_StopFanControl                ((uint8_t)0x5C)
 #define PPSMC_MSG_NoDisplay                 ((uint8_t)0x5D)
+#define PPSMC_NoDisplay                     ((uint8_t)0x5D)
 #define PPSMC_MSG_HasDisplay                ((uint8_t)0x5E)
+#define PPSMC_HasDisplay                    ((uint8_t)0x5E)
 #define PPSMC_MSG_UVDPowerOFF               ((uint8_t)0x60)
 #define PPSMC_MSG_UVDPowerON                ((uint8_t)0x61)
 #define PPSMC_MSG_EnableULV                 ((uint8_t)0x62)
@@ -108,6 +110,7 @@ typedef uint8_t PPSMC_Result;
 #define PPSMC_MSG_DisableDTE                ((uint8_t)0x88)
 #define PPSMC_MSG_ThrottleOVRDSCLKDS        ((uint8_t)0x96)
 #define PPSMC_MSG_CancelThrottleOVRDSCLKDS  ((uint8_t)0x97)
+#define PPSMC_MSG_EnableACDCGPIOInterrupt   ((uint16_t) 0x149)
 
 /* CI/KV/KB */
 #define PPSMC_MSG_UVDDPM_SetEnabledMask       ((uint16_t) 0x12D)
@@ -161,6 +164,7 @@ typedef uint8_t PPSMC_Result;
 #define PPSMC_MSG_MASTER_DeepSleep_OFF        ((uint16_t) 0x190)
 #define PPSMC_MSG_Remove_DC_Clamp             ((uint16_t) 0x191)
 #define PPSMC_MSG_SetFanPwmMax                ((uint16_t) 0x19A)
+#define PPSMC_MSG_SetFanRpmMax                ((uint16_t) 0x205)
 
 #define PPSMC_MSG_ENABLE_THERMAL_DPM          ((uint16_t) 0x19C)
 #define PPSMC_MSG_DISABLE_THERMAL_DPM         ((uint16_t) 0x19D)
diff --git a/drivers/gpu/drm/amd/amdgpu/sdma_v2_4.c b/drivers/gpu/drm/amd/amdgpu/sdma_v2_4.c
index b556bd0a8797..1351c7e834a2 100644
--- a/drivers/gpu/drm/amd/amdgpu/sdma_v2_4.c
+++ b/drivers/gpu/drm/amd/amdgpu/sdma_v2_4.c
@@ -255,19 +255,6 @@ static void sdma_v2_4_ring_emit_ib(struct amdgpu_ring *ring,
                                    unsigned vm_id, bool ctx_switch)
 {
         u32 vmid = vm_id & 0xf;
-        u32 next_rptr = ring->wptr + 5;
-
-        while ((next_rptr & 7) != 2)
-                next_rptr++;
-
-        next_rptr += 6;
-
-        amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_WRITE) |
-                          SDMA_PKT_HEADER_SUB_OP(SDMA_SUBOP_WRITE_LINEAR));
-        amdgpu_ring_write(ring, lower_32_bits(ring->next_rptr_gpu_addr) & 0xfffffffc);
-        amdgpu_ring_write(ring, upper_32_bits(ring->next_rptr_gpu_addr));
-        amdgpu_ring_write(ring, SDMA_PKT_WRITE_UNTILED_DW_3_COUNT(1));
-        amdgpu_ring_write(ring, next_rptr);
 
         /* IB packet must end on a 8 DW boundary */
         sdma_v2_4_ring_insert_nop(ring, (10 - (ring->wptr & 7)) % 8);
@@ -406,7 +393,7 @@ static void sdma_v2_4_enable(struct amdgpu_device *adev, bool enable)
         u32 f32_cntl;
         int i;
 
-        if (enable == false) {
+        if (!enable) {
                 sdma_v2_4_gfx_stop(adev);
                 sdma_v2_4_rlc_stop(adev);
         }
@@ -580,19 +567,21 @@ static int sdma_v2_4_start(struct amdgpu_device *adev)
 {
         int r;
 
-        if (!adev->firmware.smu_load) {
-                r = sdma_v2_4_load_microcode(adev);
-                if (r)
-                        return r;
-        } else {
-                r = adev->smu.smumgr_funcs->check_fw_load_finish(adev,
-                                                AMDGPU_UCODE_ID_SDMA0);
-                if (r)
-                        return -EINVAL;
-                r = adev->smu.smumgr_funcs->check_fw_load_finish(adev,
-                                                AMDGPU_UCODE_ID_SDMA1);
-                if (r)
-                        return -EINVAL;
+        if (!adev->pp_enabled) {
+                if (!adev->firmware.smu_load) {
+                        r = sdma_v2_4_load_microcode(adev);
+                        if (r)
+                                return r;
+                } else {
+                        r = adev->smu.smumgr_funcs->check_fw_load_finish(adev,
+                                                        AMDGPU_UCODE_ID_SDMA0);
+                        if (r)
+                                return -EINVAL;
+                        r = adev->smu.smumgr_funcs->check_fw_load_finish(adev,
+                                                        AMDGPU_UCODE_ID_SDMA1);
+                        if (r)
+                                return -EINVAL;
+                }
         }
 
         /* halt the engine before programing */
@@ -679,20 +668,19 @@ static int sdma_v2_4_ring_test_ring(struct amdgpu_ring *ring)
  * Test a simple IB in the DMA ring (VI).
  * Returns 0 on success, error on failure.
  */
-static int sdma_v2_4_ring_test_ib(struct amdgpu_ring *ring)
+static int sdma_v2_4_ring_test_ib(struct amdgpu_ring *ring, long timeout)
 {
         struct amdgpu_device *adev = ring->adev;
         struct amdgpu_ib ib;
         struct fence *f = NULL;
-        unsigned i;
         unsigned index;
-        int r;
         u32 tmp = 0;
         u64 gpu_addr;
+        long r;
 
         r = amdgpu_wb_get(adev, &index);
         if (r) {
-                dev_err(adev->dev, "(%d) failed to allocate wb slot\n", r);
+                dev_err(adev->dev, "(%ld) failed to allocate wb slot\n", r);
                 return r;
         }
 
@@ -702,7 +690,7 @@ static int sdma_v2_4_ring_test_ib(struct amdgpu_ring *ring)
         memset(&ib, 0, sizeof(ib));
         r = amdgpu_ib_get(adev, NULL, 256, &ib);
         if (r) {
-                DRM_ERROR("amdgpu: failed to get ib (%d).\n", r);
+                DRM_ERROR("amdgpu: failed to get ib (%ld).\n", r);
                 goto err0;
         }
 
@@ -721,28 +709,25 @@ static int sdma_v2_4_ring_test_ib(struct amdgpu_ring *ring)
         if (r)
                 goto err1;
 
-        r = fence_wait(f, false);
-        if (r) {
-                DRM_ERROR("amdgpu: fence wait failed (%d).\n", r);
+        r = fence_wait_timeout(f, false, timeout);
+        if (r == 0) {
+                DRM_ERROR("amdgpu: IB test timed out\n");
+                r = -ETIMEDOUT;
                 goto err1;
-        }
-        for (i = 0; i < adev->usec_timeout; i++) {
-                tmp = le32_to_cpu(adev->wb.wb[index]);
-                if (tmp == 0xDEADBEEF)
-                        break;
-                DRM_UDELAY(1);
-        }
-        if (i < adev->usec_timeout) {
-                DRM_INFO("ib test on ring %d succeeded in %u usecs\n",
-                         ring->idx, i);
+        } else if (r) {
+                DRM_ERROR("amdgpu: fence wait failed (%ld).\n", r);
                 goto err1;
+        }
+        tmp = le32_to_cpu(adev->wb.wb[index]);
+        if (tmp == 0xDEADBEEF) {
+                DRM_INFO("ib test on ring %d succeeded\n", ring->idx);
+                r = 0;
         } else {
                 DRM_ERROR("amdgpu: ib test failed (0x%08X)\n", tmp);
                 r = -EINVAL;
         }
 
 err1:
-        fence_put(f);
         amdgpu_ib_free(adev, &ib, NULL);
         fence_put(f);
 err0:
diff --git a/drivers/gpu/drm/amd/amdgpu/sdma_v3_0.c b/drivers/gpu/drm/amd/amdgpu/sdma_v3_0.c
index 532ea88da66a..653ce5ed55ae 100644
--- a/drivers/gpu/drm/amd/amdgpu/sdma_v3_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/sdma_v3_0.c
@@ -415,18 +415,6 @@ static void sdma_v3_0_ring_emit_ib(struct amdgpu_ring *ring,
                                    unsigned vm_id, bool ctx_switch)
 {
         u32 vmid = vm_id & 0xf;
-        u32 next_rptr = ring->wptr + 5;
-
-        while ((next_rptr & 7) != 2)
-                next_rptr++;
-        next_rptr += 6;
-
-        amdgpu_ring_write(ring, SDMA_PKT_HEADER_OP(SDMA_OP_WRITE) |
-                          SDMA_PKT_HEADER_SUB_OP(SDMA_SUBOP_WRITE_LINEAR));
-        amdgpu_ring_write(ring, lower_32_bits(ring->next_rptr_gpu_addr) & 0xfffffffc);
-        amdgpu_ring_write(ring, upper_32_bits(ring->next_rptr_gpu_addr));
-        amdgpu_ring_write(ring, SDMA_PKT_WRITE_UNTILED_DW_3_COUNT(1));
-        amdgpu_ring_write(ring, next_rptr);
 
         /* IB packet must end on a 8 DW boundary */
         sdma_v3_0_ring_insert_nop(ring, (10 - (ring->wptr & 7)) % 8);
@@ -616,7 +604,7 @@ static void sdma_v3_0_enable(struct amdgpu_device *adev, bool enable)
         u32 f32_cntl;
         int i;
 
-        if (enable == false) {
+        if (!enable) {
                 sdma_v3_0_gfx_stop(adev);
                 sdma_v3_0_rlc_stop(adev);
         }
@@ -908,20 +896,19 @@ static int sdma_v3_0_ring_test_ring(struct amdgpu_ring *ring)
  * Test a simple IB in the DMA ring (VI).
  * Returns 0 on success, error on failure.
  */
-static int sdma_v3_0_ring_test_ib(struct amdgpu_ring *ring)
+static int sdma_v3_0_ring_test_ib(struct amdgpu_ring *ring, long timeout)
 {
         struct amdgpu_device *adev = ring->adev;
         struct amdgpu_ib ib;
         struct fence *f = NULL;
-        unsigned i;
         unsigned index;
-        int r;
         u32 tmp = 0;
         u64 gpu_addr;
+        long r;
 
         r = amdgpu_wb_get(adev, &index);
         if (r) {
-                dev_err(adev->dev, "(%d) failed to allocate wb slot\n", r);
+                dev_err(adev->dev, "(%ld) failed to allocate wb slot\n", r);
                 return r;
         }
 
@@ -931,7 +918,7 @@ static int sdma_v3_0_ring_test_ib(struct amdgpu_ring *ring)
         memset(&ib, 0, sizeof(ib));
         r = amdgpu_ib_get(adev, NULL, 256, &ib);
         if (r) {
-                DRM_ERROR("amdgpu: failed to get ib (%d).\n", r);
+                DRM_ERROR("amdgpu: failed to get ib (%ld).\n", r);
                 goto err0;
         }
 
@@ -950,27 +937,24 @@ static int sdma_v3_0_ring_test_ib(struct amdgpu_ring *ring)
         if (r)
                 goto err1;
 
-        r = fence_wait(f, false);
-        if (r) {
-                DRM_ERROR("amdgpu: fence wait failed (%d).\n", r);
+        r = fence_wait_timeout(f, false, timeout);
+        if (r == 0) {
+                DRM_ERROR("amdgpu: IB test timed out\n");
+                r = -ETIMEDOUT;
                 goto err1;
-        }
-        for (i = 0; i < adev->usec_timeout; i++) {
-                tmp = le32_to_cpu(adev->wb.wb[index]);
-                if (tmp == 0xDEADBEEF)
-                        break;
-                DRM_UDELAY(1);
-        }
-        if (i < adev->usec_timeout) {
-                DRM_INFO("ib test on ring %d succeeded in %u usecs\n",
-                         ring->idx, i);
+        } else if (r < 0) {
+                DRM_ERROR("amdgpu: fence wait failed (%ld).\n", r);
                 goto err1;
+        }
+        tmp = le32_to_cpu(adev->wb.wb[index]);
+        if (tmp == 0xDEADBEEF) {
+                DRM_INFO("ib test on ring %d succeeded\n", ring->idx);
+                r = 0;
         } else {
                 DRM_ERROR("amdgpu: ib test failed (0x%08X)\n", tmp);
                 r = -EINVAL;
         }
 err1:
-        fence_put(f);
         amdgpu_ib_free(adev, &ib, NULL);
         fence_put(f);
 err0:
diff --git a/drivers/gpu/drm/amd/amdgpu/tonga_smc.c b/drivers/gpu/drm/amd/amdgpu/tonga_smc.c
index 083893dd68c0..940de1836f8f 100644
--- a/drivers/gpu/drm/amd/amdgpu/tonga_smc.c
+++ b/drivers/gpu/drm/amd/amdgpu/tonga_smc.c
@@ -173,7 +173,7 @@ static int tonga_send_msg_to_smc(struct amdgpu_device *adev, PPSMC_Msg msg)
 {
         if (!tonga_is_smc_ram_running(adev))
         {
-                return -EINVAL;;
+                return -EINVAL;
         }
 
         if (wait_smu_response(adev)) {
diff --git a/drivers/gpu/drm/amd/amdgpu/uvd_v4_2.c b/drivers/gpu/drm/amd/amdgpu/uvd_v4_2.c
index f07551476a70..132e613ed674 100644
--- a/drivers/gpu/drm/amd/amdgpu/uvd_v4_2.c
+++ b/drivers/gpu/drm/amd/amdgpu/uvd_v4_2.c
@@ -34,6 +34,8 @@
 #include "oss/oss_2_0_d.h"
 #include "oss/oss_2_0_sh_mask.h"
 
+#include "bif/bif_4_1_d.h"
+
 static void uvd_v4_2_mc_resume(struct amdgpu_device *adev);
 static void uvd_v4_2_init_cg(struct amdgpu_device *adev);
 static void uvd_v4_2_set_ring_funcs(struct amdgpu_device *adev);
@@ -439,6 +441,32 @@ static void uvd_v4_2_ring_emit_fence(struct amdgpu_ring *ring, u64 addr, u64 seq
 }
 
 /**
+ * uvd_v4_2_ring_emit_hdp_flush - emit an hdp flush
+ *
+ * @ring: amdgpu_ring pointer
+ *
+ * Emits an hdp flush.
+ */
+static void uvd_v4_2_ring_emit_hdp_flush(struct amdgpu_ring *ring)
+{
+        amdgpu_ring_write(ring, PACKET0(mmHDP_MEM_COHERENCY_FLUSH_CNTL, 0));
+        amdgpu_ring_write(ring, 0);
+}
+
+/**
+ * uvd_v4_2_ring_hdp_invalidate - emit an hdp invalidate
+ *
+ * @ring: amdgpu_ring pointer
+ *
+ * Emits an hdp invalidate.
+ */
+static void uvd_v4_2_ring_emit_hdp_invalidate(struct amdgpu_ring *ring)
+{
+        amdgpu_ring_write(ring, PACKET0(mmHDP_DEBUG0, 0));
+        amdgpu_ring_write(ring, 1);
+}
+
+/**
  * uvd_v4_2_ring_test_ring - register write test
  *
  * @ring: amdgpu_ring pointer
@@ -499,49 +527,6 @@ static void uvd_v4_2_ring_emit_ib(struct amdgpu_ring *ring,
 }
 
 /**
- * uvd_v4_2_ring_test_ib - test ib execution
- *
- * @ring: amdgpu_ring pointer
- *
- * Test if we can successfully execute an IB
- */
-static int uvd_v4_2_ring_test_ib(struct amdgpu_ring *ring)
-{
-        struct amdgpu_device *adev = ring->adev;
-        struct fence *fence = NULL;
-        int r;
-
-        r = amdgpu_asic_set_uvd_clocks(adev, 53300, 40000);
-        if (r) {
-                DRM_ERROR("amdgpu: failed to raise UVD clocks (%d).\n", r);
-                return r;
-        }
-
-        r = amdgpu_uvd_get_create_msg(ring, 1, NULL);
-        if (r) {
-                DRM_ERROR("amdgpu: failed to get create msg (%d).\n", r);
-                goto error;
-        }
-
-        r = amdgpu_uvd_get_destroy_msg(ring, 1, true, &fence);
-        if (r) {
-                DRM_ERROR("amdgpu: failed to get destroy ib (%d).\n", r);
-                goto error;
-        }
-
-        r = fence_wait(fence, false);
-        if (r) {
-                DRM_ERROR("amdgpu: fence wait failed (%d).\n", r);
-                goto error;
-        }
-        DRM_INFO("ib test on ring %d succeeded\n",  ring->idx);
-error:
-        fence_put(fence);
-        amdgpu_asic_set_uvd_clocks(adev, 0, 0);
-        return r;
-}
-
-/**
  * uvd_v4_2_mc_resume - memory controller programming
  *
  * @adev: amdgpu_device pointer
@@ -763,10 +748,14 @@ static const struct amdgpu_ring_funcs uvd_v4_2_ring_funcs = {
         .parse_cs = amdgpu_uvd_ring_parse_cs,
         .emit_ib = uvd_v4_2_ring_emit_ib,
         .emit_fence = uvd_v4_2_ring_emit_fence,
+        .emit_hdp_flush = uvd_v4_2_ring_emit_hdp_flush,
+        .emit_hdp_invalidate = uvd_v4_2_ring_emit_hdp_invalidate,
         .test_ring = uvd_v4_2_ring_test_ring,
-        .test_ib = uvd_v4_2_ring_test_ib,
+        .test_ib = amdgpu_uvd_ring_test_ib,
         .insert_nop = amdgpu_ring_insert_nop,
         .pad_ib = amdgpu_ring_generic_pad_ib,
+        .begin_use = amdgpu_uvd_ring_begin_use,
+        .end_use = amdgpu_uvd_ring_end_use,
 };
 
 static void uvd_v4_2_set_ring_funcs(struct amdgpu_device *adev)
diff --git a/drivers/gpu/drm/amd/amdgpu/uvd_v5_0.c b/drivers/gpu/drm/amd/amdgpu/uvd_v5_0.c
index e0a76a883d46..101de136ba63 100644
--- a/drivers/gpu/drm/amd/amdgpu/uvd_v5_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/uvd_v5_0.c
@@ -31,6 +31,7 @@
 #include "uvd/uvd_5_0_sh_mask.h"
 #include "oss/oss_2_0_d.h"
 #include "oss/oss_2_0_sh_mask.h"
+#include "bif/bif_5_0_d.h"
 #include "vi.h"
 
 static void uvd_v5_0_set_ring_funcs(struct amdgpu_device *adev);
@@ -489,6 +490,32 @@ static void uvd_v5_0_ring_emit_fence(struct amdgpu_ring *ring, u64 addr, u64 seq
 }
 
 /**
+ * uvd_v5_0_ring_emit_hdp_flush - emit an hdp flush
+ *
+ * @ring: amdgpu_ring pointer
+ *
+ * Emits an hdp flush.
+ */
+static void uvd_v5_0_ring_emit_hdp_flush(struct amdgpu_ring *ring)
+{
+        amdgpu_ring_write(ring, PACKET0(mmHDP_MEM_COHERENCY_FLUSH_CNTL, 0));
+        amdgpu_ring_write(ring, 0);
+}
+
+/**
+ * uvd_v5_0_ring_hdp_invalidate - emit an hdp invalidate
+ *
+ * @ring: amdgpu_ring pointer
+ *
+ * Emits an hdp invalidate.
+ */
+static void uvd_v5_0_ring_emit_hdp_invalidate(struct amdgpu_ring *ring)
+{
+        amdgpu_ring_write(ring, PACKET0(mmHDP_DEBUG0, 0));
+        amdgpu_ring_write(ring, 1);
+}
+
+/**
  * uvd_v5_0_ring_test_ring - register write test
  *
  * @ring: amdgpu_ring pointer
@@ -550,49 +577,6 @@ static void uvd_v5_0_ring_emit_ib(struct amdgpu_ring *ring,
         amdgpu_ring_write(ring, ib->length_dw);
 }
 
-/**
- * uvd_v5_0_ring_test_ib - test ib execution
- *
- * @ring: amdgpu_ring pointer
- *
- * Test if we can successfully execute an IB
- */
-static int uvd_v5_0_ring_test_ib(struct amdgpu_ring *ring)
-{
-        struct amdgpu_device *adev = ring->adev;
-        struct fence *fence = NULL;
-        int r;
-
-        r = amdgpu_asic_set_uvd_clocks(adev, 53300, 40000);
-        if (r) {
-                DRM_ERROR("amdgpu: failed to raise UVD clocks (%d).\n", r);
-                return r;
-        }
-
-        r = amdgpu_uvd_get_create_msg(ring, 1, NULL);
-        if (r) {
-                DRM_ERROR("amdgpu: failed to get create msg (%d).\n", r);
-                goto error;
-        }
-
-        r = amdgpu_uvd_get_destroy_msg(ring, 1, true, &fence);
-        if (r) {
-                DRM_ERROR("amdgpu: failed to get destroy ib (%d).\n", r);
-                goto error;
-        }
-
-        r = fence_wait(fence, false);
-        if (r) {
-                DRM_ERROR("amdgpu: fence wait failed (%d).\n", r);
-                goto error;
-        }
-        DRM_INFO("ib test on ring %d succeeded\n",  ring->idx);
-error:
-        fence_put(fence);
-        amdgpu_asic_set_uvd_clocks(adev, 0, 0);
-        return r;
-}
-
 static bool uvd_v5_0_is_idle(void *handle)
 {
         struct amdgpu_device *adev = (struct amdgpu_device *)handle;
@@ -815,10 +799,14 @@ static const struct amdgpu_ring_funcs uvd_v5_0_ring_funcs = {
         .parse_cs = amdgpu_uvd_ring_parse_cs,
         .emit_ib = uvd_v5_0_ring_emit_ib,
         .emit_fence = uvd_v5_0_ring_emit_fence,
+        .emit_hdp_flush = uvd_v5_0_ring_emit_hdp_flush,
+        .emit_hdp_invalidate = uvd_v5_0_ring_emit_hdp_invalidate,
         .test_ring = uvd_v5_0_ring_test_ring,
-        .test_ib = uvd_v5_0_ring_test_ib,
+        .test_ib = amdgpu_uvd_ring_test_ib,
         .insert_nop = amdgpu_ring_insert_nop,
         .pad_ib = amdgpu_ring_generic_pad_ib,
+        .begin_use = amdgpu_uvd_ring_begin_use,
+        .end_use = amdgpu_uvd_ring_end_use,
 };
 
 static void uvd_v5_0_set_ring_funcs(struct amdgpu_device *adev)
diff --git a/drivers/gpu/drm/amd/amdgpu/uvd_v6_0.c b/drivers/gpu/drm/amd/amdgpu/uvd_v6_0.c
index c9929d665c01..7f21102bfb99 100644
--- a/drivers/gpu/drm/amd/amdgpu/uvd_v6_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/uvd_v6_0.c
@@ -33,6 +33,8 @@
 #include "oss/oss_2_0_sh_mask.h"
 #include "smu/smu_7_1_3_d.h"
 #include "smu/smu_7_1_3_sh_mask.h"
+#include "bif/bif_5_1_d.h"
+#include "gmc/gmc_8_1_d.h"
 #include "vi.h"
 
 static void uvd_v6_0_set_ring_funcs(struct amdgpu_device *adev);
@@ -385,8 +387,8 @@ static int uvd_v6_0_start(struct amdgpu_device *adev)
         uint32_t mp_swap_cntl;
         int i, j, r;
 
-        /*disable DPG */
-        WREG32_P(mmUVD_POWER_STATUS, 0, ~(1 << 2));
+        /* disable DPG */
+        WREG32_P(mmUVD_POWER_STATUS, 0, ~UVD_POWER_STATUS__UVD_PG_MODE_MASK);
 
         /* disable byte swapping */
         lmi_swap_cntl = 0;
@@ -405,17 +407,21 @@ static int uvd_v6_0_start(struct amdgpu_device *adev)
         }
 
         /* disable interupt */
-        WREG32_P(mmUVD_MASTINT_EN, 0, ~(1 << 1));
+        WREG32_P(mmUVD_MASTINT_EN, 0, ~UVD_MASTINT_EN__VCPU_EN_MASK);
 
         /* stall UMC and register bus before resetting VCPU */
-        WREG32_P(mmUVD_LMI_CTRL2, 1 << 8, ~(1 << 8));
+        WREG32_P(mmUVD_LMI_CTRL2, UVD_LMI_CTRL2__STALL_ARB_UMC_MASK, ~UVD_LMI_CTRL2__STALL_ARB_UMC_MASK);
         mdelay(1);
 
         /* put LMI, VCPU, RBC etc... into reset */
-        WREG32(mmUVD_SOFT_RESET, UVD_SOFT_RESET__LMI_SOFT_RESET_MASK |
-                UVD_SOFT_RESET__VCPU_SOFT_RESET_MASK | UVD_SOFT_RESET__LBSI_SOFT_RESET_MASK |
-                UVD_SOFT_RESET__RBC_SOFT_RESET_MASK | UVD_SOFT_RESET__CSM_SOFT_RESET_MASK |
-                UVD_SOFT_RESET__CXW_SOFT_RESET_MASK | UVD_SOFT_RESET__TAP_SOFT_RESET_MASK |
+        WREG32(mmUVD_SOFT_RESET,
+                UVD_SOFT_RESET__LMI_SOFT_RESET_MASK |
+                UVD_SOFT_RESET__VCPU_SOFT_RESET_MASK |
+                UVD_SOFT_RESET__LBSI_SOFT_RESET_MASK |
+                UVD_SOFT_RESET__RBC_SOFT_RESET_MASK |
+                UVD_SOFT_RESET__CSM_SOFT_RESET_MASK |
+                UVD_SOFT_RESET__CXW_SOFT_RESET_MASK |
+                UVD_SOFT_RESET__TAP_SOFT_RESET_MASK |
                 UVD_SOFT_RESET__LMI_UMC_SOFT_RESET_MASK);
         mdelay(5);
 
@@ -424,8 +430,13 @@ static int uvd_v6_0_start(struct amdgpu_device *adev)
         mdelay(5);
 
         /* initialize UVD memory controller */
-        WREG32(mmUVD_LMI_CTRL, 0x40 | (1 << 8) | (1 << 13) |
-                             (1 << 21) | (1 << 9) | (1 << 20));
+        WREG32(mmUVD_LMI_CTRL,
+                (0x40 << UVD_LMI_CTRL__WRITE_CLEAN_TIMER__SHIFT) |
+                UVD_LMI_CTRL__WRITE_CLEAN_TIMER_EN_MASK |
+                UVD_LMI_CTRL__DATA_COHERENCY_EN_MASK |
+                UVD_LMI_CTRL__VCPU_DATA_COHERENCY_EN_MASK |
+                UVD_LMI_CTRL__REQ_MODE_MASK |
+                UVD_LMI_CTRL__DISABLE_ON_FWV_FAIL_MASK);
 
 #ifdef __BIG_ENDIAN
         /* swap (8 in 32) RB and IB */
@@ -447,10 +458,10 @@ static int uvd_v6_0_start(struct amdgpu_device *adev)
         mdelay(5);
 
         /* enable VCPU clock */
-        WREG32(mmUVD_VCPU_CNTL,  1 << 9);
+        WREG32(mmUVD_VCPU_CNTL, UVD_VCPU_CNTL__CLK_EN_MASK);
 
         /* enable UMC */
-        WREG32_P(mmUVD_LMI_CTRL2, 0, ~(1 << 8));
+        WREG32_P(mmUVD_LMI_CTRL2, 0, ~UVD_LMI_CTRL2__STALL_ARB_UMC_MASK);
 
         /* boot up the VCPU */
         WREG32(mmUVD_SOFT_RESET, 0);
@@ -484,10 +495,12 @@ static int uvd_v6_0_start(struct amdgpu_device *adev)
                 return r;
         }
         /* enable master interrupt */
-        WREG32_P(mmUVD_MASTINT_EN, 3 << 1, ~(3 << 1));
+        WREG32_P(mmUVD_MASTINT_EN,
+                (UVD_MASTINT_EN__VCPU_EN_MASK|UVD_MASTINT_EN__SYS_EN_MASK),
+                ~(UVD_MASTINT_EN__VCPU_EN_MASK|UVD_MASTINT_EN__SYS_EN_MASK));
 
         /* clear the bit 4 of UVD_STATUS */
-        WREG32_P(mmUVD_STATUS, 0, ~(2 << 1));
+        WREG32_P(mmUVD_STATUS, 0, ~(2 << UVD_STATUS__VCPU_REPORT__SHIFT));
 
         rb_bufsz = order_base_2(ring->ring_size);
         tmp = 0;
@@ -581,6 +594,32 @@ static void uvd_v6_0_ring_emit_fence(struct amdgpu_ring *ring, u64 addr, u64 seq
 }
 
 /**
+ * uvd_v6_0_ring_emit_hdp_flush - emit an hdp flush
+ *
+ * @ring: amdgpu_ring pointer
+ *
+ * Emits an hdp flush.
+ */
+static void uvd_v6_0_ring_emit_hdp_flush(struct amdgpu_ring *ring)
+{
+        amdgpu_ring_write(ring, PACKET0(mmHDP_MEM_COHERENCY_FLUSH_CNTL, 0));
+        amdgpu_ring_write(ring, 0);
+}
+
+/**
+ * uvd_v6_0_ring_hdp_invalidate - emit an hdp invalidate
+ *
+ * @ring: amdgpu_ring pointer
+ *
+ * Emits an hdp invalidate.
+ */
+static void uvd_v6_0_ring_emit_hdp_invalidate(struct amdgpu_ring *ring)
+{
+        amdgpu_ring_write(ring, PACKET0(mmHDP_DEBUG0, 0));
+        amdgpu_ring_write(ring, 1);
+}
+
+/**
  * uvd_v6_0_ring_test_ring - register write test
  *
  * @ring: amdgpu_ring pointer
@@ -634,6 +673,9 @@ static void uvd_v6_0_ring_emit_ib(struct amdgpu_ring *ring,
                                   struct amdgpu_ib *ib,
                                   unsigned vm_id, bool ctx_switch)
 {
+        amdgpu_ring_write(ring, PACKET0(mmUVD_LMI_RBC_IB_VMID, 0));
+        amdgpu_ring_write(ring, vm_id);
+
         amdgpu_ring_write(ring, PACKET0(mmUVD_LMI_RBC_IB_64BIT_BAR_LOW, 0));
         amdgpu_ring_write(ring, lower_32_bits(ib->gpu_addr));
         amdgpu_ring_write(ring, PACKET0(mmUVD_LMI_RBC_IB_64BIT_BAR_HIGH, 0));
@@ -642,39 +684,55 @@ static void uvd_v6_0_ring_emit_ib(struct amdgpu_ring *ring,
         amdgpu_ring_write(ring, ib->length_dw);
 }
 
-/**
- * uvd_v6_0_ring_test_ib - test ib execution
- *
- * @ring: amdgpu_ring pointer
- *
- * Test if we can successfully execute an IB
- */
-static int uvd_v6_0_ring_test_ib(struct amdgpu_ring *ring)
+static void uvd_v6_0_ring_emit_vm_flush(struct amdgpu_ring *ring,
+                                         unsigned vm_id, uint64_t pd_addr)
 {
-        struct fence *fence = NULL;
-        int r;
+        uint32_t reg;
 
-        r = amdgpu_uvd_get_create_msg(ring, 1, NULL);
-        if (r) {
-                DRM_ERROR("amdgpu: failed to get create msg (%d).\n", r);
-                goto error;
-        }
+        if (vm_id < 8)
+                reg = mmVM_CONTEXT0_PAGE_TABLE_BASE_ADDR + vm_id;
+        else
+                reg = mmVM_CONTEXT8_PAGE_TABLE_BASE_ADDR + vm_id - 8;
 
-        r = amdgpu_uvd_get_destroy_msg(ring, 1, true, &fence);
-        if (r) {
-                DRM_ERROR("amdgpu: failed to get destroy ib (%d).\n", r);
-                goto error;
-        }
+        amdgpu_ring_write(ring, PACKET0(mmUVD_GPCOM_VCPU_DATA0, 0));
+        amdgpu_ring_write(ring, reg << 2);
+        amdgpu_ring_write(ring, PACKET0(mmUVD_GPCOM_VCPU_DATA1, 0));
+        amdgpu_ring_write(ring, pd_addr >> 12);
+        amdgpu_ring_write(ring, PACKET0(mmUVD_GPCOM_VCPU_CMD, 0));
+        amdgpu_ring_write(ring, 0x8);
 
-        r = fence_wait(fence, false);
-        if (r) {
-                DRM_ERROR("amdgpu: fence wait failed (%d).\n", r);
-                goto error;
-        }
-        DRM_INFO("ib test on ring %d succeeded\n",  ring->idx);
-error:
-        fence_put(fence);
-        return r;
+        amdgpu_ring_write(ring, PACKET0(mmUVD_GPCOM_VCPU_DATA0, 0));
+        amdgpu_ring_write(ring, mmVM_INVALIDATE_REQUEST << 2);
+        amdgpu_ring_write(ring, PACKET0(mmUVD_GPCOM_VCPU_DATA1, 0));
+        amdgpu_ring_write(ring, 1 << vm_id);
+        amdgpu_ring_write(ring, PACKET0(mmUVD_GPCOM_VCPU_CMD, 0));
+        amdgpu_ring_write(ring, 0x8);
+
+        amdgpu_ring_write(ring, PACKET0(mmUVD_GPCOM_VCPU_DATA0, 0));
+        amdgpu_ring_write(ring, mmVM_INVALIDATE_REQUEST << 2);
+        amdgpu_ring_write(ring, PACKET0(mmUVD_GPCOM_VCPU_DATA1, 0));
+        amdgpu_ring_write(ring, 0);
+        amdgpu_ring_write(ring, PACKET0(mmUVD_GP_SCRATCH8, 0));
+        amdgpu_ring_write(ring, 1 << vm_id); /* mask */
+        amdgpu_ring_write(ring, PACKET0(mmUVD_GPCOM_VCPU_CMD, 0));
+        amdgpu_ring_write(ring, 0xC);
+}
+
+static void uvd_v6_0_ring_emit_pipeline_sync(struct amdgpu_ring *ring)
+{
+        uint32_t seq = ring->fence_drv.sync_seq;
+        uint64_t addr = ring->fence_drv.gpu_addr;
+
+        amdgpu_ring_write(ring, PACKET0(mmUVD_GPCOM_VCPU_DATA0, 0));
+        amdgpu_ring_write(ring, lower_32_bits(addr));
+        amdgpu_ring_write(ring, PACKET0(mmUVD_GPCOM_VCPU_DATA1, 0));
+        amdgpu_ring_write(ring, upper_32_bits(addr));
+        amdgpu_ring_write(ring, PACKET0(mmUVD_GP_SCRATCH8, 0));
+        amdgpu_ring_write(ring, 0xffffffff); /* mask */
+        amdgpu_ring_write(ring, PACKET0(mmUVD_GP_SCRATCH9, 0));
+        amdgpu_ring_write(ring, seq);
+        amdgpu_ring_write(ring, PACKET0(mmUVD_GPCOM_VCPU_CMD, 0));
+        amdgpu_ring_write(ring, 0xE);
 }
 
 static bool uvd_v6_0_is_idle(void *handle)
@@ -847,7 +905,8 @@ static int uvd_v6_0_set_clockgating_state(void *handle,
         bool enable = (state == AMD_CG_STATE_GATE) ? true : false;
         static int curstate = -1;
 
-        if (adev->asic_type == CHIP_FIJI)
+        if (adev->asic_type == CHIP_FIJI ||
+                        adev->asic_type == CHIP_POLARIS10)
                 uvd_v6_set_bypass_mode(adev, enable);
 
         if (!(adev->cg_flags & AMD_CG_SUPPORT_UVD_MGCG))
@@ -912,22 +971,51 @@ const struct amd_ip_funcs uvd_v6_0_ip_funcs = {
         .set_powergating_state = uvd_v6_0_set_powergating_state,
 };
 
-static const struct amdgpu_ring_funcs uvd_v6_0_ring_funcs = {
+static const struct amdgpu_ring_funcs uvd_v6_0_ring_phys_funcs = {
         .get_rptr = uvd_v6_0_ring_get_rptr,
         .get_wptr = uvd_v6_0_ring_get_wptr,
         .set_wptr = uvd_v6_0_ring_set_wptr,
         .parse_cs = amdgpu_uvd_ring_parse_cs,
         .emit_ib = uvd_v6_0_ring_emit_ib,
         .emit_fence = uvd_v6_0_ring_emit_fence,
+        .emit_hdp_flush = uvd_v6_0_ring_emit_hdp_flush,
+        .emit_hdp_invalidate = uvd_v6_0_ring_emit_hdp_invalidate,
+        .test_ring = uvd_v6_0_ring_test_ring,
+        .test_ib = amdgpu_uvd_ring_test_ib,
+        .insert_nop = amdgpu_ring_insert_nop,
+        .pad_ib = amdgpu_ring_generic_pad_ib,
+        .begin_use = amdgpu_uvd_ring_begin_use,
+        .end_use = amdgpu_uvd_ring_end_use,
+};
+
+static const struct amdgpu_ring_funcs uvd_v6_0_ring_vm_funcs = {
+        .get_rptr = uvd_v6_0_ring_get_rptr,
+        .get_wptr = uvd_v6_0_ring_get_wptr,
+        .set_wptr = uvd_v6_0_ring_set_wptr,
+        .parse_cs = NULL,
+        .emit_ib = uvd_v6_0_ring_emit_ib,
+        .emit_fence = uvd_v6_0_ring_emit_fence,
+        .emit_vm_flush = uvd_v6_0_ring_emit_vm_flush,
+        .emit_pipeline_sync = uvd_v6_0_ring_emit_pipeline_sync,
+        .emit_hdp_flush = uvd_v6_0_ring_emit_hdp_flush,
+        .emit_hdp_invalidate = uvd_v6_0_ring_emit_hdp_invalidate,
         .test_ring = uvd_v6_0_ring_test_ring,
-        .test_ib = uvd_v6_0_ring_test_ib,
+        .test_ib = amdgpu_uvd_ring_test_ib,
         .insert_nop = amdgpu_ring_insert_nop,
         .pad_ib = amdgpu_ring_generic_pad_ib,
+        .begin_use = amdgpu_uvd_ring_begin_use,
+        .end_use = amdgpu_uvd_ring_end_use,
 };
 
 static void uvd_v6_0_set_ring_funcs(struct amdgpu_device *adev)
 {
-        adev->uvd.ring.funcs = &uvd_v6_0_ring_funcs;
+        if (adev->asic_type >= CHIP_POLARIS10) {
+                adev->uvd.ring.funcs = &uvd_v6_0_ring_vm_funcs;
+                DRM_INFO("UVD is enabled in VM mode\n");
+        } else {
+                adev->uvd.ring.funcs = &uvd_v6_0_ring_phys_funcs;
+                DRM_INFO("UVD is enabled in physical mode\n");
+        }
 }
 
 static const struct amdgpu_irq_src_funcs uvd_v6_0_irq_funcs = {
diff --git a/drivers/gpu/drm/amd/amdgpu/vce_v2_0.c b/drivers/gpu/drm/amd/amdgpu/vce_v2_0.c
index 45d92aceb485..80a37a602181 100644
--- a/drivers/gpu/drm/amd/amdgpu/vce_v2_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/vce_v2_0.c
@@ -594,6 +594,8 @@ static const struct amdgpu_ring_funcs vce_v2_0_ring_funcs = {
         .test_ib = amdgpu_vce_ring_test_ib,
         .insert_nop = amdgpu_ring_insert_nop,
         .pad_ib = amdgpu_ring_generic_pad_ib,
+        .begin_use = amdgpu_vce_ring_begin_use,
+        .end_use = amdgpu_vce_ring_end_use,
 };
 
 static void vce_v2_0_set_ring_funcs(struct amdgpu_device *adev)
diff --git a/drivers/gpu/drm/amd/amdgpu/vce_v3_0.c b/drivers/gpu/drm/amd/amdgpu/vce_v3_0.c
index 30e8099e94c5..c271abffd8dd 100644
--- a/drivers/gpu/drm/amd/amdgpu/vce_v3_0.c
+++ b/drivers/gpu/drm/amd/amdgpu/vce_v3_0.c
@@ -43,6 +43,7 @@
 #define mmVCE_LMI_VCPU_CACHE_40BIT_BAR0 0x8616
 #define mmVCE_LMI_VCPU_CACHE_40BIT_BAR1 0x8617
 #define mmVCE_LMI_VCPU_CACHE_40BIT_BAR2 0x8618
+#define VCE_STATUS_VCPU_REPORT_FW_LOADED_MASK   0x02
 
 #define VCE_V3_0_FW_SIZE        (384 * 1024)
 #define VCE_V3_0_STACK_SIZE     (64 * 1024)
@@ -51,6 +52,7 @@
 static void vce_v3_0_mc_resume(struct amdgpu_device *adev, int idx);
 static void vce_v3_0_set_ring_funcs(struct amdgpu_device *adev);
 static void vce_v3_0_set_irq_funcs(struct amdgpu_device *adev);
+static int vce_v3_0_wait_for_idle(void *handle);
 
 /**
  * vce_v3_0_ring_get_rptr - get read pointer
@@ -205,6 +207,32 @@ static void vce_v3_0_set_vce_sw_clock_gating(struct amdgpu_device *adev,
         vce_v3_0_override_vce_clock_gating(adev, false);
 }
 
+static int vce_v3_0_firmware_loaded(struct amdgpu_device *adev)
+{
+        int i, j;
+
+        for (i = 0; i < 10; ++i) {
+                for (j = 0; j < 100; ++j) {
+                        uint32_t status = RREG32(mmVCE_STATUS);
+
+                        if (status & VCE_STATUS_VCPU_REPORT_FW_LOADED_MASK)
+                                return 0;
+                        mdelay(10);
+                }
+
+                DRM_ERROR("VCE not responding, trying to reset the ECPU!!!\n");
+                WREG32_P(mmVCE_SOFT_RESET,
+                        VCE_SOFT_RESET__ECPU_SOFT_RESET_MASK,
+                        ~VCE_SOFT_RESET__ECPU_SOFT_RESET_MASK);
+                mdelay(10);
+                WREG32_P(mmVCE_SOFT_RESET, 0,
+                        ~VCE_SOFT_RESET__ECPU_SOFT_RESET_MASK);
+                mdelay(10);
+        }
+
+        return -ETIMEDOUT;
+}
+
 /**
  * vce_v3_0_start - start VCE block
  *
@@ -215,11 +243,24 @@ static void vce_v3_0_set_vce_sw_clock_gating(struct amdgpu_device *adev,
 static int vce_v3_0_start(struct amdgpu_device *adev)
 {
         struct amdgpu_ring *ring;
-        int idx, i, j, r;
+        int idx, r;
+
+        ring = &adev->vce.ring[0];
+        WREG32(mmVCE_RB_RPTR, ring->wptr);
+        WREG32(mmVCE_RB_WPTR, ring->wptr);
+        WREG32(mmVCE_RB_BASE_LO, ring->gpu_addr);
+        WREG32(mmVCE_RB_BASE_HI, upper_32_bits(ring->gpu_addr));
+        WREG32(mmVCE_RB_SIZE, ring->ring_size / 4);
+
+        ring = &adev->vce.ring[1];
+        WREG32(mmVCE_RB_RPTR2, ring->wptr);
+        WREG32(mmVCE_RB_WPTR2, ring->wptr);
+        WREG32(mmVCE_RB_BASE_LO2, ring->gpu_addr);
+        WREG32(mmVCE_RB_BASE_HI2, upper_32_bits(ring->gpu_addr));
+        WREG32(mmVCE_RB_SIZE2, ring->ring_size / 4);
 
         mutex_lock(&adev->grbm_idx_mutex);
         for (idx = 0; idx < 2; ++idx) {
-
                 if (adev->vce.harvest_config & (1 << idx))
                         continue;
 
@@ -233,48 +274,24 @@ static int vce_v3_0_start(struct amdgpu_device *adev)
 
                 vce_v3_0_mc_resume(adev, idx);
 
-                /* set BUSY flag */
-                WREG32_P(mmVCE_STATUS, 1, ~1);
+                WREG32_P(mmVCE_STATUS, VCE_STATUS__JOB_BUSY_MASK,
+                         ~VCE_STATUS__JOB_BUSY_MASK);
+
                 if (adev->asic_type >= CHIP_STONEY)
                         WREG32_P(mmVCE_VCPU_CNTL, 1, ~0x200001);
                 else
                         WREG32_P(mmVCE_VCPU_CNTL, VCE_VCPU_CNTL__CLK_EN_MASK,
                                 ~VCE_VCPU_CNTL__CLK_EN_MASK);
 
-                WREG32_P(mmVCE_SOFT_RESET,
-                         VCE_SOFT_RESET__ECPU_SOFT_RESET_MASK,
-                         ~VCE_SOFT_RESET__ECPU_SOFT_RESET_MASK);
-
-                mdelay(100);
-
                 WREG32_P(mmVCE_SOFT_RESET, 0,
                         ~VCE_SOFT_RESET__ECPU_SOFT_RESET_MASK);
 
-                for (i = 0; i < 10; ++i) {
-                        uint32_t status;
-                        for (j = 0; j < 100; ++j) {
-                                status = RREG32(mmVCE_STATUS);
-                                if (status & 2)
-                                        break;
-                                mdelay(10);
-                        }
-                        r = 0;
-                        if (status & 2)
-                                break;
-
-                        DRM_ERROR("VCE not responding, trying to reset the ECPU!!!\n");
-                        WREG32_P(mmVCE_SOFT_RESET,
-                                VCE_SOFT_RESET__ECPU_SOFT_RESET_MASK,
-                                ~VCE_SOFT_RESET__ECPU_SOFT_RESET_MASK);
-                        mdelay(10);
-                        WREG32_P(mmVCE_SOFT_RESET, 0,
-                                ~VCE_SOFT_RESET__ECPU_SOFT_RESET_MASK);
-                        mdelay(10);
-                        r = -1;
-                }
+                mdelay(100);
+
+                r = vce_v3_0_firmware_loaded(adev);
 
                 /* clear BUSY flag */
-                WREG32_P(mmVCE_STATUS, 0, ~1);
+                WREG32_P(mmVCE_STATUS, 0, ~VCE_STATUS__JOB_BUSY_MASK);
 
                 /* Set Clock-Gating off */
                 if (adev->cg_flags & AMD_CG_SUPPORT_VCE_MGCG)
@@ -290,19 +307,46 @@ static int vce_v3_0_start(struct amdgpu_device *adev)
         WREG32_P(mmGRBM_GFX_INDEX, 0, ~GRBM_GFX_INDEX__VCE_INSTANCE_MASK);
         mutex_unlock(&adev->grbm_idx_mutex);
 
-        ring = &adev->vce.ring[0];
-        WREG32(mmVCE_RB_RPTR, ring->wptr);
-        WREG32(mmVCE_RB_WPTR, ring->wptr);
-        WREG32(mmVCE_RB_BASE_LO, ring->gpu_addr);
-        WREG32(mmVCE_RB_BASE_HI, upper_32_bits(ring->gpu_addr));
-        WREG32(mmVCE_RB_SIZE, ring->ring_size / 4);
+        return 0;
+}
 
-        ring = &adev->vce.ring[1];
-        WREG32(mmVCE_RB_RPTR2, ring->wptr);
-        WREG32(mmVCE_RB_WPTR2, ring->wptr);
-        WREG32(mmVCE_RB_BASE_LO2, ring->gpu_addr);
-        WREG32(mmVCE_RB_BASE_HI2, upper_32_bits(ring->gpu_addr));
-        WREG32(mmVCE_RB_SIZE2, ring->ring_size / 4);
+static int vce_v3_0_stop(struct amdgpu_device *adev)
+{
+        int idx;
+
+        mutex_lock(&adev->grbm_idx_mutex);
+        for (idx = 0; idx < 2; ++idx) {
+                if (adev->vce.harvest_config & (1 << idx))
+                        continue;
+
+                if (idx == 0)
+                        WREG32_P(mmGRBM_GFX_INDEX, 0,
+                                ~GRBM_GFX_INDEX__VCE_INSTANCE_MASK);
+                else
+                        WREG32_P(mmGRBM_GFX_INDEX,
+                                GRBM_GFX_INDEX__VCE_INSTANCE_MASK,
+                                ~GRBM_GFX_INDEX__VCE_INSTANCE_MASK);
+
+                if (adev->asic_type >= CHIP_STONEY)
+                        WREG32_P(mmVCE_VCPU_CNTL, 0, ~0x200001);
+                else
+                        WREG32_P(mmVCE_VCPU_CNTL, 0,
+                                ~VCE_VCPU_CNTL__CLK_EN_MASK);
+                /* hold on ECPU */
+                WREG32_P(mmVCE_SOFT_RESET,
+                         VCE_SOFT_RESET__ECPU_SOFT_RESET_MASK,
+                         ~VCE_SOFT_RESET__ECPU_SOFT_RESET_MASK);
+
+                /* clear BUSY flag */
+                WREG32_P(mmVCE_STATUS, 0, ~VCE_STATUS__JOB_BUSY_MASK);
+
+                /* Set Clock-Gating off */
+                if (adev->cg_flags & AMD_CG_SUPPORT_VCE_MGCG)
+                        vce_v3_0_set_vce_sw_clock_gating(adev, false);
+        }
+
+        WREG32_P(mmGRBM_GFX_INDEX, 0, ~GRBM_GFX_INDEX__VCE_INSTANCE_MASK);
+        mutex_unlock(&adev->grbm_idx_mutex);
 
         return 0;
 }
@@ -441,7 +485,14 @@ static int vce_v3_0_hw_init(void *handle)
 
 static int vce_v3_0_hw_fini(void *handle)
 {
-        return 0;
+        int r;
+        struct amdgpu_device *adev = (struct amdgpu_device *)handle;
+
+        r = vce_v3_0_wait_for_idle(handle);
+        if (r)
+                return r;
+
+        return vce_v3_0_stop(adev);
 }
 
 static int vce_v3_0_suspend(void *handle)
@@ -604,6 +655,18 @@ static int vce_v3_0_process_interrupt(struct amdgpu_device *adev,
         return 0;
 }
 
+static void vce_v3_set_bypass_mode(struct amdgpu_device *adev, bool enable)
+{
+        u32 tmp = RREG32_SMC(ixGCK_DFS_BYPASS_CNTL);
+
+        if (enable)
+                tmp |= GCK_DFS_BYPASS_CNTL__BYPASSECLK_MASK;
+        else
+                tmp &= ~GCK_DFS_BYPASS_CNTL__BYPASSECLK_MASK;
+
+        WREG32_SMC(ixGCK_DFS_BYPASS_CNTL, tmp);
+}
+
 static int vce_v3_0_set_clockgating_state(void *handle,
                                           enum amd_clockgating_state state)
 {
@@ -611,6 +674,9 @@ static int vce_v3_0_set_clockgating_state(void *handle,
         bool enable = (state == AMD_CG_STATE_GATE) ? true : false;
         int i;
 
+        if (adev->asic_type == CHIP_POLARIS10)
+                vce_v3_set_bypass_mode(adev, enable);
+
         if (!(adev->cg_flags & AMD_CG_SUPPORT_VCE_MGCG))
                 return 0;
 
@@ -701,6 +767,8 @@ static const struct amdgpu_ring_funcs vce_v3_0_ring_funcs = {
         .test_ib = amdgpu_vce_ring_test_ib,
         .insert_nop = amdgpu_ring_insert_nop,
         .pad_ib = amdgpu_ring_generic_pad_ib,
+        .begin_use = amdgpu_vce_ring_begin_use,
+        .end_use = amdgpu_vce_ring_end_use,
 };
 
 static void vce_v3_0_set_ring_funcs(struct amdgpu_device *adev)
diff --git a/drivers/gpu/drm/amd/amdgpu/vi.c b/drivers/gpu/drm/amd/amdgpu/vi.c
index a65c96029476..03a31c53aec3 100644
--- a/drivers/gpu/drm/amd/amdgpu/vi.c
+++ b/drivers/gpu/drm/amd/amdgpu/vi.c
@@ -203,6 +203,29 @@ static void vi_didt_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
         spin_unlock_irqrestore(&adev->didt_idx_lock, flags);
 }
 
+static u32 vi_gc_cac_rreg(struct amdgpu_device *adev, u32 reg)
+{
+        unsigned long flags;
+        u32 r;
+
+        spin_lock_irqsave(&adev->gc_cac_idx_lock, flags);
+        WREG32(mmGC_CAC_IND_INDEX, (reg));
+        r = RREG32(mmGC_CAC_IND_DATA);
+        spin_unlock_irqrestore(&adev->gc_cac_idx_lock, flags);
+        return r;
+}
+
+static void vi_gc_cac_wreg(struct amdgpu_device *adev, u32 reg, u32 v)
+{
+        unsigned long flags;
+
+        spin_lock_irqsave(&adev->gc_cac_idx_lock, flags);
+        WREG32(mmGC_CAC_IND_INDEX, (reg));
+        WREG32(mmGC_CAC_IND_DATA, (v));
+        spin_unlock_irqrestore(&adev->gc_cac_idx_lock, flags);
+}
+
+
 static const u32 tonga_mgcg_cgcg_init[] =
 {
         mmCGTT_DRM_CLK_CTRL0, 0xffffffff, 0x00600100,
@@ -533,12 +556,12 @@ static uint32_t vi_read_indexed_register(struct amdgpu_device *adev, u32 se_num,
 
         mutex_lock(&adev->grbm_idx_mutex);
         if (se_num != 0xffffffff || sh_num != 0xffffffff)
-                gfx_v8_0_select_se_sh(adev, se_num, sh_num);
+                amdgpu_gfx_select_se_sh(adev, se_num, sh_num, 0xffffffff);
 
         val = RREG32(reg_offset);
 
         if (se_num != 0xffffffff || sh_num != 0xffffffff)
-                gfx_v8_0_select_se_sh(adev, 0xffffffff, 0xffffffff);
+                amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
         mutex_unlock(&adev->grbm_idx_mutex);
         return val;
 }
@@ -597,7 +620,7 @@ static int vi_read_register(struct amdgpu_device *adev, u32 se_num,
         return -EINVAL;
 }
 
-static void vi_gpu_pci_config_reset(struct amdgpu_device *adev)
+static int vi_gpu_pci_config_reset(struct amdgpu_device *adev)
 {
         u32 i;
 
@@ -612,11 +635,14 @@ static void vi_gpu_pci_config_reset(struct amdgpu_device *adev)
 
         /* wait for asic to come out of reset */
         for (i = 0; i < adev->usec_timeout; i++) {
-                if (RREG32(mmCONFIG_MEMSIZE) != 0xffffffff)
-                        break;
+                if (RREG32(mmCONFIG_MEMSIZE) != 0xffffffff) {
+                        /* enable BM */
+                        pci_set_master(adev->pdev);
+                        return 0;
+                }
                 udelay(1);
         }
-
+        return -EINVAL;
 }
 
 static void vi_set_bios_scratch_engine_hung(struct amdgpu_device *adev, bool hung)
@@ -642,13 +668,15 @@ static void vi_set_bios_scratch_engine_hung(struct amdgpu_device *adev, bool hun
  */
 static int vi_asic_reset(struct amdgpu_device *adev)
 {
+        int r;
+
         vi_set_bios_scratch_engine_hung(adev, true);
 
-        vi_gpu_pci_config_reset(adev);
+        r = vi_gpu_pci_config_reset(adev);
 
         vi_set_bios_scratch_engine_hung(adev, false);
 
-        return 0;
+        return r;
 }
 
 static int vi_set_uvd_clock(struct amdgpu_device *adev, u32 clock,
@@ -1133,9 +1161,6 @@ static const struct amdgpu_asic_funcs vi_asic_funcs =
         .set_uvd_clocks = &vi_set_uvd_clocks,
         .set_vce_clocks = &vi_set_vce_clocks,
         .get_virtual_caps = &vi_get_virtual_caps,
-        /* these should be moved to their own ip modules */
-        .get_gpu_clock_counter = &gfx_v8_0_get_gpu_clock_counter,
-        .wait_for_mc_idle = &gmc_v8_0_mc_wait_for_idle,
 };
 
 static int vi_common_early_init(void *handle)
@@ -1156,6 +1181,8 @@ static int vi_common_early_init(void *handle)
         adev->uvd_ctx_wreg = &vi_uvd_ctx_wreg;
         adev->didt_rreg = &vi_didt_rreg;
         adev->didt_wreg = &vi_didt_wreg;
+        adev->gc_cac_rreg = &vi_gc_cac_rreg;
+        adev->gc_cac_wreg = &vi_gc_cac_wreg;
 
         adev->asic_funcs = &vi_asic_funcs;
 
@@ -1229,12 +1256,18 @@ static int vi_common_early_init(void *handle)
                 adev->cg_flags = AMD_CG_SUPPORT_UVD_MGCG |
                         AMD_CG_SUPPORT_GFX_MGCG |
                         AMD_CG_SUPPORT_GFX_MGLS |
+                        AMD_CG_SUPPORT_GFX_RLC_LS |
+                        AMD_CG_SUPPORT_GFX_CP_LS |
+                        AMD_CG_SUPPORT_GFX_CGTS |
+                        AMD_CG_SUPPORT_GFX_MGLS |
+                        AMD_CG_SUPPORT_GFX_CGTS_LS |
+                        AMD_CG_SUPPORT_GFX_CGCG |
+                        AMD_CG_SUPPORT_GFX_CGLS |
                         AMD_CG_SUPPORT_BIF_LS |
                         AMD_CG_SUPPORT_HDP_MGCG |
                         AMD_CG_SUPPORT_HDP_LS |
                         AMD_CG_SUPPORT_SDMA_MGCG |
                         AMD_CG_SUPPORT_SDMA_LS;
-                adev->pg_flags = 0;
                 adev->external_rev_id = adev->rev_id + 0x1;
                 break;
         default:
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.h b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.h
index ec4036a09f3e..a625b9137da2 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.h
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_device_queue_manager.h
@@ -187,12 +187,12 @@ int init_pipelines(struct device_queue_manager *dqm,
 unsigned int get_first_pipe(struct device_queue_manager *dqm);
 unsigned int get_pipes_num(struct device_queue_manager *dqm);
 
-extern inline unsigned int get_sh_mem_bases_32(struct kfd_process_device *pdd)
+static inline unsigned int get_sh_mem_bases_32(struct kfd_process_device *pdd)
 {
         return (pdd->lds_base >> 16) & 0xFF;
 }
 
-extern inline unsigned int
+static inline unsigned int
 get_sh_mem_bases_nybble_64(struct kfd_process_device *pdd)
 {
         return (pdd->lds_base >> 60) & 0x0E;
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_priv.h b/drivers/gpu/drm/amd/amdkfd/kfd_priv.h
index d0d5f4baf72d..80113c335966 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_priv.h
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_priv.h
@@ -617,10 +617,7 @@ int kgd2kfd_resume(struct kfd_dev *kfd);
 int kfd_init_apertures(struct kfd_process *process);
 
 /* Queue Context Management */
-inline uint32_t lower_32(uint64_t x);
-inline uint32_t upper_32(uint64_t x);
 struct cik_sdma_rlc_registers *get_sdma_mqd(void *mqd);
-inline uint32_t get_sdma_base_addr(struct cik_sdma_rlc_registers *m);
 
 int init_queue(struct queue **q, struct queue_properties properties);
 void uninit_queue(struct queue *q);
diff --git a/drivers/gpu/drm/amd/amdkfd/kfd_process.c b/drivers/gpu/drm/amd/amdkfd/kfd_process.c
index 7708d90b9da9..4f3849ac8c07 100644
--- a/drivers/gpu/drm/amd/amdkfd/kfd_process.c
+++ b/drivers/gpu/drm/amd/amdkfd/kfd_process.c
@@ -63,13 +63,12 @@ static struct kfd_process *create_process(const struct task_struct *thread);
 void kfd_process_create_wq(void)
 {
         if (!kfd_process_wq)
-                kfd_process_wq = create_workqueue("kfd_process_wq");
+                kfd_process_wq = alloc_workqueue("kfd_process_wq", 0, 0);
 }
 
 void kfd_process_destroy_wq(void)
 {
         if (kfd_process_wq) {
-                flush_workqueue(kfd_process_wq);
                 destroy_workqueue(kfd_process_wq);
                 kfd_process_wq = NULL;
         }
@@ -330,6 +329,7 @@ err_process_pqm_init:
         synchronize_rcu();
         mmu_notifier_unregister_no_release(&process->mmu_notifier, process->mm);
 err_mmu_notifier:
+        mutex_destroy(&process->mutex);
         kfd_pasid_free(process->pasid);
 err_alloc_pasid:
         kfree(process->queues);
diff --git a/drivers/gpu/drm/amd/include/amd_pcie.h b/drivers/gpu/drm/amd/include/amd_pcie.h
index 7c2a916c1e63..5eb895fd98bf 100644
--- a/drivers/gpu/drm/amd/include/amd_pcie.h
+++ b/drivers/gpu/drm/amd/include/amd_pcie.h
@@ -37,6 +37,13 @@
 #define CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_MASK   0x0000FFFF
 #define CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_SHIFT  0
 
+/* gen: chipset 1/2, asic 1/2/3 */
+#define AMDGPU_DEFAULT_PCIE_GEN_MASK (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1 \
+                                      | CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2 \
+                                      | CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1 \
+                                      | CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN2 \
+                                      | CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN3)
+
 /* Following flags shows PCIe lane width switch supported in driver which are decided by chipset and ASIC */
 #define CAIL_PCIE_LINK_WIDTH_SUPPORT_X1          0x00010000
 #define CAIL_PCIE_LINK_WIDTH_SUPPORT_X2          0x00020000
@@ -47,4 +54,11 @@
 #define CAIL_PCIE_LINK_WIDTH_SUPPORT_X32         0x00400000
 #define CAIL_PCIE_LINK_WIDTH_SUPPORT_SHIFT       16
 
+/* 1/2/4/8/16 lanes */
+#define AMDGPU_DEFAULT_PCIE_MLW_MASK (CAIL_PCIE_LINK_WIDTH_SUPPORT_X1 \
+                                      | CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 \
+                                      | CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 \
+                                      | CAIL_PCIE_LINK_WIDTH_SUPPORT_X8 \
+                                      | CAIL_PCIE_LINK_WIDTH_SUPPORT_X16)
+
 #endif
diff --git a/drivers/gpu/drm/amd/include/amd_shared.h b/drivers/gpu/drm/amd/include/amd_shared.h
index afce1edbe250..a74a0d2ff1ca 100644
--- a/drivers/gpu/drm/amd/include/amd_shared.h
+++ b/drivers/gpu/drm/amd/include/amd_shared.h
@@ -26,15 +26,6 @@
 #define AMD_MAX_USEC_TIMEOUT            100000  /* 100 ms */
 
 /*
-* Supported GPU families (aligned with amdgpu_drm.h)
-*/
-#define AMD_FAMILY_UNKNOWN              0
-#define AMD_FAMILY_CI                   120 /* Bonaire, Hawaii */
-#define AMD_FAMILY_KV                   125 /* Kaveri, Kabini, Mullins */
-#define AMD_FAMILY_VI                   130 /* Iceland, Tonga */
-#define AMD_FAMILY_CZ                   135 /* Carrizo */
-
-/*
  * Supported ASIC types
  */
 enum amd_asic_type {
@@ -120,6 +111,8 @@ enum amd_powergating_state {
 #define AMD_PG_SUPPORT_SDMA                     (1 << 8)
 #define AMD_PG_SUPPORT_ACP                      (1 << 9)
 #define AMD_PG_SUPPORT_SAMU                     (1 << 10)
+#define AMD_PG_SUPPORT_GFX_QUICK_MG             (1 << 11)
+#define AMD_PG_SUPPORT_GFX_PIPELINE             (1 << 12)
 
 enum amd_pm_state_type {
         /* not used for dpm */
diff --git a/drivers/gpu/drm/amd/include/asic_reg/bif/bif_5_0_d.h b/drivers/gpu/drm/amd/include/asic_reg/bif/bif_5_0_d.h
index 293329719bba..809759f7bb81 100644
--- a/drivers/gpu/drm/amd/include/asic_reg/bif/bif_5_0_d.h
+++ b/drivers/gpu/drm/amd/include/asic_reg/bif/bif_5_0_d.h
@@ -27,6 +27,7 @@
 #define mmMM_INDEX                                                              0x0
 #define mmMM_INDEX_HI                                                           0x6
 #define mmMM_DATA                                                               0x1
+#define mmCC_BIF_BX_STRAP2                                                      0x152A
 #define mmBIF_MM_INDACCESS_CNTL                                                 0x1500
 #define mmBIF_DOORBELL_APER_EN                                                  0x1501
 #define mmBUS_CNTL                                                              0x1508
diff --git a/drivers/gpu/drm/amd/include/asic_reg/gca/gfx_8_0_d.h b/drivers/gpu/drm/amd/include/asic_reg/gca/gfx_8_0_d.h
index ebaf67bb1589..90ff7c8a6011 100644
--- a/drivers/gpu/drm/amd/include/asic_reg/gca/gfx_8_0_d.h
+++ b/drivers/gpu/drm/amd/include/asic_reg/gca/gfx_8_0_d.h
@@ -2823,4 +2823,7 @@
 #define mmDC_EDC_CSINVOC_CNT                                                    0x3192
 #define mmDC_EDC_RESTORE_CNT                                                    0x3193
 
+#define mmGC_CAC_IND_INDEX                                                      0x129a
+#define mmGC_CAC_IND_DATA                                                       0x129b
+
 #endif /* GFX_8_0_D_H */
diff --git a/drivers/gpu/drm/amd/include/asic_reg/gca/gfx_8_0_sh_mask.h b/drivers/gpu/drm/amd/include/asic_reg/gca/gfx_8_0_sh_mask.h
index 7d722458d9f5..4070ca3a68eb 100644
--- a/drivers/gpu/drm/amd/include/asic_reg/gca/gfx_8_0_sh_mask.h
+++ b/drivers/gpu/drm/amd/include/asic_reg/gca/gfx_8_0_sh_mask.h
@@ -8730,8 +8730,6 @@
 #define RLC_GPM_STAT__DYN_CU_POWERING_DOWN__SHIFT 0x10
 #define RLC_GPM_STAT__ABORTED_PD_SEQUENCE_MASK 0x20000
 #define RLC_GPM_STAT__ABORTED_PD_SEQUENCE__SHIFT 0x11
-#define RLC_GPM_STAT__RESERVED_MASK 0xfc0000
-#define RLC_GPM_STAT__RESERVED__SHIFT 0x12
 #define RLC_GPM_STAT__PG_ERROR_STATUS_MASK 0xff000000
 #define RLC_GPM_STAT__PG_ERROR_STATUS__SHIFT 0x18
 #define RLC_GPU_CLOCK_32_RES_SEL__RES_SEL_MASK 0x3f
@@ -8764,8 +8762,10 @@
 #define RLC_PG_CNTL__SMU_CLK_SLOWDOWN_ON_PD_ENABLE__SHIFT 0x12
 #define RLC_PG_CNTL__SMU_HANDSHAKE_ENABLE_MASK 0x80000
 #define RLC_PG_CNTL__SMU_HANDSHAKE_ENABLE__SHIFT 0x13
-#define RLC_PG_CNTL__RESERVED1_MASK 0xf00000
-#define RLC_PG_CNTL__RESERVED1__SHIFT 0x14
+#define RLC_PG_CNTL__QUICK_PG_ENABLE_MASK 0x100000
+#define RLC_PG_CNTL__QUICK_PG_ENABLE__SHIFT 0x14
+#define RLC_PG_CNTL__RESERVED1_MASK 0xe00000
+#define RLC_PG_CNTL__RESERVED1__SHIFT 0x15
 #define RLC_GPM_THREAD_PRIORITY__THREAD0_PRIORITY_MASK 0xff
 #define RLC_GPM_THREAD_PRIORITY__THREAD0_PRIORITY__SHIFT 0x0
 #define RLC_GPM_THREAD_PRIORITY__THREAD1_PRIORITY_MASK 0xff00
@@ -9102,8 +9102,6 @@
 #define RLC_GPM_LOG_CONT__CONT__SHIFT 0x0
 #define RLC_PG_DELAY_3__CGCG_ACTIVE_BEFORE_CGPG_MASK 0xff
 #define RLC_PG_DELAY_3__CGCG_ACTIVE_BEFORE_CGPG__SHIFT 0x0
-#define RLC_PG_DELAY_3__RESERVED_MASK 0xffffff00
-#define RLC_PG_DELAY_3__RESERVED__SHIFT 0x8
 #define RLC_GPM_INT_DISABLE_TH0__DISABLE_MASK 0xffffffff
 #define RLC_GPM_INT_DISABLE_TH0__DISABLE__SHIFT 0x0
 #define RLC_GPM_INT_DISABLE_TH1__DISABLE_MASK 0xffffffff
@@ -9124,14 +9122,8 @@
 #define RLC_SRM_DEBUG_SELECT__RESERVED__SHIFT 0x8
 #define RLC_SRM_DEBUG__DATA_MASK 0xffffffff
 #define RLC_SRM_DEBUG__DATA__SHIFT 0x0
-#define RLC_SRM_ARAM_ADDR__ADDR_MASK 0x3ff
-#define RLC_SRM_ARAM_ADDR__ADDR__SHIFT 0x0
-#define RLC_SRM_ARAM_ADDR__RESERVED_MASK 0xfffffc00
-#define RLC_SRM_ARAM_ADDR__RESERVED__SHIFT 0xa
 #define RLC_SRM_ARAM_DATA__DATA_MASK 0xffffffff
 #define RLC_SRM_ARAM_DATA__DATA__SHIFT 0x0
-#define RLC_SRM_DRAM_ADDR__ADDR_MASK 0x3ff
-#define RLC_SRM_DRAM_ADDR__ADDR__SHIFT 0x0
 #define RLC_SRM_DRAM_ADDR__RESERVED_MASK 0xfffffc00
 #define RLC_SRM_DRAM_ADDR__RESERVED__SHIFT 0xa
 #define RLC_SRM_DRAM_DATA__DATA_MASK 0xffffffff
@@ -17946,8 +17938,6 @@
 #define VGT_TESS_DISTRIBUTION__ACCUM_TRI__SHIFT 0x8
 #define VGT_TESS_DISTRIBUTION__ACCUM_QUAD_MASK 0xff0000
 #define VGT_TESS_DISTRIBUTION__ACCUM_QUAD__SHIFT 0x10
-#define VGT_TESS_DISTRIBUTION__DONUT_SPLIT_MASK 0xff000000
-#define VGT_TESS_DISTRIBUTION__DONUT_SPLIT__SHIFT 0x18
 #define VGT_TF_RING_SIZE__SIZE_MASK 0xffff
 #define VGT_TF_RING_SIZE__SIZE__SHIFT 0x0
 #define VGT_SYS_CONFIG__DUAL_CORE_EN_MASK 0x1
@@ -20502,8 +20492,6 @@
 #define DIDT_SQ_CTRL0__DIDT_CTRL_RST__SHIFT 0x4
 #define DIDT_SQ_CTRL0__DIDT_CLK_EN_OVERRIDE_MASK 0x20
 #define DIDT_SQ_CTRL0__DIDT_CLK_EN_OVERRIDE__SHIFT 0x5
-#define DIDT_SQ_CTRL0__UNUSED_0_MASK 0xffffffc0
-#define DIDT_SQ_CTRL0__UNUSED_0__SHIFT 0x6
 #define DIDT_SQ_CTRL1__MIN_POWER_MASK 0xffff
 #define DIDT_SQ_CTRL1__MIN_POWER__SHIFT 0x0
 #define DIDT_SQ_CTRL1__MAX_POWER_MASK 0xffff0000
@@ -20558,8 +20546,6 @@
 #define DIDT_DB_CTRL0__DIDT_CTRL_RST__SHIFT 0x4
 #define DIDT_DB_CTRL0__DIDT_CLK_EN_OVERRIDE_MASK 0x20
 #define DIDT_DB_CTRL0__DIDT_CLK_EN_OVERRIDE__SHIFT 0x5
-#define DIDT_DB_CTRL0__UNUSED_0_MASK 0xffffffc0
-#define DIDT_DB_CTRL0__UNUSED_0__SHIFT 0x6
 #define DIDT_DB_CTRL1__MIN_POWER_MASK 0xffff
 #define DIDT_DB_CTRL1__MIN_POWER__SHIFT 0x0
 #define DIDT_DB_CTRL1__MAX_POWER_MASK 0xffff0000
@@ -20614,8 +20600,6 @@
 #define DIDT_TD_CTRL0__DIDT_CTRL_RST__SHIFT 0x4
 #define DIDT_TD_CTRL0__DIDT_CLK_EN_OVERRIDE_MASK 0x20
 #define DIDT_TD_CTRL0__DIDT_CLK_EN_OVERRIDE__SHIFT 0x5
-#define DIDT_TD_CTRL0__UNUSED_0_MASK 0xffffffc0
-#define DIDT_TD_CTRL0__UNUSED_0__SHIFT 0x6
 #define DIDT_TD_CTRL1__MIN_POWER_MASK 0xffff
 #define DIDT_TD_CTRL1__MIN_POWER__SHIFT 0x0
 #define DIDT_TD_CTRL1__MAX_POWER_MASK 0xffff0000
@@ -20670,8 +20654,6 @@
 #define DIDT_TCP_CTRL0__DIDT_CTRL_RST__SHIFT 0x4
 #define DIDT_TCP_CTRL0__DIDT_CLK_EN_OVERRIDE_MASK 0x20
 #define DIDT_TCP_CTRL0__DIDT_CLK_EN_OVERRIDE__SHIFT 0x5
-#define DIDT_TCP_CTRL0__UNUSED_0_MASK 0xffffffc0
-#define DIDT_TCP_CTRL0__UNUSED_0__SHIFT 0x6
 #define DIDT_TCP_CTRL1__MIN_POWER_MASK 0xffff
 #define DIDT_TCP_CTRL1__MIN_POWER__SHIFT 0x0
 #define DIDT_TCP_CTRL1__MAX_POWER_MASK 0xffff0000
@@ -20726,8 +20708,6 @@
 #define DIDT_DBR_CTRL0__DIDT_CTRL_RST__SHIFT 0x4
 #define DIDT_DBR_CTRL0__DIDT_CLK_EN_OVERRIDE_MASK 0x20
 #define DIDT_DBR_CTRL0__DIDT_CLK_EN_OVERRIDE__SHIFT 0x5
-#define DIDT_DBR_CTRL0__UNUSED_0_MASK 0xffffffc0
-#define DIDT_DBR_CTRL0__UNUSED_0__SHIFT 0x6
 #define DIDT_DBR_CTRL1__MIN_POWER_MASK 0xffff
 #define DIDT_DBR_CTRL1__MIN_POWER__SHIFT 0x0
 #define DIDT_DBR_CTRL1__MAX_POWER_MASK 0xffff0000
@@ -20773,4 +20753,84 @@
 #define DIDT_DBR_WEIGHT8_11__WEIGHT11_MASK 0xff000000
 #define DIDT_DBR_WEIGHT8_11__WEIGHT11__SHIFT 0x18
 
+#define DIDT_SQ_STALL_CTRL__DIDT_STALL_CTRL_ENABLE_MASK    0x00000001
+#define DIDT_SQ_STALL_CTRL__DIDT_STALL_CTRL_ENABLE__SHIFT  0x00000000
+
+#define DIDT_SQ_STALL_CTRL__DIDT_STALL_DELAY_HI_MASK       0x0000007e
+#define DIDT_SQ_STALL_CTRL__DIDT_STALL_DELAY_LO_MASK       0x00001f80L
+#define DIDT_SQ_STALL_CTRL__DIDT_STALL_DELAY_HI__SHIFT     0x00000001
+#define DIDT_SQ_STALL_CTRL__DIDT_STALL_DELAY_LO__SHIFT     0x00000007
+
+#define DIDT_SQ_STALL_CTRL__DIDT_HI_POWER_THRESHOLD_MASK   0x1fffe000L
+#define DIDT_SQ_STALL_CTRL__DIDT_HI_POWER_THRESHOLD__SHIFT 0x0000000d
+
+#define DIDT_SQ_STALL_CTRL__UNUSED_0_MASK                  0xe0000000L
+#define DIDT_SQ_STALL_CTRL__UNUSED_0__SHIFT                0x0000001d
+
+#define DIDT_SQ_TUNING_CTRL__DIDT_TUNING_ENABLE_MASK       0x00000001L
+#define DIDT_SQ_TUNING_CTRL__DIDT_TUNING_ENABLE__SHIFT     0x00000000
+
+#define DIDT_SQ_TUNING_CTRL__MAX_POWER_DELTA_HI_MASK       0x00007ffeL
+#define DIDT_SQ_TUNING_CTRL__MAX_POWER_DELTA_HI__SHIFT     0x00000001
+#define DIDT_SQ_TUNING_CTRL__MAX_POWER_DELTA_LO_MASK       0x1fff8000L
+#define DIDT_SQ_TUNING_CTRL__MAX_POWER_DELTA_LO__SHIFT     0x0000000f
+
+#define DIDT_TD_STALL_CTRL__DIDT_STALL_CTRL_ENABLE_MASK    0x00000001L
+#define DIDT_TD_STALL_CTRL__DIDT_STALL_CTRL_ENABLE__SHIFT  0x00000000
+
+#define DIDT_TD_STALL_CTRL__DIDT_STALL_DELAY_HI_MASK       0x0000007eL
+#define DIDT_TD_STALL_CTRL__DIDT_STALL_DELAY_LO_MASK       0x00001f80L
+#define DIDT_TD_STALL_CTRL__DIDT_STALL_DELAY_HI__SHIFT     0x00000001
+#define DIDT_TD_STALL_CTRL__DIDT_STALL_DELAY_LO__SHIFT     0x00000007
+
+#define DIDT_TD_STALL_CTRL__DIDT_HI_POWER_THRESHOLD_MASK   0x1fffe000L
+#define DIDT_TD_STALL_CTRL__DIDT_HI_POWER_THRESHOLD__SHIFT 0x0000000d
+
+#define DIDT_SQ_CTRL0__DIDT_MAX_STALLS_ALLOWED_HI_MASK     0x00000fc0L
+#define DIDT_SQ_CTRL0__DIDT_MAX_STALLS_ALLOWED_LO_MASK     0x0003f000L
+#define DIDT_SQ_CTRL0__DIDT_MAX_STALLS_ALLOWED_HI__SHIFT   0x00000006
+#define DIDT_SQ_CTRL0__DIDT_MAX_STALLS_ALLOWED_LO__SHIFT   0x0000000c
+
+#define DIDT_TD_TUNING_CTRL__DIDT_TUNING_ENABLE_MASK       0x00000001L
+#define DIDT_TD_TUNING_CTRL__MAX_POWER_DELTA_HI_MASK       0x00007ffeL
+#define DIDT_TD_TUNING_CTRL__MAX_POWER_DELTA_LO_MASK       0x1fff8000L
+
+#define DIDT_TD_TUNING_CTRL__DIDT_TUNING_ENABLE__SHIFT     0x00000000
+#define DIDT_TD_TUNING_CTRL__MAX_POWER_DELTA_HI__SHIFT     0x00000001
+#define DIDT_TD_TUNING_CTRL__MAX_POWER_DELTA_LO__SHIFT     0x0000000f
+
+#define DIDT_TD_STALL_CTRL__UNUSED_0_MASK                  0xe0000000L
+#define DIDT_TD_STALL_CTRL__UNUSED_0__SHIFT                0x0000001d
+
+#define DIDT_TD_CTRL0__DIDT_MAX_STALLS_ALLOWED_HI_MASK     0x00000fc0L
+#define DIDT_TD_CTRL0__DIDT_MAX_STALLS_ALLOWED_LO_MASK     0x0003f000L
+#define DIDT_TD_CTRL0__DIDT_MAX_STALLS_ALLOWED_HI__SHIFT   0x00000006
+#define DIDT_TD_CTRL0__DIDT_MAX_STALLS_ALLOWED_LO__SHIFT   0x0000000c
+
+#define DIDT_TCP_STALL_CTRL__DIDT_STALL_CTRL_ENABLE_MASK   0x00000001L
+#define DIDT_TCP_STALL_CTRL__DIDT_STALL_CTRL_ENABLE__SHIFT 0x00000000
+
+#define DIDT_TCP_STALL_CTRL__DIDT_STALL_DELAY_HI_MASK      0x0000007eL
+#define DIDT_TCP_STALL_CTRL__DIDT_STALL_DELAY_LO_MASK      0x00001f80L
+#define DIDT_TCP_STALL_CTRL__DIDT_STALL_DELAY_HI__SHIFT    0x00000001
+#define DIDT_TCP_STALL_CTRL__DIDT_STALL_DELAY_LO__SHIFT    0x00000007
+
+#define DIDT_TCP_STALL_CTRL__DIDT_HI_POWER_THRESHOLD_MASK  0x1fffe000L
+#define DIDT_TCP_STALL_CTRL__DIDT_HI_POWER_THRESHOLD__SHIFT 0x0000000d
+
+#define DIDT_TCP_STALL_CTRL__UNUSED_0_MASK                 0xe0000000L
+#define DIDT_TCP_STALL_CTRL__UNUSED_0__SHIFT               0x0000001d
+
+#define DIDT_TCP_TUNING_CTRL__DIDT_TUNING_ENABLE_MASK      0x00000001L
+#define DIDT_TCP_TUNING_CTRL__MAX_POWER_DELTA_HI_MASK      0x00007ffeL
+#define DIDT_TCP_TUNING_CTRL__MAX_POWER_DELTA_LO_MASK      0x1fff8000L
+#define DIDT_TCP_TUNING_CTRL__DIDT_TUNING_ENABLE__SHIFT    0x00000000
+#define DIDT_TCP_TUNING_CTRL__MAX_POWER_DELTA_HI__SHIFT    0x00000001
+#define DIDT_TCP_TUNING_CTRL__MAX_POWER_DELTA_LO__SHIFT    0x0000000f
+
+#define DIDT_TCP_CTRL0__DIDT_MAX_STALLS_ALLOWED_HI_MASK    0x00000fc0L
+#define DIDT_TCP_CTRL0__DIDT_MAX_STALLS_ALLOWED_LO_MASK    0x0003f000L
+#define DIDT_TCP_CTRL0__DIDT_MAX_STALLS_ALLOWED_HI__SHIFT  0x00000006
+#define DIDT_TCP_CTRL0__DIDT_MAX_STALLS_ALLOWED_LO__SHIFT  0x0000000c
+
 #endif /* GFX_8_0_SH_MASK_H */
diff --git a/drivers/gpu/drm/amd/include/asic_reg/uvd/uvd_6_0_d.h b/drivers/gpu/drm/amd/include/asic_reg/uvd/uvd_6_0_d.h
index 6f6fb34742d2..ec69869c55ff 100644
--- a/drivers/gpu/drm/amd/include/asic_reg/uvd/uvd_6_0_d.h
+++ b/drivers/gpu/drm/amd/include/asic_reg/uvd/uvd_6_0_d.h
@@ -111,6 +111,8 @@
 #define mmUVD_MIF_RECON1_ADDR_CONFIG                                            0x39c5
 #define ixUVD_MIF_SCLR_ADDR_CONFIG                                              0x4
 #define mmUVD_JPEG_ADDR_CONFIG                                                  0x3a1f
+#define mmUVD_GP_SCRATCH8                                                       0x3c0a
+#define mmUVD_GP_SCRATCH9                                                       0x3c0b
 #define mmUVD_GP_SCRATCH4                                                       0x3d38
 
 #endif /* UVD_6_0_D_H */
diff --git a/drivers/gpu/drm/amd/include/cgs_common.h b/drivers/gpu/drm/amd/include/cgs_common.h
index 7464daf89ca1..b86aba9d019f 100644
--- a/drivers/gpu/drm/amd/include/cgs_common.h
+++ b/drivers/gpu/drm/amd/include/cgs_common.h
@@ -49,6 +49,7 @@ enum cgs_ind_reg {
         CGS_IND_REG__SMC,
         CGS_IND_REG__UVD_CTX,
         CGS_IND_REG__DIDT,
+        CGS_IND_REG_GC_CAC,
         CGS_IND_REG__AUDIO_ENDPT
 };
 
@@ -112,20 +113,23 @@ enum cgs_system_info_id {
         CGS_SYSTEM_INFO_ADAPTER_BDF_ID = 1,
         CGS_SYSTEM_INFO_PCIE_GEN_INFO,
         CGS_SYSTEM_INFO_PCIE_MLW,
+        CGS_SYSTEM_INFO_PCIE_DEV,
+        CGS_SYSTEM_INFO_PCIE_REV,
         CGS_SYSTEM_INFO_CG_FLAGS,
         CGS_SYSTEM_INFO_PG_FLAGS,
         CGS_SYSTEM_INFO_GFX_CU_INFO,
+        CGS_SYSTEM_INFO_GFX_SE_INFO,
         CGS_SYSTEM_INFO_ID_MAXIMUM,
 };
 
 struct cgs_system_info {
-        uint64_t       size;
-        uint64_t       info_id;
+        uint64_t                        size;
+        enum cgs_system_info_id         info_id;
         union {
-                void           *ptr;
-                uint64_t        value;
+                void                    *ptr;
+                uint64_t                value;
         };
-        uint64_t               padding[13];
+        uint64_t                        padding[13];
 };
 
 /*
@@ -158,6 +162,10 @@ struct cgs_firmware_info {
         uint16_t                feature_version;
         uint32_t                image_size;
         uint64_t                mc_addr;
+
+        /* only for smc firmware */
+        uint32_t                ucode_start_address;
+
         void                    *kptr;
 };
 
@@ -189,7 +197,6 @@ typedef unsigned long cgs_handle_t;
 
 struct cgs_acpi_method_argument {
         uint32_t type;
-        uint32_t method_length;
         uint32_t data_length;
         union{
                 uint32_t value;
diff --git a/drivers/gpu/drm/amd/powerplay/amd_powerplay.c b/drivers/gpu/drm/amd/powerplay/amd_powerplay.c
index e629f8a9fe93..abbb658bdc1e 100644
--- a/drivers/gpu/drm/amd/powerplay/amd_powerplay.c
+++ b/drivers/gpu/drm/amd/powerplay/amd_powerplay.c
@@ -176,7 +176,7 @@ static int pp_hw_fini(void *handle)
 
 static bool pp_is_idle(void *handle)
 {
-        return 0;
+        return false;
 }
 
 static int pp_wait_for_idle(void *handle)
@@ -536,6 +536,10 @@ int pp_dpm_dispatch_tasks(void *handle, enum amd_pp_event event_id, void *input,
         case AMD_PP_EVENT_COMPLETE_INIT:
                 ret = pem_handle_event(pp_handle->eventmgr, event_id, &data);
                 break;
+        case AMD_PP_EVENT_READJUST_POWER_STATE:
+                pp_handle->hwmgr->current_ps = pp_handle->hwmgr->boot_ps;
+                ret = pem_handle_event(pp_handle->eventmgr, event_id, &data);
+                break;
         default:
                 break;
         }
@@ -740,12 +744,12 @@ static int pp_dpm_get_pp_table(void *handle, char **table)
 
         PP_CHECK_HW(hwmgr);
 
-        if (hwmgr->hwmgr_func->get_pp_table == NULL) {
-                printk(KERN_INFO "%s was not implemented.\n", __func__);
-                return 0;
-        }
+        if (!hwmgr->soft_pp_table)
+                return -EINVAL;
 
-        return hwmgr->hwmgr_func->get_pp_table(hwmgr, table);
+        *table = (char *)hwmgr->soft_pp_table;
+
+        return hwmgr->soft_pp_table_size;
 }
 
 static int pp_dpm_set_pp_table(void *handle, const char *buf, size_t size)
@@ -759,12 +763,23 @@ static int pp_dpm_set_pp_table(void *handle, const char *buf, size_t size)
 
         PP_CHECK_HW(hwmgr);
 
-        if (hwmgr->hwmgr_func->set_pp_table == NULL) {
-                printk(KERN_INFO "%s was not implemented.\n", __func__);
-                return 0;
+        if (!hwmgr->hardcode_pp_table) {
+                hwmgr->hardcode_pp_table =
+                                kzalloc(hwmgr->soft_pp_table_size, GFP_KERNEL);
+
+                if (!hwmgr->hardcode_pp_table)
+                        return -ENOMEM;
+
+                /* to avoid powerplay crash when hardcode pptable is empty */
+                memcpy(hwmgr->hardcode_pp_table, hwmgr->soft_pp_table,
+                                hwmgr->soft_pp_table_size);
         }
 
-        return hwmgr->hwmgr_func->set_pp_table(hwmgr, buf, size);
+        memcpy(hwmgr->hardcode_pp_table, buf, size);
+
+        hwmgr->soft_pp_table = hwmgr->hardcode_pp_table;
+
+        return amd_powerplay_reset(handle);
 }
 
 static int pp_dpm_force_clock_level(void *handle,
@@ -806,6 +821,82 @@ static int pp_dpm_print_clock_levels(void *handle,
         return hwmgr->hwmgr_func->print_clock_levels(hwmgr, type, buf);
 }
 
+static int pp_dpm_get_sclk_od(void *handle)
+{
+        struct pp_hwmgr *hwmgr;
+
+        if (!handle)
+                return -EINVAL;
+
+        hwmgr = ((struct pp_instance *)handle)->hwmgr;
+
+        PP_CHECK_HW(hwmgr);
+
+        if (hwmgr->hwmgr_func->get_sclk_od == NULL) {
+                printk(KERN_INFO "%s was not implemented.\n", __func__);
+                return 0;
+        }
+
+        return hwmgr->hwmgr_func->get_sclk_od(hwmgr);
+}
+
+static int pp_dpm_set_sclk_od(void *handle, uint32_t value)
+{
+        struct pp_hwmgr *hwmgr;
+
+        if (!handle)
+                return -EINVAL;
+
+        hwmgr = ((struct pp_instance *)handle)->hwmgr;
+
+        PP_CHECK_HW(hwmgr);
+
+        if (hwmgr->hwmgr_func->set_sclk_od == NULL) {
+                printk(KERN_INFO "%s was not implemented.\n", __func__);
+                return 0;
+        }
+
+        return hwmgr->hwmgr_func->set_sclk_od(hwmgr, value);
+}
+
+static int pp_dpm_get_mclk_od(void *handle)
+{
+        struct pp_hwmgr *hwmgr;
+
+        if (!handle)
+                return -EINVAL;
+
+        hwmgr = ((struct pp_instance *)handle)->hwmgr;
+
+        PP_CHECK_HW(hwmgr);
+
+        if (hwmgr->hwmgr_func->get_mclk_od == NULL) {
+                printk(KERN_INFO "%s was not implemented.\n", __func__);
+                return 0;
+        }
+
+        return hwmgr->hwmgr_func->get_mclk_od(hwmgr);
+}
+
+static int pp_dpm_set_mclk_od(void *handle, uint32_t value)
+{
+        struct pp_hwmgr *hwmgr;
+
+        if (!handle)
+                return -EINVAL;
+
+        hwmgr = ((struct pp_instance *)handle)->hwmgr;
+
+        PP_CHECK_HW(hwmgr);
+
+        if (hwmgr->hwmgr_func->set_mclk_od == NULL) {
+                printk(KERN_INFO "%s was not implemented.\n", __func__);
+                return 0;
+        }
+
+        return hwmgr->hwmgr_func->set_mclk_od(hwmgr, value);
+}
+
 const struct amd_powerplay_funcs pp_dpm_funcs = {
         .get_temperature = pp_dpm_get_temperature,
         .load_firmware = pp_dpm_load_fw,
@@ -828,6 +919,10 @@ const struct amd_powerplay_funcs pp_dpm_funcs = {
         .set_pp_table = pp_dpm_set_pp_table,
         .force_clock_level = pp_dpm_force_clock_level,
         .print_clock_levels = pp_dpm_print_clock_levels,
+        .get_sclk_od = pp_dpm_get_sclk_od,
+        .set_sclk_od = pp_dpm_set_sclk_od,
+        .get_mclk_od = pp_dpm_get_mclk_od,
+        .set_mclk_od = pp_dpm_set_mclk_od,
 };
 
 static int amd_pp_instance_init(struct amd_pp_init *pp_init,
@@ -909,6 +1004,44 @@ int amd_powerplay_fini(void *handle)
         return 0;
 }
 
+int amd_powerplay_reset(void *handle)
+{
+        struct pp_instance *instance = (struct pp_instance *)handle;
+        struct pp_eventmgr *eventmgr;
+        struct pem_event_data event_data = { {0} };
+        int ret;
+
+        if (instance == NULL)
+                return -EINVAL;
+
+        eventmgr = instance->eventmgr;
+        if (!eventmgr || !eventmgr->pp_eventmgr_fini)
+                return -EINVAL;
+
+        eventmgr->pp_eventmgr_fini(eventmgr);
+
+        ret = pp_sw_fini(handle);
+        if (ret)
+                return ret;
+
+        kfree(instance->hwmgr->ps);
+
+        ret = pp_sw_init(handle);
+        if (ret)
+                return ret;
+
+        hw_init_power_state_table(instance->hwmgr);
+
+        if (eventmgr == NULL || eventmgr->pp_eventmgr_init == NULL)
+                return -EINVAL;
+
+        ret = eventmgr->pp_eventmgr_init(eventmgr);
+        if (ret)
+                return ret;
+
+        return pem_handle_event(eventmgr, AMD_PP_EVENT_COMPLETE_INIT, &event_data);
+}
+
 /* export this function to DAL */
 
 int amd_powerplay_display_configuration_change(void *handle,
diff --git a/drivers/gpu/drm/amd/powerplay/eventmgr/eventactionchains.c b/drivers/gpu/drm/amd/powerplay/eventmgr/eventactionchains.c
index d6635cc4b0fc..635fc4b48184 100644
--- a/drivers/gpu/drm/amd/powerplay/eventmgr/eventactionchains.c
+++ b/drivers/gpu/drm/amd/powerplay/eventmgr/eventactionchains.c
@@ -30,7 +30,6 @@ static const pem_event_action * const initialize_event[] = {
         system_config_tasks,
         setup_asic_tasks,
         enable_dynamic_state_management_tasks,
-        enable_clock_power_gatings_tasks,
         get_2d_performance_state_tasks,
         set_performance_state_tasks,
         initialize_thermal_controller_tasks,
@@ -140,7 +139,6 @@ static const pem_event_action * const resume_event[] = {
         setup_asic_tasks,
         enable_stutter_mode_tasks, /*must do this in boot state and before SMC is started */
         enable_dynamic_state_management_tasks,
-        enable_clock_power_gatings_tasks,
         enable_disable_bapm_tasks,
         initialize_thermal_controller_tasks,
         get_2d_performance_state_tasks,
diff --git a/drivers/gpu/drm/amd/powerplay/eventmgr/eventtasks.c b/drivers/gpu/drm/amd/powerplay/eventmgr/eventtasks.c
index 5cd123472db4..b6f45fd01fa6 100644
--- a/drivers/gpu/drm/amd/powerplay/eventmgr/eventtasks.c
+++ b/drivers/gpu/drm/amd/powerplay/eventmgr/eventtasks.c
@@ -132,8 +132,7 @@ int pem_task_enable_dynamic_state_management(struct pp_eventmgr *eventmgr, struc
 
 int pem_task_disable_dynamic_state_management(struct pp_eventmgr *eventmgr, struct pem_event_data *event_data)
 {
-        /* TODO */
-        return 0;
+        return phm_disable_dynamic_state_management(eventmgr->hwmgr);
 }
 
 int pem_task_enable_clock_power_gatings_tasks(struct pp_eventmgr *eventmgr, struct pem_event_data *event_data)
diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/cz_clockpowergating.c b/drivers/gpu/drm/amd/powerplay/hwmgr/cz_clockpowergating.c
index 436fc16dabb6..2028980f1ed4 100644
--- a/drivers/gpu/drm/amd/powerplay/hwmgr/cz_clockpowergating.c
+++ b/drivers/gpu/drm/amd/powerplay/hwmgr/cz_clockpowergating.c
@@ -177,12 +177,12 @@ int cz_dpm_powergate_uvd(struct pp_hwmgr *hwmgr, bool bgate)
                 cz_dpm_powerdown_uvd(hwmgr);
         } else {
                 cz_dpm_powerup_uvd(hwmgr);
-                cgs_set_clockgating_state(hwmgr->device,
-                                                AMD_IP_BLOCK_TYPE_UVD,
-                                                AMD_PG_STATE_GATE);
                 cgs_set_powergating_state(hwmgr->device,
                                                 AMD_IP_BLOCK_TYPE_UVD,
                                                 AMD_CG_STATE_UNGATE);
+                cgs_set_clockgating_state(hwmgr->device,
+                                                AMD_IP_BLOCK_TYPE_UVD,
+                                                AMD_PG_STATE_GATE);
                 cz_dpm_update_uvd_dpm(hwmgr, false);
         }
 
@@ -206,25 +206,26 @@ int cz_dpm_powergate_vce(struct pp_hwmgr *hwmgr, bool bgate)
                                                         AMD_IP_BLOCK_TYPE_VCE,
                                                         AMD_PG_STATE_GATE);
                                 cz_enable_disable_vce_dpm(hwmgr, false);
-                        /* TODO: to figure out why vce can't be poweroff*/
+                                cz_dpm_powerdown_vce(hwmgr);
                                 cz_hwmgr->vce_power_gated = true;
                         } else {
                                 cz_dpm_powerup_vce(hwmgr);
                                 cz_hwmgr->vce_power_gated = false;
-                                cgs_set_clockgating_state(
-                                                        hwmgr->device,
-                                                        AMD_IP_BLOCK_TYPE_VCE,
-                                                        AMD_PG_STATE_GATE);
                                 cgs_set_powergating_state(
                                                         hwmgr->device,
                                                         AMD_IP_BLOCK_TYPE_VCE,
                                                         AMD_CG_STATE_UNGATE);
+                                cgs_set_clockgating_state(
+                                                        hwmgr->device,
+                                                        AMD_IP_BLOCK_TYPE_VCE,
+                                                        AMD_PG_STATE_GATE);
                                 cz_dpm_update_vce_dpm(hwmgr);
                                 cz_enable_disable_vce_dpm(hwmgr, true);
                                 return 0;
                         }
                 }
         } else {
+                cz_hwmgr->vce_power_gated = bgate;
                 cz_dpm_update_vce_dpm(hwmgr);
                 cz_enable_disable_vce_dpm(hwmgr, !bgate);
                 return 0;
diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/cz_hwmgr.c b/drivers/gpu/drm/amd/powerplay/hwmgr/cz_hwmgr.c
index 1f14c477d15d..8cc0df9b534a 100644
--- a/drivers/gpu/drm/amd/powerplay/hwmgr/cz_hwmgr.c
+++ b/drivers/gpu/drm/amd/powerplay/hwmgr/cz_hwmgr.c
@@ -1167,9 +1167,9 @@ static int cz_apply_state_adjust_rules(struct pp_hwmgr *hwmgr,
 
         cz_ps->action = cz_current_ps->action;
 
-        if ((force_high == false) && (cz_ps->action == FORCE_HIGH))
+        if (!force_high && (cz_ps->action == FORCE_HIGH))
                 cz_ps->action = CANCEL_FORCE_HIGH;
-        else if ((force_high == true) && (cz_ps->action != FORCE_HIGH))
+        else if (force_high && (cz_ps->action != FORCE_HIGH))
                 cz_ps->action = FORCE_HIGH;
         else
                 cz_ps->action = DO_NOTHING;
@@ -1180,6 +1180,13 @@ static int cz_apply_state_adjust_rules(struct pp_hwmgr *hwmgr,
 static int cz_hwmgr_backend_init(struct pp_hwmgr *hwmgr)
 {
         int result = 0;
+        struct cz_hwmgr *data;
+
+        data = kzalloc(sizeof(struct cz_hwmgr), GFP_KERNEL);
+        if (data == NULL)
+                return -ENOMEM;
+
+        hwmgr->backend = data;
 
         result = cz_initialize_dpm_defaults(hwmgr);
         if (result != 0) {
@@ -1649,7 +1656,7 @@ static void cz_hw_print_display_cfg(
         struct cz_hwmgr *hw_data = (struct cz_hwmgr *)(hwmgr->backend);
         uint32_t data = 0;
 
-        if (hw_data->cc6_settings.cc6_setting_changed == true) {
+        if (hw_data->cc6_settings.cc6_setting_changed) {
 
                 hw_data->cc6_settings.cc6_setting_changed = false;
 
@@ -1909,15 +1916,7 @@ static const struct pp_hwmgr_func cz_hwmgr_funcs = {
 
 int cz_hwmgr_init(struct pp_hwmgr *hwmgr)
 {
-        struct cz_hwmgr *cz_hwmgr;
-        int ret = 0;
-
-        cz_hwmgr = kzalloc(sizeof(struct cz_hwmgr), GFP_KERNEL);
-        if (cz_hwmgr == NULL)
-                return -ENOMEM;
-
-        hwmgr->backend = cz_hwmgr;
         hwmgr->hwmgr_func = &cz_hwmgr_funcs;
         hwmgr->pptable_func = &pptable_funcs;
-        return ret;
+        return 0;
 }
diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/fiji_clockpowergating.c b/drivers/gpu/drm/amd/powerplay/hwmgr/fiji_clockpowergating.c
index e1b649bd5344..5afe82068b29 100644
--- a/drivers/gpu/drm/amd/powerplay/hwmgr/fiji_clockpowergating.c
+++ b/drivers/gpu/drm/amd/powerplay/hwmgr/fiji_clockpowergating.c
@@ -56,7 +56,7 @@ int fiji_phm_powergate_uvd(struct pp_hwmgr *hwmgr, bool bgate)
                 fiji_update_uvd_dpm(hwmgr, false);
                 cgs_set_clockgating_state(hwmgr->device,
                                           AMD_IP_BLOCK_TYPE_UVD,
-                                          AMD_PG_STATE_UNGATE);
+                                          AMD_CG_STATE_UNGATE);
         }
 
         return 0;
diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/fiji_hwmgr.c b/drivers/gpu/drm/amd/powerplay/hwmgr/fiji_hwmgr.c
index 92912ab20944..120a9e2c3152 100644
--- a/drivers/gpu/drm/amd/powerplay/hwmgr/fiji_hwmgr.c
+++ b/drivers/gpu/drm/amd/powerplay/hwmgr/fiji_hwmgr.c
@@ -581,25 +581,24 @@ static int fiji_patch_boot_state(struct pp_hwmgr *hwmgr,
 
 static int fiji_hwmgr_backend_fini(struct pp_hwmgr *hwmgr)
 {
-        struct fiji_hwmgr *data = (struct fiji_hwmgr *)(hwmgr->backend);
-
-        if (data->soft_pp_table) {
-                kfree(data->soft_pp_table);
-                data->soft_pp_table = NULL;
-        }
-
         return phm_hwmgr_backend_fini(hwmgr);
 }
 
 static int fiji_hwmgr_backend_init(struct pp_hwmgr *hwmgr)
 {
-        struct fiji_hwmgr *data = (struct fiji_hwmgr *)(hwmgr->backend);
+        struct fiji_hwmgr *data;
         uint32_t i;
         struct phm_ppt_v1_information *table_info =
                         (struct phm_ppt_v1_information *)(hwmgr->pptable);
         bool stay_in_boot;
         int result;
 
+        data = kzalloc(sizeof(struct fiji_hwmgr), GFP_KERNEL);
+        if (data == NULL)
+                return -ENOMEM;
+
+        hwmgr->backend = data;
+
         data->dll_default_on = false;
         data->sram_end = SMC_RAM_END;
 
@@ -699,7 +698,7 @@ static int fiji_hwmgr_backend_init(struct pp_hwmgr *hwmgr)
         if (0 == result) {
                 struct cgs_system_info sys_info = {0};
 
-                data->is_tlu_enabled = 0;
+                data->is_tlu_enabled = false;
                 hwmgr->platform_descriptor.hardwareActivityPerformanceLevels =
                                 FIJI_MAX_HARDWARE_POWERLEVELS;
                 hwmgr->platform_descriptor.hardwarePerformanceLevels = 2;
@@ -734,7 +733,7 @@ static int fiji_hwmgr_backend_init(struct pp_hwmgr *hwmgr)
                 sys_info.info_id = CGS_SYSTEM_INFO_PCIE_GEN_INFO;
                 result = cgs_query_system_info(hwmgr->device, &sys_info);
                 if (result)
-                        data->pcie_gen_cap = 0x30007;
+                        data->pcie_gen_cap = AMDGPU_DEFAULT_PCIE_GEN_MASK;
                 else
                         data->pcie_gen_cap = (uint32_t)sys_info.value;
                 if (data->pcie_gen_cap & CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3)
@@ -743,7 +742,7 @@ static int fiji_hwmgr_backend_init(struct pp_hwmgr *hwmgr)
                 sys_info.info_id = CGS_SYSTEM_INFO_PCIE_MLW;
                 result = cgs_query_system_info(hwmgr->device, &sys_info);
                 if (result)
-                        data->pcie_lane_cap = 0x2f0000;
+                        data->pcie_lane_cap = AMDGPU_DEFAULT_PCIE_MLW_MASK;
                 else
                         data->pcie_lane_cap = (uint32_t)sys_info.value;
         } else {
@@ -1236,6 +1235,34 @@ static int fiji_program_voting_clients(struct pp_hwmgr *hwmgr)
         return 0;
 }
 
+static int fiji_clear_voting_clients(struct pp_hwmgr *hwmgr)
+{
+        /* Reset voting clients before disabling DPM */
+        PHM_WRITE_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC,
+                        SCLK_PWRMGT_CNTL, RESET_SCLK_CNT, 1);
+        PHM_WRITE_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC,
+                        SCLK_PWRMGT_CNTL, RESET_BUSY_CNT, 1);
+
+        cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC,
+                        ixCG_FREQ_TRAN_VOTING_0, 0);
+        cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC,
+                        ixCG_FREQ_TRAN_VOTING_1, 0);
+        cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC,
+                        ixCG_FREQ_TRAN_VOTING_2, 0);
+        cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC,
+                        ixCG_FREQ_TRAN_VOTING_3, 0);
+        cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC,
+                        ixCG_FREQ_TRAN_VOTING_4, 0);
+        cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC,
+                        ixCG_FREQ_TRAN_VOTING_5, 0);
+        cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC,
+                        ixCG_FREQ_TRAN_VOTING_6, 0);
+        cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC,
+                        ixCG_FREQ_TRAN_VOTING_7, 0);
+
+        return 0;
+}
+
 /**
 * Get the location of various tables inside the FW image.
 *
@@ -1363,6 +1390,17 @@ static int fiji_copy_and_switch_arb_sets(struct pp_hwmgr *hwmgr,
 }
 
 /**
+* Call SMC to reset S0/S1 to S1 and Reset SMIO to initial value
+*
+* @param    hwmgr  the address of the powerplay hardware manager.
+* @return   if success then 0;
+*/
+static int fiji_reset_to_default(struct pp_hwmgr *hwmgr)
+{
+        return smum_send_msg_to_smc(hwmgr->smumgr, PPSMC_MSG_ResetToDefaults);
+}
+
+/**
 * Initial switch from ARB F0->F1
 *
 * @param    hwmgr  the address of the powerplay hardware manager.
@@ -1375,6 +1413,21 @@ static int fiji_initial_switch_from_arbf0_to_f1(struct pp_hwmgr *hwmgr)
                         MC_CG_ARB_FREQ_F0, MC_CG_ARB_FREQ_F1);
 }
 
+static int fiji_force_switch_to_arbf0(struct pp_hwmgr *hwmgr)
+{
+        uint32_t tmp;
+
+        tmp = (cgs_read_ind_register(hwmgr->device,
+                        CGS_IND_REG__SMC, ixSMC_SCRATCH9) &
+                        0x0000ff00) >> 8;
+
+        if (tmp == MC_CG_ARB_FREQ_F0)
+                return 0;
+
+        return fiji_copy_and_switch_arb_sets(hwmgr,
+                        tmp, MC_CG_ARB_FREQ_F0);
+}
+
 static int fiji_reset_single_dpm_table(struct pp_hwmgr *hwmgr,
                 struct fiji_single_dpm_table *dpm_table, uint32_t count)
 {
@@ -1397,7 +1450,7 @@ static void fiji_setup_pcie_table_entry(
 {
         dpm_table->dpm_levels[index].value = pcie_gen;
         dpm_table->dpm_levels[index].param1 = pcie_lanes;
-        dpm_table->dpm_levels[index].enabled = 1;
+        dpm_table->dpm_levels[index].enabled = true;
 }
 
 static int fiji_setup_default_pcie_table(struct pp_hwmgr *hwmgr)
@@ -1609,7 +1662,6 @@ static int fiji_populate_cac_table(struct pp_hwmgr *hwmgr,
 {
         uint32_t count;
         uint8_t index;
-        int result = 0;
         struct fiji_hwmgr *data = (struct fiji_hwmgr *)(hwmgr->backend);
         struct phm_ppt_v1_information *table_info =
                         (struct phm_ppt_v1_information *)(hwmgr->pptable);
@@ -1631,7 +1683,7 @@ static int fiji_populate_cac_table(struct pp_hwmgr *hwmgr,
                                                 VOLTAGE_SCALE)) / 25);
         }
 
-        return result;
+        return 0;
 }
 
 /**
@@ -3177,6 +3229,17 @@ static int fiji_enable_ulv(struct pp_hwmgr *hwmgr)
         return 0;
 }
 
+static int fiji_disable_ulv(struct pp_hwmgr *hwmgr)
+{
+        struct fiji_hwmgr *data = (struct fiji_hwmgr *)(hwmgr->backend);
+        struct fiji_ulv_parm *ulv = &(data->ulv);
+
+        if (ulv->ulv_supported)
+                return smum_send_msg_to_smc(hwmgr->smumgr, PPSMC_MSG_DisableULV);
+
+        return 0;
+}
+
 static int fiji_enable_deep_sleep_master_switch(struct pp_hwmgr *hwmgr)
 {
         if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
@@ -3197,6 +3260,21 @@ static int fiji_enable_deep_sleep_master_switch(struct pp_hwmgr *hwmgr)
         return 0;
 }
 
+static int fiji_disable_deep_sleep_master_switch(struct pp_hwmgr *hwmgr)
+{
+        if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
+                        PHM_PlatformCaps_SclkDeepSleep)) {
+                if (smum_send_msg_to_smc(hwmgr->smumgr,
+                                PPSMC_MSG_MASTER_DeepSleep_OFF)) {
+                        PP_ASSERT_WITH_CODE(false,
+                                        "Attempt to disable Master Deep Sleep switch failed!",
+                                        return -1);
+                }
+        }
+
+        return 0;
+}
+
 static int fiji_enable_sclk_mclk_dpm(struct pp_hwmgr *hwmgr)
 {
         struct fiji_hwmgr *data = (struct fiji_hwmgr *)(hwmgr->backend);
@@ -3357,6 +3435,70 @@ static int fiji_start_dpm(struct pp_hwmgr *hwmgr)
         return 0;
 }
 
+static int fiji_disable_sclk_mclk_dpm(struct pp_hwmgr *hwmgr)
+{
+        struct fiji_hwmgr *data = (struct fiji_hwmgr *)(hwmgr->backend);
+
+        /* disable SCLK dpm */
+        if (!data->sclk_dpm_key_disabled)
+                PP_ASSERT_WITH_CODE(
+                                (smum_send_msg_to_smc(hwmgr->smumgr,
+                                                PPSMC_MSG_DPM_Disable) == 0),
+                                "Failed to disable SCLK DPM!",
+                                return -1);
+
+        /* disable MCLK dpm */
+        if (!data->mclk_dpm_key_disabled) {
+                PP_ASSERT_WITH_CODE(
+                                (smum_send_msg_to_smc_with_parameter(hwmgr->smumgr,
+                                PPSMC_MSG_MCLKDPM_SetEnabledMask, 1) == 0),
+                                "Failed to force MCLK DPM0!",
+                                return -1);
+
+                PP_ASSERT_WITH_CODE(
+                                (smum_send_msg_to_smc(hwmgr->smumgr,
+                                                PPSMC_MSG_MCLKDPM_Disable) == 0),
+                                "Failed to disable MCLK DPM!",
+                                return -1);
+        }
+
+        return 0;
+}
+
+static int fiji_stop_dpm(struct pp_hwmgr *hwmgr)
+{
+        struct fiji_hwmgr *data = (struct fiji_hwmgr *)(hwmgr->backend);
+
+        /* disable general power management */
+        PHM_WRITE_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC, GENERAL_PWRMGT,
+                        GLOBAL_PWRMGT_EN, 0);
+        /* disable sclk deep sleep */
+        PHM_WRITE_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC, SCLK_PWRMGT_CNTL,
+                        DYNAMIC_PM_EN, 0);
+
+        /* disable PCIE dpm */
+        if (!data->pcie_dpm_key_disabled) {
+                PP_ASSERT_WITH_CODE(
+                                (smum_send_msg_to_smc(hwmgr->smumgr,
+                                                PPSMC_MSG_PCIeDPM_Disable) == 0),
+                                "Failed to disable pcie DPM during DPM Stop Function!",
+                                return -1);
+        }
+
+        if (fiji_disable_sclk_mclk_dpm(hwmgr)) {
+                printk(KERN_ERR "Failed to disable Sclk DPM and Mclk DPM!");
+                return -1;
+        }
+
+        PP_ASSERT_WITH_CODE(
+                        (smum_send_msg_to_smc(hwmgr->smumgr,
+                                        PPSMC_MSG_Voltage_Cntl_Disable) == 0),
+                        "Failed to disable voltage DPM during DPM Stop Function!",
+                        return -1);
+
+        return 0;
+}
+
 static void fiji_set_dpm_event_sources(struct pp_hwmgr *hwmgr,
                 uint32_t sources)
 {
@@ -3415,6 +3557,23 @@ static int fiji_enable_thermal_auto_throttle(struct pp_hwmgr *hwmgr)
         return fiji_enable_auto_throttle_source(hwmgr, PHM_AutoThrottleSource_Thermal);
 }
 
+static int fiji_disable_auto_throttle_source(struct pp_hwmgr *hwmgr,
+                PHM_AutoThrottleSource source)
+{
+        struct fiji_hwmgr *data = (struct fiji_hwmgr *)(hwmgr->backend);
+
+        if (data->active_auto_throttle_sources & (1 << source)) {
+                data->active_auto_throttle_sources &= ~(1 << source);
+                fiji_set_dpm_event_sources(hwmgr, data->active_auto_throttle_sources);
+        }
+        return 0;
+}
+
+static int fiji_disable_thermal_auto_throttle(struct pp_hwmgr *hwmgr)
+{
+        return fiji_disable_auto_throttle_source(hwmgr, PHM_AutoThrottleSource_Thermal);
+}
+
 static int fiji_enable_dpm_tasks(struct pp_hwmgr *hwmgr)
 {
         int tmp_result, result = 0;
@@ -3529,6 +3688,64 @@ static int fiji_enable_dpm_tasks(struct pp_hwmgr *hwmgr)
         return result;
 }
 
+static int fiji_disable_dpm_tasks(struct pp_hwmgr *hwmgr)
+{
+        int tmp_result, result = 0;
+
+        tmp_result = (fiji_is_dpm_running(hwmgr)) ? 0 : -1;
+        PP_ASSERT_WITH_CODE(tmp_result == 0,
+                        "DPM is not running right now, no need to disable DPM!",
+                        return 0);
+
+        if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
+                        PHM_PlatformCaps_ThermalController))
+                PHM_WRITE_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC,
+                                GENERAL_PWRMGT, THERMAL_PROTECTION_DIS, 1);
+
+        tmp_result = fiji_disable_power_containment(hwmgr);
+        PP_ASSERT_WITH_CODE((tmp_result == 0),
+                        "Failed to disable power containment!", result = tmp_result);
+
+        tmp_result = fiji_disable_smc_cac(hwmgr);
+        PP_ASSERT_WITH_CODE((tmp_result == 0),
+                        "Failed to disable SMC CAC!", result = tmp_result);
+
+        PHM_WRITE_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC,
+                        CG_SPLL_SPREAD_SPECTRUM, SSEN, 0);
+        PHM_WRITE_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC,
+                        GENERAL_PWRMGT, DYN_SPREAD_SPECTRUM_EN, 0);
+
+        tmp_result = fiji_disable_thermal_auto_throttle(hwmgr);
+        PP_ASSERT_WITH_CODE((tmp_result == 0),
+                        "Failed to disable thermal auto throttle!", result = tmp_result);
+
+        tmp_result = fiji_stop_dpm(hwmgr);
+        PP_ASSERT_WITH_CODE((tmp_result == 0),
+                        "Failed to stop DPM!", result = tmp_result);
+
+        tmp_result = fiji_disable_deep_sleep_master_switch(hwmgr);
+        PP_ASSERT_WITH_CODE((tmp_result == 0),
+                        "Failed to disable deep sleep master switch!", result = tmp_result);
+
+        tmp_result = fiji_disable_ulv(hwmgr);
+        PP_ASSERT_WITH_CODE((tmp_result == 0),
+                        "Failed to disable ULV!", result = tmp_result);
+
+        tmp_result = fiji_clear_voting_clients(hwmgr);
+        PP_ASSERT_WITH_CODE((tmp_result == 0),
+                        "Failed to clear voting clients!", result = tmp_result);
+
+        tmp_result = fiji_reset_to_default(hwmgr);
+        PP_ASSERT_WITH_CODE((tmp_result == 0),
+                        "Failed to reset to default!", result = tmp_result);
+
+        tmp_result = fiji_force_switch_to_arbf0(hwmgr);
+        PP_ASSERT_WITH_CODE((tmp_result == 0),
+                        "Failed to force to switch arbf0!", result = tmp_result);
+
+        return result;
+}
+
 static int fiji_force_dpm_highest(struct pp_hwmgr *hwmgr)
 {
         struct fiji_hwmgr *data = (struct fiji_hwmgr *)(hwmgr->backend);
@@ -4171,8 +4388,9 @@ static int fiji_freeze_sclk_mclk_dpm(struct pp_hwmgr *hwmgr)
         if ((0 == data->sclk_dpm_key_disabled) &&
                 (data->need_update_smu7_dpm_table &
                         (DPMTABLE_OD_UPDATE_SCLK + DPMTABLE_UPDATE_SCLK))) {
-                PP_ASSERT_WITH_CODE(true == fiji_is_dpm_running(hwmgr),
-                                "Trying to freeze SCLK DPM when DPM is disabled",);
+                PP_ASSERT_WITH_CODE(fiji_is_dpm_running(hwmgr),
+                                    "Trying to freeze SCLK DPM when DPM is disabled",
+                                    );
                 PP_ASSERT_WITH_CODE(0 == smum_send_msg_to_smc(hwmgr->smumgr,
                                 PPSMC_MSG_SCLKDPM_FreezeLevel),
                                 "Failed to freeze SCLK DPM during FreezeSclkMclkDPM Function!",
@@ -4182,8 +4400,9 @@ static int fiji_freeze_sclk_mclk_dpm(struct pp_hwmgr *hwmgr)
         if ((0 == data->mclk_dpm_key_disabled) &&
                 (data->need_update_smu7_dpm_table &
                  DPMTABLE_OD_UPDATE_MCLK)) {
-                PP_ASSERT_WITH_CODE(true == fiji_is_dpm_running(hwmgr),
-                                "Trying to freeze MCLK DPM when DPM is disabled",);
+                PP_ASSERT_WITH_CODE(fiji_is_dpm_running(hwmgr),
+                                    "Trying to freeze MCLK DPM when DPM is disabled",
+                                    );
                 PP_ASSERT_WITH_CODE(0 == smum_send_msg_to_smc(hwmgr->smumgr,
                                 PPSMC_MSG_MCLKDPM_FreezeLevel),
                                 "Failed to freeze MCLK DPM during FreezeSclkMclkDPM Function!",
@@ -4353,7 +4572,6 @@ static int fiji_trim_single_dpm_states(struct pp_hwmgr *hwmgr,
 static int fiji_trim_dpm_states(struct pp_hwmgr *hwmgr,
                 const struct fiji_power_state *fiji_ps)
 {
-        int result = 0;
         struct fiji_hwmgr *data = (struct fiji_hwmgr *)(hwmgr->backend);
         uint32_t high_limit_count;
 
@@ -4373,7 +4591,7 @@ static int fiji_trim_dpm_states(struct pp_hwmgr *hwmgr,
                         fiji_ps->performance_levels[0].memory_clock,
                         fiji_ps->performance_levels[high_limit_count].memory_clock);
 
-        return result;
+        return 0;
 }
 
 static int fiji_generate_dpm_level_enable_mask(
@@ -4632,8 +4850,9 @@ static int fiji_unfreeze_sclk_mclk_dpm(struct pp_hwmgr *hwmgr)
                 (data->need_update_smu7_dpm_table &
                 (DPMTABLE_OD_UPDATE_SCLK + DPMTABLE_UPDATE_SCLK))) {
 
-                PP_ASSERT_WITH_CODE(true == fiji_is_dpm_running(hwmgr),
-                                "Trying to Unfreeze SCLK DPM when DPM is disabled",);
+                PP_ASSERT_WITH_CODE(fiji_is_dpm_running(hwmgr),
+                                    "Trying to Unfreeze SCLK DPM when DPM is disabled",
+                                    );
                 PP_ASSERT_WITH_CODE(0 == smum_send_msg_to_smc(hwmgr->smumgr,
                                 PPSMC_MSG_SCLKDPM_UnfreezeLevel),
                         "Failed to unfreeze SCLK DPM during UnFreezeSclkMclkDPM Function!",
@@ -4643,8 +4862,9 @@ static int fiji_unfreeze_sclk_mclk_dpm(struct pp_hwmgr *hwmgr)
         if ((0 == data->mclk_dpm_key_disabled) &&
                 (data->need_update_smu7_dpm_table & DPMTABLE_OD_UPDATE_MCLK)) {
 
-                PP_ASSERT_WITH_CODE(true == fiji_is_dpm_running(hwmgr),
-                                "Trying to Unfreeze MCLK DPM when DPM is disabled",);
+                PP_ASSERT_WITH_CODE(fiji_is_dpm_running(hwmgr),
+                                    "Trying to Unfreeze MCLK DPM when DPM is disabled",
+                                    );
                 PP_ASSERT_WITH_CODE(0 == smum_send_msg_to_smc(hwmgr->smumgr,
                                 PPSMC_MSG_SCLKDPM_UnfreezeLevel),
                     "Failed to unfreeze MCLK DPM during UnFreezeSclkMclkDPM Function!",
@@ -5071,42 +5291,6 @@ static int fiji_get_fan_control_mode(struct pp_hwmgr *hwmgr)
                                 CG_FDO_CTRL2, FDO_PWM_MODE);
 }
 
-static int fiji_get_pp_table(struct pp_hwmgr *hwmgr, char **table)
-{
-        struct fiji_hwmgr *data = (struct fiji_hwmgr *)(hwmgr->backend);
-
-        if (!data->soft_pp_table) {
-                data->soft_pp_table = kmemdup(hwmgr->soft_pp_table,
-                                              hwmgr->soft_pp_table_size,
-                                              GFP_KERNEL);
-                if (!data->soft_pp_table)
-                        return -ENOMEM;
-        }
-
-        *table = (char *)&data->soft_pp_table;
-
-        return hwmgr->soft_pp_table_size;
-}
-
-static int fiji_set_pp_table(struct pp_hwmgr *hwmgr, const char *buf, size_t size)
-{
-        struct fiji_hwmgr *data = (struct fiji_hwmgr *)(hwmgr->backend);
-
-        if (!data->soft_pp_table) {
-                data->soft_pp_table = kzalloc(hwmgr->soft_pp_table_size, GFP_KERNEL);
-                if (!data->soft_pp_table)
-                        return -ENOMEM;
-        }
-
-        memcpy(data->soft_pp_table, buf, size);
-
-        hwmgr->soft_pp_table = data->soft_pp_table;
-
-        /* TODO: re-init powerplay to implement modified pptable */
-
-        return 0;
-}
-
 static int fiji_force_clock_level(struct pp_hwmgr *hwmgr,
                 enum pp_clock_type type, uint32_t mask)
 {
@@ -5276,12 +5460,96 @@ bool fiji_check_smc_update_required_for_display_configuration(struct pp_hwmgr *h
         return is_update_required;
 }
 
+static int fiji_get_sclk_od(struct pp_hwmgr *hwmgr)
+{
+        struct fiji_hwmgr *data = (struct fiji_hwmgr *)(hwmgr->backend);
+        struct fiji_single_dpm_table *sclk_table = &(data->dpm_table.sclk_table);
+        struct fiji_single_dpm_table *golden_sclk_table =
+                        &(data->golden_dpm_table.sclk_table);
+        int value;
+
+        value = (sclk_table->dpm_levels[sclk_table->count - 1].value -
+                        golden_sclk_table->dpm_levels[golden_sclk_table->count - 1].value) *
+                        100 /
+                        golden_sclk_table->dpm_levels[golden_sclk_table->count - 1].value;
+
+        return value;
+}
+
+static int fiji_set_sclk_od(struct pp_hwmgr *hwmgr, uint32_t value)
+{
+        struct fiji_hwmgr *data = (struct fiji_hwmgr *)(hwmgr->backend);
+        struct fiji_single_dpm_table *golden_sclk_table =
+                        &(data->golden_dpm_table.sclk_table);
+        struct pp_power_state  *ps;
+        struct fiji_power_state  *fiji_ps;
+
+        if (value > 20)
+                value = 20;
+
+        ps = hwmgr->request_ps;
+
+        if (ps == NULL)
+                return -EINVAL;
+
+        fiji_ps = cast_phw_fiji_power_state(&ps->hardware);
+
+        fiji_ps->performance_levels[fiji_ps->performance_level_count - 1].engine_clock =
+                        golden_sclk_table->dpm_levels[golden_sclk_table->count - 1].value *
+                        value / 100 +
+                        golden_sclk_table->dpm_levels[golden_sclk_table->count - 1].value;
+
+        return 0;
+}
+
+static int fiji_get_mclk_od(struct pp_hwmgr *hwmgr)
+{
+        struct fiji_hwmgr *data = (struct fiji_hwmgr *)(hwmgr->backend);
+        struct fiji_single_dpm_table *mclk_table = &(data->dpm_table.mclk_table);
+        struct fiji_single_dpm_table *golden_mclk_table =
+                        &(data->golden_dpm_table.mclk_table);
+        int value;
+
+        value = (mclk_table->dpm_levels[mclk_table->count - 1].value -
+                        golden_mclk_table->dpm_levels[golden_mclk_table->count - 1].value) *
+                        100 /
+                        golden_mclk_table->dpm_levels[golden_mclk_table->count - 1].value;
+
+        return value;
+}
+
+static int fiji_set_mclk_od(struct pp_hwmgr *hwmgr, uint32_t value)
+{
+        struct fiji_hwmgr *data = (struct fiji_hwmgr *)(hwmgr->backend);
+        struct fiji_single_dpm_table *golden_mclk_table =
+                        &(data->golden_dpm_table.mclk_table);
+        struct pp_power_state  *ps;
+        struct fiji_power_state  *fiji_ps;
+
+        if (value > 20)
+                value = 20;
+
+        ps = hwmgr->request_ps;
+
+        if (ps == NULL)
+                return -EINVAL;
+
+        fiji_ps = cast_phw_fiji_power_state(&ps->hardware);
+
+        fiji_ps->performance_levels[fiji_ps->performance_level_count - 1].memory_clock =
+                        golden_mclk_table->dpm_levels[golden_mclk_table->count - 1].value *
+                        value / 100 +
+                        golden_mclk_table->dpm_levels[golden_mclk_table->count - 1].value;
+
+        return 0;
+}
 
 static const struct pp_hwmgr_func fiji_hwmgr_funcs = {
         .backend_init = &fiji_hwmgr_backend_init,
         .backend_fini = &fiji_hwmgr_backend_fini,
         .asic_setup = &fiji_setup_asic_task,
         .dynamic_state_management_enable = &fiji_enable_dpm_tasks,
+        .dynamic_state_management_disable = &fiji_disable_dpm_tasks,
         .force_dpm_level = &fiji_dpm_force_dpm_level,
         .get_num_of_pp_table_entries = &tonga_get_number_of_powerplay_table_entries,
         .get_power_state_size = &fiji_get_power_state_size,
@@ -5314,24 +5582,18 @@ static const struct pp_hwmgr_func fiji_hwmgr_funcs = {
         .get_fan_control_mode = fiji_get_fan_control_mode,
         .check_states_equal = fiji_check_states_equal,
         .check_smc_update_required_for_display_configuration = fiji_check_smc_update_required_for_display_configuration,
-        .get_pp_table = fiji_get_pp_table,
-        .set_pp_table = fiji_set_pp_table,
         .force_clock_level = fiji_force_clock_level,
         .print_clock_levels = fiji_print_clock_levels,
+        .get_sclk_od = fiji_get_sclk_od,
+        .set_sclk_od = fiji_set_sclk_od,
+        .get_mclk_od = fiji_get_mclk_od,
+        .set_mclk_od = fiji_set_mclk_od,
 };
 
 int fiji_hwmgr_init(struct pp_hwmgr *hwmgr)
 {
-        struct fiji_hwmgr  *data;
-        int ret = 0;
-
-        data = kzalloc(sizeof(struct fiji_hwmgr), GFP_KERNEL);
-        if (data == NULL)
-                return -ENOMEM;
-
-        hwmgr->backend = data;
         hwmgr->hwmgr_func = &fiji_hwmgr_funcs;
         hwmgr->pptable_func = &tonga_pptable_funcs;
         pp_fiji_thermal_initialize(hwmgr);
-        return ret;
+        return 0;
 }
diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/fiji_hwmgr.h b/drivers/gpu/drm/amd/powerplay/hwmgr/fiji_hwmgr.h
index 170edf5a772d..bf67c2a92c68 100644
--- a/drivers/gpu/drm/amd/powerplay/hwmgr/fiji_hwmgr.h
+++ b/drivers/gpu/drm/amd/powerplay/hwmgr/fiji_hwmgr.h
@@ -302,9 +302,6 @@ struct fiji_hwmgr {
         bool                           pg_acp_init;
         bool                           frtc_enabled;
         bool                           frtc_status_changed;
-
-        /* soft pptable for re-uploading into smu */
-        void *soft_pp_table;
 };
 
 /* To convert to Q8.8 format for firmware */
diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/fiji_powertune.c b/drivers/gpu/drm/amd/powerplay/hwmgr/fiji_powertune.c
index db23a4068baf..44658451a8d2 100644
--- a/drivers/gpu/drm/amd/powerplay/hwmgr/fiji_powertune.c
+++ b/drivers/gpu/drm/amd/powerplay/hwmgr/fiji_powertune.c
@@ -73,17 +73,18 @@ void fiji_initialize_power_tune_defaults(struct pp_hwmgr *hwmgr)
 
         if (!tmp) {
                 phm_cap_set(hwmgr->platform_descriptor.platformCaps,
-                                PHM_PlatformCaps_PowerContainment);
-
-                phm_cap_set(hwmgr->platform_descriptor.platformCaps,
                                 PHM_PlatformCaps_CAC);
 
                 fiji_hwmgr->fast_watermark_threshold = 100;
 
-                tmp = 1;
-                fiji_hwmgr->enable_dte_feature = tmp ? false : true;
-                fiji_hwmgr->enable_tdc_limit_feature = tmp ? true : false;
-                fiji_hwmgr->enable_pkg_pwr_tracking_feature = tmp ? true : false;
+                if (hwmgr->powercontainment_enabled) {
+                        phm_cap_set(hwmgr->platform_descriptor.platformCaps,
+                                    PHM_PlatformCaps_PowerContainment);
+                        tmp = 1;
+                        fiji_hwmgr->enable_dte_feature = tmp ? false : true;
+                        fiji_hwmgr->enable_tdc_limit_feature = tmp ? true : false;
+                        fiji_hwmgr->enable_pkg_pwr_tracking_feature = tmp ? true : false;
+                }
         }
 }
 
@@ -459,6 +460,23 @@ int fiji_enable_smc_cac(struct pp_hwmgr *hwmgr)
         return result;
 }
 
+int fiji_disable_smc_cac(struct pp_hwmgr *hwmgr)
+{
+        struct fiji_hwmgr *data = (struct fiji_hwmgr *)(hwmgr->backend);
+        int result = 0;
+
+        if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
+                        PHM_PlatformCaps_CAC) && data->cac_enabled) {
+                int smc_result = smum_send_msg_to_smc(hwmgr->smumgr,
+                                (uint16_t)(PPSMC_MSG_DisableCac));
+                PP_ASSERT_WITH_CODE((smc_result == 0),
+                                "Failed to disable CAC in SMC.", result = -1);
+
+                data->cac_enabled = false;
+        }
+        return result;
+}
+
 int fiji_set_power_limit(struct pp_hwmgr *hwmgr, uint32_t n)
 {
         struct fiji_hwmgr *data = (struct fiji_hwmgr *)(hwmgr->backend);
@@ -528,6 +546,48 @@ int fiji_enable_power_containment(struct pp_hwmgr *hwmgr)
         return result;
 }
 
+int fiji_disable_power_containment(struct pp_hwmgr *hwmgr)
+{
+        struct fiji_hwmgr *data = (struct fiji_hwmgr *)(hwmgr->backend);
+        int result = 0;
+
+        if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
+                        PHM_PlatformCaps_PowerContainment) &&
+                        data->power_containment_features) {
+                int smc_result;
+
+                if (data->power_containment_features &
+                                POWERCONTAINMENT_FEATURE_TDCLimit) {
+                        smc_result = smum_send_msg_to_smc(hwmgr->smumgr,
+                                        (uint16_t)(PPSMC_MSG_TDCLimitDisable));
+                        PP_ASSERT_WITH_CODE((smc_result == 0),
+                                        "Failed to disable TDCLimit in SMC.",
+                                        result = smc_result);
+                }
+
+                if (data->power_containment_features &
+                                POWERCONTAINMENT_FEATURE_DTE) {
+                        smc_result = smum_send_msg_to_smc(hwmgr->smumgr,
+                                        (uint16_t)(PPSMC_MSG_DisableDTE));
+                        PP_ASSERT_WITH_CODE((smc_result == 0),
+                                        "Failed to disable DTE in SMC.",
+                                        result = smc_result);
+                }
+
+                if (data->power_containment_features &
+                                POWERCONTAINMENT_FEATURE_PkgPwrLimit) {
+                        smc_result = smum_send_msg_to_smc(hwmgr->smumgr,
+                                        (uint16_t)(PPSMC_MSG_PkgPwrLimitDisable));
+                        PP_ASSERT_WITH_CODE((smc_result == 0),
+                                        "Failed to disable PkgPwrTracking in SMC.",
+                                        result = smc_result);
+                }
+                data->power_containment_features = 0;
+        }
+
+        return result;
+}
+
 int fiji_power_control_set_level(struct pp_hwmgr *hwmgr)
 {
         struct phm_ppt_v1_information *table_info =
diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/fiji_powertune.h b/drivers/gpu/drm/amd/powerplay/hwmgr/fiji_powertune.h
index 55e58200f33a..fec772421733 100644
--- a/drivers/gpu/drm/amd/powerplay/hwmgr/fiji_powertune.h
+++ b/drivers/gpu/drm/amd/powerplay/hwmgr/fiji_powertune.h
@@ -36,6 +36,19 @@ enum fiji_pt_config_reg_type {
 #define POWERCONTAINMENT_FEATURE_TDCLimit        0x00000002
 #define POWERCONTAINMENT_FEATURE_PkgPwrLimit     0x00000004
 
+#define DIDT_SQ_CTRL0__UNUSED_0_MASK             0xffffffc0
+#define DIDT_SQ_CTRL0__UNUSED_0__SHIFT           0x6
+#define DIDT_TD_CTRL0__UNUSED_0_MASK             0xffffffc0
+#define DIDT_TD_CTRL0__UNUSED_0__SHIFT           0x6
+#define DIDT_TCP_CTRL0__UNUSED_0_MASK            0xffffffc0
+#define DIDT_TCP_CTRL0__UNUSED_0__SHIFT          0x6
+#define DIDT_SQ_TUNING_CTRL__UNUSED_0_MASK                 0xe0000000
+#define DIDT_SQ_TUNING_CTRL__UNUSED_0__SHIFT               0x0000001d
+#define DIDT_TD_TUNING_CTRL__UNUSED_0_MASK                 0xe0000000
+#define DIDT_TD_TUNING_CTRL__UNUSED_0__SHIFT               0x0000001d
+#define DIDT_TCP_TUNING_CTRL__UNUSED_0_MASK                0xe0000000
+#define DIDT_TCP_TUNING_CTRL__UNUSED_0__SHIFT              0x0000001d
+
 struct fiji_pt_config_reg {
         uint32_t                           offset;
         uint32_t                           mask;
@@ -58,7 +71,9 @@ void fiji_initialize_power_tune_defaults(struct pp_hwmgr *hwmgr);
 int fiji_populate_bapm_parameters_in_dpm_table(struct pp_hwmgr *hwmgr);
 int fiji_populate_pm_fuses(struct pp_hwmgr *hwmgr);
 int fiji_enable_smc_cac(struct pp_hwmgr *hwmgr);
+int fiji_disable_smc_cac(struct pp_hwmgr *hwmgr);
 int fiji_enable_power_containment(struct pp_hwmgr *hwmgr);
+int fiji_disable_power_containment(struct pp_hwmgr *hwmgr);
 int fiji_set_power_limit(struct pp_hwmgr *hwmgr, uint32_t n);
 int fiji_power_control_set_level(struct pp_hwmgr *hwmgr);
 
diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/functiontables.c b/drivers/gpu/drm/amd/powerplay/hwmgr/functiontables.c
index 7a705cee0cc2..a6abe81bc843 100644
--- a/drivers/gpu/drm/amd/powerplay/hwmgr/functiontables.c
+++ b/drivers/gpu/drm/amd/powerplay/hwmgr/functiontables.c
@@ -59,8 +59,8 @@ int phm_dispatch_table(struct pp_hwmgr *hwmgr,
                        struct phm_runtime_table_header *rt_table,
                        void *input, void *output)
 {
-        int result = 0;
-        void *temp_storage = NULL;
+        int result;
+        void *temp_storage;
 
         if (hwmgr == NULL || rt_table == NULL) {
                 printk(KERN_ERR "[ powerplay ] Invalid Parameter!\n");
@@ -73,12 +73,13 @@ int phm_dispatch_table(struct pp_hwmgr *hwmgr,
                         printk(KERN_ERR "[ powerplay ] Could not allocate table temporary storage\n");
                         return -ENOMEM;
                 }
+        } else {
+                temp_storage = NULL;
         }
 
         result = phm_run_table(hwmgr, rt_table, input, output, temp_storage);
 
-        if (NULL != temp_storage)
-                kfree(temp_storage);
+        kfree(temp_storage);
 
         return result;
 }
diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/hardwaremanager.c b/drivers/gpu/drm/amd/powerplay/hwmgr/hardwaremanager.c
index efb77eda7508..789f98ad2615 100644
--- a/drivers/gpu/drm/amd/powerplay/hwmgr/hardwaremanager.c
+++ b/drivers/gpu/drm/amd/powerplay/hwmgr/hardwaremanager.c
@@ -154,6 +154,30 @@ int phm_enable_dynamic_state_management(struct pp_hwmgr *hwmgr)
         return ret;
 }
 
+int phm_disable_dynamic_state_management(struct pp_hwmgr *hwmgr)
+{
+        int ret = -1;
+        bool enabled;
+
+        PHM_FUNC_CHECK(hwmgr);
+
+        if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
+                PHM_PlatformCaps_TablelessHardwareInterface)) {
+                if (hwmgr->hwmgr_func->dynamic_state_management_disable)
+                        ret = hwmgr->hwmgr_func->dynamic_state_management_disable(hwmgr);
+        } else {
+                ret = phm_dispatch_table(hwmgr,
+                                &(hwmgr->disable_dynamic_state_management),
+                                NULL, NULL);
+        }
+
+        enabled = ret == 0 ? false : true;
+
+        cgs_notify_dpm_enabled(hwmgr->device, enabled);
+
+        return ret;
+}
+
 int phm_force_dpm_levels(struct pp_hwmgr *hwmgr, enum amd_dpm_forced_level level)
 {
         PHM_FUNC_CHECK(hwmgr);
@@ -314,7 +338,7 @@ int phm_store_dal_configuration_data(struct pp_hwmgr *hwmgr,
         if (hwmgr->hwmgr_func->store_cc6_data == NULL)
                 return -EINVAL;
 
-        /* to do pass other display configuration in furture */
+        /* TODO: pass other display configuration in the future */
 
         if (hwmgr->hwmgr_func->store_cc6_data)
                 hwmgr->hwmgr_func->store_cc6_data(hwmgr,
diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/hwmgr.c b/drivers/gpu/drm/amd/powerplay/hwmgr/hwmgr.c
index 20f20e075588..27e07624ac28 100644
--- a/drivers/gpu/drm/amd/powerplay/hwmgr/hwmgr.c
+++ b/drivers/gpu/drm/amd/powerplay/hwmgr/hwmgr.c
@@ -24,6 +24,7 @@
 #include <linux/types.h>
 #include <linux/kernel.h>
 #include <linux/slab.h>
+#include <drm/amdgpu_drm.h>
 #include "cgs_common.h"
 #include "power_state.h"
 #include "hwmgr.h"
@@ -58,12 +59,13 @@ int hwmgr_init(struct amd_pp_init *pp_init, struct pp_instance *handle)
         hwmgr->hw_revision = pp_init->rev_id;
         hwmgr->usec_timeout = AMD_MAX_USEC_TIMEOUT;
         hwmgr->power_source = PP_PowerSource_AC;
+        hwmgr->powercontainment_enabled = pp_init->powercontainment_enabled;
 
         switch (hwmgr->chip_family) {
-        case AMD_FAMILY_CZ:
+        case AMDGPU_FAMILY_CZ:
                 cz_hwmgr_init(hwmgr);
                 break;
-        case AMD_FAMILY_VI:
+        case AMDGPU_FAMILY_VI:
                 switch (hwmgr->chip_id) {
                 case CHIP_TONGA:
                         tonga_hwmgr_init(hwmgr);
@@ -94,6 +96,8 @@ int hwmgr_fini(struct pp_hwmgr *hwmgr)
                 return -EINVAL;
 
         /* do hwmgr finish*/
+        kfree(hwmgr->hardcode_pp_table);
+
         kfree(hwmgr->backend);
 
         kfree(hwmgr->start_thermal_controller.function_list);
@@ -530,7 +534,7 @@ int phm_initializa_dynamic_state_adjustment_rule_settings(struct pp_hwmgr *hwmgr
 
         /* initialize vddc_dep_on_dal_pwrl table */
         table_size = sizeof(uint32_t) + 4 * sizeof(struct phm_clock_voltage_dependency_record);
-        table_clk_vlt = (struct phm_clock_voltage_dependency_table *)kzalloc(table_size, GFP_KERNEL);
+        table_clk_vlt = kzalloc(table_size, GFP_KERNEL);
 
         if (NULL == table_clk_vlt) {
                 printk(KERN_ERR "[ powerplay ] Can not allocate space for vddc_dep_on_dal_pwrl! \n");
diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/polaris10_clockpowergating.c b/drivers/gpu/drm/amd/powerplay/hwmgr/polaris10_clockpowergating.c
index 8f142a74ad08..b5edb5105986 100644
--- a/drivers/gpu/drm/amd/powerplay/hwmgr/polaris10_clockpowergating.c
+++ b/drivers/gpu/drm/amd/powerplay/hwmgr/polaris10_clockpowergating.c
@@ -106,11 +106,17 @@ int polaris10_phm_powergate_uvd(struct pp_hwmgr *hwmgr, bool bgate)
         data->uvd_power_gated = bgate;
 
         if (bgate) {
+                cgs_set_clockgating_state(hwmgr->device,
+                                AMD_IP_BLOCK_TYPE_UVD,
+                                AMD_CG_STATE_GATE);
                 polaris10_update_uvd_dpm(hwmgr, true);
                 polaris10_phm_powerdown_uvd(hwmgr);
         } else {
                 polaris10_phm_powerup_uvd(hwmgr);
                 polaris10_update_uvd_dpm(hwmgr, false);
+                cgs_set_clockgating_state(hwmgr->device,
+                                AMD_IP_BLOCK_TYPE_UVD,
+                                AMD_CG_STATE_UNGATE);
         }
 
         return 0;
@@ -125,11 +131,19 @@ int polaris10_phm_powergate_vce(struct pp_hwmgr *hwmgr, bool bgate)
 
         data->vce_power_gated = bgate;
 
-        if (bgate)
+        if (bgate) {
+                cgs_set_clockgating_state(hwmgr->device,
+                                AMD_IP_BLOCK_TYPE_VCE,
+                                AMD_CG_STATE_GATE);
+                polaris10_update_vce_dpm(hwmgr, true);
                 polaris10_phm_powerdown_vce(hwmgr);
-        else
+        } else {
                 polaris10_phm_powerup_vce(hwmgr);
-
+                polaris10_update_vce_dpm(hwmgr, false);
+                cgs_set_clockgating_state(hwmgr->device,
+                                AMD_IP_BLOCK_TYPE_VCE,
+                                AMD_CG_STATE_UNGATE);
+        }
         return 0;
 }
 
diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/polaris10_hwmgr.c b/drivers/gpu/drm/amd/powerplay/hwmgr/polaris10_hwmgr.c
index 91e25f942d90..769636a0c5b5 100644
--- a/drivers/gpu/drm/amd/powerplay/hwmgr/polaris10_hwmgr.c
+++ b/drivers/gpu/drm/amd/powerplay/hwmgr/polaris10_hwmgr.c
@@ -389,6 +389,34 @@ static int polaris10_program_voting_clients(struct pp_hwmgr *hwmgr)
         return 0;
 }
 
+static int polaris10_clear_voting_clients(struct pp_hwmgr *hwmgr)
+{
+        /* Reset voting clients before disabling DPM */
+        PHM_WRITE_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC,
+                        SCLK_PWRMGT_CNTL, RESET_SCLK_CNT, 1);
+        PHM_WRITE_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC,
+                        SCLK_PWRMGT_CNTL, RESET_BUSY_CNT, 1);
+
+        cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC,
+                        ixCG_FREQ_TRAN_VOTING_0, 0);
+        cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC,
+                        ixCG_FREQ_TRAN_VOTING_1, 0);
+        cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC,
+                        ixCG_FREQ_TRAN_VOTING_2, 0);
+        cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC,
+                        ixCG_FREQ_TRAN_VOTING_3, 0);
+        cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC,
+                        ixCG_FREQ_TRAN_VOTING_4, 0);
+        cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC,
+                        ixCG_FREQ_TRAN_VOTING_5, 0);
+        cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC,
+                        ixCG_FREQ_TRAN_VOTING_6, 0);
+        cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC,
+                        ixCG_FREQ_TRAN_VOTING_7, 0);
+
+        return 0;
+}
+
 /**
 * Get the location of various tables inside the FW image.
 *
@@ -515,6 +543,11 @@ static int polaris10_copy_and_switch_arb_sets(struct pp_hwmgr *hwmgr,
         return 0;
 }
 
+static int polaris10_reset_to_default(struct pp_hwmgr *hwmgr)
+{
+        return smum_send_msg_to_smc(hwmgr->smumgr, PPSMC_MSG_ResetToDefaults);
+}
+
 /**
 * Initial switch from ARB F0->F1
 *
@@ -528,6 +561,21 @@ static int polaris10_initial_switch_from_arbf0_to_f1(struct pp_hwmgr *hwmgr)
                         MC_CG_ARB_FREQ_F0, MC_CG_ARB_FREQ_F1);
 }
 
+static int polaris10_force_switch_to_arbf0(struct pp_hwmgr *hwmgr)
+{
+        uint32_t tmp;
+
+        tmp = (cgs_read_ind_register(hwmgr->device,
+                        CGS_IND_REG__SMC, ixSMC_SCRATCH9) &
+                        0x0000ff00) >> 8;
+
+        if (tmp == MC_CG_ARB_FREQ_F0)
+                return 0;
+
+        return polaris10_copy_and_switch_arb_sets(hwmgr,
+                        tmp, MC_CG_ARB_FREQ_F0);
+}
+
 static int polaris10_setup_default_pcie_table(struct pp_hwmgr *hwmgr)
 {
         struct polaris10_hwmgr *data = (struct polaris10_hwmgr *)(hwmgr->backend);
@@ -1356,9 +1404,9 @@ static int polaris10_populate_all_memory_levels(struct pp_hwmgr *hwmgr)
                         return result;
         }
 
-        /* in order to prevent MC activity from stutter mode to push DPM up.
+        /* In order to prevent MC activity from stutter mode to push DPM up,
          * the UVD change complements this by putting the MCLK in
-         * a higher state by default such that we are not effected by
+         * a higher state by default such that we are not affected by
          * up threshold or and MCLK DPM latency.
          */
         levels[0].ActivityLevel = 0x1f;
@@ -1425,7 +1473,7 @@ static int polaris10_populate_smc_acpi_level(struct pp_hwmgr *hwmgr,
 
         /* Get MinVoltage and Frequency from DPM0,
          * already converted to SMC_UL */
-        sclk_frequency = data->dpm_table.sclk_table.dpm_levels[0].value;
+        sclk_frequency = data->vbios_boot_state.sclk_bootup_value;
         result = polaris10_get_dependency_volt_by_clk(hwmgr,
                         table_info->vdd_dep_on_sclk,
                         sclk_frequency,
@@ -1461,8 +1509,7 @@ static int polaris10_populate_smc_acpi_level(struct pp_hwmgr *hwmgr,
 
 
         /* Get MinVoltage and Frequency from DPM0, already converted to SMC_UL */
-        table->MemoryACPILevel.MclkFrequency =
-                        data->dpm_table.mclk_table.dpm_levels[0].value;
+        table->MemoryACPILevel.MclkFrequency = data->vbios_boot_state.mclk_bootup_value;
         result = polaris10_get_dependency_volt_by_clk(hwmgr,
                         table_info->vdd_dep_on_mclk,
                         table->MemoryACPILevel.MclkFrequency,
@@ -1780,7 +1827,7 @@ static int polaris10_populate_clock_stretcher_data_table(struct pp_hwmgr *hwmgr)
 {
         uint32_t ro, efuse, volt_without_cks, volt_with_cks, value, max, min;
         struct polaris10_hwmgr *data = (struct polaris10_hwmgr *)(hwmgr->backend);
-        uint8_t i, stretch_amount, stretch_amount2, volt_offset = 0;
+        uint8_t i, stretch_amount, volt_offset = 0;
         struct phm_ppt_v1_information *table_info =
                         (struct phm_ppt_v1_information *)(hwmgr->pptable);
         struct phm_ppt_v1_clock_voltage_dependency_table *sclk_table =
@@ -1831,11 +1878,8 @@ static int polaris10_populate_clock_stretcher_data_table(struct pp_hwmgr *hwmgr)
 
         data->smc_state_table.LdoRefSel = (table_info->cac_dtp_table->ucCKS_LDO_REFSEL != 0) ? table_info->cac_dtp_table->ucCKS_LDO_REFSEL : 6;
         /* Populate CKS Lookup Table */
-        if (stretch_amount == 1 || stretch_amount == 2 || stretch_amount == 5)
-                stretch_amount2 = 0;
-        else if (stretch_amount == 3 || stretch_amount == 4)
-                stretch_amount2 = 1;
-        else {
+        if (stretch_amount != 1 && stretch_amount != 2 && stretch_amount != 3 &&
+                        stretch_amount != 4 && stretch_amount != 5) {
                 phm_cap_unset(hwmgr->platform_descriptor.platformCaps,
                                 PHM_PlatformCaps_ClockStretcher);
                 PP_ASSERT_WITH_CODE(false,
@@ -1890,9 +1934,8 @@ static int polaris10_populate_vr_config(struct pp_hwmgr *hwmgr,
         if (POLARIS10_VOLTAGE_CONTROL_BY_SVID2 == data->mvdd_control) {
                 config = VR_SVI2_PLANE_2;
                 table->VRConfig |= (config << VRCONF_MVDD_SHIFT);
-        } else if (POLARIS10_VOLTAGE_CONTROL_BY_GPIO == data->mvdd_control) {
-                config = VR_SMIO_PATTERN_2;
-                table->VRConfig |= (config << VRCONF_MVDD_SHIFT);
+                cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC, data->soft_regs_start +
+                        offsetof(SMU74_SoftRegisters, AllowMvddSwitch), 0x1);
         } else {
                 config = VR_STATIC_VOLTAGE;
                 table->VRConfig |= (config << VRCONF_MVDD_SHIFT);
@@ -2262,6 +2305,17 @@ static int polaris10_enable_ulv(struct pp_hwmgr *hwmgr)
         return 0;
 }
 
+static int polaris10_disable_ulv(struct pp_hwmgr *hwmgr)
+{
+        struct polaris10_hwmgr *data = (struct polaris10_hwmgr *)(hwmgr->backend);
+        struct polaris10_ulv_parm *ulv = &(data->ulv);
+
+        if (ulv->ulv_supported)
+                return smum_send_msg_to_smc(hwmgr->smumgr, PPSMC_MSG_DisableULV);
+
+        return 0;
+}
+
 static int polaris10_enable_deep_sleep_master_switch(struct pp_hwmgr *hwmgr)
 {
         if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
@@ -2282,6 +2336,21 @@ static int polaris10_enable_deep_sleep_master_switch(struct pp_hwmgr *hwmgr)
         return 0;
 }
 
+static int polaris10_disable_deep_sleep_master_switch(struct pp_hwmgr *hwmgr)
+{
+        if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
+                        PHM_PlatformCaps_SclkDeepSleep)) {
+                if (smum_send_msg_to_smc(hwmgr->smumgr,
+                                PPSMC_MSG_MASTER_DeepSleep_OFF)) {
+                        PP_ASSERT_WITH_CODE(false,
+                                        "Attempt to disable Master Deep Sleep switch failed!",
+                                        return -1);
+                }
+        }
+
+        return 0;
+}
+
 static int polaris10_enable_sclk_mclk_dpm(struct pp_hwmgr *hwmgr)
 {
         struct polaris10_hwmgr *data = (struct polaris10_hwmgr *)(hwmgr->backend);
@@ -2379,6 +2448,58 @@ static int polaris10_start_dpm(struct pp_hwmgr *hwmgr)
         return 0;
 }
 
+static int polaris10_disable_sclk_mclk_dpm(struct pp_hwmgr *hwmgr)
+{
+        struct polaris10_hwmgr *data = (struct polaris10_hwmgr *)(hwmgr->backend);
+
+        /* disable SCLK dpm */
+        if (!data->sclk_dpm_key_disabled)
+                PP_ASSERT_WITH_CODE(
+                                (smum_send_msg_to_smc(hwmgr->smumgr,
+                                                PPSMC_MSG_DPM_Disable) == 0),
+                                "Failed to disable SCLK DPM!",
+                                return -1);
+
+        /* disable MCLK dpm */
+        if (!data->mclk_dpm_key_disabled) {
+                PP_ASSERT_WITH_CODE(
+                                (smum_send_msg_to_smc(hwmgr->smumgr,
+                                                PPSMC_MSG_MCLKDPM_Disable) == 0),
+                                "Failed to disable MCLK DPM!",
+                                return -1);
+        }
+
+        return 0;
+}
+
+static int polaris10_stop_dpm(struct pp_hwmgr *hwmgr)
+{
+        struct polaris10_hwmgr *data = (struct polaris10_hwmgr *)(hwmgr->backend);
+
+        /* disable general power management */
+        PHM_WRITE_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC, GENERAL_PWRMGT,
+                        GLOBAL_PWRMGT_EN, 0);
+        /* disable sclk deep sleep */
+        PHM_WRITE_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC, SCLK_PWRMGT_CNTL,
+                        DYNAMIC_PM_EN, 0);
+
+        /* disable PCIE dpm */
+        if (!data->pcie_dpm_key_disabled) {
+                PP_ASSERT_WITH_CODE(
+                                (smum_send_msg_to_smc(hwmgr->smumgr,
+                                                PPSMC_MSG_PCIeDPM_Disable) == 0),
+                                "Failed to disable pcie DPM during DPM Stop Function!",
+                                return -1);
+        }
+
+        if (polaris10_disable_sclk_mclk_dpm(hwmgr)) {
+                printk(KERN_ERR "Failed to disable Sclk DPM and Mclk DPM!");
+                return -1;
+        }
+
+        return 0;
+}
+
 static void polaris10_set_dpm_event_sources(struct pp_hwmgr *hwmgr, uint32_t sources)
 {
         bool protection;
@@ -2436,6 +2557,23 @@ static int polaris10_enable_thermal_auto_throttle(struct pp_hwmgr *hwmgr)
         return polaris10_enable_auto_throttle_source(hwmgr, PHM_AutoThrottleSource_Thermal);
 }
 
+static int polaris10_disable_auto_throttle_source(struct pp_hwmgr *hwmgr,
+                PHM_AutoThrottleSource source)
+{
+        struct polaris10_hwmgr *data = (struct polaris10_hwmgr *)(hwmgr->backend);
+
+        if (data->active_auto_throttle_sources & (1 << source)) {
+                data->active_auto_throttle_sources &= ~(1 << source);
+                polaris10_set_dpm_event_sources(hwmgr, data->active_auto_throttle_sources);
+        }
+        return 0;
+}
+
+static int polaris10_disable_thermal_auto_throttle(struct pp_hwmgr *hwmgr)
+{
+        return polaris10_disable_auto_throttle_source(hwmgr, PHM_AutoThrottleSource_Thermal);
+}
+
 int polaris10_pcie_performance_request(struct pp_hwmgr *hwmgr)
 {
         struct polaris10_hwmgr *data = (struct polaris10_hwmgr *)(hwmgr->backend);
@@ -2530,6 +2668,10 @@ int polaris10_enable_dpm_tasks(struct pp_hwmgr *hwmgr)
         PP_ASSERT_WITH_CODE((0 == tmp_result),
                         "Failed to enable deep sleep master switch!", result = tmp_result);
 
+        tmp_result = polaris10_enable_didt_config(hwmgr);
+        PP_ASSERT_WITH_CODE((tmp_result == 0),
+                        "Failed to enable deep sleep master switch!", result = tmp_result);
+
         tmp_result = polaris10_start_dpm(hwmgr);
         PP_ASSERT_WITH_CODE((0 == tmp_result),
                         "Failed to start DPM!", result = tmp_result);
@@ -2559,8 +2701,60 @@ int polaris10_enable_dpm_tasks(struct pp_hwmgr *hwmgr)
 
 int polaris10_disable_dpm_tasks(struct pp_hwmgr *hwmgr)
 {
+        int tmp_result, result = 0;
 
-        return 0;
+        tmp_result = (polaris10_is_dpm_running(hwmgr)) ? 0 : -1;
+        PP_ASSERT_WITH_CODE(tmp_result == 0,
+                        "DPM is not running right now, no need to disable DPM!",
+                        return 0);
+
+        if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
+                        PHM_PlatformCaps_ThermalController))
+                PHM_WRITE_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC,
+                                GENERAL_PWRMGT, THERMAL_PROTECTION_DIS, 1);
+
+        tmp_result = polaris10_disable_power_containment(hwmgr);
+        PP_ASSERT_WITH_CODE((tmp_result == 0),
+                        "Failed to disable power containment!", result = tmp_result);
+
+        tmp_result = polaris10_disable_smc_cac(hwmgr);
+        PP_ASSERT_WITH_CODE((tmp_result == 0),
+                        "Failed to disable SMC CAC!", result = tmp_result);
+
+        PHM_WRITE_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC,
+                        CG_SPLL_SPREAD_SPECTRUM, SSEN, 0);
+        PHM_WRITE_INDIRECT_FIELD(hwmgr->device, CGS_IND_REG__SMC,
+                        GENERAL_PWRMGT, DYN_SPREAD_SPECTRUM_EN, 0);
+
+        tmp_result = polaris10_disable_thermal_auto_throttle(hwmgr);
+        PP_ASSERT_WITH_CODE((tmp_result == 0),
+                        "Failed to disable thermal auto throttle!", result = tmp_result);
+
+        tmp_result = polaris10_stop_dpm(hwmgr);
+        PP_ASSERT_WITH_CODE((tmp_result == 0),
+                        "Failed to stop DPM!", result = tmp_result);
+
+        tmp_result = polaris10_disable_deep_sleep_master_switch(hwmgr);
+        PP_ASSERT_WITH_CODE((tmp_result == 0),
+                        "Failed to disable deep sleep master switch!", result = tmp_result);
+
+        tmp_result = polaris10_disable_ulv(hwmgr);
+        PP_ASSERT_WITH_CODE((tmp_result == 0),
+                        "Failed to disable ULV!", result = tmp_result);
+
+        tmp_result = polaris10_clear_voting_clients(hwmgr);
+        PP_ASSERT_WITH_CODE((tmp_result == 0),
+                        "Failed to clear voting clients!", result = tmp_result);
+
+        tmp_result = polaris10_reset_to_default(hwmgr);
+        PP_ASSERT_WITH_CODE((tmp_result == 0),
+                        "Failed to reset to default!", result = tmp_result);
+
+        tmp_result = polaris10_force_switch_to_arbf0(hwmgr);
+        PP_ASSERT_WITH_CODE((tmp_result == 0),
+                        "Failed to force to switch arbf0!", result = tmp_result);
+
+        return result;
 }
 
 int polaris10_reset_asic_tasks(struct pp_hwmgr *hwmgr)
@@ -2571,13 +2765,6 @@ int polaris10_reset_asic_tasks(struct pp_hwmgr *hwmgr)
 
 int polaris10_hwmgr_backend_fini(struct pp_hwmgr *hwmgr)
 {
-        struct polaris10_hwmgr *data = (struct polaris10_hwmgr *)(hwmgr->backend);
-
-        if (data->soft_pp_table) {
-                kfree(data->soft_pp_table);
-                data->soft_pp_table = NULL;
-        }
-
         return phm_hwmgr_backend_fini(hwmgr);
 }
 
@@ -2624,17 +2811,22 @@ int polaris10_set_features_platform_caps(struct pp_hwmgr *hwmgr)
                                         PHM_PlatformCaps_DynamicUVDState);
 
         /* power tune caps Assume disabled */
-        phm_cap_unset(hwmgr->platform_descriptor.platformCaps,
+        phm_cap_set(hwmgr->platform_descriptor.platformCaps,
                                                 PHM_PlatformCaps_SQRamping);
-        phm_cap_unset(hwmgr->platform_descriptor.platformCaps,
+        phm_cap_set(hwmgr->platform_descriptor.platformCaps,
                                                 PHM_PlatformCaps_DBRamping);
-        phm_cap_unset(hwmgr->platform_descriptor.platformCaps,
+        phm_cap_set(hwmgr->platform_descriptor.platformCaps,
                                                 PHM_PlatformCaps_TDRamping);
-        phm_cap_unset(hwmgr->platform_descriptor.platformCaps,
+        phm_cap_set(hwmgr->platform_descriptor.platformCaps,
                                                 PHM_PlatformCaps_TCPRamping);
 
-        phm_cap_set(hwmgr->platform_descriptor.platformCaps,
-                                        PHM_PlatformCaps_PowerContainment);
+        if (hwmgr->powercontainment_enabled)
+                phm_cap_set(hwmgr->platform_descriptor.platformCaps,
+                            PHM_PlatformCaps_PowerContainment);
+        else
+                phm_cap_unset(hwmgr->platform_descriptor.platformCaps,
+                            PHM_PlatformCaps_PowerContainment);
+
         phm_cap_set(hwmgr->platform_descriptor.platformCaps,
                                                         PHM_PlatformCaps_CAC);
 
@@ -2706,12 +2898,12 @@ static int polaris10_get_evv_voltages(struct pp_hwmgr *hwmgr)
                                 }
                         }
 
-
-                        PP_ASSERT_WITH_CODE(0 == atomctrl_get_voltage_evv_on_sclk_ai(hwmgr,
-                                                        VOLTAGE_TYPE_VDDC, sclk, vv_id, &vddc),
-                                                "Error retrieving EVV voltage value!",
-                                                continue);
-
+                        if (atomctrl_get_voltage_evv_on_sclk_ai(hwmgr,
+                                                VOLTAGE_TYPE_VDDC,
+                                                sclk, vv_id, &vddc) != 0) {
+                                printk(KERN_WARNING "failed to retrieving EVV voltage!\n");
+                                continue;
+                        }
 
                         /* need to make sure vddc is less than 2v or else, it could burn the ASIC.
                          * real voltage level in unit of 0.01mv */
@@ -2968,13 +3160,19 @@ int polaris10_patch_voltage_workaround(struct pp_hwmgr *hwmgr)
 
 int polaris10_hwmgr_backend_init(struct pp_hwmgr *hwmgr)
 {
-        struct polaris10_hwmgr *data = (struct polaris10_hwmgr *)(hwmgr->backend);
+        struct polaris10_hwmgr *data;
         struct pp_atomctrl_gpio_pin_assignment gpio_pin_assignment;
         uint32_t temp_reg;
         int result;
         struct phm_ppt_v1_information *table_info =
                         (struct phm_ppt_v1_information *)(hwmgr->pptable);
 
+        data = kzalloc(sizeof(struct polaris10_hwmgr), GFP_KERNEL);
+        if (data == NULL)
+                return -ENOMEM;
+
+        hwmgr->backend = data;
+
         data->dll_default_on = false;
         data->sram_end = SMC_RAM_END;
         data->mclk_dpm0_activity_target = 0xa;
@@ -3063,7 +3261,7 @@ int polaris10_hwmgr_backend_init(struct pp_hwmgr *hwmgr)
         if (0 == result) {
                 struct cgs_system_info sys_info = {0};
 
-                data->is_tlu_enabled = 0;
+                data->is_tlu_enabled = false;
 
                 hwmgr->platform_descriptor.hardwareActivityPerformanceLevels =
                                                         POLARIS10_MAX_HARDWARE_POWERLEVELS;
@@ -3148,7 +3346,7 @@ int polaris10_hwmgr_backend_init(struct pp_hwmgr *hwmgr)
                 sys_info.info_id = CGS_SYSTEM_INFO_PCIE_GEN_INFO;
                 result = cgs_query_system_info(hwmgr->device, &sys_info);
                 if (result)
-                        data->pcie_gen_cap = 0x30007;
+                        data->pcie_gen_cap = AMDGPU_DEFAULT_PCIE_GEN_MASK;
                 else
                         data->pcie_gen_cap = (uint32_t)sys_info.value;
                 if (data->pcie_gen_cap & CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3)
@@ -3157,7 +3355,7 @@ int polaris10_hwmgr_backend_init(struct pp_hwmgr *hwmgr)
                 sys_info.info_id = CGS_SYSTEM_INFO_PCIE_MLW;
                 result = cgs_query_system_info(hwmgr->device, &sys_info);
                 if (result)
-                        data->pcie_lane_cap = 0x2f0000;
+                        data->pcie_lane_cap = AMDGPU_DEFAULT_PCIE_MLW_MASK;
                 else
                         data->pcie_lane_cap = (uint32_t)sys_info.value;
 
@@ -3446,6 +3644,7 @@ static int polaris10_apply_state_adjust_rules(struct pp_hwmgr *hwmgr,
                                     hwmgr->platform_descriptor.platformCaps,
                                     PHM_PlatformCaps_DisableMclkSwitchingForFrameLock);
 
+
         disable_mclk_switching = (1 < info.display_count) ||
                                     disable_mclk_switching_for_frame_lock;
 
@@ -3950,8 +4149,8 @@ static int polaris10_freeze_sclk_mclk_dpm(struct pp_hwmgr *hwmgr)
         if ((0 == data->sclk_dpm_key_disabled) &&
                 (data->need_update_smu7_dpm_table &
                         (DPMTABLE_OD_UPDATE_SCLK + DPMTABLE_UPDATE_SCLK))) {
-                PP_ASSERT_WITH_CODE(true == polaris10_is_dpm_running(hwmgr),
-                                "Trying to freeze SCLK DPM when DPM is disabled",
+                PP_ASSERT_WITH_CODE(polaris10_is_dpm_running(hwmgr),
+                                    "Trying to freeze SCLK DPM when DPM is disabled",
                                 );
                 PP_ASSERT_WITH_CODE(0 == smum_send_msg_to_smc(hwmgr->smumgr,
                                 PPSMC_MSG_SCLKDPM_FreezeLevel),
@@ -3962,8 +4161,8 @@ static int polaris10_freeze_sclk_mclk_dpm(struct pp_hwmgr *hwmgr)
         if ((0 == data->mclk_dpm_key_disabled) &&
                 (data->need_update_smu7_dpm_table &
                  DPMTABLE_OD_UPDATE_MCLK)) {
-                PP_ASSERT_WITH_CODE(true == polaris10_is_dpm_running(hwmgr),
-                                "Trying to freeze MCLK DPM when DPM is disabled",
+                PP_ASSERT_WITH_CODE(polaris10_is_dpm_running(hwmgr),
+                                    "Trying to freeze MCLK DPM when DPM is disabled",
                                 );
                 PP_ASSERT_WITH_CODE(0 == smum_send_msg_to_smc(hwmgr->smumgr,
                                 PPSMC_MSG_MCLKDPM_FreezeLevel),
@@ -4123,7 +4322,6 @@ static int polaris10_trim_single_dpm_states(struct pp_hwmgr *hwmgr,
 static int polaris10_trim_dpm_states(struct pp_hwmgr *hwmgr,
                 const struct polaris10_power_state *polaris10_ps)
 {
-        int result = 0;
         struct polaris10_hwmgr *data = (struct polaris10_hwmgr *)(hwmgr->backend);
         uint32_t high_limit_count;
 
@@ -4143,7 +4341,7 @@ static int polaris10_trim_dpm_states(struct pp_hwmgr *hwmgr,
                         polaris10_ps->performance_levels[0].memory_clock,
                         polaris10_ps->performance_levels[high_limit_count].memory_clock);
 
-        return result;
+        return 0;
 }
 
 static int polaris10_generate_dpm_level_enable_mask(
@@ -4226,25 +4424,20 @@ int polaris10_update_uvd_dpm(struct pp_hwmgr *hwmgr, bool bgate)
         return polaris10_enable_disable_uvd_dpm(hwmgr, !bgate);
 }
 
-static int polaris10_update_vce_dpm(struct pp_hwmgr *hwmgr, const void *input)
+int polaris10_update_vce_dpm(struct pp_hwmgr *hwmgr, bool bgate)
 {
-        const struct phm_set_power_state_input *states =
-                        (const struct phm_set_power_state_input *)input;
         struct polaris10_hwmgr *data = (struct polaris10_hwmgr *)(hwmgr->backend);
-        const struct polaris10_power_state *polaris10_nps =
-                        cast_const_phw_polaris10_power_state(states->pnew_state);
-        const struct polaris10_power_state *polaris10_cps =
-                        cast_const_phw_polaris10_power_state(states->pcurrent_state);
-
         uint32_t mm_boot_level_offset, mm_boot_level_value;
         struct phm_ppt_v1_information *table_info =
                         (struct phm_ppt_v1_information *)(hwmgr->pptable);
 
-        if (polaris10_nps->vce_clks.evclk > 0 &&
-        (polaris10_cps == NULL || polaris10_cps->vce_clks.evclk == 0)) {
-
-                data->smc_state_table.VceBootLevel =
+        if (!bgate) {
+                if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
+                                                PHM_PlatformCaps_StablePState))
+                        data->smc_state_table.VceBootLevel =
                                 (uint8_t) (table_info->mm_dep_table->count - 1);
+                else
+                        data->smc_state_table.VceBootLevel = 0;
 
                 mm_boot_level_offset = data->dpm_table_start +
                                 offsetof(SMU74_Discrete_DpmTable, VceBootLevel);
@@ -4257,18 +4450,14 @@ static int polaris10_update_vce_dpm(struct pp_hwmgr *hwmgr, const void *input)
                 cgs_write_ind_register(hwmgr->device,
                                 CGS_IND_REG__SMC, mm_boot_level_offset, mm_boot_level_value);
 
-                if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps, PHM_PlatformCaps_StablePState)) {
+                if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps, PHM_PlatformCaps_StablePState))
                         smum_send_msg_to_smc_with_parameter(hwmgr->smumgr,
                                         PPSMC_MSG_VCEDPM_SetEnabledMask,
                                         (uint32_t)1 << data->smc_state_table.VceBootLevel);
-
-                        polaris10_enable_disable_vce_dpm(hwmgr, true);
-                } else if (polaris10_nps->vce_clks.evclk == 0 &&
-                                polaris10_cps != NULL &&
-                                polaris10_cps->vce_clks.evclk > 0)
-                        polaris10_enable_disable_vce_dpm(hwmgr, false);
         }
 
+        polaris10_enable_disable_vce_dpm(hwmgr, !bgate);
+
         return 0;
 }
 
@@ -4353,8 +4542,8 @@ static int polaris10_unfreeze_sclk_mclk_dpm(struct pp_hwmgr *hwmgr)
                 (data->need_update_smu7_dpm_table &
                 (DPMTABLE_OD_UPDATE_SCLK + DPMTABLE_UPDATE_SCLK))) {
 
-                PP_ASSERT_WITH_CODE(true == polaris10_is_dpm_running(hwmgr),
-                                "Trying to Unfreeze SCLK DPM when DPM is disabled",
+                PP_ASSERT_WITH_CODE(polaris10_is_dpm_running(hwmgr),
+                                    "Trying to Unfreeze SCLK DPM when DPM is disabled",
                                 );
                 PP_ASSERT_WITH_CODE(0 == smum_send_msg_to_smc(hwmgr->smumgr,
                                 PPSMC_MSG_SCLKDPM_UnfreezeLevel),
@@ -4365,8 +4554,8 @@ static int polaris10_unfreeze_sclk_mclk_dpm(struct pp_hwmgr *hwmgr)
         if ((0 == data->mclk_dpm_key_disabled) &&
                 (data->need_update_smu7_dpm_table & DPMTABLE_OD_UPDATE_MCLK)) {
 
-                PP_ASSERT_WITH_CODE(true == polaris10_is_dpm_running(hwmgr),
-                                "Trying to Unfreeze MCLK DPM when DPM is disabled",
+                PP_ASSERT_WITH_CODE(polaris10_is_dpm_running(hwmgr),
+                                    "Trying to Unfreeze MCLK DPM when DPM is disabled",
                                 );
                 PP_ASSERT_WITH_CODE(0 == smum_send_msg_to_smc(hwmgr->smumgr,
                                 PPSMC_MSG_SCLKDPM_UnfreezeLevel),
@@ -4422,6 +4611,8 @@ static int polaris10_notify_smc_display(struct pp_hwmgr *hwmgr)
         return (smum_send_msg_to_smc(hwmgr->smumgr, (PPSMC_Msg)PPSMC_HasDisplay) == 0) ?  0 : -EINVAL;
 }
 
+
+
 static int polaris10_set_power_state_tasks(struct pp_hwmgr *hwmgr, const void *input)
 {
         int tmp_result, result = 0;
@@ -4455,11 +4646,6 @@ static int polaris10_set_power_state_tasks(struct pp_hwmgr *hwmgr, const void *i
                         "Failed to generate DPM level enabled mask!",
                         result = tmp_result);
 
-        tmp_result = polaris10_update_vce_dpm(hwmgr, input);
-        PP_ASSERT_WITH_CODE((0 == tmp_result),
-                        "Failed to update VCE DPM!",
-                        result = tmp_result);
-
         tmp_result = polaris10_update_sclk_threshold(hwmgr);
         PP_ASSERT_WITH_CODE((0 == tmp_result),
                         "Failed to update SCLK threshold!",
@@ -4530,6 +4716,7 @@ int polaris10_notify_smc_display_config_after_ps_adjustment(struct pp_hwmgr *hwm
         if (num_active_displays > 1)  /* to do && (pHwMgr->pPECI->displayConfiguration.bMultiMonitorInSync != TRUE)) */
                 polaris10_notify_smc_display_change(hwmgr, false);
 
+
         return 0;
 }
 
@@ -4579,6 +4766,7 @@ int polaris10_program_display_gap(struct pp_hwmgr *hwmgr)
 
         cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC, data->soft_regs_start + offsetof(SMU74_SoftRegisters, VBlankTimeout), (frame_time_in_us - pre_vbi_time_in_us));
 
+
         return 0;
 }
 
@@ -4820,42 +5008,6 @@ int polaris10_setup_asic_task(struct pp_hwmgr *hwmgr)
         return result;
 }
 
-static int polaris10_get_pp_table(struct pp_hwmgr *hwmgr, char **table)
-{
-        struct polaris10_hwmgr *data = (struct polaris10_hwmgr *)(hwmgr->backend);
-
-        if (!data->soft_pp_table) {
-                data->soft_pp_table = kmemdup(hwmgr->soft_pp_table,
-                                              hwmgr->soft_pp_table_size,
-                                              GFP_KERNEL);
-                if (!data->soft_pp_table)
-                        return -ENOMEM;
-        }
-
-        *table = (char *)&data->soft_pp_table;
-
-        return hwmgr->soft_pp_table_size;
-}
-
-static int polaris10_set_pp_table(struct pp_hwmgr *hwmgr, const char *buf, size_t size)
-{
-        struct polaris10_hwmgr *data = (struct polaris10_hwmgr *)(hwmgr->backend);
-
-        if (!data->soft_pp_table) {
-                data->soft_pp_table = kzalloc(hwmgr->soft_pp_table_size, GFP_KERNEL);
-                if (!data->soft_pp_table)
-                        return -ENOMEM;
-        }
-
-        memcpy(data->soft_pp_table, buf, size);
-
-        hwmgr->soft_pp_table = data->soft_pp_table;
-
-        /* TODO: re-init powerplay to implement modified pptable */
-
-        return 0;
-}
-
 static int polaris10_force_clock_level(struct pp_hwmgr *hwmgr,
                 enum pp_clock_type type, uint32_t mask)
 {
@@ -4998,6 +5150,89 @@ static int polaris10_get_fan_control_mode(struct pp_hwmgr *hwmgr)
                                 CG_FDO_CTRL2, FDO_PWM_MODE);
 }
 
+static int polaris10_get_sclk_od(struct pp_hwmgr *hwmgr)
+{
+        struct polaris10_hwmgr *data = (struct polaris10_hwmgr *)(hwmgr->backend);
+        struct polaris10_single_dpm_table *sclk_table = &(data->dpm_table.sclk_table);
+        struct polaris10_single_dpm_table *golden_sclk_table =
+                        &(data->golden_dpm_table.sclk_table);
+        int value;
+
+        value = (sclk_table->dpm_levels[sclk_table->count - 1].value -
+                        golden_sclk_table->dpm_levels[golden_sclk_table->count - 1].value) *
+                        100 /
+                        golden_sclk_table->dpm_levels[golden_sclk_table->count - 1].value;
+
+        return value;
+}
+
+static int polaris10_set_sclk_od(struct pp_hwmgr *hwmgr, uint32_t value)
+{
+        struct polaris10_hwmgr *data = (struct polaris10_hwmgr *)(hwmgr->backend);
+        struct polaris10_single_dpm_table *golden_sclk_table =
+                        &(data->golden_dpm_table.sclk_table);
+        struct pp_power_state  *ps;
+        struct polaris10_power_state  *polaris10_ps;
+
+        if (value > 20)
+                value = 20;
+
+        ps = hwmgr->request_ps;
+
+        if (ps == NULL)
+                return -EINVAL;
+
+        polaris10_ps = cast_phw_polaris10_power_state(&ps->hardware);
+
+        polaris10_ps->performance_levels[polaris10_ps->performance_level_count - 1].engine_clock =
+                        golden_sclk_table->dpm_levels[golden_sclk_table->count - 1].value *
+                        value / 100 +
+                        golden_sclk_table->dpm_levels[golden_sclk_table->count - 1].value;
+
+        return 0;
+}
+
+static int polaris10_get_mclk_od(struct pp_hwmgr *hwmgr)
+{
+        struct polaris10_hwmgr *data = (struct polaris10_hwmgr *)(hwmgr->backend);
+        struct polaris10_single_dpm_table *mclk_table = &(data->dpm_table.mclk_table);
+        struct polaris10_single_dpm_table *golden_mclk_table =
+                        &(data->golden_dpm_table.mclk_table);
+        int value;
+
+        value = (mclk_table->dpm_levels[mclk_table->count - 1].value -
+                        golden_mclk_table->dpm_levels[golden_mclk_table->count - 1].value) *
+                        100 /
+                        golden_mclk_table->dpm_levels[golden_mclk_table->count - 1].value;
+
+        return value;
+}
+
+static int polaris10_set_mclk_od(struct pp_hwmgr *hwmgr, uint32_t value)
+{
+        struct polaris10_hwmgr *data = (struct polaris10_hwmgr *)(hwmgr->backend);
+        struct polaris10_single_dpm_table *golden_mclk_table =
+                        &(data->golden_dpm_table.mclk_table);
+        struct pp_power_state  *ps;
+        struct polaris10_power_state  *polaris10_ps;
+
+        if (value > 20)
+                value = 20;
+
+        ps = hwmgr->request_ps;
+
+        if (ps == NULL)
+                return -EINVAL;
+
+        polaris10_ps = cast_phw_polaris10_power_state(&ps->hardware);
+
+        polaris10_ps->performance_levels[polaris10_ps->performance_level_count - 1].memory_clock =
+                        golden_mclk_table->dpm_levels[golden_mclk_table->count - 1].value *
+                        value / 100 +
+                        golden_mclk_table->dpm_levels[golden_mclk_table->count - 1].value;
+
+        return 0;
+}
 static const struct pp_hwmgr_func polaris10_hwmgr_funcs = {
         .backend_init = &polaris10_hwmgr_backend_init,
         .backend_fini = &polaris10_hwmgr_backend_fini,
@@ -5036,22 +5271,17 @@ static const struct pp_hwmgr_func polaris10_hwmgr_funcs = {
         .check_states_equal = polaris10_check_states_equal,
         .set_fan_control_mode = polaris10_set_fan_control_mode,
         .get_fan_control_mode = polaris10_get_fan_control_mode,
-        .get_pp_table = polaris10_get_pp_table,
-        .set_pp_table = polaris10_set_pp_table,
         .force_clock_level = polaris10_force_clock_level,
         .print_clock_levels = polaris10_print_clock_levels,
         .enable_per_cu_power_gating = polaris10_phm_enable_per_cu_power_gating,
+        .get_sclk_od = polaris10_get_sclk_od,
+        .set_sclk_od = polaris10_set_sclk_od,
+        .get_mclk_od = polaris10_get_mclk_od,
+        .set_mclk_od = polaris10_set_mclk_od,
 };
 
 int polaris10_hwmgr_init(struct pp_hwmgr *hwmgr)
 {
-        struct polaris10_hwmgr  *data;
-
-        data = kzalloc (sizeof(struct polaris10_hwmgr), GFP_KERNEL);
-        if (data == NULL)
-                return -ENOMEM;
-
-        hwmgr->backend = data;
         hwmgr->hwmgr_func = &polaris10_hwmgr_funcs;
         hwmgr->pptable_func = &tonga_pptable_funcs;
         pp_polaris10_thermal_initialize(hwmgr);
diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/polaris10_hwmgr.h b/drivers/gpu/drm/amd/powerplay/hwmgr/polaris10_hwmgr.h
index afc3434822d1..33c33947e827 100644
--- a/drivers/gpu/drm/amd/powerplay/hwmgr/polaris10_hwmgr.h
+++ b/drivers/gpu/drm/amd/powerplay/hwmgr/polaris10_hwmgr.h
@@ -309,10 +309,6 @@ struct polaris10_hwmgr {
         uint32_t                           up_hyst;
         uint32_t disable_dpm_mask;
         bool apply_optimized_settings;
-
-        /* soft pptable for re-uploading into smu */
-        void *soft_pp_table;
-
         uint32_t                              avfs_vdroop_override_setting;
         bool                                  apply_avfs_cks_off_voltage;
         uint32_t                              frame_time_x2;
@@ -356,6 +352,6 @@ int polaris10_hwmgr_init(struct pp_hwmgr *hwmgr);
 int polaris10_update_uvd_dpm(struct pp_hwmgr *hwmgr, bool bgate);
 int polaris10_update_samu_dpm(struct pp_hwmgr *hwmgr, bool bgate);
 int polaris10_enable_disable_vce_dpm(struct pp_hwmgr *hwmgr, bool enable);
-
+int polaris10_update_vce_dpm(struct pp_hwmgr *hwmgr, bool bgate);
 #endif
 
diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/polaris10_powertune.c b/drivers/gpu/drm/amd/powerplay/hwmgr/polaris10_powertune.c
index ae96f14b827c..b9cb240a135d 100644
--- a/drivers/gpu/drm/amd/powerplay/hwmgr/polaris10_powertune.c
+++ b/drivers/gpu/drm/amd/powerplay/hwmgr/polaris10_powertune.c
@@ -28,10 +28,360 @@
 #include "polaris10_smumgr.h"
 #include "smu74_discrete.h"
 #include "pp_debug.h"
+#include "gca/gfx_8_0_d.h"
+#include "gca/gfx_8_0_sh_mask.h"
+#include "oss/oss_3_0_sh_mask.h"
 
 #define VOLTAGE_SCALE  4
 #define POWERTUNE_DEFAULT_SET_MAX    1
 
+uint32_t DIDTBlock_Info = SQ_IR_MASK | TCP_IR_MASK | TD_PCC_MASK;
+
+struct polaris10_pt_config_reg GCCACConfig_Polaris10[] = {
+/* ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
+ *      Offset                             Mask                                                Shift                                               Value       Type
+ * ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
+ */
+        {   ixGC_CAC_CNTL,                     0xFFFFFFFF,                                         0,                                                  0x00060013, POLARIS10_CONFIGREG_GC_CAC_IND },
+        {   ixGC_CAC_CNTL,                     0xFFFFFFFF,                                         0,                                                  0x00860013, POLARIS10_CONFIGREG_GC_CAC_IND },
+        {   ixGC_CAC_CNTL,                     0xFFFFFFFF,                                         0,                                                  0x01060013, POLARIS10_CONFIGREG_GC_CAC_IND },
+        {   ixGC_CAC_CNTL,                     0xFFFFFFFF,                                         0,                                                  0x01860013, POLARIS10_CONFIGREG_GC_CAC_IND },
+        {   ixGC_CAC_CNTL,                     0xFFFFFFFF,                                         0,                                                  0x02060013, POLARIS10_CONFIGREG_GC_CAC_IND },
+        {   ixGC_CAC_CNTL,                     0xFFFFFFFF,                                         0,                                                  0x02860013, POLARIS10_CONFIGREG_GC_CAC_IND },
+        {   ixGC_CAC_CNTL,                     0xFFFFFFFF,                                         0,                                                  0x03060013, POLARIS10_CONFIGREG_GC_CAC_IND },
+        {   ixGC_CAC_CNTL,                     0xFFFFFFFF,                                         0,                                                  0x03860013, POLARIS10_CONFIGREG_GC_CAC_IND },
+        {   ixGC_CAC_CNTL,                     0xFFFFFFFF,                                         0,                                                  0x04060013, POLARIS10_CONFIGREG_GC_CAC_IND },
+
+        {   ixGC_CAC_CNTL,                     0xFFFFFFFF,                                         0,                                                  0x000E0013, POLARIS10_CONFIGREG_GC_CAC_IND },
+        {   ixGC_CAC_CNTL,                     0xFFFFFFFF,                                         0,                                                  0x008E0013, POLARIS10_CONFIGREG_GC_CAC_IND },
+        {   ixGC_CAC_CNTL,                     0xFFFFFFFF,                                         0,                                                  0x010E0013, POLARIS10_CONFIGREG_GC_CAC_IND },
+        {   ixGC_CAC_CNTL,                     0xFFFFFFFF,                                         0,                                                  0x018E0013, POLARIS10_CONFIGREG_GC_CAC_IND },
+        {   ixGC_CAC_CNTL,                     0xFFFFFFFF,                                         0,                                                  0x020E0013, POLARIS10_CONFIGREG_GC_CAC_IND },
+
+        {   ixGC_CAC_CNTL,                     0xFFFFFFFF,                                         0,                                                  0x00100013, POLARIS10_CONFIGREG_GC_CAC_IND },
+        {   ixGC_CAC_CNTL,                     0xFFFFFFFF,                                         0,                                                  0x00900013, POLARIS10_CONFIGREG_GC_CAC_IND },
+        {   ixGC_CAC_CNTL,                     0xFFFFFFFF,                                         0,                                                  0x01100013, POLARIS10_CONFIGREG_GC_CAC_IND },
+        {   ixGC_CAC_CNTL,                     0xFFFFFFFF,                                         0,                                                  0x01900013, POLARIS10_CONFIGREG_GC_CAC_IND },
+        {   ixGC_CAC_CNTL,                     0xFFFFFFFF,                                         0,                                                  0x02100013, POLARIS10_CONFIGREG_GC_CAC_IND },
+        {   ixGC_CAC_CNTL,                     0xFFFFFFFF,                                         0,                                                  0x02900013, POLARIS10_CONFIGREG_GC_CAC_IND },
+
+        {   0xFFFFFFFF  }
+};
+
+struct polaris10_pt_config_reg GCCACConfig_Polaris11[] = {
+/* ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
+ *      Offset                             Mask                                                Shift                                               Value       Type
+ * ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
+ */
+        {   ixGC_CAC_CNTL,                     0xFFFFFFFF,                                         0,                                                  0x00060011, POLARIS10_CONFIGREG_GC_CAC_IND },
+        {   ixGC_CAC_CNTL,                     0xFFFFFFFF,                                         0,                                                  0x00860011, POLARIS10_CONFIGREG_GC_CAC_IND },
+        {   ixGC_CAC_CNTL,                     0xFFFFFFFF,                                         0,                                                  0x01060011, POLARIS10_CONFIGREG_GC_CAC_IND },
+        {   ixGC_CAC_CNTL,                     0xFFFFFFFF,                                         0,                                                  0x01860011, POLARIS10_CONFIGREG_GC_CAC_IND },
+        {   ixGC_CAC_CNTL,                     0xFFFFFFFF,                                         0,                                                  0x02060011, POLARIS10_CONFIGREG_GC_CAC_IND },
+        {   ixGC_CAC_CNTL,                     0xFFFFFFFF,                                         0,                                                  0x02860011, POLARIS10_CONFIGREG_GC_CAC_IND },
+        {   ixGC_CAC_CNTL,                     0xFFFFFFFF,                                         0,                                                  0x03060011, POLARIS10_CONFIGREG_GC_CAC_IND },
+        {   ixGC_CAC_CNTL,                     0xFFFFFFFF,                                         0,                                                  0x03860011, POLARIS10_CONFIGREG_GC_CAC_IND },
+        {   ixGC_CAC_CNTL,                     0xFFFFFFFF,                                         0,                                                  0x04060011, POLARIS10_CONFIGREG_GC_CAC_IND },
+
+        {   ixGC_CAC_CNTL,                     0xFFFFFFFF,                                         0,                                                  0x000E0011, POLARIS10_CONFIGREG_GC_CAC_IND },
+        {   ixGC_CAC_CNTL,                     0xFFFFFFFF,                                         0,                                                  0x008E0011, POLARIS10_CONFIGREG_GC_CAC_IND },
+        {   ixGC_CAC_CNTL,                     0xFFFFFFFF,                                         0,                                                  0x010E0011, POLARIS10_CONFIGREG_GC_CAC_IND },
+        {   ixGC_CAC_CNTL,                     0xFFFFFFFF,                                         0,                                                  0x018E0011, POLARIS10_CONFIGREG_GC_CAC_IND },
+        {   ixGC_CAC_CNTL,                     0xFFFFFFFF,                                         0,                                                  0x020E0011, POLARIS10_CONFIGREG_GC_CAC_IND },
+
+        {   ixGC_CAC_CNTL,                     0xFFFFFFFF,                                         0,                                                  0x00100011, POLARIS10_CONFIGREG_GC_CAC_IND },
+        {   ixGC_CAC_CNTL,                     0xFFFFFFFF,                                         0,                                                  0x00900011, POLARIS10_CONFIGREG_GC_CAC_IND },
+        {   ixGC_CAC_CNTL,                     0xFFFFFFFF,                                         0,                                                  0x01100011, POLARIS10_CONFIGREG_GC_CAC_IND },
+        {   ixGC_CAC_CNTL,                     0xFFFFFFFF,                                         0,                                                  0x01900011, POLARIS10_CONFIGREG_GC_CAC_IND },
+        {   ixGC_CAC_CNTL,                     0xFFFFFFFF,                                         0,                                                  0x02100011, POLARIS10_CONFIGREG_GC_CAC_IND },
+        {   ixGC_CAC_CNTL,                     0xFFFFFFFF,                                         0,                                                  0x02900011, POLARIS10_CONFIGREG_GC_CAC_IND },
+
+        {   0xFFFFFFFF  }
+};
+
+struct polaris10_pt_config_reg DIDTConfig_Polaris10[] = {
+/* ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
+ *      Offset                             Mask                                                Shift                                               Value       Type
+ * ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
+ */
+        {   ixDIDT_SQ_WEIGHT0_3,               DIDT_SQ_WEIGHT0_3__WEIGHT0_MASK,                    DIDT_SQ_WEIGHT0_3__WEIGHT0__SHIFT,                  0x0073,     POLARIS10_CONFIGREG_DIDT_IND },
+        {   ixDIDT_SQ_WEIGHT0_3,               DIDT_SQ_WEIGHT0_3__WEIGHT1_MASK,                    DIDT_SQ_WEIGHT0_3__WEIGHT1__SHIFT,                  0x00ab,     POLARIS10_CONFIGREG_DIDT_IND },
+        {   ixDIDT_SQ_WEIGHT0_3,               DIDT_SQ_WEIGHT0_3__WEIGHT2_MASK,                    DIDT_SQ_WEIGHT0_3__WEIGHT2__SHIFT,                  0x0084,     POLARIS10_CONFIGREG_DIDT_IND },
+        {   ixDIDT_SQ_WEIGHT0_3,               DIDT_SQ_WEIGHT0_3__WEIGHT3_MASK,                    DIDT_SQ_WEIGHT0_3__WEIGHT3__SHIFT,                  0x005a,     POLARIS10_CONFIGREG_DIDT_IND },
+
+        {   ixDIDT_SQ_WEIGHT4_7,               DIDT_SQ_WEIGHT4_7__WEIGHT4_MASK,                    DIDT_SQ_WEIGHT4_7__WEIGHT4__SHIFT,                  0x0067,     POLARIS10_CONFIGREG_DIDT_IND },
+        {   ixDIDT_SQ_WEIGHT4_7,               DIDT_SQ_WEIGHT4_7__WEIGHT5_MASK,                    DIDT_SQ_WEIGHT4_7__WEIGHT5__SHIFT,                  0x0084,     POLARIS10_CONFIGREG_DIDT_IND },
+        {   ixDIDT_SQ_WEIGHT4_7,               DIDT_SQ_WEIGHT4_7__WEIGHT6_MASK,                    DIDT_SQ_WEIGHT4_7__WEIGHT6__SHIFT,                  0x0027,     POLARIS10_CONFIGREG_DIDT_IND },
+        {   ixDIDT_SQ_WEIGHT4_7,               DIDT_SQ_WEIGHT4_7__WEIGHT7_MASK,                    DIDT_SQ_WEIGHT4_7__WEIGHT7__SHIFT,                  0x0046,     POLARIS10_CONFIGREG_DIDT_IND },
+
+        {   ixDIDT_SQ_WEIGHT8_11,              DIDT_SQ_WEIGHT8_11__WEIGHT8_MASK,                   DIDT_SQ_WEIGHT8_11__WEIGHT8__SHIFT,                 0x00aa,     POLARIS10_CONFIGREG_DIDT_IND },
+        {   ixDIDT_SQ_WEIGHT8_11,              DIDT_SQ_WEIGHT8_11__WEIGHT9_MASK,                   DIDT_SQ_WEIGHT8_11__WEIGHT9__SHIFT,                 0x0000,     POLARIS10_CONFIGREG_DIDT_IND },
+        {   ixDIDT_SQ_WEIGHT8_11,              DIDT_SQ_WEIGHT8_11__WEIGHT10_MASK,                  DIDT_SQ_WEIGHT8_11__WEIGHT10__SHIFT,                0x0000,     POLARIS10_CONFIGREG_DIDT_IND },
+        {   ixDIDT_SQ_WEIGHT8_11,              DIDT_SQ_WEIGHT8_11__WEIGHT11_MASK,                  DIDT_SQ_WEIGHT8_11__WEIGHT11__SHIFT,                0x0000,     POLARIS10_CONFIGREG_DIDT_IND },
+
+        {   ixDIDT_SQ_CTRL1,                   DIDT_SQ_CTRL1__MIN_POWER_MASK,                      DIDT_SQ_CTRL1__MIN_POWER__SHIFT,                    0x0000,     POLARIS10_CONFIGREG_DIDT_IND },
+        {   ixDIDT_SQ_CTRL1,                   DIDT_SQ_CTRL1__MAX_POWER_MASK,                      DIDT_SQ_CTRL1__MAX_POWER__SHIFT,                    0xffff,     POLARIS10_CONFIGREG_DIDT_IND },
+
+        {   ixDIDT_SQ_CTRL_OCP,                DIDT_SQ_CTRL_OCP__UNUSED_0_MASK,                    DIDT_SQ_CTRL_OCP__UNUSED_0__SHIFT,                  0x0000,     POLARIS10_CONFIGREG_DIDT_IND },
+        {   ixDIDT_SQ_CTRL_OCP,                DIDT_SQ_CTRL_OCP__OCP_MAX_POWER_MASK,               DIDT_SQ_CTRL_OCP__OCP_MAX_POWER__SHIFT,             0xffff,     POLARIS10_CONFIGREG_DIDT_IND },
+
+        {   ixDIDT_SQ_CTRL2,                   DIDT_SQ_CTRL2__MAX_POWER_DELTA_MASK,                DIDT_SQ_CTRL2__MAX_POWER_DELTA__SHIFT,              0x3853,     POLARIS10_CONFIGREG_DIDT_IND },
+        {   ixDIDT_SQ_CTRL2,                   DIDT_SQ_CTRL2__UNUSED_0_MASK,                       DIDT_SQ_CTRL2__UNUSED_0__SHIFT,                     0x0000,     POLARIS10_CONFIGREG_DIDT_IND },
+        {   ixDIDT_SQ_CTRL2,                   DIDT_SQ_CTRL2__SHORT_TERM_INTERVAL_SIZE_MASK,       DIDT_SQ_CTRL2__SHORT_TERM_INTERVAL_SIZE__SHIFT,     0x005a,     POLARIS10_CONFIGREG_DIDT_IND },
+        {   ixDIDT_SQ_CTRL2,                   DIDT_SQ_CTRL2__UNUSED_1_MASK,                       DIDT_SQ_CTRL2__UNUSED_1__SHIFT,                     0x0000,     POLARIS10_CONFIGREG_DIDT_IND },
+        {   ixDIDT_SQ_CTRL2,                   DIDT_SQ_CTRL2__LONG_TERM_INTERVAL_RATIO_MASK,       DIDT_SQ_CTRL2__LONG_TERM_INTERVAL_RATIO__SHIFT,     0x0000,     POLARIS10_CONFIGREG_DIDT_IND },
+        {   ixDIDT_SQ_CTRL2,                   DIDT_SQ_CTRL2__UNUSED_2_MASK,                       DIDT_SQ_CTRL2__UNUSED_2__SHIFT,                     0x0000,     POLARIS10_CONFIGREG_DIDT_IND },
+
+        {   ixDIDT_SQ_STALL_CTRL,              DIDT_SQ_STALL_CTRL__DIDT_STALL_CTRL_ENABLE_MASK,    DIDT_SQ_STALL_CTRL__DIDT_STALL_CTRL_ENABLE__SHIFT,  0x0001,     POLARIS10_CONFIGREG_DIDT_IND },
+        {   ixDIDT_SQ_STALL_CTRL,              DIDT_SQ_STALL_CTRL__DIDT_STALL_DELAY_HI_MASK,       DIDT_SQ_STALL_CTRL__DIDT_STALL_DELAY_HI__SHIFT,     0x0001,     POLARIS10_CONFIGREG_DIDT_IND },
+        {   ixDIDT_SQ_STALL_CTRL,              DIDT_SQ_STALL_CTRL__DIDT_STALL_DELAY_LO_MASK,       DIDT_SQ_STALL_CTRL__DIDT_STALL_DELAY_LO__SHIFT,     0x0001,     POLARIS10_CONFIGREG_DIDT_IND },
+        {   ixDIDT_SQ_STALL_CTRL,              DIDT_SQ_STALL_CTRL__DIDT_HI_POWER_THRESHOLD_MASK,   DIDT_SQ_STALL_CTRL__DIDT_HI_POWER_THRESHOLD__SHIFT, 0x0ebb,     POLARIS10_CONFIGREG_DIDT_IND },
+        {   ixDIDT_SQ_STALL_CTRL,              DIDT_SQ_STALL_CTRL__UNUSED_0_MASK,                  DIDT_SQ_STALL_CTRL__UNUSED_0__SHIFT,                0x0000,     POLARIS10_CONFIGREG_DIDT_IND },
+
+        {   ixDIDT_SQ_TUNING_CTRL,             DIDT_SQ_TUNING_CTRL__DIDT_TUNING_ENABLE_MASK,       DIDT_SQ_TUNING_CTRL__DIDT_TUNING_ENABLE__SHIFT,     0x0001,     POLARIS10_CONFIGREG_DIDT_IND },
+        {   ixDIDT_SQ_TUNING_CTRL,             DIDT_SQ_TUNING_CTRL__MAX_POWER_DELTA_HI_MASK,       DIDT_SQ_TUNING_CTRL__MAX_POWER_DELTA_HI__SHIFT,     0x3853,     POLARIS10_CONFIGREG_DIDT_IND },
+        {   ixDIDT_SQ_TUNING_CTRL,             DIDT_SQ_TUNING_CTRL__MAX_POWER_DELTA_LO_MASK,       DIDT_SQ_TUNING_CTRL__MAX_POWER_DELTA_LO__SHIFT,     0x3153,     POLARIS10_CONFIGREG_DIDT_IND },
+        {   ixDIDT_SQ_TUNING_CTRL,             DIDT_SQ_TUNING_CTRL__UNUSED_0_MASK,                 DIDT_SQ_TUNING_CTRL__UNUSED_0__SHIFT,               0x0000,     POLARIS10_CONFIGREG_DIDT_IND },
+
+        {   ixDIDT_SQ_CTRL0,                   DIDT_SQ_CTRL0__DIDT_CTRL_EN_MASK,                   DIDT_SQ_CTRL0__DIDT_CTRL_EN__SHIFT,                 0x0001,     POLARIS10_CONFIGREG_DIDT_IND },
+        {   ixDIDT_SQ_CTRL0,                   DIDT_SQ_CTRL0__USE_REF_CLOCK_MASK,                  DIDT_SQ_CTRL0__USE_REF_CLOCK__SHIFT,                0x0000,     POLARIS10_CONFIGREG_DIDT_IND },
+        {   ixDIDT_SQ_CTRL0,                   DIDT_SQ_CTRL0__PHASE_OFFSET_MASK,                   DIDT_SQ_CTRL0__PHASE_OFFSET__SHIFT,                 0x0000,     POLARIS10_CONFIGREG_DIDT_IND },
+        {   ixDIDT_SQ_CTRL0,                   DIDT_SQ_CTRL0__DIDT_CTRL_RST_MASK,                  DIDT_SQ_CTRL0__DIDT_CTRL_RST__SHIFT,                0x0000,     POLARIS10_CONFIGREG_DIDT_IND },
+        {   ixDIDT_SQ_CTRL0,                   DIDT_SQ_CTRL0__DIDT_CLK_EN_OVERRIDE_MASK,           DIDT_SQ_CTRL0__DIDT_CLK_EN_OVERRIDE__SHIFT,         0x0000,     POLARIS10_CONFIGREG_DIDT_IND },
+        {   ixDIDT_SQ_CTRL0,                   DIDT_SQ_CTRL0__DIDT_MAX_STALLS_ALLOWED_HI_MASK,     DIDT_SQ_CTRL0__DIDT_MAX_STALLS_ALLOWED_HI__SHIFT,   0x0010,     POLARIS10_CONFIGREG_DIDT_IND },
+        {   ixDIDT_SQ_CTRL0,                   DIDT_SQ_CTRL0__DIDT_MAX_STALLS_ALLOWED_LO_MASK,     DIDT_SQ_CTRL0__DIDT_MAX_STALLS_ALLOWED_LO__SHIFT,   0x0010,     POLARIS10_CONFIGREG_DIDT_IND },
+        {   ixDIDT_SQ_CTRL0,                   DIDT_SQ_CTRL0__UNUSED_0_MASK,                       DIDT_SQ_CTRL0__UNUSED_0__SHIFT,                     0x0000,     POLARIS10_CONFIGREG_DIDT_IND },
+
+        {   ixDIDT_TD_WEIGHT0_3,               DIDT_TD_WEIGHT0_3__WEIGHT0_MASK,                    DIDT_TD_WEIGHT0_3__WEIGHT0__SHIFT,                  0x000a,     POLARIS10_CONFIGREG_DIDT_IND },
+        {   ixDIDT_TD_WEIGHT0_3,               DIDT_TD_WEIGHT0_3__WEIGHT1_MASK,                    DIDT_TD_WEIGHT0_3__WEIGHT1__SHIFT,                  0x0010,     POLARIS10_CONFIGREG_DIDT_IND },
+        {   ixDIDT_TD_WEIGHT0_3,               DIDT_TD_WEIGHT0_3__WEIGHT2_MASK,                    DIDT_TD_WEIGHT0_3__WEIGHT2__SHIFT,                  0x0017,     POLARIS10_CONFIGREG_DIDT_IND },
+        {   ixDIDT_TD_WEIGHT0_3,               DIDT_TD_WEIGHT0_3__WEIGHT3_MASK,                    DIDT_TD_WEIGHT0_3__WEIGHT3__SHIFT,                  0x002f,     POLARIS10_CONFIGREG_DIDT_IND },
+
+        {   ixDIDT_TD_WEIGHT4_7,               DIDT_TD_WEIGHT4_7__WEIGHT4_MASK,                    DIDT_TD_WEIGHT4_7__WEIGHT4__SHIFT,                  0x0046,     POLARIS10_CONFIGREG_DIDT_IND },
+        {   ixDIDT_TD_WEIGHT4_7,               DIDT_TD_WEIGHT4_7__WEIGHT5_MASK,                    DIDT_TD_WEIGHT4_7__WEIGHT5__SHIFT,                  0x005d,     POLARIS10_CONFIGREG_DIDT_IND },
+        {   ixDIDT_TD_WEIGHT4_7,               DIDT_TD_WEIGHT4_7__WEIGHT6_MASK,                    DIDT_TD_WEIGHT4_7__WEIGHT6__SHIFT,                  0x0000,     POLARIS10_CONFIGREG_DIDT_IND },
+        {   ixDIDT_TD_WEIGHT4_7,               DIDT_TD_WEIGHT4_7__WEIGHT7_MASK,                    DIDT_TD_WEIGHT4_7__WEIGHT7__SHIFT,                  0x0000,     POLARIS10_CONFIGREG_DIDT_IND },
+
+        {   ixDIDT_TD_CTRL1,                   DIDT_TD_CTRL1__MIN_POWER_MASK,                      DIDT_TD_CTRL1__MIN_POWER__SHIFT,                    0x0000,     POLARIS10_CONFIGREG_DIDT_IND },
+        {   ixDIDT_TD_CTRL1,                   DIDT_TD_CTRL1__MAX_POWER_MASK,                      DIDT_TD_CTRL1__MAX_POWER__SHIFT,                    0xffff,     POLARIS10_CONFIGREG_DIDT_IND },
+
+        {   ixDIDT_TD_CTRL_OCP,                DIDT_TD_CTRL_OCP__UNUSED_0_MASK,                    DIDT_TD_CTRL_OCP__UNUSED_0__SHIFT,                  0x0000,     POLARIS10_CONFIGREG_DIDT_IND },
+        {   ixDIDT_TD_CTRL_OCP,                DIDT_TD_CTRL_OCP__OCP_MAX_POWER_MASK,               DIDT_TD_CTRL_OCP__OCP_MAX_POWER__SHIFT,             0x00ff,     POLARIS10_CONFIGREG_DIDT_IND },
+
+        {   ixDIDT_TD_CTRL2,                   DIDT_TD_CTRL2__MAX_POWER_DELTA_MASK,                DIDT_TD_CTRL2__MAX_POWER_DELTA__SHIFT,              0x3fff,     POLARIS10_CONFIGREG_DIDT_IND },
+        {   ixDIDT_TD_CTRL2,                   DIDT_TD_CTRL2__UNUSED_0_MASK,                       DIDT_TD_CTRL2__UNUSED_0__SHIFT,                     0x0000,     POLARIS10_CONFIGREG_DIDT_IND },
+        {   ixDIDT_TD_CTRL2,                   DIDT_TD_CTRL2__SHORT_TERM_INTERVAL_SIZE_MASK,       DIDT_TD_CTRL2__SHORT_TERM_INTERVAL_SIZE__SHIFT,     0x000f,     POLARIS10_CONFIGREG_DIDT_IND },
+        {   ixDIDT_TD_CTRL2,                   DIDT_TD_CTRL2__UNUSED_1_MASK,                       DIDT_TD_CTRL2__UNUSED_1__SHIFT,                     0x0000,     POLARIS10_CONFIGREG_DIDT_IND },
+        {   ixDIDT_TD_CTRL2,                   DIDT_TD_CTRL2__LONG_TERM_INTERVAL_RATIO_MASK,       DIDT_TD_CTRL2__LONG_TERM_INTERVAL_RATIO__SHIFT,     0x0000,     POLARIS10_CONFIGREG_DIDT_IND },
+        {   ixDIDT_TD_CTRL2,                   DIDT_TD_CTRL2__UNUSED_2_MASK,                       DIDT_TD_CTRL2__UNUSED_2__SHIFT,                     0x0000,     POLARIS10_CONFIGREG_DIDT_IND },
+
+        {   ixDIDT_TD_STALL_CTRL,              DIDT_TD_STALL_CTRL__DIDT_STALL_CTRL_ENABLE_MASK,    DIDT_TD_STALL_CTRL__DIDT_STALL_CTRL_ENABLE__SHIFT,  0x0001,     POLARIS10_CONFIGREG_DIDT_IND },
+        {   ixDIDT_TD_STALL_CTRL,              DIDT_TD_STALL_CTRL__DIDT_STALL_DELAY_HI_MASK,       DIDT_TD_STALL_CTRL__DIDT_STALL_DELAY_HI__SHIFT,     0x0001,     POLARIS10_CONFIGREG_DIDT_IND },
+        {   ixDIDT_TD_STALL_CTRL,              DIDT_TD_STALL_CTRL__DIDT_STALL_DELAY_LO_MASK,       DIDT_TD_STALL_CTRL__DIDT_STALL_DELAY_LO__SHIFT,     0x0001,     POLARIS10_CONFIGREG_DIDT_IND },
+        {   ixDIDT_TD_STALL_CTRL,              DIDT_TD_STALL_CTRL__DIDT_HI_POWER_THRESHOLD_MASK,   DIDT_TD_STALL_CTRL__DIDT_HI_POWER_THRESHOLD__SHIFT, 0x01aa,     POLARIS10_CONFIGREG_DIDT_IND },
+        {   ixDIDT_TD_STALL_CTRL,              DIDT_TD_STALL_CTRL__UNUSED_0_MASK,                  DIDT_TD_STALL_CTRL__UNUSED_0__SHIFT,                0x0000,     POLARIS10_CONFIGREG_DIDT_IND },
+
+        {   ixDIDT_TD_TUNING_CTRL,             DIDT_TD_TUNING_CTRL__DIDT_TUNING_ENABLE_MASK,       DIDT_TD_TUNING_CTRL__DIDT_TUNING_ENABLE__SHIFT,     0x0000,     POLARIS10_CONFIGREG_DIDT_IND },
+        {   ixDIDT_TD_TUNING_CTRL,             DIDT_TD_TUNING_CTRL__MAX_POWER_DELTA_HI_MASK,       DIDT_TD_TUNING_CTRL__MAX_POWER_DELTA_HI__SHIFT,     0x0dde,     POLARIS10_CONFIGREG_DIDT_IND },
+        {   ixDIDT_TD_TUNING_CTRL,             DIDT_TD_TUNING_CTRL__MAX_POWER_DELTA_LO_MASK,       DIDT_TD_TUNING_CTRL__MAX_POWER_DELTA_LO__SHIFT,     0x0dde,     POLARIS10_CONFIGREG_DIDT_IND },
+        {   ixDIDT_TD_TUNING_CTRL,             DIDT_TD_TUNING_CTRL__UNUSED_0_MASK,                 DIDT_TD_TUNING_CTRL__UNUSED_0__SHIFT,               0x0000,     POLARIS10_CONFIGREG_DIDT_IND },
+
+        {   ixDIDT_TD_CTRL0,                   DIDT_TD_CTRL0__DIDT_CTRL_EN_MASK,                   DIDT_TD_CTRL0__DIDT_CTRL_EN__SHIFT,                 0x0001,     POLARIS10_CONFIGREG_DIDT_IND },
+        {   ixDIDT_TD_CTRL0,                   DIDT_TD_CTRL0__USE_REF_CLOCK_MASK,                  DIDT_TD_CTRL0__USE_REF_CLOCK__SHIFT,                0x0000,     POLARIS10_CONFIGREG_DIDT_IND },
+        {   ixDIDT_TD_CTRL0,                   DIDT_TD_CTRL0__PHASE_OFFSET_MASK,                   DIDT_TD_CTRL0__PHASE_OFFSET__SHIFT,                 0x0000,     POLARIS10_CONFIGREG_DIDT_IND },
+        {   ixDIDT_TD_CTRL0,                   DIDT_TD_CTRL0__DIDT_CTRL_RST_MASK,                  DIDT_TD_CTRL0__DIDT_CTRL_RST__SHIFT,                0x0000,     POLARIS10_CONFIGREG_DIDT_IND },
+        {   ixDIDT_TD_CTRL0,                   DIDT_TD_CTRL0__DIDT_CLK_EN_OVERRIDE_MASK,           DIDT_TD_CTRL0__DIDT_CLK_EN_OVERRIDE__SHIFT,         0x0000,     POLARIS10_CONFIGREG_DIDT_IND },
+        {   ixDIDT_TD_CTRL0,                   DIDT_TD_CTRL0__DIDT_MAX_STALLS_ALLOWED_HI_MASK,     DIDT_TD_CTRL0__DIDT_MAX_STALLS_ALLOWED_HI__SHIFT,   0x0009,     POLARIS10_CONFIGREG_DIDT_IND },
+        {   ixDIDT_TD_CTRL0,                   DIDT_TD_CTRL0__DIDT_MAX_STALLS_ALLOWED_LO_MASK,     DIDT_TD_CTRL0__DIDT_MAX_STALLS_ALLOWED_LO__SHIFT,   0x0009,     POLARIS10_CONFIGREG_DIDT_IND },
+        {   ixDIDT_TD_CTRL0,                   DIDT_TD_CTRL0__UNUSED_0_MASK,                       DIDT_TD_CTRL0__UNUSED_0__SHIFT,                     0x0000,     POLARIS10_CONFIGREG_DIDT_IND },
+
+        {   ixDIDT_TCP_WEIGHT0_3,              DIDT_TCP_WEIGHT0_3__WEIGHT0_MASK,                   DIDT_TCP_WEIGHT0_3__WEIGHT0__SHIFT,                 0x0004,     POLARIS10_CONFIGREG_DIDT_IND },
+        {   ixDIDT_TCP_WEIGHT0_3,              DIDT_TCP_WEIGHT0_3__WEIGHT1_MASK,                   DIDT_TCP_WEIGHT0_3__WEIGHT1__SHIFT,                 0x0037,     POLARIS10_CONFIGREG_DIDT_IND },
+        {   ixDIDT_TCP_WEIGHT0_3,              DIDT_TCP_WEIGHT0_3__WEIGHT2_MASK,                   DIDT_TCP_WEIGHT0_3__WEIGHT2__SHIFT,                 0x0001,     POLARIS10_CONFIGREG_DIDT_IND },
+        {   ixDIDT_TCP_WEIGHT0_3,              DIDT_TCP_WEIGHT0_3__WEIGHT3_MASK,                   DIDT_TCP_WEIGHT0_3__WEIGHT3__SHIFT,                 0x00ff,     POLARIS10_CONFIGREG_DIDT_IND },
+
+        {   ixDIDT_TCP_WEIGHT4_7,              DIDT_TCP_WEIGHT4_7__WEIGHT4_MASK,                   DIDT_TCP_WEIGHT4_7__WEIGHT4__SHIFT,                 0x0054,     POLARIS10_CONFIGREG_DIDT_IND },
+        {   ixDIDT_TCP_WEIGHT4_7,              DIDT_TCP_WEIGHT4_7__WEIGHT5_MASK,                   DIDT_TCP_WEIGHT4_7__WEIGHT5__SHIFT,                 0x0000,     POLARIS10_CONFIGREG_DIDT_IND },
+        {   ixDIDT_TCP_WEIGHT4_7,              DIDT_TCP_WEIGHT4_7__WEIGHT6_MASK,                   DIDT_TCP_WEIGHT4_7__WEIGHT6__SHIFT,                 0x0000,     POLARIS10_CONFIGREG_DIDT_IND },
+        {   ixDIDT_TCP_WEIGHT4_7,              DIDT_TCP_WEIGHT4_7__WEIGHT7_MASK,                   DIDT_TCP_WEIGHT4_7__WEIGHT7__SHIFT,                 0x0000,     POLARIS10_CONFIGREG_DIDT_IND },
+
+        {   ixDIDT_TCP_CTRL1,                  DIDT_TCP_CTRL1__MIN_POWER_MASK,                     DIDT_TCP_CTRL1__MIN_POWER__SHIFT,                   0x0000,     POLARIS10_CONFIGREG_DIDT_IND },
+        {   ixDIDT_TCP_CTRL1,                  DIDT_TCP_CTRL1__MAX_POWER_MASK,                     DIDT_TCP_CTRL1__MAX_POWER__SHIFT,                   0xffff,     POLARIS10_CONFIGREG_DIDT_IND },
+
+        {   ixDIDT_TCP_CTRL_OCP,               DIDT_TCP_CTRL_OCP__UNUSED_0_MASK,                   DIDT_TCP_CTRL_OCP__UNUSED_0__SHIFT,                 0x0000,     POLARIS10_CONFIGREG_DIDT_IND },
+        {   ixDIDT_TCP_CTRL_OCP,               DIDT_TCP_CTRL_OCP__OCP_MAX_POWER_MASK,              DIDT_TCP_CTRL_OCP__OCP_MAX_POWER__SHIFT,            0xffff,     POLARIS10_CONFIGREG_DIDT_IND },
+
+        {   ixDIDT_TCP_CTRL2,                  DIDT_TCP_CTRL2__MAX_POWER_DELTA_MASK,               DIDT_TCP_CTRL2__MAX_POWER_DELTA__SHIFT,             0x3dde,     POLARIS10_CONFIGREG_DIDT_IND },
+        {   ixDIDT_TCP_CTRL2,                  DIDT_TCP_CTRL2__UNUSED_0_MASK,                      DIDT_TCP_CTRL2__UNUSED_0__SHIFT,                    0x0000,     POLARIS10_CONFIGREG_DIDT_IND },
+        {   ixDIDT_TCP_CTRL2,                  DIDT_TCP_CTRL2__SHORT_TERM_INTERVAL_SIZE_MASK,      DIDT_TCP_CTRL2__SHORT_TERM_INTERVAL_SIZE__SHIFT,    0x0032,     POLARIS10_CONFIGREG_DIDT_IND },
+        {   ixDIDT_TCP_CTRL2,                  DIDT_TCP_CTRL2__UNUSED_1_MASK,                      DIDT_TCP_CTRL2__UNUSED_1__SHIFT,                    0x0000,     POLARIS10_CONFIGREG_DIDT_IND },
+        {   ixDIDT_TCP_CTRL2,                  DIDT_TCP_CTRL2__LONG_TERM_INTERVAL_RATIO_MASK,      DIDT_TCP_CTRL2__LONG_TERM_INTERVAL_RATIO__SHIFT,    0x0000,     POLARIS10_CONFIGREG_DIDT_IND },
+        {   ixDIDT_TCP_CTRL2,                  DIDT_TCP_CTRL2__UNUSED_2_MASK,                      DIDT_TCP_CTRL2__UNUSED_2__SHIFT,                    0x0000,     POLARIS10_CONFIGREG_DIDT_IND },
+
+        {   ixDIDT_TCP_STALL_CTRL,             DIDT_TCP_STALL_CTRL__DIDT_STALL_CTRL_ENABLE_MASK,   DIDT_TCP_STALL_CTRL__DIDT_STALL_CTRL_ENABLE__SHIFT, 0x0001,     POLARIS10_CONFIGREG_DIDT_IND },
+        {   ixDIDT_TCP_STALL_CTRL,             DIDT_TCP_STALL_CTRL__DIDT_STALL_DELAY_HI_MASK,      DIDT_TCP_STALL_CTRL__DIDT_STALL_DELAY_HI__SHIFT,    0x0001,     POLARIS10_CONFIGREG_DIDT_IND },
+        {   ixDIDT_TCP_STALL_CTRL,             DIDT_TCP_STALL_CTRL__DIDT_STALL_DELAY_LO_MASK,      DIDT_TCP_STALL_CTRL__DIDT_STALL_DELAY_LO__SHIFT,    0x0001,     POLARIS10_CONFIGREG_DIDT_IND },
+        {   ixDIDT_TCP_STALL_CTRL,             DIDT_TCP_STALL_CTRL__DIDT_HI_POWER_THRESHOLD_MASK,  DIDT_TCP_STALL_CTRL__DIDT_HI_POWER_THRESHOLD__SHIFT, 0x01aa,     POLARIS10_CONFIGREG_DIDT_IND },
+        {   ixDIDT_TCP_STALL_CTRL,             DIDT_TCP_STALL_CTRL__UNUSED_0_MASK,                 DIDT_TCP_STALL_CTRL__UNUSED_0__SHIFT,               0x0000,     POLARIS10_CONFIGREG_DIDT_IND },
+
+        {   ixDIDT_TCP_TUNING_CTRL,            DIDT_TCP_TUNING_CTRL__DIDT_TUNING_ENABLE_MASK,      DIDT_TCP_TUNING_CTRL__DIDT_TUNING_ENABLE__SHIFT,    0x0001,     POLARIS10_CONFIGREG_DIDT_IND },
+        {   ixDIDT_TCP_TUNING_CTRL,            DIDT_TCP_TUNING_CTRL__MAX_POWER_DELTA_HI_MASK,      DIDT_TCP_TUNING_CTRL__MAX_POWER_DELTA_HI__SHIFT,    0x3dde,     POLARIS10_CONFIGREG_DIDT_IND },
+        {   ixDIDT_TCP_TUNING_CTRL,            DIDT_TCP_TUNING_CTRL__MAX_POWER_DELTA_LO_MASK,      DIDT_TCP_TUNING_CTRL__MAX_POWER_DELTA_LO__SHIFT,    0x3dde,     POLARIS10_CONFIGREG_DIDT_IND },
+        {   ixDIDT_TCP_TUNING_CTRL,            DIDT_TCP_TUNING_CTRL__UNUSED_0_MASK,                DIDT_TCP_TUNING_CTRL__UNUSED_0__SHIFT,              0x0000,     POLARIS10_CONFIGREG_DIDT_IND },
+
+        {   ixDIDT_TCP_CTRL0,                   DIDT_TCP_CTRL0__DIDT_CTRL_EN_MASK,                   DIDT_TCP_CTRL0__DIDT_CTRL_EN__SHIFT,                 0x0001,     POLARIS10_CONFIGREG_DIDT_IND },
+        {   ixDIDT_TCP_CTRL0,                   DIDT_TCP_CTRL0__USE_REF_CLOCK_MASK,                  DIDT_TCP_CTRL0__USE_REF_CLOCK__SHIFT,                0x0000,     POLARIS10_CONFIGREG_DIDT_IND },
+        {   ixDIDT_TCP_CTRL0,                   DIDT_TCP_CTRL0__PHASE_OFFSET_MASK,                   DIDT_TCP_CTRL0__PHASE_OFFSET__SHIFT,                 0x0000,     POLARIS10_CONFIGREG_DIDT_IND },
+        {   ixDIDT_TCP_CTRL0,                   DIDT_TCP_CTRL0__DIDT_CTRL_RST_MASK,                  DIDT_TCP_CTRL0__DIDT_CTRL_RST__SHIFT,                0x0000,     POLARIS10_CONFIGREG_DIDT_IND },
+        {   ixDIDT_TCP_CTRL0,                   DIDT_TCP_CTRL0__DIDT_CLK_EN_OVERRIDE_MASK,           DIDT_TCP_CTRL0__DIDT_CLK_EN_OVERRIDE__SHIFT,         0x0000,     POLARIS10_CONFIGREG_DIDT_IND },
+        {   ixDIDT_TCP_CTRL0,                   DIDT_TCP_CTRL0__DIDT_MAX_STALLS_ALLOWED_HI_MASK,     DIDT_TCP_CTRL0__DIDT_MAX_STALLS_ALLOWED_HI__SHIFT,   0x0010,     POLARIS10_CONFIGREG_DIDT_IND },
+        {   ixDIDT_TCP_CTRL0,                   DIDT_TCP_CTRL0__DIDT_MAX_STALLS_ALLOWED_LO_MASK,     DIDT_TCP_CTRL0__DIDT_MAX_STALLS_ALLOWED_LO__SHIFT,   0x0010,     POLARIS10_CONFIGREG_DIDT_IND },
+        {   ixDIDT_TCP_CTRL0,                   DIDT_TCP_CTRL0__UNUSED_0_MASK,                       DIDT_TCP_CTRL0__UNUSED_0__SHIFT,                     0x0000,     POLARIS10_CONFIGREG_DIDT_IND },
+
+        {   0xFFFFFFFF  }
+};
+
+struct polaris10_pt_config_reg DIDTConfig_Polaris11[] = {
+/* ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
+ *      Offset                             Mask                                                Shift                                               Value       Type
+ * ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
+ */
+        {   ixDIDT_SQ_WEIGHT0_3,               DIDT_SQ_WEIGHT0_3__WEIGHT0_MASK,                    DIDT_SQ_WEIGHT0_3__WEIGHT0__SHIFT,                  0x0073,     POLARIS10_CONFIGREG_DIDT_IND },
+        {   ixDIDT_SQ_WEIGHT0_3,               DIDT_SQ_WEIGHT0_3__WEIGHT1_MASK,                    DIDT_SQ_WEIGHT0_3__WEIGHT1__SHIFT,                  0x00ab,     POLARIS10_CONFIGREG_DIDT_IND },
+        {   ixDIDT_SQ_WEIGHT0_3,               DIDT_SQ_WEIGHT0_3__WEIGHT2_MASK,                    DIDT_SQ_WEIGHT0_3__WEIGHT2__SHIFT,                  0x0084,     POLARIS10_CONFIGREG_DIDT_IND },
+        {   ixDIDT_SQ_WEIGHT0_3,               DIDT_SQ_WEIGHT0_3__WEIGHT3_MASK,                    DIDT_SQ_WEIGHT0_3__WEIGHT3__SHIFT,                  0x005a,     POLARIS10_CONFIGREG_DIDT_IND },
+
+        {   ixDIDT_SQ_WEIGHT4_7,               DIDT_SQ_WEIGHT4_7__WEIGHT4_MASK,                    DIDT_SQ_WEIGHT4_7__WEIGHT4__SHIFT,                  0x0067,     POLARIS10_CONFIGREG_DIDT_IND },
+        {   ixDIDT_SQ_WEIGHT4_7,               DIDT_SQ_WEIGHT4_7__WEIGHT5_MASK,                    DIDT_SQ_WEIGHT4_7__WEIGHT5__SHIFT,                  0x0084,     POLARIS10_CONFIGREG_DIDT_IND },
+        {   ixDIDT_SQ_WEIGHT4_7,               DIDT_SQ_WEIGHT4_7__WEIGHT6_MASK,                    DIDT_SQ_WEIGHT4_7__WEIGHT6__SHIFT,                  0x0027,     POLARIS10_CONFIGREG_DIDT_IND },
+        {   ixDIDT_SQ_WEIGHT4_7,               DIDT_SQ_WEIGHT4_7__WEIGHT7_MASK,                    DIDT_SQ_WEIGHT4_7__WEIGHT7__SHIFT,                  0x0046,     POLARIS10_CONFIGREG_DIDT_IND },
+
+        {   ixDIDT_SQ_WEIGHT8_11,              DIDT_SQ_WEIGHT8_11__WEIGHT8_MASK,                   DIDT_SQ_WEIGHT8_11__WEIGHT8__SHIFT,                 0x00aa,     POLARIS10_CONFIGREG_DIDT_IND },
+        {   ixDIDT_SQ_WEIGHT8_11,              DIDT_SQ_WEIGHT8_11__WEIGHT9_MASK,                   DIDT_SQ_WEIGHT8_11__WEIGHT9__SHIFT,                 0x0000,     POLARIS10_CONFIGREG_DIDT_IND },
+        {   ixDIDT_SQ_WEIGHT8_11,              DIDT_SQ_WEIGHT8_11__WEIGHT10_MASK,                  DIDT_SQ_WEIGHT8_11__WEIGHT10__SHIFT,                0x0000,     POLARIS10_CONFIGREG_DIDT_IND },
+        {   ixDIDT_SQ_WEIGHT8_11,              DIDT_SQ_WEIGHT8_11__WEIGHT11_MASK,                  DIDT_SQ_WEIGHT8_11__WEIGHT11__SHIFT,                0x0000,     POLARIS10_CONFIGREG_DIDT_IND },
+
+        {   ixDIDT_SQ_CTRL1,                   DIDT_SQ_CTRL1__MIN_POWER_MASK,                      DIDT_SQ_CTRL1__MIN_POWER__SHIFT,                    0x0000,     POLARIS10_CONFIGREG_DIDT_IND },
+        {   ixDIDT_SQ_CTRL1,                   DIDT_SQ_CTRL1__MAX_POWER_MASK,                      DIDT_SQ_CTRL1__MAX_POWER__SHIFT,                    0xffff,     POLARIS10_CONFIGREG_DIDT_IND },
+
+        {   ixDIDT_SQ_CTRL_OCP,                DIDT_SQ_CTRL_OCP__UNUSED_0_MASK,                    DIDT_SQ_CTRL_OCP__UNUSED_0__SHIFT,                  0x0000,     POLARIS10_CONFIGREG_DIDT_IND },
+        {   ixDIDT_SQ_CTRL_OCP,                DIDT_SQ_CTRL_OCP__OCP_MAX_POWER_MASK,               DIDT_SQ_CTRL_OCP__OCP_MAX_POWER__SHIFT,             0xffff,     POLARIS10_CONFIGREG_DIDT_IND },
+
+        {   ixDIDT_SQ_CTRL2,                   DIDT_SQ_CTRL2__MAX_POWER_DELTA_MASK,                DIDT_SQ_CTRL2__MAX_POWER_DELTA__SHIFT,              0x3853,     POLARIS10_CONFIGREG_DIDT_IND },
+        {   ixDIDT_SQ_CTRL2,                   DIDT_SQ_CTRL2__UNUSED_0_MASK,                       DIDT_SQ_CTRL2__UNUSED_0__SHIFT,                     0x0000,     POLARIS10_CONFIGREG_DIDT_IND },
+        {   ixDIDT_SQ_CTRL2,                   DIDT_SQ_CTRL2__SHORT_TERM_INTERVAL_SIZE_MASK,       DIDT_SQ_CTRL2__SHORT_TERM_INTERVAL_SIZE__SHIFT,     0x005a,     POLARIS10_CONFIGREG_DIDT_IND },
+        {   ixDIDT_SQ_CTRL2,                   DIDT_SQ_CTRL2__UNUSED_1_MASK,                       DIDT_SQ_CTRL2__UNUSED_1__SHIFT,                     0x0000,     POLARIS10_CONFIGREG_DIDT_IND },
+        {   ixDIDT_SQ_CTRL2,                   DIDT_SQ_CTRL2__LONG_TERM_INTERVAL_RATIO_MASK,       DIDT_SQ_CTRL2__LONG_TERM_INTERVAL_RATIO__SHIFT,     0x0000,     POLARIS10_CONFIGREG_DIDT_IND },
+        {   ixDIDT_SQ_CTRL2,                   DIDT_SQ_CTRL2__UNUSED_2_MASK,                       DIDT_SQ_CTRL2__UNUSED_2__SHIFT,                     0x0000,     POLARIS10_CONFIGREG_DIDT_IND },
+
+        {   ixDIDT_SQ_STALL_CTRL,              DIDT_SQ_STALL_CTRL__DIDT_STALL_CTRL_ENABLE_MASK,    DIDT_SQ_STALL_CTRL__DIDT_STALL_CTRL_ENABLE__SHIFT,  0x0001,     POLARIS10_CONFIGREG_DIDT_IND },
+        {   ixDIDT_SQ_STALL_CTRL,              DIDT_SQ_STALL_CTRL__DIDT_STALL_DELAY_HI_MASK,       DIDT_SQ_STALL_CTRL__DIDT_STALL_DELAY_HI__SHIFT,     0x0001,     POLARIS10_CONFIGREG_DIDT_IND },
+        {   ixDIDT_SQ_STALL_CTRL,              DIDT_SQ_STALL_CTRL__DIDT_STALL_DELAY_LO_MASK,       DIDT_SQ_STALL_CTRL__DIDT_STALL_DELAY_LO__SHIFT,     0x0001,     POLARIS10_CONFIGREG_DIDT_IND },
+        {   ixDIDT_SQ_STALL_CTRL,              DIDT_SQ_STALL_CTRL__DIDT_HI_POWER_THRESHOLD_MASK,   DIDT_SQ_STALL_CTRL__DIDT_HI_POWER_THRESHOLD__SHIFT, 0x0ebb,     POLARIS10_CONFIGREG_DIDT_IND },
+        {   ixDIDT_SQ_STALL_CTRL,              DIDT_SQ_STALL_CTRL__UNUSED_0_MASK,                  DIDT_SQ_STALL_CTRL__UNUSED_0__SHIFT,                0x0000,     POLARIS10_CONFIGREG_DIDT_IND },
+
+        {   ixDIDT_SQ_TUNING_CTRL,             DIDT_SQ_TUNING_CTRL__DIDT_TUNING_ENABLE_MASK,       DIDT_SQ_TUNING_CTRL__DIDT_TUNING_ENABLE__SHIFT,     0x0001,     POLARIS10_CONFIGREG_DIDT_IND },
+        {   ixDIDT_SQ_TUNING_CTRL,             DIDT_SQ_TUNING_CTRL__MAX_POWER_DELTA_HI_MASK,       DIDT_SQ_TUNING_CTRL__MAX_POWER_DELTA_HI__SHIFT,     0x3853,     POLARIS10_CONFIGREG_DIDT_IND },
+        {   ixDIDT_SQ_TUNING_CTRL,             DIDT_SQ_TUNING_CTRL__MAX_POWER_DELTA_LO_MASK,       DIDT_SQ_TUNING_CTRL__MAX_POWER_DELTA_LO__SHIFT,     0x3153,     POLARIS10_CONFIGREG_DIDT_IND },
+        {   ixDIDT_SQ_TUNING_CTRL,             DIDT_SQ_TUNING_CTRL__UNUSED_0_MASK,                 DIDT_SQ_TUNING_CTRL__UNUSED_0__SHIFT,               0x0000,     POLARIS10_CONFIGREG_DIDT_IND },
+
+        {   ixDIDT_SQ_CTRL0,                   DIDT_SQ_CTRL0__DIDT_CTRL_EN_MASK,                   DIDT_SQ_CTRL0__DIDT_CTRL_EN__SHIFT,                 0x0001,     POLARIS10_CONFIGREG_DIDT_IND },
+        {   ixDIDT_SQ_CTRL0,                   DIDT_SQ_CTRL0__USE_REF_CLOCK_MASK,                  DIDT_SQ_CTRL0__USE_REF_CLOCK__SHIFT,                0x0000,     POLARIS10_CONFIGREG_DIDT_IND },
+        {   ixDIDT_SQ_CTRL0,                   DIDT_SQ_CTRL0__PHASE_OFFSET_MASK,                   DIDT_SQ_CTRL0__PHASE_OFFSET__SHIFT,                 0x0000,     POLARIS10_CONFIGREG_DIDT_IND },
+        {   ixDIDT_SQ_CTRL0,                   DIDT_SQ_CTRL0__DIDT_CTRL_RST_MASK,                  DIDT_SQ_CTRL0__DIDT_CTRL_RST__SHIFT,                0x0000,     POLARIS10_CONFIGREG_DIDT_IND },
+        {   ixDIDT_SQ_CTRL0,                   DIDT_SQ_CTRL0__DIDT_CLK_EN_OVERRIDE_MASK,           DIDT_SQ_CTRL0__DIDT_CLK_EN_OVERRIDE__SHIFT,         0x0000,     POLARIS10_CONFIGREG_DIDT_IND },
+        {   ixDIDT_SQ_CTRL0,                   DIDT_SQ_CTRL0__DIDT_MAX_STALLS_ALLOWED_HI_MASK,     DIDT_SQ_CTRL0__DIDT_MAX_STALLS_ALLOWED_HI__SHIFT,   0x0010,     POLARIS10_CONFIGREG_DIDT_IND },
+        {   ixDIDT_SQ_CTRL0,                   DIDT_SQ_CTRL0__DIDT_MAX_STALLS_ALLOWED_LO_MASK,     DIDT_SQ_CTRL0__DIDT_MAX_STALLS_ALLOWED_LO__SHIFT,   0x0010,     POLARIS10_CONFIGREG_DIDT_IND },
+        {   ixDIDT_SQ_CTRL0,                   DIDT_SQ_CTRL0__UNUSED_0_MASK,                       DIDT_SQ_CTRL0__UNUSED_0__SHIFT,                     0x0000,     POLARIS10_CONFIGREG_DIDT_IND },
+
+        {   ixDIDT_TD_WEIGHT0_3,               DIDT_TD_WEIGHT0_3__WEIGHT0_MASK,                    DIDT_TD_WEIGHT0_3__WEIGHT0__SHIFT,                  0x000a,     POLARIS10_CONFIGREG_DIDT_IND },
+        {   ixDIDT_TD_WEIGHT0_3,               DIDT_TD_WEIGHT0_3__WEIGHT1_MASK,                    DIDT_TD_WEIGHT0_3__WEIGHT1__SHIFT,                  0x0010,     POLARIS10_CONFIGREG_DIDT_IND },
+        {   ixDIDT_TD_WEIGHT0_3,               DIDT_TD_WEIGHT0_3__WEIGHT2_MASK,                    DIDT_TD_WEIGHT0_3__WEIGHT2__SHIFT,                  0x0017,     POLARIS10_CONFIGREG_DIDT_IND },
+        {   ixDIDT_TD_WEIGHT0_3,               DIDT_TD_WEIGHT0_3__WEIGHT3_MASK,                    DIDT_TD_WEIGHT0_3__WEIGHT3__SHIFT,                  0x002f,     POLARIS10_CONFIGREG_DIDT_IND },
+
+        {   ixDIDT_TD_WEIGHT4_7,               DIDT_TD_WEIGHT4_7__WEIGHT4_MASK,                    DIDT_TD_WEIGHT4_7__WEIGHT4__SHIFT,                  0x0046,     POLARIS10_CONFIGREG_DIDT_IND },
+        {   ixDIDT_TD_WEIGHT4_7,               DIDT_TD_WEIGHT4_7__WEIGHT5_MASK,                    DIDT_TD_WEIGHT4_7__WEIGHT5__SHIFT,                  0x005d,     POLARIS10_CONFIGREG_DIDT_IND },
+        {   ixDIDT_TD_WEIGHT4_7,               DIDT_TD_WEIGHT4_7__WEIGHT6_MASK,                    DIDT_TD_WEIGHT4_7__WEIGHT6__SHIFT,                  0x0000,     POLARIS10_CONFIGREG_DIDT_IND },
+        {   ixDIDT_TD_WEIGHT4_7,               DIDT_TD_WEIGHT4_7__WEIGHT7_MASK,                    DIDT_TD_WEIGHT4_7__WEIGHT7__SHIFT,                  0x0000,     POLARIS10_CONFIGREG_DIDT_IND },
+
+        {   ixDIDT_TD_CTRL1,                   DIDT_TD_CTRL1__MIN_POWER_MASK,                      DIDT_TD_CTRL1__MIN_POWER__SHIFT,                    0x0000,     POLARIS10_CONFIGREG_DIDT_IND },
+        {   ixDIDT_TD_CTRL1,                   DIDT_TD_CTRL1__MAX_POWER_MASK,                      DIDT_TD_CTRL1__MAX_POWER__SHIFT,                    0xffff,     POLARIS10_CONFIGREG_DIDT_IND },
+
+        {   ixDIDT_TD_CTRL_OCP,                DIDT_TD_CTRL_OCP__UNUSED_0_MASK,                    DIDT_TD_CTRL_OCP__UNUSED_0__SHIFT,                  0x0000,     POLARIS10_CONFIGREG_DIDT_IND },
+        {   ixDIDT_TD_CTRL_OCP,                DIDT_TD_CTRL_OCP__OCP_MAX_POWER_MASK,               DIDT_TD_CTRL_OCP__OCP_MAX_POWER__SHIFT,             0x00ff,     POLARIS10_CONFIGREG_DIDT_IND },
+
+        {   ixDIDT_TD_CTRL2,                   DIDT_TD_CTRL2__MAX_POWER_DELTA_MASK,                DIDT_TD_CTRL2__MAX_POWER_DELTA__SHIFT,              0x3fff,     POLARIS10_CONFIGREG_DIDT_IND },
+        {   ixDIDT_TD_CTRL2,                   DIDT_TD_CTRL2__UNUSED_0_MASK,                       DIDT_TD_CTRL2__UNUSED_0__SHIFT,                     0x0000,     POLARIS10_CONFIGREG_DIDT_IND },
+        {   ixDIDT_TD_CTRL2,                   DIDT_TD_CTRL2__SHORT_TERM_INTERVAL_SIZE_MASK,       DIDT_TD_CTRL2__SHORT_TERM_INTERVAL_SIZE__SHIFT,     0x000f,     POLARIS10_CONFIGREG_DIDT_IND },
+        {   ixDIDT_TD_CTRL2,                   DIDT_TD_CTRL2__UNUSED_1_MASK,                       DIDT_TD_CTRL2__UNUSED_1__SHIFT,                     0x0000,     POLARIS10_CONFIGREG_DIDT_IND },
+        {   ixDIDT_TD_CTRL2,                   DIDT_TD_CTRL2__LONG_TERM_INTERVAL_RATIO_MASK,       DIDT_TD_CTRL2__LONG_TERM_INTERVAL_RATIO__SHIFT,     0x0000,     POLARIS10_CONFIGREG_DIDT_IND },
+        {   ixDIDT_TD_CTRL2,                   DIDT_TD_CTRL2__UNUSED_2_MASK,                       DIDT_TD_CTRL2__UNUSED_2__SHIFT,                     0x0000,     POLARIS10_CONFIGREG_DIDT_IND },
+
+        {   ixDIDT_TD_STALL_CTRL,              DIDT_TD_STALL_CTRL__DIDT_STALL_CTRL_ENABLE_MASK,    DIDT_TD_STALL_CTRL__DIDT_STALL_CTRL_ENABLE__SHIFT,  0x0001,     POLARIS10_CONFIGREG_DIDT_IND },
+        {   ixDIDT_TD_STALL_CTRL,              DIDT_TD_STALL_CTRL__DIDT_STALL_DELAY_HI_MASK,       DIDT_TD_STALL_CTRL__DIDT_STALL_DELAY_HI__SHIFT,     0x0001,     POLARIS10_CONFIGREG_DIDT_IND },
+        {   ixDIDT_TD_STALL_CTRL,              DIDT_TD_STALL_CTRL__DIDT_STALL_DELAY_LO_MASK,       DIDT_TD_STALL_CTRL__DIDT_STALL_DELAY_LO__SHIFT,     0x0001,     POLARIS10_CONFIGREG_DIDT_IND },
+        {   ixDIDT_TD_STALL_CTRL,              DIDT_TD_STALL_CTRL__DIDT_HI_POWER_THRESHOLD_MASK,   DIDT_TD_STALL_CTRL__DIDT_HI_POWER_THRESHOLD__SHIFT, 0x01aa,     POLARIS10_CONFIGREG_DIDT_IND },
+        {   ixDIDT_TD_STALL_CTRL,              DIDT_TD_STALL_CTRL__UNUSED_0_MASK,                  DIDT_TD_STALL_CTRL__UNUSED_0__SHIFT,                0x0000,     POLARIS10_CONFIGREG_DIDT_IND },
+
+        {   ixDIDT_TD_TUNING_CTRL,             DIDT_TD_TUNING_CTRL__DIDT_TUNING_ENABLE_MASK,       DIDT_TD_TUNING_CTRL__DIDT_TUNING_ENABLE__SHIFT,     0x0000,     POLARIS10_CONFIGREG_DIDT_IND },
+        {   ixDIDT_TD_TUNING_CTRL,             DIDT_TD_TUNING_CTRL__MAX_POWER_DELTA_HI_MASK,       DIDT_TD_TUNING_CTRL__MAX_POWER_DELTA_HI__SHIFT,     0x0dde,     POLARIS10_CONFIGREG_DIDT_IND },
+        {   ixDIDT_TD_TUNING_CTRL,             DIDT_TD_TUNING_CTRL__MAX_POWER_DELTA_LO_MASK,       DIDT_TD_TUNING_CTRL__MAX_POWER_DELTA_LO__SHIFT,     0x0dde,     POLARIS10_CONFIGREG_DIDT_IND },
+        {   ixDIDT_TD_TUNING_CTRL,             DIDT_TD_TUNING_CTRL__UNUSED_0_MASK,                 DIDT_TD_TUNING_CTRL__UNUSED_0__SHIFT,               0x0000,     POLARIS10_CONFIGREG_DIDT_IND },
+
+        {   ixDIDT_TD_CTRL0,                   DIDT_TD_CTRL0__DIDT_CTRL_EN_MASK,                   DIDT_TD_CTRL0__DIDT_CTRL_EN__SHIFT,                 0x0001,     POLARIS10_CONFIGREG_DIDT_IND },
+        {   ixDIDT_TD_CTRL0,                   DIDT_TD_CTRL0__USE_REF_CLOCK_MASK,                  DIDT_TD_CTRL0__USE_REF_CLOCK__SHIFT,                0x0000,     POLARIS10_CONFIGREG_DIDT_IND },
+        {   ixDIDT_TD_CTRL0,                   DIDT_TD_CTRL0__PHASE_OFFSET_MASK,                   DIDT_TD_CTRL0__PHASE_OFFSET__SHIFT,                 0x0000,     POLARIS10_CONFIGREG_DIDT_IND },
+        {   ixDIDT_TD_CTRL0,                   DIDT_TD_CTRL0__DIDT_CTRL_RST_MASK,                  DIDT_TD_CTRL0__DIDT_CTRL_RST__SHIFT,                0x0000,     POLARIS10_CONFIGREG_DIDT_IND },
+        {   ixDIDT_TD_CTRL0,                   DIDT_TD_CTRL0__DIDT_CLK_EN_OVERRIDE_MASK,           DIDT_TD_CTRL0__DIDT_CLK_EN_OVERRIDE__SHIFT,         0x0000,     POLARIS10_CONFIGREG_DIDT_IND },
+        {   ixDIDT_TD_CTRL0,                   DIDT_TD_CTRL0__DIDT_MAX_STALLS_ALLOWED_HI_MASK,     DIDT_TD_CTRL0__DIDT_MAX_STALLS_ALLOWED_HI__SHIFT,   0x0008,     POLARIS10_CONFIGREG_DIDT_IND },
+        {   ixDIDT_TD_CTRL0,                   DIDT_TD_CTRL0__DIDT_MAX_STALLS_ALLOWED_LO_MASK,     DIDT_TD_CTRL0__DIDT_MAX_STALLS_ALLOWED_LO__SHIFT,   0x0008,     POLARIS10_CONFIGREG_DIDT_IND },
+        {   ixDIDT_TD_CTRL0,                   DIDT_TD_CTRL0__UNUSED_0_MASK,                       DIDT_TD_CTRL0__UNUSED_0__SHIFT,                     0x0000,     POLARIS10_CONFIGREG_DIDT_IND },
+
+        {   ixDIDT_TCP_WEIGHT0_3,              DIDT_TCP_WEIGHT0_3__WEIGHT0_MASK,                   DIDT_TCP_WEIGHT0_3__WEIGHT0__SHIFT,                 0x0004,     POLARIS10_CONFIGREG_DIDT_IND },
+        {   ixDIDT_TCP_WEIGHT0_3,              DIDT_TCP_WEIGHT0_3__WEIGHT1_MASK,                   DIDT_TCP_WEIGHT0_3__WEIGHT1__SHIFT,                 0x0037,     POLARIS10_CONFIGREG_DIDT_IND },
+        {   ixDIDT_TCP_WEIGHT0_3,              DIDT_TCP_WEIGHT0_3__WEIGHT2_MASK,                   DIDT_TCP_WEIGHT0_3__WEIGHT2__SHIFT,                 0x0001,     POLARIS10_CONFIGREG_DIDT_IND },
+        {   ixDIDT_TCP_WEIGHT0_3,              DIDT_TCP_WEIGHT0_3__WEIGHT3_MASK,                   DIDT_TCP_WEIGHT0_3__WEIGHT3__SHIFT,                 0x00ff,     POLARIS10_CONFIGREG_DIDT_IND },
+
+        {   ixDIDT_TCP_WEIGHT4_7,              DIDT_TCP_WEIGHT4_7__WEIGHT4_MASK,                   DIDT_TCP_WEIGHT4_7__WEIGHT4__SHIFT,                 0x0054,     POLARIS10_CONFIGREG_DIDT_IND },
+        {   ixDIDT_TCP_WEIGHT4_7,              DIDT_TCP_WEIGHT4_7__WEIGHT5_MASK,                   DIDT_TCP_WEIGHT4_7__WEIGHT5__SHIFT,                 0x0000,     POLARIS10_CONFIGREG_DIDT_IND },
+        {   ixDIDT_TCP_WEIGHT4_7,              DIDT_TCP_WEIGHT4_7__WEIGHT6_MASK,                   DIDT_TCP_WEIGHT4_7__WEIGHT6__SHIFT,                 0x0000,     POLARIS10_CONFIGREG_DIDT_IND },
+        {   ixDIDT_TCP_WEIGHT4_7,              DIDT_TCP_WEIGHT4_7__WEIGHT7_MASK,                   DIDT_TCP_WEIGHT4_7__WEIGHT7__SHIFT,                 0x0000,     POLARIS10_CONFIGREG_DIDT_IND },
+
+        {   ixDIDT_TCP_CTRL1,                  DIDT_TCP_CTRL1__MIN_POWER_MASK,                     DIDT_TCP_CTRL1__MIN_POWER__SHIFT,                   0x0000,     POLARIS10_CONFIGREG_DIDT_IND },
+        {   ixDIDT_TCP_CTRL1,                  DIDT_TCP_CTRL1__MAX_POWER_MASK,                     DIDT_TCP_CTRL1__MAX_POWER__SHIFT,                   0xffff,     POLARIS10_CONFIGREG_DIDT_IND },
+
+        {   ixDIDT_TCP_CTRL_OCP,               DIDT_TCP_CTRL_OCP__UNUSED_0_MASK,                   DIDT_TCP_CTRL_OCP__UNUSED_0__SHIFT,                 0x0000,     POLARIS10_CONFIGREG_DIDT_IND },
+        {   ixDIDT_TCP_CTRL_OCP,               DIDT_TCP_CTRL_OCP__OCP_MAX_POWER_MASK,              DIDT_TCP_CTRL_OCP__OCP_MAX_POWER__SHIFT,            0xffff,     POLARIS10_CONFIGREG_DIDT_IND },
+
+        {   ixDIDT_TCP_CTRL2,                  DIDT_TCP_CTRL2__MAX_POWER_DELTA_MASK,               DIDT_TCP_CTRL2__MAX_POWER_DELTA__SHIFT,             0x3dde,     POLARIS10_CONFIGREG_DIDT_IND },
+        {   ixDIDT_TCP_CTRL2,                  DIDT_TCP_CTRL2__UNUSED_0_MASK,                      DIDT_TCP_CTRL2__UNUSED_0__SHIFT,                    0x0000,     POLARIS10_CONFIGREG_DIDT_IND },
+        {   ixDIDT_TCP_CTRL2,                  DIDT_TCP_CTRL2__SHORT_TERM_INTERVAL_SIZE_MASK,      DIDT_TCP_CTRL2__SHORT_TERM_INTERVAL_SIZE__SHIFT,    0x0032,     POLARIS10_CONFIGREG_DIDT_IND },
+        {   ixDIDT_TCP_CTRL2,                  DIDT_TCP_CTRL2__UNUSED_1_MASK,                      DIDT_TCP_CTRL2__UNUSED_1__SHIFT,                    0x0000,     POLARIS10_CONFIGREG_DIDT_IND },
+        {   ixDIDT_TCP_CTRL2,                  DIDT_TCP_CTRL2__LONG_TERM_INTERVAL_RATIO_MASK,      DIDT_TCP_CTRL2__LONG_TERM_INTERVAL_RATIO__SHIFT,    0x0000,     POLARIS10_CONFIGREG_DIDT_IND },
+        {   ixDIDT_TCP_CTRL2,                  DIDT_TCP_CTRL2__UNUSED_2_MASK,                      DIDT_TCP_CTRL2__UNUSED_2__SHIFT,                    0x0000,     POLARIS10_CONFIGREG_DIDT_IND },
+
+        {   ixDIDT_TCP_STALL_CTRL,             DIDT_TCP_STALL_CTRL__DIDT_STALL_CTRL_ENABLE_MASK,   DIDT_TCP_STALL_CTRL__DIDT_STALL_CTRL_ENABLE__SHIFT, 0x0001,     POLARIS10_CONFIGREG_DIDT_IND },
+        {   ixDIDT_TCP_STALL_CTRL,             DIDT_TCP_STALL_CTRL__DIDT_STALL_DELAY_HI_MASK,      DIDT_TCP_STALL_CTRL__DIDT_STALL_DELAY_HI__SHIFT,    0x0001,     POLARIS10_CONFIGREG_DIDT_IND },
+        {   ixDIDT_TCP_STALL_CTRL,             DIDT_TCP_STALL_CTRL__DIDT_STALL_DELAY_LO_MASK,      DIDT_TCP_STALL_CTRL__DIDT_STALL_DELAY_LO__SHIFT,    0x0001,     POLARIS10_CONFIGREG_DIDT_IND },
+        {   ixDIDT_TCP_STALL_CTRL,             DIDT_TCP_STALL_CTRL__DIDT_HI_POWER_THRESHOLD_MASK,  DIDT_TCP_STALL_CTRL__DIDT_HI_POWER_THRESHOLD__SHIFT, 0x01aa,     POLARIS10_CONFIGREG_DIDT_IND },
+        {   ixDIDT_TCP_STALL_CTRL,             DIDT_TCP_STALL_CTRL__UNUSED_0_MASK,                 DIDT_TCP_STALL_CTRL__UNUSED_0__SHIFT,               0x0000,     POLARIS10_CONFIGREG_DIDT_IND },
+
+        {   ixDIDT_TCP_TUNING_CTRL,            DIDT_TCP_TUNING_CTRL__DIDT_TUNING_ENABLE_MASK,      DIDT_TCP_TUNING_CTRL__DIDT_TUNING_ENABLE__SHIFT,    0x0001,     POLARIS10_CONFIGREG_DIDT_IND },
+        {   ixDIDT_TCP_TUNING_CTRL,            DIDT_TCP_TUNING_CTRL__MAX_POWER_DELTA_HI_MASK,      DIDT_TCP_TUNING_CTRL__MAX_POWER_DELTA_HI__SHIFT,    0x3dde,     POLARIS10_CONFIGREG_DIDT_IND },
+        {   ixDIDT_TCP_TUNING_CTRL,            DIDT_TCP_TUNING_CTRL__MAX_POWER_DELTA_LO_MASK,      DIDT_TCP_TUNING_CTRL__MAX_POWER_DELTA_LO__SHIFT,    0x3dde,     POLARIS10_CONFIGREG_DIDT_IND },
+        {   ixDIDT_TCP_TUNING_CTRL,            DIDT_TCP_TUNING_CTRL__UNUSED_0_MASK,                DIDT_TCP_TUNING_CTRL__UNUSED_0__SHIFT,              0x0000,     POLARIS10_CONFIGREG_DIDT_IND },
+
+        {   ixDIDT_TCP_CTRL0,                   DIDT_TCP_CTRL0__DIDT_CTRL_EN_MASK,                   DIDT_TCP_CTRL0__DIDT_CTRL_EN__SHIFT,                 0x0001,     POLARIS10_CONFIGREG_DIDT_IND },
+        {   ixDIDT_TCP_CTRL0,                   DIDT_TCP_CTRL0__USE_REF_CLOCK_MASK,                  DIDT_TCP_CTRL0__USE_REF_CLOCK__SHIFT,                0x0000,     POLARIS10_CONFIGREG_DIDT_IND },
+        {   ixDIDT_TCP_CTRL0,                   DIDT_TCP_CTRL0__PHASE_OFFSET_MASK,                   DIDT_TCP_CTRL0__PHASE_OFFSET__SHIFT,                 0x0000,     POLARIS10_CONFIGREG_DIDT_IND },
+        {   ixDIDT_TCP_CTRL0,                   DIDT_TCP_CTRL0__DIDT_CTRL_RST_MASK,                  DIDT_TCP_CTRL0__DIDT_CTRL_RST__SHIFT,                0x0000,     POLARIS10_CONFIGREG_DIDT_IND },
+        {   ixDIDT_TCP_CTRL0,                   DIDT_TCP_CTRL0__DIDT_CLK_EN_OVERRIDE_MASK,           DIDT_TCP_CTRL0__DIDT_CLK_EN_OVERRIDE__SHIFT,         0x0000,     POLARIS10_CONFIGREG_DIDT_IND },
+        {   ixDIDT_TCP_CTRL0,                   DIDT_TCP_CTRL0__DIDT_MAX_STALLS_ALLOWED_HI_MASK,     DIDT_TCP_CTRL0__DIDT_MAX_STALLS_ALLOWED_HI__SHIFT,   0x0010,     POLARIS10_CONFIGREG_DIDT_IND },
+        {   ixDIDT_TCP_CTRL0,                   DIDT_TCP_CTRL0__DIDT_MAX_STALLS_ALLOWED_LO_MASK,     DIDT_TCP_CTRL0__DIDT_MAX_STALLS_ALLOWED_LO__SHIFT,   0x0010,     POLARIS10_CONFIGREG_DIDT_IND },
+        {   ixDIDT_TCP_CTRL0,                   DIDT_TCP_CTRL0__UNUSED_0_MASK,                       DIDT_TCP_CTRL0__UNUSED_0__SHIFT,                     0x0000,     POLARIS10_CONFIGREG_DIDT_IND },
+        {   0xFFFFFFFF  }
+};
+
 static const struct polaris10_pt_defaults polaris10_power_tune_data_set_array[POWERTUNE_DEFAULT_SET_MAX] = {
         /* sviLoadLIneEn, SviLoadLineVddC, TDC_VDDC_ThrottleReleaseLimitPerc, TDC_MAWt,
          * TdcWaterfallCtl, DTEAmbientTempBase, DisplayCac, BAPM_TEMP_GRADIENT */
@@ -209,6 +559,187 @@ static int polaris10_min_max_vgnb_lpml_id_from_bapm_vddc(struct pp_hwmgr *hwmgr)
         return 0;
 }
 
+static int polaris10_enable_didt(struct pp_hwmgr *hwmgr, const bool enable)
+{
+
+        uint32_t en = enable ? 1 : 0;
+        int32_t result = 0;
+        uint32_t data;
+
+        if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps, PHM_PlatformCaps_SQRamping)) {
+                data = cgs_read_ind_register(hwmgr->device, CGS_IND_REG__DIDT, ixDIDT_SQ_CTRL0);
+                data &= ~DIDT_SQ_CTRL0__DIDT_CTRL_EN_MASK;
+                data |= ((en << DIDT_SQ_CTRL0__DIDT_CTRL_EN__SHIFT) & DIDT_SQ_CTRL0__DIDT_CTRL_EN_MASK);
+                cgs_write_ind_register(hwmgr->device, CGS_IND_REG__DIDT, ixDIDT_SQ_CTRL0, data);
+                DIDTBlock_Info &= ~SQ_Enable_MASK;
+                DIDTBlock_Info |= en << SQ_Enable_SHIFT;
+        }
+
+        if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps, PHM_PlatformCaps_DBRamping)) {
+                data = cgs_read_ind_register(hwmgr->device, CGS_IND_REG__DIDT, ixDIDT_DB_CTRL0);
+                data &= ~DIDT_DB_CTRL0__DIDT_CTRL_EN_MASK;
+                data |= ((en << DIDT_DB_CTRL0__DIDT_CTRL_EN__SHIFT) & DIDT_DB_CTRL0__DIDT_CTRL_EN_MASK);
+                cgs_write_ind_register(hwmgr->device, CGS_IND_REG__DIDT, ixDIDT_DB_CTRL0, data);
+                DIDTBlock_Info &= ~DB_Enable_MASK;
+                DIDTBlock_Info |= en << DB_Enable_SHIFT;
+        }
+
+        if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps, PHM_PlatformCaps_TDRamping)) {
+                data = cgs_read_ind_register(hwmgr->device, CGS_IND_REG__DIDT, ixDIDT_TD_CTRL0);
+                data &= ~DIDT_TD_CTRL0__DIDT_CTRL_EN_MASK;
+                data |= ((en << DIDT_TD_CTRL0__DIDT_CTRL_EN__SHIFT) & DIDT_TD_CTRL0__DIDT_CTRL_EN_MASK);
+                cgs_write_ind_register(hwmgr->device, CGS_IND_REG__DIDT, ixDIDT_TD_CTRL0, data);
+                DIDTBlock_Info &= ~TD_Enable_MASK;
+                DIDTBlock_Info |= en << TD_Enable_SHIFT;
+        }
+
+        if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps, PHM_PlatformCaps_TCPRamping)) {
+                data = cgs_read_ind_register(hwmgr->device, CGS_IND_REG__DIDT, ixDIDT_TCP_CTRL0);
+                data &= ~DIDT_TCP_CTRL0__DIDT_CTRL_EN_MASK;
+                data |= ((en << DIDT_TCP_CTRL0__DIDT_CTRL_EN__SHIFT) & DIDT_TCP_CTRL0__DIDT_CTRL_EN_MASK);
+                cgs_write_ind_register(hwmgr->device, CGS_IND_REG__DIDT, ixDIDT_TCP_CTRL0, data);
+                DIDTBlock_Info &= ~TCP_Enable_MASK;
+                DIDTBlock_Info |= en << TCP_Enable_SHIFT;
+        }
+
+        if (enable)
+                result = smum_send_msg_to_smc_with_parameter(hwmgr->smumgr, PPSMC_MSG_Didt_Block_Function, DIDTBlock_Info);
+
+        return result;
+}
+
+static int polaris10_program_pt_config_registers(struct pp_hwmgr *hwmgr,
+                                struct polaris10_pt_config_reg *cac_config_regs)
+{
+        struct polaris10_pt_config_reg *config_regs = cac_config_regs;
+        uint32_t cache = 0;
+        uint32_t data = 0;
+
+        PP_ASSERT_WITH_CODE((config_regs != NULL), "Invalid config register table.", return -EINVAL);
+
+        while (config_regs->offset != 0xFFFFFFFF) {
+                if (config_regs->type == POLARIS10_CONFIGREG_CACHE)
+                        cache |= ((config_regs->value << config_regs->shift) & config_regs->mask);
+                else {
+                        switch (config_regs->type) {
+                        case POLARIS10_CONFIGREG_SMC_IND:
+                                data = cgs_read_ind_register(hwmgr->device, CGS_IND_REG__SMC, config_regs->offset);
+                                break;
+
+                        case POLARIS10_CONFIGREG_DIDT_IND:
+                                data = cgs_read_ind_register(hwmgr->device, CGS_IND_REG__DIDT, config_regs->offset);
+                                break;
+
+                        case POLARIS10_CONFIGREG_GC_CAC_IND:
+                                data = cgs_read_ind_register(hwmgr->device, CGS_IND_REG_GC_CAC, config_regs->offset);
+                                break;
+
+                        default:
+                                data = cgs_read_register(hwmgr->device, config_regs->offset);
+                                break;
+                        }
+
+                        data &= ~config_regs->mask;
+                        data |= ((config_regs->value << config_regs->shift) & config_regs->mask);
+                        data |= cache;
+
+                        switch (config_regs->type) {
+                        case POLARIS10_CONFIGREG_SMC_IND:
+                                cgs_write_ind_register(hwmgr->device, CGS_IND_REG__SMC, config_regs->offset, data);
+                                break;
+
+                        case POLARIS10_CONFIGREG_DIDT_IND:
+                                cgs_write_ind_register(hwmgr->device, CGS_IND_REG__DIDT, config_regs->offset, data);
+                                break;
+
+                        case POLARIS10_CONFIGREG_GC_CAC_IND:
+                                cgs_write_ind_register(hwmgr->device, CGS_IND_REG_GC_CAC, config_regs->offset, data);
+                                break;
+
+                        default:
+                                cgs_write_register(hwmgr->device, config_regs->offset, data);
+                                break;
+                        }
+                        cache = 0;
+                }
+
+                config_regs++;
+        }
+
+        return 0;
+}
+
+int polaris10_enable_didt_config(struct pp_hwmgr *hwmgr)
+{
+        int result;
+        uint32_t num_se = 0;
+        uint32_t count, value, value2;
+        struct cgs_system_info sys_info = {0};
+
+        sys_info.size = sizeof(struct cgs_system_info);
+        sys_info.info_id = CGS_SYSTEM_INFO_GFX_SE_INFO;
+        result = cgs_query_system_info(hwmgr->device, &sys_info);
+
+
+        if (result == 0)
+                num_se = sys_info.value;
+
+        if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps, PHM_PlatformCaps_SQRamping) ||
+                phm_cap_enabled(hwmgr->platform_descriptor.platformCaps, PHM_PlatformCaps_DBRamping) ||
+                phm_cap_enabled(hwmgr->platform_descriptor.platformCaps, PHM_PlatformCaps_TDRamping) ||
+                phm_cap_enabled(hwmgr->platform_descriptor.platformCaps, PHM_PlatformCaps_TCPRamping)) {
+
+                /* TO DO Pre DIDT disable clock gating */
+                value = 0;
+                value2 = cgs_read_register(hwmgr->device, mmGRBM_GFX_INDEX);
+                for (count = 0; count < num_se; count++) {
+                        value = SYS_GRBM_GFX_INDEX_DATA__INSTANCE_BROADCAST_WRITES_MASK
+                                | SYS_GRBM_GFX_INDEX_DATA__SH_BROADCAST_WRITES_MASK
+                                | (count << SYS_GRBM_GFX_INDEX_DATA__SE_INDEX__SHIFT);
+                        cgs_write_register(hwmgr->device, mmGRBM_GFX_INDEX, value);
+
+                        if (hwmgr->chip_id == CHIP_POLARIS10) {
+                                result = polaris10_program_pt_config_registers(hwmgr, GCCACConfig_Polaris10);
+                                PP_ASSERT_WITH_CODE((result == 0), "DIDT Config failed.", return result);
+                                result = polaris10_program_pt_config_registers(hwmgr, DIDTConfig_Polaris10);
+                                PP_ASSERT_WITH_CODE((result == 0), "DIDT Config failed.", return result);
+                        } else if (hwmgr->chip_id == CHIP_POLARIS11) {
+                                result = polaris10_program_pt_config_registers(hwmgr, GCCACConfig_Polaris11);
+                                PP_ASSERT_WITH_CODE((result == 0), "DIDT Config failed.", return result);
+                                result = polaris10_program_pt_config_registers(hwmgr, DIDTConfig_Polaris11);
+                                PP_ASSERT_WITH_CODE((result == 0), "DIDT Config failed.", return result);
+                        }
+                }
+                cgs_write_register(hwmgr->device, mmGRBM_GFX_INDEX, value2);
+
+                result = polaris10_enable_didt(hwmgr, true);
+                PP_ASSERT_WITH_CODE((result == 0), "EnableDiDt failed.", return result);
+
+                /* TO DO Post DIDT enable clock gating */
+        }
+
+        return 0;
+}
+
+int polaris10_disable_didt_config(struct pp_hwmgr *hwmgr)
+{
+        int result;
+
+        if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps, PHM_PlatformCaps_SQRamping) ||
+                phm_cap_enabled(hwmgr->platform_descriptor.platformCaps, PHM_PlatformCaps_DBRamping) ||
+                phm_cap_enabled(hwmgr->platform_descriptor.platformCaps, PHM_PlatformCaps_TDRamping) ||
+                phm_cap_enabled(hwmgr->platform_descriptor.platformCaps, PHM_PlatformCaps_TCPRamping)) {
+                /* TO DO Pre DIDT disable clock gating */
+
+                result = polaris10_enable_didt(hwmgr, false);
+                PP_ASSERT_WITH_CODE((result == 0), "Post DIDT enable clock gating failed.", return result);
+                /* TO DO Post DIDT enable clock gating */
+        }
+
+        return 0;
+}
+
+
 static int polaris10_populate_bapm_vddc_base_leakage_sidd(struct pp_hwmgr *hwmgr)
 {
         struct polaris10_hwmgr *data = (struct polaris10_hwmgr *)(hwmgr->backend);
@@ -312,6 +843,23 @@ int polaris10_enable_smc_cac(struct pp_hwmgr *hwmgr)
         return result;
 }
 
+int polaris10_disable_smc_cac(struct pp_hwmgr *hwmgr)
+{
+        struct polaris10_hwmgr *data = (struct polaris10_hwmgr *)(hwmgr->backend);
+        int result = 0;
+
+        if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
+                        PHM_PlatformCaps_CAC) && data->cac_enabled) {
+                int smc_result = smum_send_msg_to_smc(hwmgr->smumgr,
+                                (uint16_t)(PPSMC_MSG_DisableCac));
+                PP_ASSERT_WITH_CODE((smc_result == 0),
+                                "Failed to disable CAC in SMC.", result = -1);
+
+                data->cac_enabled = false;
+        }
+        return result;
+}
+
 int polaris10_set_power_limit(struct pp_hwmgr *hwmgr, uint32_t n)
 {
         struct polaris10_hwmgr *data = (struct polaris10_hwmgr *)(hwmgr->backend);
@@ -373,6 +921,48 @@ int polaris10_enable_power_containment(struct pp_hwmgr *hwmgr)
         return result;
 }
 
+int polaris10_disable_power_containment(struct pp_hwmgr *hwmgr)
+{
+        struct polaris10_hwmgr *data = (struct polaris10_hwmgr *)(hwmgr->backend);
+        int result = 0;
+
+        if (phm_cap_enabled(hwmgr->platform_descriptor.platformCaps,
+                        PHM_PlatformCaps_PowerContainment) &&
+                        data->power_containment_features) {
+                int smc_result;
+
+                if (data->power_containment_features &
+                                POWERCONTAINMENT_FEATURE_TDCLimit) {
+                        smc_result = smum_send_msg_to_smc(hwmgr->smumgr,
+                                        (uint16_t)(PPSMC_MSG_TDCLimitDisable));
+                        PP_ASSERT_WITH_CODE((smc_result == 0),
+                                        "Failed to disable TDCLimit in SMC.",
+                                        result = smc_result);
+                }
+
+                if (data->power_containment_features &
+                                POWERCONTAINMENT_FEATURE_DTE) {
+                        smc_result = smum_send_msg_to_smc(hwmgr->smumgr,
+                                        (uint16_t)(PPSMC_MSG_DisableDTE));
+                        PP_ASSERT_WITH_CODE((smc_result == 0),
+                                        "Failed to disable DTE in SMC.",
+                                        result = smc_result);
+                }
+
+                if (data->power_containment_features &
+                                POWERCONTAINMENT_FEATURE_PkgPwrLimit) {
+                        smc_result = smum_send_msg_to_smc(hwmgr->smumgr,
+                                        (uint16_t)(PPSMC_MSG_PkgPwrLimitDisable));
+                        PP_ASSERT_WITH_CODE((smc_result == 0),
+                                        "Failed to disable PkgPwrTracking in SMC.",
+                                        result = smc_result);
+                }
+                data->power_containment_features = 0;
+        }
+
+        return result;
+}
+
 int polaris10_power_control_set_level(struct pp_hwmgr *hwmgr)
 {
         struct phm_ppt_v1_information *table_info =
diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/polaris10_powertune.h b/drivers/gpu/drm/amd/powerplay/hwmgr/polaris10_powertune.h
index 68bc1cb6d40c..bc78e28f010d 100644
--- a/drivers/gpu/drm/amd/powerplay/hwmgr/polaris10_powertune.h
+++ b/drivers/gpu/drm/amd/powerplay/hwmgr/polaris10_powertune.h
@@ -27,15 +27,37 @@ enum polaris10_pt_config_reg_type {
         POLARIS10_CONFIGREG_MMR = 0,
         POLARIS10_CONFIGREG_SMC_IND,
         POLARIS10_CONFIGREG_DIDT_IND,
+        POLARIS10_CONFIGREG_GC_CAC_IND,
         POLARIS10_CONFIGREG_CACHE,
         POLARIS10_CONFIGREG_MAX
 };
 
+#define DIDT_SQ_CTRL0__UNUSED_0_MASK    0xfffc0000
+#define DIDT_SQ_CTRL0__UNUSED_0__SHIFT  0x12
+#define DIDT_TD_CTRL0__UNUSED_0_MASK    0xfffc0000
+#define DIDT_TD_CTRL0__UNUSED_0__SHIFT  0x12
+#define DIDT_TCP_CTRL0__UNUSED_0_MASK   0xfffc0000
+#define DIDT_TCP_CTRL0__UNUSED_0__SHIFT 0x12
+#define DIDT_SQ_TUNING_CTRL__UNUSED_0_MASK                 0xc0000000
+#define DIDT_SQ_TUNING_CTRL__UNUSED_0__SHIFT               0x0000001e
+#define DIDT_TD_TUNING_CTRL__UNUSED_0_MASK                 0xc0000000
+#define DIDT_TD_TUNING_CTRL__UNUSED_0__SHIFT               0x0000001e
+#define DIDT_TCP_TUNING_CTRL__UNUSED_0_MASK                0xc0000000
+#define DIDT_TCP_TUNING_CTRL__UNUSED_0__SHIFT              0x0000001e
+
 /* PowerContainment Features */
 #define POWERCONTAINMENT_FEATURE_DTE             0x00000001
 #define POWERCONTAINMENT_FEATURE_TDCLimit        0x00000002
 #define POWERCONTAINMENT_FEATURE_PkgPwrLimit     0x00000004
 
+#define ixGC_CAC_CNTL 0x0000
+#define ixDIDT_SQ_STALL_CTRL 0x0004
+#define  ixDIDT_SQ_TUNING_CTRL 0x0005
+#define ixDIDT_TD_STALL_CTRL 0x0044
+#define ixDIDT_TD_TUNING_CTRL 0x0045
+#define ixDIDT_TCP_STALL_CTRL 0x0064
+#define ixDIDT_TCP_TUNING_CTRL 0x0065
+
 struct polaris10_pt_config_reg {
         uint32_t                           offset;
         uint32_t                           mask;
@@ -62,9 +84,11 @@ void polaris10_initialize_power_tune_defaults(struct pp_hwmgr *hwmgr);
 int polaris10_populate_bapm_parameters_in_dpm_table(struct pp_hwmgr *hwmgr);
 int polaris10_populate_pm_fuses(struct pp_hwmgr *hwmgr);
 int polaris10_enable_smc_cac(struct pp_hwmgr *hwmgr);
+int polaris10_disable_smc_cac(struct pp_hwmgr *hwmgr);
 int polaris10_enable_power_containment(struct pp_hwmgr *hwmgr);
+int polaris10_disable_power_containment(struct pp_hwmgr *hwmgr);
 int polaris10_set_power_limit(struct pp_hwmgr *hwmgr, uint32_t n);
 int polaris10_power_control_set_level(struct pp_hwmgr *hwmgr);
-
+int polaris10_enable_didt_config(struct pp_hwmgr *hwmgr);
 #endif  /* POLARIS10_POWERTUNE_H */
 
diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/pp_acpi.c b/drivers/gpu/drm/amd/powerplay/hwmgr/pp_acpi.c
index a3c38bbd1e94..1944d289f846 100644
--- a/drivers/gpu/drm/amd/powerplay/hwmgr/pp_acpi.c
+++ b/drivers/gpu/drm/amd/powerplay/hwmgr/pp_acpi.c
@@ -66,7 +66,7 @@ int acpi_pcie_perf_request(void *device, uint8_t perf_req, bool advertise)
         int result;
         struct cgs_system_info info = {0};
 
-        if( 0 != acpi_atcs_notify_pcie_device_ready(device))
+        if (acpi_atcs_notify_pcie_device_ready(device))
                 return -EINVAL;
 
         info.size = sizeof(struct cgs_system_info);
diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/ppatomctrl.c b/drivers/gpu/drm/amd/powerplay/hwmgr/ppatomctrl.c
index 90b35c5c10a4..26f3e30d0fef 100644
--- a/drivers/gpu/drm/amd/powerplay/hwmgr/ppatomctrl.c
+++ b/drivers/gpu/drm/amd/powerplay/hwmgr/ppatomctrl.c
@@ -179,13 +179,12 @@ int atomctrl_set_engine_dram_timings_rv770(
 
         /* They are both in 10KHz Units. */
         engine_clock_parameters.ulTargetEngineClock =
-                (uint32_t) engine_clock & SET_CLOCK_FREQ_MASK;
-        engine_clock_parameters.ulTargetEngineClock |=
-                (COMPUTE_ENGINE_PLL_PARAM << 24);
+                cpu_to_le32((engine_clock & SET_CLOCK_FREQ_MASK) |
+                            ((COMPUTE_ENGINE_PLL_PARAM << 24)));
 
         /* in 10 khz units.*/
         engine_clock_parameters.sReserved.ulClock =
-                (uint32_t) memory_clock & SET_CLOCK_FREQ_MASK;
+                cpu_to_le32(memory_clock & SET_CLOCK_FREQ_MASK);
         return cgs_atom_exec_cmd_table(hwmgr->device,
                         GetIndexIntoMasterTable(COMMAND, DynamicMemorySettings),
                         &engine_clock_parameters);
@@ -252,7 +251,7 @@ int atomctrl_get_memory_pll_dividers_si(
         COMPUTE_MEMORY_CLOCK_PARAM_PARAMETERS_V2_1 mpll_parameters;
         int result;
 
-        mpll_parameters.ulClock = (uint32_t) clock_value;
+        mpll_parameters.ulClock = cpu_to_le32(clock_value);
         mpll_parameters.ucInputFlag = (uint8_t)((strobe_mode) ? 1 : 0);
 
         result = cgs_atom_exec_cmd_table
@@ -262,9 +261,9 @@ int atomctrl_get_memory_pll_dividers_si(
 
         if (0 == result) {
                 mpll_param->mpll_fb_divider.clk_frac =
-                        mpll_parameters.ulFbDiv.usFbDivFrac;
+                        le16_to_cpu(mpll_parameters.ulFbDiv.usFbDivFrac);
                 mpll_param->mpll_fb_divider.cl_kf =
-                        mpll_parameters.ulFbDiv.usFbDiv;
+                        le16_to_cpu(mpll_parameters.ulFbDiv.usFbDiv);
                 mpll_param->mpll_post_divider =
                         (uint32_t)mpll_parameters.ucPostDiv;
                 mpll_param->vco_mode =
@@ -300,7 +299,7 @@ int atomctrl_get_memory_pll_dividers_vi(struct pp_hwmgr *hwmgr,
         COMPUTE_MEMORY_CLOCK_PARAM_PARAMETERS_V2_2 mpll_parameters;
         int result;
 
-        mpll_parameters.ulClock.ulClock = (uint32_t)clock_value;
+        mpll_parameters.ulClock.ulClock = cpu_to_le32(clock_value);
 
         result = cgs_atom_exec_cmd_table(hwmgr->device,
                         GetIndexIntoMasterTable(COMMAND, ComputeMemoryClockParam),
@@ -320,7 +319,7 @@ int atomctrl_get_engine_pll_dividers_kong(struct pp_hwmgr *hwmgr,
         COMPUTE_MEMORY_ENGINE_PLL_PARAMETERS_V4 pll_parameters;
         int result;
 
-        pll_parameters.ulClock = clock_value;
+        pll_parameters.ulClock = cpu_to_le32(clock_value);
 
         result = cgs_atom_exec_cmd_table
                 (hwmgr->device,
@@ -329,7 +328,7 @@ int atomctrl_get_engine_pll_dividers_kong(struct pp_hwmgr *hwmgr,
 
         if (0 == result) {
                 dividers->pll_post_divider = pll_parameters.ucPostDiv;
-                dividers->real_clock = pll_parameters.ulClock;
+                dividers->real_clock = le32_to_cpu(pll_parameters.ulClock);
         }
 
         return result;
@@ -343,7 +342,7 @@ int atomctrl_get_engine_pll_dividers_vi(
         COMPUTE_GPU_CLOCK_OUTPUT_PARAMETERS_V1_6 pll_patameters;
         int result;
 
-        pll_patameters.ulClock.ulClock = clock_value;
+        pll_patameters.ulClock.ulClock = cpu_to_le32(clock_value);
         pll_patameters.ulClock.ucPostDiv = COMPUTE_GPUCLK_INPUT_FLAG_SCLK;
 
         result = cgs_atom_exec_cmd_table
@@ -355,12 +354,12 @@ int atomctrl_get_engine_pll_dividers_vi(
                 dividers->pll_post_divider =
                         pll_patameters.ulClock.ucPostDiv;
                 dividers->real_clock =
-                        pll_patameters.ulClock.ulClock;
+                        le32_to_cpu(pll_patameters.ulClock.ulClock);
 
                 dividers->ul_fb_div.ul_fb_div_frac =
-                        pll_patameters.ulFbDiv.usFbDivFrac;
+                        le16_to_cpu(pll_patameters.ulFbDiv.usFbDivFrac);
                 dividers->ul_fb_div.ul_fb_div =
-                        pll_patameters.ulFbDiv.usFbDiv;
+                        le16_to_cpu(pll_patameters.ulFbDiv.usFbDiv);
 
                 dividers->uc_pll_ref_div =
                         pll_patameters.ucPllRefDiv;
@@ -380,7 +379,7 @@ int atomctrl_get_engine_pll_dividers_ai(struct pp_hwmgr *hwmgr,
         COMPUTE_GPU_CLOCK_OUTPUT_PARAMETERS_V1_7 pll_patameters;
         int result;
 
-        pll_patameters.ulClock.ulClock = clock_value;
+        pll_patameters.ulClock.ulClock = cpu_to_le32(clock_value);
         pll_patameters.ulClock.ucPostDiv = COMPUTE_GPUCLK_INPUT_FLAG_SCLK;
 
         result = cgs_atom_exec_cmd_table
@@ -412,7 +411,7 @@ int atomctrl_get_dfs_pll_dividers_vi(
         COMPUTE_GPU_CLOCK_OUTPUT_PARAMETERS_V1_6 pll_patameters;
         int result;
 
-        pll_patameters.ulClock.ulClock = clock_value;
+        pll_patameters.ulClock.ulClock = cpu_to_le32(clock_value);
         pll_patameters.ulClock.ucPostDiv =
                 COMPUTE_GPUCLK_INPUT_FLAG_DEFAULT_GPUCLK;
 
@@ -425,12 +424,12 @@ int atomctrl_get_dfs_pll_dividers_vi(
                 dividers->pll_post_divider =
                         pll_patameters.ulClock.ucPostDiv;
                 dividers->real_clock =
-                        pll_patameters.ulClock.ulClock;
+                        le32_to_cpu(pll_patameters.ulClock.ulClock);
 
                 dividers->ul_fb_div.ul_fb_div_frac =
-                        pll_patameters.ulFbDiv.usFbDivFrac;
+                        le16_to_cpu(pll_patameters.ulFbDiv.usFbDivFrac);
                 dividers->ul_fb_div.ul_fb_div =
-                        pll_patameters.ulFbDiv.usFbDiv;
+                        le16_to_cpu(pll_patameters.ulFbDiv.usFbDiv);
 
                 dividers->uc_pll_ref_div =
                         pll_patameters.ucPllRefDiv;
@@ -519,13 +518,13 @@ int atomctrl_get_voltage_table_v3(
 
         for (i = 0; i < voltage_object->asGpioVoltageObj.ucGpioEntryNum; i++) {
                 voltage_table->entries[i].value =
-                        voltage_object->asGpioVoltageObj.asVolGpioLut[i].usVoltageValue;
+                        le16_to_cpu(voltage_object->asGpioVoltageObj.asVolGpioLut[i].usVoltageValue);
                 voltage_table->entries[i].smio_low =
-                        voltage_object->asGpioVoltageObj.asVolGpioLut[i].ulVoltageId;
+                        le32_to_cpu(voltage_object->asGpioVoltageObj.asVolGpioLut[i].ulVoltageId);
         }
 
         voltage_table->mask_low    =
-                voltage_object->asGpioVoltageObj.ulGpioMaskVal;
+                le32_to_cpu(voltage_object->asGpioVoltageObj.ulGpioMaskVal);
         voltage_table->count      =
                 voltage_object->asGpioVoltageObj.ucGpioEntryNum;
         voltage_table->phase_delay =
@@ -552,13 +551,13 @@ static bool atomctrl_lookup_gpio_pin(
                                 pin_assignment->ucGpioPinBitShift;
                         gpio_pin_assignment->us_gpio_pin_aindex =
                                 le16_to_cpu(pin_assignment->usGpioPin_AIndex);
-                        return false;
+                        return true;
                 }
 
                 offset += offsetof(ATOM_GPIO_PIN_ASSIGNMENT, ucGPIO_ID) + 1;
         }
 
-        return true;
+        return false;
 }
 
 /**
@@ -592,12 +591,12 @@ bool atomctrl_get_pp_assign_pin(
                 const uint32_t pinId,
                 pp_atomctrl_gpio_pin_assignment *gpio_pin_assignment)
 {
-        bool bRet = 0;
+        bool bRet = false;
         ATOM_GPIO_PIN_LUT *gpio_lookup_table =
                 get_gpio_lookup_table(hwmgr->device);
 
         PP_ASSERT_WITH_CODE((NULL != gpio_lookup_table),
-                        "Could not find GPIO lookup Table in BIOS.", return -1);
+                        "Could not find GPIO lookup Table in BIOS.", return false);
 
         bRet = atomctrl_lookup_gpio_pin(gpio_lookup_table, pinId,
                 gpio_pin_assignment);
@@ -650,8 +649,8 @@ int atomctrl_calculate_voltage_evv_on_sclk(
                 return -1;
 
         if (getASICProfilingInfo->asHeader.ucTableFormatRevision < 3 ||
-                        (getASICProfilingInfo->asHeader.ucTableFormatRevision == 3 &&
-                        getASICProfilingInfo->asHeader.ucTableContentRevision < 4))
+            (getASICProfilingInfo->asHeader.ucTableFormatRevision == 3 &&
+             getASICProfilingInfo->asHeader.ucTableContentRevision < 4))
                 return -1;
 
         /*-----------------------------------------------------------
@@ -662,37 +661,37 @@ int atomctrl_calculate_voltage_evv_on_sclk(
 
         switch (dpm_level) {
         case 1:
-                fPowerDPMx = Convert_ULONG_ToFraction(getASICProfilingInfo->usPowerDpm1);
-                fDerateTDP = GetScaledFraction(getASICProfilingInfo->ulTdpDerateDPM1, 1000);
+                fPowerDPMx = Convert_ULONG_ToFraction(le16_to_cpu(getASICProfilingInfo->usPowerDpm1));
+                fDerateTDP = GetScaledFraction(le32_to_cpu(getASICProfilingInfo->ulTdpDerateDPM1), 1000);
                 break;
         case 2:
-                fPowerDPMx = Convert_ULONG_ToFraction(getASICProfilingInfo->usPowerDpm2);
-                fDerateTDP = GetScaledFraction(getASICProfilingInfo->ulTdpDerateDPM2, 1000);
+                fPowerDPMx = Convert_ULONG_ToFraction(le16_to_cpu(getASICProfilingInfo->usPowerDpm2));
+                fDerateTDP = GetScaledFraction(le32_to_cpu(getASICProfilingInfo->ulTdpDerateDPM2), 1000);
                 break;
         case 3:
-                fPowerDPMx = Convert_ULONG_ToFraction(getASICProfilingInfo->usPowerDpm3);
-                fDerateTDP = GetScaledFraction(getASICProfilingInfo->ulTdpDerateDPM3, 1000);
+                fPowerDPMx = Convert_ULONG_ToFraction(le16_to_cpu(getASICProfilingInfo->usPowerDpm3));
+                fDerateTDP = GetScaledFraction(le32_to_cpu(getASICProfilingInfo->ulTdpDerateDPM3), 1000);
                 break;
         case 4:
-                fPowerDPMx = Convert_ULONG_ToFraction(getASICProfilingInfo->usPowerDpm4);
-                fDerateTDP = GetScaledFraction(getASICProfilingInfo->ulTdpDerateDPM4, 1000);
+                fPowerDPMx = Convert_ULONG_ToFraction(le16_to_cpu(getASICProfilingInfo->usPowerDpm4));
+                fDerateTDP = GetScaledFraction(le32_to_cpu(getASICProfilingInfo->ulTdpDerateDPM4), 1000);
                 break;
         case 5:
-                fPowerDPMx = Convert_ULONG_ToFraction(getASICProfilingInfo->usPowerDpm5);
-                fDerateTDP = GetScaledFraction(getASICProfilingInfo->ulTdpDerateDPM5, 1000);
+                fPowerDPMx = Convert_ULONG_ToFraction(le16_to_cpu(getASICProfilingInfo->usPowerDpm5));
+                fDerateTDP = GetScaledFraction(le32_to_cpu(getASICProfilingInfo->ulTdpDerateDPM5), 1000);
                 break;
         case 6:
-                fPowerDPMx = Convert_ULONG_ToFraction(getASICProfilingInfo->usPowerDpm6);
-                fDerateTDP = GetScaledFraction(getASICProfilingInfo->ulTdpDerateDPM6, 1000);
+                fPowerDPMx = Convert_ULONG_ToFraction(le16_to_cpu(getASICProfilingInfo->usPowerDpm6));
+                fDerateTDP = GetScaledFraction(le32_to_cpu(getASICProfilingInfo->ulTdpDerateDPM6), 1000);
                 break;
         case 7:
-                fPowerDPMx = Convert_ULONG_ToFraction(getASICProfilingInfo->usPowerDpm7);
-                fDerateTDP = GetScaledFraction(getASICProfilingInfo->ulTdpDerateDPM7, 1000);
+                fPowerDPMx = Convert_ULONG_ToFraction(le16_to_cpu(getASICProfilingInfo->usPowerDpm7));
+                fDerateTDP = GetScaledFraction(le32_to_cpu(getASICProfilingInfo->ulTdpDerateDPM7), 1000);
                 break;
         default:
                 printk(KERN_ERR "DPM Level not supported\n");
                 fPowerDPMx = Convert_ULONG_ToFraction(1);
-                fDerateTDP = GetScaledFraction(getASICProfilingInfo->ulTdpDerateDPM0, 1000);
+                fDerateTDP = GetScaledFraction(le32_to_cpu(getASICProfilingInfo->ulTdpDerateDPM0), 1000);
         }
 
         /*-------------------------
@@ -716,9 +715,9 @@ int atomctrl_calculate_voltage_evv_on_sclk(
                 return result;
 
         /* Finally, the actual fuse value */
-        ul_RO_fused = sOutput_FuseValues.ulEfuseValue;
-        fMin = GetScaledFraction(sRO_fuse.ulEfuseMin, 1);
-        fRange = GetScaledFraction(sRO_fuse.ulEfuseEncodeRange, 1);
+        ul_RO_fused = le32_to_cpu(sOutput_FuseValues.ulEfuseValue);
+        fMin = GetScaledFraction(le32_to_cpu(sRO_fuse.ulEfuseMin), 1);
+        fRange = GetScaledFraction(le32_to_cpu(sRO_fuse.ulEfuseEncodeRange), 1);
         fRO_fused = fDecodeLinearFuse(ul_RO_fused, fMin, fRange, sRO_fuse.ucEfuseLength);
 
         sCACm_fuse = getASICProfilingInfo->sCACm;
@@ -736,9 +735,9 @@ int atomctrl_calculate_voltage_evv_on_sclk(
         if (result)
                 return result;
 
-        ul_CACm_fused = sOutput_FuseValues.ulEfuseValue;
-        fMin = GetScaledFraction(sCACm_fuse.ulEfuseMin, 1000);
-        fRange = GetScaledFraction(sCACm_fuse.ulEfuseEncodeRange, 1000);
+        ul_CACm_fused = le32_to_cpu(sOutput_FuseValues.ulEfuseValue);
+        fMin = GetScaledFraction(le32_to_cpu(sCACm_fuse.ulEfuseMin), 1000);
+        fRange = GetScaledFraction(le32_to_cpu(sCACm_fuse.ulEfuseEncodeRange), 1000);
 
         fCACm_fused = fDecodeLinearFuse(ul_CACm_fused, fMin, fRange, sCACm_fuse.ucEfuseLength);
 
@@ -756,9 +755,9 @@ int atomctrl_calculate_voltage_evv_on_sclk(
         if (result)
                 return result;
 
-        ul_CACb_fused = sOutput_FuseValues.ulEfuseValue;
-        fMin = GetScaledFraction(sCACb_fuse.ulEfuseMin, 1000);
-        fRange = GetScaledFraction(sCACb_fuse.ulEfuseEncodeRange, 1000);
+        ul_CACb_fused = le32_to_cpu(sOutput_FuseValues.ulEfuseValue);
+        fMin = GetScaledFraction(le32_to_cpu(sCACb_fuse.ulEfuseMin), 1000);
+        fRange = GetScaledFraction(le32_to_cpu(sCACb_fuse.ulEfuseEncodeRange), 1000);
 
         fCACb_fused = fDecodeLinearFuse(ul_CACb_fused, fMin, fRange, sCACb_fuse.ucEfuseLength);
 
@@ -777,9 +776,9 @@ int atomctrl_calculate_voltage_evv_on_sclk(
         if (result)
                 return result;
 
-        ul_Kt_Beta_fused = sOutput_FuseValues.ulEfuseValue;
-        fAverage = GetScaledFraction(sKt_Beta_fuse.ulEfuseEncodeAverage, 1000);
-        fRange = GetScaledFraction(sKt_Beta_fuse.ulEfuseEncodeRange, 1000);
+        ul_Kt_Beta_fused = le32_to_cpu(sOutput_FuseValues.ulEfuseValue);
+        fAverage = GetScaledFraction(le32_to_cpu(sKt_Beta_fuse.ulEfuseEncodeAverage), 1000);
+        fRange = GetScaledFraction(le32_to_cpu(sKt_Beta_fuse.ulEfuseEncodeRange), 1000);
 
         fKt_Beta_fused = fDecodeLogisticFuse(ul_Kt_Beta_fused,
                         fAverage, fRange, sKt_Beta_fuse.ucEfuseLength);
@@ -798,9 +797,9 @@ int atomctrl_calculate_voltage_evv_on_sclk(
         if (result)
                 return result;
 
-        ul_Kv_m_fused = sOutput_FuseValues.ulEfuseValue;
-        fAverage = GetScaledFraction(sKv_m_fuse.ulEfuseEncodeAverage, 1000);
-        fRange = GetScaledFraction((sKv_m_fuse.ulEfuseEncodeRange & 0x7fffffff), 1000);
+        ul_Kv_m_fused = le32_to_cpu(sOutput_FuseValues.ulEfuseValue);
+        fAverage = GetScaledFraction(le32_to_cpu(sKv_m_fuse.ulEfuseEncodeAverage), 1000);
+        fRange = GetScaledFraction((le32_to_cpu(sKv_m_fuse.ulEfuseEncodeRange) & 0x7fffffff), 1000);
         fRange = fMultiply(fRange, ConvertToFraction(-1));
 
         fKv_m_fused = fDecodeLogisticFuse(ul_Kv_m_fused,
@@ -820,9 +819,9 @@ int atomctrl_calculate_voltage_evv_on_sclk(
         if (result)
                 return result;
 
-        ul_Kv_b_fused = sOutput_FuseValues.ulEfuseValue;
-        fAverage = GetScaledFraction(sKv_b_fuse.ulEfuseEncodeAverage, 1000);
-        fRange = GetScaledFraction(sKv_b_fuse.ulEfuseEncodeRange, 1000);
+        ul_Kv_b_fused = le32_to_cpu(sOutput_FuseValues.ulEfuseValue);
+        fAverage = GetScaledFraction(le32_to_cpu(sKv_b_fuse.ulEfuseEncodeAverage), 1000);
+        fRange = GetScaledFraction(le32_to_cpu(sKv_b_fuse.ulEfuseEncodeRange), 1000);
 
         fKv_b_fused = fDecodeLogisticFuse(ul_Kv_b_fused,
                         fAverage, fRange, sKv_b_fuse.ucEfuseLength);
@@ -851,9 +850,9 @@ int atomctrl_calculate_voltage_evv_on_sclk(
         if (result)
                 return result;
 
-        ul_FT_Lkg_V0NORM = sOutput_FuseValues.ulEfuseValue;
-        fLn_MaxDivMin = GetScaledFraction(getASICProfilingInfo->ulLkgEncodeLn_MaxDivMin, 10000);
-        fMin = GetScaledFraction(getASICProfilingInfo->ulLkgEncodeMin, 10000);
+        ul_FT_Lkg_V0NORM = le32_to_cpu(sOutput_FuseValues.ulEfuseValue);
+        fLn_MaxDivMin = GetScaledFraction(le32_to_cpu(getASICProfilingInfo->ulLkgEncodeLn_MaxDivMin), 10000);
+        fMin = GetScaledFraction(le32_to_cpu(getASICProfilingInfo->ulLkgEncodeMin), 10000);
 
         fFT_Lkg_V0NORM = fDecodeLeakageID(ul_FT_Lkg_V0NORM,
                         fLn_MaxDivMin, fMin, getASICProfilingInfo->ucLkgEfuseLength);
@@ -863,40 +862,40 @@ int atomctrl_calculate_voltage_evv_on_sclk(
          * PART 2 - Grabbing all required values
          *-------------------------------------------
          */
-        fSM_A0 = fMultiply(GetScaledFraction(getASICProfilingInfo->ulSM_A0, 1000000),
+        fSM_A0 = fMultiply(GetScaledFraction(le32_to_cpu(getASICProfilingInfo->ulSM_A0), 1000000),
                         ConvertToFraction(uPow(-1, getASICProfilingInfo->ucSM_A0_sign)));
-        fSM_A1 = fMultiply(GetScaledFraction(getASICProfilingInfo->ulSM_A1, 1000000),
+        fSM_A1 = fMultiply(GetScaledFraction(le32_to_cpu(getASICProfilingInfo->ulSM_A1), 1000000),
                         ConvertToFraction(uPow(-1, getASICProfilingInfo->ucSM_A1_sign)));
-        fSM_A2 = fMultiply(GetScaledFraction(getASICProfilingInfo->ulSM_A2, 100000),
+        fSM_A2 = fMultiply(GetScaledFraction(le32_to_cpu(getASICProfilingInfo->ulSM_A2), 100000),
                         ConvertToFraction(uPow(-1, getASICProfilingInfo->ucSM_A2_sign)));
-        fSM_A3 = fMultiply(GetScaledFraction(getASICProfilingInfo->ulSM_A3, 1000000),
+        fSM_A3 = fMultiply(GetScaledFraction(le32_to_cpu(getASICProfilingInfo->ulSM_A3), 1000000),
                         ConvertToFraction(uPow(-1, getASICProfilingInfo->ucSM_A3_sign)));
-        fSM_A4 = fMultiply(GetScaledFraction(getASICProfilingInfo->ulSM_A4, 1000000),
+        fSM_A4 = fMultiply(GetScaledFraction(le32_to_cpu(getASICProfilingInfo->ulSM_A4), 1000000),
                         ConvertToFraction(uPow(-1, getASICProfilingInfo->ucSM_A4_sign)));
-        fSM_A5 = fMultiply(GetScaledFraction(getASICProfilingInfo->ulSM_A5, 1000),
+        fSM_A5 = fMultiply(GetScaledFraction(le32_to_cpu(getASICProfilingInfo->ulSM_A5), 1000),
                         ConvertToFraction(uPow(-1, getASICProfilingInfo->ucSM_A5_sign)));
-        fSM_A6 = fMultiply(GetScaledFraction(getASICProfilingInfo->ulSM_A6, 1000),
+        fSM_A6 = fMultiply(GetScaledFraction(le32_to_cpu(getASICProfilingInfo->ulSM_A6), 1000),
                         ConvertToFraction(uPow(-1, getASICProfilingInfo->ucSM_A6_sign)));
-        fSM_A7 = fMultiply(GetScaledFraction(getASICProfilingInfo->ulSM_A7, 1000),
+        fSM_A7 = fMultiply(GetScaledFraction(le32_to_cpu(getASICProfilingInfo->ulSM_A7), 1000),
                         ConvertToFraction(uPow(-1, getASICProfilingInfo->ucSM_A7_sign)));
 
-        fMargin_RO_a = ConvertToFraction(getASICProfilingInfo->ulMargin_RO_a);
-        fMargin_RO_b = ConvertToFraction(getASICProfilingInfo->ulMargin_RO_b);
-        fMargin_RO_c = ConvertToFraction(getASICProfilingInfo->ulMargin_RO_c);
+        fMargin_RO_a = ConvertToFraction(le32_to_cpu(getASICProfilingInfo->ulMargin_RO_a));
+        fMargin_RO_b = ConvertToFraction(le32_to_cpu(getASICProfilingInfo->ulMargin_RO_b));
+        fMargin_RO_c = ConvertToFraction(le32_to_cpu(getASICProfilingInfo->ulMargin_RO_c));
 
-        fMargin_fixed = ConvertToFraction(getASICProfilingInfo->ulMargin_fixed);
+        fMargin_fixed = ConvertToFraction(le32_to_cpu(getASICProfilingInfo->ulMargin_fixed));
 
         fMargin_FMAX_mean = GetScaledFraction(
-                        getASICProfilingInfo->ulMargin_Fmax_mean, 10000);
+                le32_to_cpu(getASICProfilingInfo->ulMargin_Fmax_mean), 10000);
         fMargin_Plat_mean = GetScaledFraction(
-                        getASICProfilingInfo->ulMargin_plat_mean, 10000);
+                le32_to_cpu(getASICProfilingInfo->ulMargin_plat_mean), 10000);
         fMargin_FMAX_sigma = GetScaledFraction(
-                        getASICProfilingInfo->ulMargin_Fmax_sigma, 10000);
+                le32_to_cpu(getASICProfilingInfo->ulMargin_Fmax_sigma), 10000);
         fMargin_Plat_sigma = GetScaledFraction(
-                        getASICProfilingInfo->ulMargin_plat_sigma, 10000);
+                le32_to_cpu(getASICProfilingInfo->ulMargin_plat_sigma), 10000);
 
         fMargin_DC_sigma = GetScaledFraction(
-                        getASICProfilingInfo->ulMargin_DC_sigma, 100);
+                le32_to_cpu(getASICProfilingInfo->ulMargin_DC_sigma), 100);
         fMargin_DC_sigma = fDivide(fMargin_DC_sigma, ConvertToFraction(1000));
 
         fCACm_fused = fDivide(fCACm_fused, ConvertToFraction(100));
@@ -908,14 +907,14 @@ int atomctrl_calculate_voltage_evv_on_sclk(
         fSclk = GetScaledFraction(sclk, 100);
 
         fV_max = fDivide(GetScaledFraction(
-                        getASICProfilingInfo->ulMaxVddc, 1000), ConvertToFraction(4));
-        fT_prod = GetScaledFraction(getASICProfilingInfo->ulBoardCoreTemp, 10);
-        fLKG_Factor = GetScaledFraction(getASICProfilingInfo->ulEvvLkgFactor, 100);
-        fT_FT = GetScaledFraction(getASICProfilingInfo->ulLeakageTemp, 10);
+                                 le32_to_cpu(getASICProfilingInfo->ulMaxVddc), 1000), ConvertToFraction(4));
+        fT_prod = GetScaledFraction(le32_to_cpu(getASICProfilingInfo->ulBoardCoreTemp), 10);
+        fLKG_Factor = GetScaledFraction(le32_to_cpu(getASICProfilingInfo->ulEvvLkgFactor), 100);
+        fT_FT = GetScaledFraction(le32_to_cpu(getASICProfilingInfo->ulLeakageTemp), 10);
         fV_FT = fDivide(GetScaledFraction(
-                        getASICProfilingInfo->ulLeakageVoltage, 1000), ConvertToFraction(4));
+                                le32_to_cpu(getASICProfilingInfo->ulLeakageVoltage), 1000), ConvertToFraction(4));
         fV_min = fDivide(GetScaledFraction(
-                        getASICProfilingInfo->ulMinVddc, 1000), ConvertToFraction(4));
+                                 le32_to_cpu(getASICProfilingInfo->ulMinVddc), 1000), ConvertToFraction(4));
 
         /*-----------------------
          * PART 3
@@ -925,7 +924,7 @@ int atomctrl_calculate_voltage_evv_on_sclk(
         fA_Term = fAdd(fMargin_RO_a, fAdd(fMultiply(fSM_A4, fSclk), fSM_A5));
         fB_Term = fAdd(fAdd(fMultiply(fSM_A2, fSclk), fSM_A6), fMargin_RO_b);
         fC_Term = fAdd(fMargin_RO_c,
-                        fAdd(fMultiply(fSM_A0,fLkg_FT),
+                        fAdd(fMultiply(fSM_A0, fLkg_FT),
                         fAdd(fMultiply(fSM_A1, fMultiply(fLkg_FT, fSclk)),
                         fAdd(fMultiply(fSM_A3, fSclk),
                         fSubtract(fSM_A7, fRO_fused)))));
@@ -1063,9 +1062,55 @@ int atomctrl_get_voltage_evv_on_sclk(
         get_voltage_info_param_space.ucVoltageMode   =
                 ATOM_GET_VOLTAGE_EVV_VOLTAGE;
         get_voltage_info_param_space.usVoltageLevel  =
-                virtual_voltage_Id;
+                cpu_to_le16(virtual_voltage_Id);
         get_voltage_info_param_space.ulSCLKFreq      =
-                sclk;
+                cpu_to_le32(sclk);
+
+        result = cgs_atom_exec_cmd_table(hwmgr->device,
+                        GetIndexIntoMasterTable(COMMAND, GetVoltageInfo),
+                        &get_voltage_info_param_space);
+
+        if (0 != result)
+                return result;
+
+        *voltage = le16_to_cpu(((GET_EVV_VOLTAGE_INFO_OUTPUT_PARAMETER_V1_2 *)
+                                (&get_voltage_info_param_space))->usVoltageLevel);
+
+        return result;
+}
+
+/**
+ * atomctrl_get_voltage_evv gets voltage via call to ATOM COMMAND table.
+ * @param hwmgr input: pointer to hwManager
+ * @param virtual_voltage_id      input: voltage id which match per voltage DPM state: 0xff01, 0xff02.. 0xff08
+ * @param voltage                      output: real voltage level in unit of mv
+ */
+int atomctrl_get_voltage_evv(struct pp_hwmgr *hwmgr,
+                             uint16_t virtual_voltage_id,
+                             uint16_t *voltage)
+{
+        int result;
+        int entry_id;
+        GET_VOLTAGE_INFO_INPUT_PARAMETER_V1_2 get_voltage_info_param_space;
+
+        /* search for leakage voltage ID 0xff01 ~ 0xff08 and sckl */
+        for (entry_id = 0; entry_id < hwmgr->dyn_state.vddc_dependency_on_sclk->count; entry_id++) {
+                if (hwmgr->dyn_state.vddc_dependency_on_sclk->entries[entry_id].v == virtual_voltage_id) {
+                        /* found */
+                        break;
+                }
+        }
+
+        PP_ASSERT_WITH_CODE(entry_id < hwmgr->dyn_state.vddc_dependency_on_sclk->count,
+                "Can't find requested voltage id in vddc_dependency_on_sclk table!",
+                return -EINVAL;
+        );
+
+        get_voltage_info_param_space.ucVoltageType = VOLTAGE_TYPE_VDDC;
+        get_voltage_info_param_space.ucVoltageMode = ATOM_GET_VOLTAGE_EVV_VOLTAGE;
+        get_voltage_info_param_space.usVoltageLevel = virtual_voltage_id;
+        get_voltage_info_param_space.ulSCLKFreq =
+                cpu_to_le32(hwmgr->dyn_state.vddc_dependency_on_sclk->entries[entry_id].clk);
 
         result = cgs_atom_exec_cmd_table(hwmgr->device,
                         GetIndexIntoMasterTable(COMMAND, GetVoltageInfo),
@@ -1074,8 +1119,8 @@ int atomctrl_get_voltage_evv_on_sclk(
         if (0 != result)
                 return result;
 
-        *voltage = ((GET_EVV_VOLTAGE_INFO_OUTPUT_PARAMETER_V1_2 *)
-                        (&get_voltage_info_param_space))->usVoltageLevel;
+        *voltage = le16_to_cpu(((GET_EVV_VOLTAGE_INFO_OUTPUT_PARAMETER_V1_2 *)
+                                (&get_voltage_info_param_space))->usVoltageLevel);
 
         return result;
 }
@@ -1165,8 +1210,8 @@ static int asic_internal_ss_get_ss_asignment(struct pp_hwmgr *hwmgr,
 
         if (entry_found) {
                 ssEntry->speed_spectrum_percentage =
-                        ssInfo->usSpreadSpectrumPercentage;
-                ssEntry->speed_spectrum_rate = ssInfo->usSpreadRateInKhz;
+                        le16_to_cpu(ssInfo->usSpreadSpectrumPercentage);
+                ssEntry->speed_spectrum_rate = le16_to_cpu(ssInfo->usSpreadRateInKhz);
 
                 if (((GET_DATA_TABLE_MAJOR_REVISION(table) == 2) &&
                         (GET_DATA_TABLE_MINOR_REVISION(table) >= 2)) ||
@@ -1222,7 +1267,7 @@ int atomctrl_read_efuse(void *device, uint16_t start_index,
         int result;
         READ_EFUSE_VALUE_PARAMETER efuse_param;
 
-        efuse_param.sEfuse.usEfuseIndex = (start_index / 32) * 4;
+        efuse_param.sEfuse.usEfuseIndex = cpu_to_le16((start_index / 32) * 4);
         efuse_param.sEfuse.ucBitShift = (uint8_t)
                         (start_index - ((start_index / 32) * 32));
         efuse_param.sEfuse.ucBitLength  = (uint8_t)
@@ -1232,19 +1277,21 @@ int atomctrl_read_efuse(void *device, uint16_t start_index,
                         GetIndexIntoMasterTable(COMMAND, ReadEfuseValue),
                         &efuse_param);
         if (!result)
-                *efuse = efuse_param.ulEfuseValue & mask;
+                *efuse = le32_to_cpu(efuse_param.ulEfuseValue) & mask;
 
         return result;
 }
 
 int atomctrl_set_ac_timing_ai(struct pp_hwmgr *hwmgr, uint32_t memory_clock,
-                                                                uint8_t level)
+                              uint8_t level)
 {
         DYNAMICE_MEMORY_SETTINGS_PARAMETER_V2_1 memory_clock_parameters;
         int result;
 
-        memory_clock_parameters.asDPMMCReg.ulClock.ulClockFreq = memory_clock & SET_CLOCK_FREQ_MASK;
-        memory_clock_parameters.asDPMMCReg.ulClock.ulComputeClockFlag = ADJUST_MC_SETTING_PARAM;
+        memory_clock_parameters.asDPMMCReg.ulClock.ulClockFreq =
+                memory_clock & SET_CLOCK_FREQ_MASK;
+        memory_clock_parameters.asDPMMCReg.ulClock.ulComputeClockFlag =
+                ADJUST_MC_SETTING_PARAM;
         memory_clock_parameters.asDPMMCReg.ucMclkDPMState = level;
 
         result = cgs_atom_exec_cmd_table
@@ -1264,8 +1311,8 @@ int atomctrl_get_voltage_evv_on_sclk_ai(struct pp_hwmgr *hwmgr, uint8_t voltage_
 
         get_voltage_info_param_space.ucVoltageType = voltage_type;
         get_voltage_info_param_space.ucVoltageMode = ATOM_GET_VOLTAGE_EVV_VOLTAGE;
-        get_voltage_info_param_space.usVoltageLevel = virtual_voltage_Id;
-        get_voltage_info_param_space.ulSCLKFreq = sclk;
+        get_voltage_info_param_space.usVoltageLevel = cpu_to_le16(virtual_voltage_Id);
+        get_voltage_info_param_space.ulSCLKFreq = cpu_to_le32(sclk);
 
         result = cgs_atom_exec_cmd_table(hwmgr->device,
                         GetIndexIntoMasterTable(COMMAND, GetVoltageInfo),
@@ -1274,7 +1321,7 @@ int atomctrl_get_voltage_evv_on_sclk_ai(struct pp_hwmgr *hwmgr, uint8_t voltage_
         if (0 != result)
                 return result;
 
-        *voltage = ((GET_EVV_VOLTAGE_INFO_OUTPUT_PARAMETER_V1_3 *)(&get_voltage_info_param_space))->ulVoltageLevel;
+        *voltage = le32_to_cpu(((GET_EVV_VOLTAGE_INFO_OUTPUT_PARAMETER_V1_3 *)(&get_voltage_info_param_space))->ulVoltageLevel);
 
         return result;
 }
@@ -1295,15 +1342,19 @@ int atomctrl_get_smc_sclk_range_table(struct pp_hwmgr *hwmgr, struct pp_atom_ctr
         for (i = 0; i < psmu_info->ucSclkEntryNum; i++) {
                 table->entry[i].ucVco_setting = psmu_info->asSclkFcwRangeEntry[i].ucVco_setting;
                 table->entry[i].ucPostdiv = psmu_info->asSclkFcwRangeEntry[i].ucPostdiv;
-                table->entry[i].usFcw_pcc = psmu_info->asSclkFcwRangeEntry[i].ucFcw_pcc;
-                table->entry[i].usFcw_trans_upper = psmu_info->asSclkFcwRangeEntry[i].ucFcw_trans_upper;
-                table->entry[i].usRcw_trans_lower = psmu_info->asSclkFcwRangeEntry[i].ucRcw_trans_lower;
+                table->entry[i].usFcw_pcc =
+                        le16_to_cpu(psmu_info->asSclkFcwRangeEntry[i].ucFcw_pcc);
+                table->entry[i].usFcw_trans_upper =
+                        le16_to_cpu(psmu_info->asSclkFcwRangeEntry[i].ucFcw_trans_upper);
+                table->entry[i].usRcw_trans_lower =
+                        le16_to_cpu(psmu_info->asSclkFcwRangeEntry[i].ucRcw_trans_lower);
         }
 
         return 0;
 }
 
-int atomctrl_get_avfs_information(struct pp_hwmgr *hwmgr, struct pp_atom_ctrl__avfs_parameters *param)
+int atomctrl_get_avfs_information(struct pp_hwmgr *hwmgr,
+                                  struct pp_atom_ctrl__avfs_parameters *param)
 {
         ATOM_ASIC_PROFILING_INFO_V3_6 *profile = NULL;
 
@@ -1317,30 +1368,30 @@ int atomctrl_get_avfs_information(struct pp_hwmgr *hwmgr, struct pp_atom_ctrl__a
         if (!profile)
                 return -1;
 
-        param->ulAVFS_meanNsigma_Acontant0 = profile->ulAVFS_meanNsigma_Acontant0;
-        param->ulAVFS_meanNsigma_Acontant1 = profile->ulAVFS_meanNsigma_Acontant1;
-        param->ulAVFS_meanNsigma_Acontant2 = profile->ulAVFS_meanNsigma_Acontant2;
-        param->usAVFS_meanNsigma_DC_tol_sigma = profile->usAVFS_meanNsigma_DC_tol_sigma;
-        param->usAVFS_meanNsigma_Platform_mean = profile->usAVFS_meanNsigma_Platform_mean;
-        param->usAVFS_meanNsigma_Platform_sigma = profile->usAVFS_meanNsigma_Platform_sigma;
-        param->ulGB_VDROOP_TABLE_CKSOFF_a0 = profile->ulGB_VDROOP_TABLE_CKSOFF_a0;
-        param->ulGB_VDROOP_TABLE_CKSOFF_a1 = profile->ulGB_VDROOP_TABLE_CKSOFF_a1;
-        param->ulGB_VDROOP_TABLE_CKSOFF_a2 = profile->ulGB_VDROOP_TABLE_CKSOFF_a2;
-        param->ulGB_VDROOP_TABLE_CKSON_a0 = profile->ulGB_VDROOP_TABLE_CKSON_a0;
-        param->ulGB_VDROOP_TABLE_CKSON_a1 = profile->ulGB_VDROOP_TABLE_CKSON_a1;
-        param->ulGB_VDROOP_TABLE_CKSON_a2 = profile->ulGB_VDROOP_TABLE_CKSON_a2;
-        param->ulAVFSGB_FUSE_TABLE_CKSOFF_m1 = profile->ulAVFSGB_FUSE_TABLE_CKSOFF_m1;
-        param->usAVFSGB_FUSE_TABLE_CKSOFF_m2 = profile->usAVFSGB_FUSE_TABLE_CKSOFF_m2;
-        param->ulAVFSGB_FUSE_TABLE_CKSOFF_b = profile->ulAVFSGB_FUSE_TABLE_CKSOFF_b;
-        param->ulAVFSGB_FUSE_TABLE_CKSON_m1 = profile->ulAVFSGB_FUSE_TABLE_CKSON_m1;
-        param->usAVFSGB_FUSE_TABLE_CKSON_m2 = profile->usAVFSGB_FUSE_TABLE_CKSON_m2;
-        param->ulAVFSGB_FUSE_TABLE_CKSON_b = profile->ulAVFSGB_FUSE_TABLE_CKSON_b;
-        param->usMaxVoltage_0_25mv = profile->usMaxVoltage_0_25mv;
+        param->ulAVFS_meanNsigma_Acontant0 = le32_to_cpu(profile->ulAVFS_meanNsigma_Acontant0);
+        param->ulAVFS_meanNsigma_Acontant1 = le32_to_cpu(profile->ulAVFS_meanNsigma_Acontant1);
+        param->ulAVFS_meanNsigma_Acontant2 = le32_to_cpu(profile->ulAVFS_meanNsigma_Acontant2);
+        param->usAVFS_meanNsigma_DC_tol_sigma = le16_to_cpu(profile->usAVFS_meanNsigma_DC_tol_sigma);
+        param->usAVFS_meanNsigma_Platform_mean = le16_to_cpu(profile->usAVFS_meanNsigma_Platform_mean);
+        param->usAVFS_meanNsigma_Platform_sigma = le16_to_cpu(profile->usAVFS_meanNsigma_Platform_sigma);
+        param->ulGB_VDROOP_TABLE_CKSOFF_a0 = le32_to_cpu(profile->ulGB_VDROOP_TABLE_CKSOFF_a0);
+        param->ulGB_VDROOP_TABLE_CKSOFF_a1 = le32_to_cpu(profile->ulGB_VDROOP_TABLE_CKSOFF_a1);
+        param->ulGB_VDROOP_TABLE_CKSOFF_a2 = le32_to_cpu(profile->ulGB_VDROOP_TABLE_CKSOFF_a2);
+        param->ulGB_VDROOP_TABLE_CKSON_a0 = le32_to_cpu(profile->ulGB_VDROOP_TABLE_CKSON_a0);
+        param->ulGB_VDROOP_TABLE_CKSON_a1 = le32_to_cpu(profile->ulGB_VDROOP_TABLE_CKSON_a1);
+        param->ulGB_VDROOP_TABLE_CKSON_a2 = le32_to_cpu(profile->ulGB_VDROOP_TABLE_CKSON_a2);
+        param->ulAVFSGB_FUSE_TABLE_CKSOFF_m1 = le32_to_cpu(profile->ulAVFSGB_FUSE_TABLE_CKSOFF_m1);
+        param->usAVFSGB_FUSE_TABLE_CKSOFF_m2 = le16_to_cpu(profile->usAVFSGB_FUSE_TABLE_CKSOFF_m2);
+        param->ulAVFSGB_FUSE_TABLE_CKSOFF_b = le32_to_cpu(profile->ulAVFSGB_FUSE_TABLE_CKSOFF_b);
+        param->ulAVFSGB_FUSE_TABLE_CKSON_m1 = le32_to_cpu(profile->ulAVFSGB_FUSE_TABLE_CKSON_m1);
+        param->usAVFSGB_FUSE_TABLE_CKSON_m2 = le16_to_cpu(profile->usAVFSGB_FUSE_TABLE_CKSON_m2);
+        param->ulAVFSGB_FUSE_TABLE_CKSON_b = le32_to_cpu(profile->ulAVFSGB_FUSE_TABLE_CKSON_b);
+        param->usMaxVoltage_0_25mv = le16_to_cpu(profile->usMaxVoltage_0_25mv);
         param->ucEnableGB_VDROOP_TABLE_CKSOFF = profile->ucEnableGB_VDROOP_TABLE_CKSOFF;
         param->ucEnableGB_VDROOP_TABLE_CKSON = profile->ucEnableGB_VDROOP_TABLE_CKSON;
         param->ucEnableGB_FUSE_TABLE_CKSOFF = profile->ucEnableGB_FUSE_TABLE_CKSOFF;
         param->ucEnableGB_FUSE_TABLE_CKSON = profile->ucEnableGB_FUSE_TABLE_CKSON;
-        param->usPSM_Age_ComFactor = profile->usPSM_Age_ComFactor;
+        param->usPSM_Age_ComFactor = le16_to_cpu(profile->usPSM_Age_ComFactor);
         param->ucEnableApplyAVFS_CKS_OFF_Voltage = profile->ucEnableApplyAVFS_CKS_OFF_Voltage;
 
         return 0;
diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/ppatomctrl.h b/drivers/gpu/drm/amd/powerplay/hwmgr/ppatomctrl.h
index 1e35a9625baf..fc898afce002 100644
--- a/drivers/gpu/drm/amd/powerplay/hwmgr/ppatomctrl.h
+++ b/drivers/gpu/drm/amd/powerplay/hwmgr/ppatomctrl.h
@@ -281,6 +281,7 @@ struct pp_atom_ctrl__avfs_parameters {
 
 extern bool atomctrl_get_pp_assign_pin(struct pp_hwmgr *hwmgr, const uint32_t pinId, pp_atomctrl_gpio_pin_assignment *gpio_pin_assignment);
 extern int atomctrl_get_voltage_evv_on_sclk(struct pp_hwmgr *hwmgr, uint8_t voltage_type, uint32_t sclk, uint16_t virtual_voltage_Id, uint16_t *voltage);
+extern int atomctrl_get_voltage_evv(struct pp_hwmgr *hwmgr, uint16_t virtual_voltage_id, uint16_t *voltage);
 extern uint32_t atomctrl_get_mpll_reference_clock(struct pp_hwmgr *hwmgr);
 extern int atomctrl_get_memory_clock_spread_spectrum(struct pp_hwmgr *hwmgr, const uint32_t memory_clock, pp_atomctrl_internal_ss_info *ssInfo);
 extern int atomctrl_get_engine_clock_spread_spectrum(struct pp_hwmgr *hwmgr, const uint32_t engine_clock, pp_atomctrl_internal_ss_info *ssInfo);
diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/ppevvmath.h b/drivers/gpu/drm/amd/powerplay/hwmgr/ppevvmath.h
index 009bd5963ed8..8f50a038396c 100644
--- a/drivers/gpu/drm/amd/powerplay/hwmgr/ppevvmath.h
+++ b/drivers/gpu/drm/amd/powerplay/hwmgr/ppevvmath.h
@@ -50,55 +50,45 @@ typedef union _fInt {
  * Function Declarations
  *  -------------------------------------------------------------------------------
  */
-fInt ConvertToFraction(int);                       /* Use this to convert an INT to a FINT */
-fInt Convert_ULONG_ToFraction(uint32_t);              /* Use this to convert an uint32_t to a FINT */
-fInt GetScaledFraction(int, int);                  /* Use this to convert an INT to a FINT after scaling it by a factor */
-int ConvertBackToInteger(fInt);                    /* Convert a FINT back to an INT that is scaled by 1000 (i.e. last 3 digits are the decimal digits) */
-
-fInt fNegate(fInt);                                /* Returns -1 * input fInt value */
-fInt fAdd (fInt, fInt);                            /* Returns the sum of two fInt numbers */
-fInt fSubtract (fInt A, fInt B);                   /* Returns A-B - Sometimes easier than Adding negative numbers */
-fInt fMultiply (fInt, fInt);                       /* Returns the product of two fInt numbers */
-fInt fDivide (fInt A, fInt B);                     /* Returns A/B */
-fInt fGetSquare(fInt);                             /* Returns the square of a fInt number */
-fInt fSqrt(fInt);                                  /* Returns the Square Root of a fInt number */
-
-int uAbs(int);                                     /* Returns the Absolute value of the Int */
-fInt fAbs(fInt);                                   /* Returns the Absolute value of the fInt */
-int uPow(int base, int exponent);                  /* Returns base^exponent an INT */
-
-void SolveQuadracticEqn(fInt, fInt, fInt, fInt[]); /* Returns the 2 roots via the array */
-bool Equal(fInt, fInt);                         /* Returns true if two fInts are equal to each other */
-bool GreaterThan(fInt A, fInt B);               /* Returns true if A > B */
-
-fInt fExponential(fInt exponent);                  /* Can be used to calculate e^exponent */
-fInt fNaturalLog(fInt value);                      /* Can be used to calculate ln(value) */
+static fInt ConvertToFraction(int);                       /* Use this to convert an INT to a FINT */
+static fInt Convert_ULONG_ToFraction(uint32_t);           /* Use this to convert an uint32_t to a FINT */
+static fInt GetScaledFraction(int, int);                  /* Use this to convert an INT to a FINT after scaling it by a factor */
+static int ConvertBackToInteger(fInt);                    /* Convert a FINT back to an INT that is scaled by 1000 (i.e. last 3 digits are the decimal digits) */
+
+static fInt fNegate(fInt);                                /* Returns -1 * input fInt value */
+static fInt fAdd (fInt, fInt);                            /* Returns the sum of two fInt numbers */
+static fInt fSubtract (fInt A, fInt B);                   /* Returns A-B - Sometimes easier than Adding negative numbers */
+static fInt fMultiply (fInt, fInt);                       /* Returns the product of two fInt numbers */
+static fInt fDivide (fInt A, fInt B);                     /* Returns A/B */
+static fInt fGetSquare(fInt);                             /* Returns the square of a fInt number */
+static fInt fSqrt(fInt);                                  /* Returns the Square Root of a fInt number */
+
+static int uAbs(int);                                     /* Returns the Absolute value of the Int */
+static int uPow(int base, int exponent);                  /* Returns base^exponent an INT */
+
+static void SolveQuadracticEqn(fInt, fInt, fInt, fInt[]); /* Returns the 2 roots via the array */
+static bool Equal(fInt, fInt);                            /* Returns true if two fInts are equal to each other */
+static bool GreaterThan(fInt A, fInt B);                  /* Returns true if A > B */
+
+static fInt fExponential(fInt exponent);                  /* Can be used to calculate e^exponent */
+static fInt fNaturalLog(fInt value);                      /* Can be used to calculate ln(value) */
 
 /* Fuse decoding functions
  * -------------------------------------------------------------------------------------
  */
-fInt fDecodeLinearFuse(uint32_t fuse_value, fInt f_min, fInt f_range, uint32_t bitlength);
-fInt fDecodeLogisticFuse(uint32_t fuse_value, fInt f_average, fInt f_range, uint32_t bitlength);
-fInt fDecodeLeakageID (uint32_t leakageID_fuse, fInt ln_max_div_min, fInt f_min, uint32_t bitlength);
+static fInt fDecodeLinearFuse(uint32_t fuse_value, fInt f_min, fInt f_range, uint32_t bitlength);
+static fInt fDecodeLogisticFuse(uint32_t fuse_value, fInt f_average, fInt f_range, uint32_t bitlength);
+static fInt fDecodeLeakageID (uint32_t leakageID_fuse, fInt ln_max_div_min, fInt f_min, uint32_t bitlength);
 
 /* Internal Support Functions - Use these ONLY for testing or adding to internal functions
  * -------------------------------------------------------------------------------------
  * Some of the following functions take two INTs as their input - This is unsafe for a variety of reasons.
  */
-fInt Add (int, int);                               /* Add two INTs and return Sum as FINT */
-fInt Multiply (int, int);                          /* Multiply two INTs and return Product as FINT */
-fInt Divide (int, int);                            /* You get the idea... */
-fInt fNegate(fInt);
+static fInt Divide (int, int);                            /* Divide two INTs and return result as FINT */
+static fInt fNegate(fInt);
 
-int uGetScaledDecimal (fInt);                      /* Internal function */
-int GetReal (fInt A);                              /* Internal function */
-
-/* Future Additions and Incomplete Functions
- * -------------------------------------------------------------------------------------
- */
-int GetRoundedValue(fInt);                         /* Incomplete function - Useful only when Precision is lacking */
-                                                   /* Let us say we have 2.126 but can only handle 2 decimal points. We could */
-                                                   /* either chop of 6 and keep 2.12 or use this function to get 2.13, which is more accurate */
+static int uGetScaledDecimal (fInt);                      /* Internal function */
+static int GetReal (fInt A);                              /* Internal function */
 
 /* -------------------------------------------------------------------------------------
  * TROUBLESHOOTING INFORMATION
@@ -115,7 +105,7 @@ int GetRoundedValue(fInt);                         /* Incomplete function - Usef
  * START OF CODE
  * -------------------------------------------------------------------------------------
  */
-fInt fExponential(fInt exponent)        /*Can be used to calculate e^exponent*/
+static fInt fExponential(fInt exponent)        /*Can be used to calculate e^exponent*/
 {
         uint32_t i;
         bool bNegated = false;
@@ -154,7 +144,7 @@ fInt fExponential(fInt exponent)        /*Can be used to calculate e^exponent*/
         return solution;
 }
 
-fInt fNaturalLog(fInt value)
+static fInt fNaturalLog(fInt value)
 {
         uint32_t i;
         fInt upper_bound = Divide(8, 1000);
@@ -179,7 +169,7 @@ fInt fNaturalLog(fInt value)
         return (fAdd(solution, error_term));
 }
 
-fInt fDecodeLinearFuse(uint32_t fuse_value, fInt f_min, fInt f_range, uint32_t bitlength)
+static fInt fDecodeLinearFuse(uint32_t fuse_value, fInt f_min, fInt f_range, uint32_t bitlength)
 {
         fInt f_fuse_value = Convert_ULONG_ToFraction(fuse_value);
         fInt f_bit_max_value = Convert_ULONG_ToFraction((uPow(2, bitlength)) - 1);
@@ -194,7 +184,7 @@ fInt fDecodeLinearFuse(uint32_t fuse_value, fInt f_min, fInt f_range, uint32_t b
 }
 
 
-fInt fDecodeLogisticFuse(uint32_t fuse_value, fInt f_average, fInt f_range, uint32_t bitlength)
+static fInt fDecodeLogisticFuse(uint32_t fuse_value, fInt f_average, fInt f_range, uint32_t bitlength)
 {
         fInt f_fuse_value = Convert_ULONG_ToFraction(fuse_value);
         fInt f_bit_max_value = Convert_ULONG_ToFraction((uPow(2, bitlength)) - 1);
@@ -212,7 +202,7 @@ fInt fDecodeLogisticFuse(uint32_t fuse_value, fInt f_average, fInt f_range, uint
         return f_decoded_value;
 }
 
-fInt fDecodeLeakageID (uint32_t leakageID_fuse, fInt ln_max_div_min, fInt f_min, uint32_t bitlength)
+static fInt fDecodeLeakageID (uint32_t leakageID_fuse, fInt ln_max_div_min, fInt f_min, uint32_t bitlength)
 {
         fInt fLeakage;
         fInt f_bit_max_value = Convert_ULONG_ToFraction((uPow(2, bitlength)) - 1);
@@ -225,7 +215,7 @@ fInt fDecodeLeakageID (uint32_t leakageID_fuse, fInt ln_max_div_min, fInt f_min,
         return fLeakage;
 }
 
-fInt ConvertToFraction(int X) /*Add all range checking here. Is it possible to make fInt a private declaration? */
+static fInt ConvertToFraction(int X) /*Add all range checking here. Is it possible to make fInt a private declaration? */
 {
         fInt temp;
 
@@ -237,13 +227,13 @@ fInt ConvertToFraction(int X) /*Add all range checking here. Is it possible to m
         return temp;
 }
 
-fInt fNegate(fInt X)
+static fInt fNegate(fInt X)
 {
         fInt CONSTANT_NEGONE = ConvertToFraction(-1);
         return (fMultiply(X, CONSTANT_NEGONE));
 }
 
-fInt Convert_ULONG_ToFraction(uint32_t X)
+static fInt Convert_ULONG_ToFraction(uint32_t X)
 {
         fInt temp;
 
@@ -255,7 +245,7 @@ fInt Convert_ULONG_ToFraction(uint32_t X)
         return temp;
 }
 
-fInt GetScaledFraction(int X, int factor)
+static fInt GetScaledFraction(int X, int factor)
 {
         int times_shifted, factor_shifted;
         bool bNEGATED;
@@ -304,7 +294,7 @@ fInt GetScaledFraction(int X, int factor)
 }
 
 /* Addition using two fInts */
-fInt fAdd (fInt X, fInt Y)
+static fInt fAdd (fInt X, fInt Y)
 {
         fInt Sum;
 
@@ -314,7 +304,7 @@ fInt fAdd (fInt X, fInt Y)
 }
 
 /* Addition using two fInts */
-fInt fSubtract (fInt X, fInt Y)
+static fInt fSubtract (fInt X, fInt Y)
 {
         fInt Difference;
 
@@ -323,7 +313,7 @@ fInt fSubtract (fInt X, fInt Y)
         return Difference;
 }
 
-bool Equal(fInt A, fInt B)
+static bool Equal(fInt A, fInt B)
 {
         if (A.full == B.full)
                 return true;
@@ -331,7 +321,7 @@ bool Equal(fInt A, fInt B)
                 return false;
 }
 
-bool GreaterThan(fInt A, fInt B)
+static bool GreaterThan(fInt A, fInt B)
 {
         if (A.full > B.full)
                 return true;
@@ -339,7 +329,7 @@ bool GreaterThan(fInt A, fInt B)
                 return false;
 }
 
-fInt fMultiply (fInt X, fInt Y) /* Uses 64-bit integers (int64_t) */
+static fInt fMultiply (fInt X, fInt Y) /* Uses 64-bit integers (int64_t) */
 {
         fInt Product;
         int64_t tempProduct;
@@ -363,7 +353,7 @@ fInt fMultiply (fInt X, fInt Y) /* Uses 64-bit integers (int64_t) */
         return Product;
 }
 
-fInt fDivide (fInt X, fInt Y)
+static fInt fDivide (fInt X, fInt Y)
 {
         fInt fZERO, fQuotient;
         int64_t longlongX, longlongY;
@@ -384,7 +374,7 @@ fInt fDivide (fInt X, fInt Y)
         return fQuotient;
 }
 
-int ConvertBackToInteger (fInt A) /*THIS is the function that will be used to check with the Golden settings table*/
+static int ConvertBackToInteger (fInt A) /*THIS is the function that will be used to check with the Golden settings table*/
 {
         fInt fullNumber, scaledDecimal, scaledReal;
 
@@ -397,13 +387,13 @@ int ConvertBackToInteger (fInt A) /*THIS is the function that will be used to ch
         return fullNumber.full;
 }
 
-fInt fGetSquare(fInt A)
+static fInt fGetSquare(fInt A)
 {
         return fMultiply(A,A);
 }
 
 /* x_new = x_old - (x_old^2 - C) / (2 * x_old) */
-fInt fSqrt(fInt num)
+static fInt fSqrt(fInt num)
 {
         fInt F_divide_Fprime, Fprime;
         fInt test;
@@ -460,7 +450,7 @@ fInt fSqrt(fInt num)
         return (x_new);
 }
 
-void SolveQuadracticEqn(fInt A, fInt B, fInt C, fInt Roots[])
+static void SolveQuadracticEqn(fInt A, fInt B, fInt C, fInt Roots[])
 {
         fInt *pRoots = &Roots[0];
         fInt temp, root_first, root_second;
@@ -498,52 +488,13 @@ void SolveQuadracticEqn(fInt A, fInt B, fInt C, fInt Roots[])
  * -----------------------------------------------------------------------------
  */
 
-/* Addition using two normal ints - Temporary - Use only for testing purposes?. */
-fInt Add (int X, int Y)
-{
-        fInt A, B, Sum;
-
-        A.full = (X << SHIFT_AMOUNT);
-        B.full = (Y << SHIFT_AMOUNT);
-
-        Sum.full = A.full + B.full;
-
-        return Sum;
-}
-
 /* Conversion Functions */
-int GetReal (fInt A)
+static int GetReal (fInt A)
 {
         return (A.full >> SHIFT_AMOUNT);
 }
 
-/* Temporarily Disabled */
-int GetRoundedValue(fInt A) /*For now, round the 3rd decimal place */
-{
-        /* ROUNDING TEMPORARLY DISABLED
-        int temp = A.full;
-        int decimal_cutoff, decimal_mask = 0x000001FF;
-        decimal_cutoff = temp & decimal_mask;
-        if (decimal_cutoff > 0x147) {
-                temp += 673;
-        }*/
-
-        return ConvertBackToInteger(A)/10000; /*Temporary - in case this was used somewhere else */
-}
-
-fInt Multiply (int X, int Y)
-{
-        fInt A, B, Product;
-
-        A.full = X << SHIFT_AMOUNT;
-        B.full = Y << SHIFT_AMOUNT;
-
-        Product = fMultiply(A, B);
-
-        return Product;
-}
-
-fInt Divide (int X, int Y)
+static fInt Divide (int X, int Y)
 {
         fInt A, B, Quotient;
 
@@ -555,7 +506,7 @@ fInt Divide (int X, int Y)
         return Quotient;
 }
 
-int uGetScaledDecimal (fInt A) /*Converts the fractional portion to whole integers - Costly function */
+static int uGetScaledDecimal (fInt A) /*Converts the fractional portion to whole integers - Costly function */
 {
         int dec[PRECISION];
         int i, scaledDecimal = 0, tmp = A.partial.decimal;
@@ -570,7 +521,7 @@ int uGetScaledDecimal (fInt A) /*Converts the fractional portion to whole intege
         return scaledDecimal;
 }
 
-int uPow(int base, int power)
+static int uPow(int base, int power)
 {
         if (power == 0)
                 return 1;
@@ -578,15 +529,7 @@ int uPow(int base, int power)
                 return (base)*uPow(base, power - 1);
 }
 
-fInt fAbs(fInt A)
-{
-        if (A.partial.real < 0)
-                return (fMultiply(A, ConvertToFraction(-1)));
-        else
-                return A;
-}
-
-int uAbs(int X)
+static int uAbs(int X)
 {
         if (X < 0)
                 return (X * -1);
@@ -594,7 +537,7 @@ int uAbs(int X)
                 return X;
 }
 
-fInt fRoundUpByStepSize(fInt A, fInt fStepSize, bool error_term)
+static fInt fRoundUpByStepSize(fInt A, fInt fStepSize, bool error_term)
 {
         fInt solution;
 
diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/processpptables.c b/drivers/gpu/drm/amd/powerplay/hwmgr/processpptables.c
index 2f1a14fe05b1..6c321b0d8a1e 100644
--- a/drivers/gpu/drm/amd/powerplay/hwmgr/processpptables.c
+++ b/drivers/gpu/drm/amd/powerplay/hwmgr/processpptables.c
@@ -794,19 +794,35 @@ static const ATOM_PPLIB_STATE_V2 *get_state_entry_v2(
 static const ATOM_PPLIB_POWERPLAYTABLE *get_powerplay_table(
                                      struct pp_hwmgr *hwmgr)
 {
-        const void *table_addr = NULL;
+        const void *table_addr = hwmgr->soft_pp_table;
         uint8_t frev, crev;
         uint16_t size;
 
-        table_addr = cgs_atom_get_data_table(hwmgr->device,
-                        GetIndexIntoMasterTable(DATA, PowerPlayInfo),
-                        &size, &frev, &crev);
+        if (!table_addr) {
+                table_addr = cgs_atom_get_data_table(hwmgr->device,
+                                GetIndexIntoMasterTable(DATA, PowerPlayInfo),
+                                &size, &frev, &crev);
 
-        hwmgr->soft_pp_table = table_addr;
+                hwmgr->soft_pp_table = table_addr;
+                hwmgr->soft_pp_table_size = size;
+        }
 
         return (const ATOM_PPLIB_POWERPLAYTABLE *)table_addr;
 }
 
+int pp_tables_get_response_times(struct pp_hwmgr *hwmgr,
+                                uint32_t *vol_rep_time, uint32_t *bb_rep_time)
+{
+        const ATOM_PPLIB_POWERPLAYTABLE *powerplay_tab = get_powerplay_table(hwmgr);
+
+        PP_ASSERT_WITH_CODE(NULL != powerplay_tab,
+                            "Missing PowerPlay Table!", return -EINVAL);
+
+        *vol_rep_time = (uint32_t)le16_to_cpu(powerplay_tab->usVoltageTime);
+        *bb_rep_time = (uint32_t)le16_to_cpu(powerplay_tab->usBackbiasTime);
+
+        return 0;
+}
 
 int pp_tables_get_num_of_entries(struct pp_hwmgr *hwmgr,
                                      unsigned long *num_of_entries)
@@ -1499,7 +1515,7 @@ int get_number_of_vce_state_table_entries(
         const ATOM_PPLIB_VCE_State_Table *vce_table =
                                     get_vce_state_table(hwmgr, table);
 
-        if (vce_table > 0)
+        if (vce_table)
                 return vce_table->numEntries;
 
         return 0;
@@ -1589,11 +1605,6 @@ static int pp_tables_initialize(struct pp_hwmgr *hwmgr)
 
 static int pp_tables_uninitialize(struct pp_hwmgr *hwmgr)
 {
-        if (NULL != hwmgr->soft_pp_table) {
-                kfree(hwmgr->soft_pp_table);
-                hwmgr->soft_pp_table = NULL;
-        }
-
         if (NULL != hwmgr->dyn_state.vddc_dependency_on_sclk) {
                 kfree(hwmgr->dyn_state.vddc_dependency_on_sclk);
                 hwmgr->dyn_state.vddc_dependency_on_sclk = NULL;
diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/processpptables.h b/drivers/gpu/drm/amd/powerplay/hwmgr/processpptables.h
index 30434802417e..baddaa75693b 100644
--- a/drivers/gpu/drm/amd/powerplay/hwmgr/processpptables.h
+++ b/drivers/gpu/drm/amd/powerplay/hwmgr/processpptables.h
@@ -32,16 +32,19 @@ struct pp_hw_power_state;
 extern const struct pp_table_func pptable_funcs;
 
 typedef int (*pp_tables_hw_clock_info_callback)(struct pp_hwmgr *hwmgr,
-                                        struct pp_hw_power_state *hw_ps,
-                                                        unsigned int index,
-                                                 const void *clock_info);
+                                                struct pp_hw_power_state *hw_ps,
+                                                unsigned int index,
+                                                const void *clock_info);
 
 int pp_tables_get_num_of_entries(struct pp_hwmgr *hwmgr,
-                                     unsigned long *num_of_entries);
+                                 unsigned long *num_of_entries);
 
 int pp_tables_get_entry(struct pp_hwmgr *hwmgr,
-                                                unsigned long entry_index,
-                                                struct pp_power_state *ps,
-                                pp_tables_hw_clock_info_callback func);
+                        unsigned long entry_index,
+                        struct pp_power_state *ps,
+                        pp_tables_hw_clock_info_callback func);
+
+int pp_tables_get_response_times(struct pp_hwmgr *hwmgr,
+                                 uint32_t *vol_rep_time, uint32_t *bb_rep_time);
 
 #endif
diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/tonga_hwmgr.c b/drivers/gpu/drm/amd/powerplay/hwmgr/tonga_hwmgr.c
index 5d0f655bf160..c7dc111221c2 100644
--- a/drivers/gpu/drm/amd/powerplay/hwmgr/tonga_hwmgr.c
+++ b/drivers/gpu/drm/amd/powerplay/hwmgr/tonga_hwmgr.c
@@ -571,7 +571,7 @@ int tonga_disable_sclk_mclk_dpm(struct pp_hwmgr *hwmgr)
         if (0 == data->sclk_dpm_key_disabled) {
                 /* Checking if DPM is running.  If we discover hang because of this, we should skip this message.*/
                 PP_ASSERT_WITH_CODE(
-                                (0 == tonga_is_dpm_running(hwmgr)),
+                                !tonga_is_dpm_running(hwmgr),
                                 "Trying to Disable SCLK DPM when DPM is disabled",
                                 return -1
                                 );
@@ -587,7 +587,7 @@ int tonga_disable_sclk_mclk_dpm(struct pp_hwmgr *hwmgr)
         if (0 == data->mclk_dpm_key_disabled) {
                 /* Checking if DPM is running.  If we discover hang because of this, we should skip this message. */
                 PP_ASSERT_WITH_CODE(
-                                (0 == tonga_is_dpm_running(hwmgr)),
+                                !tonga_is_dpm_running(hwmgr),
                                 "Trying to Disable MCLK DPM when DPM is disabled",
                                 return -1
                                 );
@@ -614,7 +614,7 @@ int tonga_stop_dpm(struct pp_hwmgr *hwmgr)
         if (0 == data->pcie_dpm_key_disabled) {
                 /* Checking if DPM is running.  If we discover hang because of this, we should skip this message.*/
                 PP_ASSERT_WITH_CODE(
-                                (0 == tonga_is_dpm_running(hwmgr)),
+                                !tonga_is_dpm_running(hwmgr),
                                 "Trying to Disable PCIE DPM when DPM is disabled",
                                 return -1
                                 );
@@ -630,7 +630,7 @@ int tonga_stop_dpm(struct pp_hwmgr *hwmgr)
 
         /* Checking if DPM is running.  If we discover hang because of this, we should skip this message.*/
         PP_ASSERT_WITH_CODE(
-                        (0 == tonga_is_dpm_running(hwmgr)),
+                        !tonga_is_dpm_running(hwmgr),
                         "Trying to Disable Voltage CNTL when DPM is disabled",
                         return -1
                         );
@@ -688,8 +688,9 @@ int tonga_dpm_force_state(struct pp_hwmgr *hwmgr, uint32_t n)
         uint32_t level_mask = 1 << n;
 
         /* Checking if DPM is running.  If we discover hang because of this, we should skip this message. */
-        PP_ASSERT_WITH_CODE(0 == tonga_is_dpm_running(hwmgr),
-                        "Trying to force SCLK when DPM is disabled", return -1;);
+        PP_ASSERT_WITH_CODE(!tonga_is_dpm_running(hwmgr),
+                            "Trying to force SCLK when DPM is disabled",
+                            return -1;);
         if (0 == data->sclk_dpm_key_disabled)
                 return (0 == smum_send_msg_to_smc_with_parameter(
                                                              hwmgr->smumgr,
@@ -712,8 +713,9 @@ int tonga_dpm_force_state_mclk(struct pp_hwmgr *hwmgr, uint32_t n)
         uint32_t level_mask = 1 << n;
 
         /* Checking if DPM is running.  If we discover hang because of this, we should skip this message. */
-        PP_ASSERT_WITH_CODE(0 == tonga_is_dpm_running(hwmgr),
-                        "Trying to Force MCLK when DPM is disabled", return -1;);
+        PP_ASSERT_WITH_CODE(!tonga_is_dpm_running(hwmgr),
+                            "Trying to Force MCLK when DPM is disabled",
+                            return -1;);
         if (0 == data->mclk_dpm_key_disabled)
                 return (0 == smum_send_msg_to_smc_with_parameter(
                                                                 hwmgr->smumgr,
@@ -735,8 +737,9 @@ int tonga_dpm_force_state_pcie(struct pp_hwmgr *hwmgr, uint32_t n)
         tonga_hwmgr *data = (tonga_hwmgr *)(hwmgr->backend);
 
         /* Checking if DPM is running.  If we discover hang because of this, we should skip this message.*/
-        PP_ASSERT_WITH_CODE(0 == tonga_is_dpm_running(hwmgr),
-                        "Trying to Force PCIE level when DPM is disabled", return -1;);
+        PP_ASSERT_WITH_CODE(!tonga_is_dpm_running(hwmgr),
+                            "Trying to Force PCIE level when DPM is disabled",
+                            return -1;);
         if (0 == data->pcie_dpm_key_disabled)
                 return (0 == smum_send_msg_to_smc_with_parameter(
                                                              hwmgr->smumgr,
@@ -774,7 +777,7 @@ int tonga_process_firmware_header(struct pp_hwmgr *hwmgr)
 
         uint32_t tmp;
         int result;
-        bool error = 0;
+        bool error = false;
 
         result = tonga_read_smc_sram_dword(hwmgr->smumgr,
                                 SMU72_FIRMWARE_HEADER_LOCATION +
@@ -933,11 +936,11 @@ int tonga_init_power_gate_state(struct pp_hwmgr *hwmgr)
 {
         tonga_hwmgr *data = (tonga_hwmgr *)(hwmgr->backend);
 
-        data->uvd_power_gated = 0;
-        data->vce_power_gated = 0;
-        data->samu_power_gated = 0;
-        data->acp_power_gated = 0;
-        data->pg_acp_init = 1;
+        data->uvd_power_gated = false;
+        data->vce_power_gated = false;
+        data->samu_power_gated = false;
+        data->acp_power_gated = false;
+        data->pg_acp_init = true;
 
         return 0;
 }
@@ -955,7 +958,7 @@ int tonga_check_for_dpm_running(struct pp_hwmgr *hwmgr)
          * because we may have test scenarios that need us intentionly disable SCLK/MCLK DPM,
          * whereas voltage control is a fundemental change that will not be disabled
          */
-        return (0 == tonga_is_dpm_running(hwmgr) ? 0 : 1);
+        return (!tonga_is_dpm_running(hwmgr) ? 0 : 1);
 }
 
 /**
@@ -968,7 +971,7 @@ int tonga_check_for_dpm_stopped(struct pp_hwmgr *hwmgr)
 {
         tonga_hwmgr *data = (tonga_hwmgr *)(hwmgr->backend);
 
-        if (0 != tonga_is_dpm_running(hwmgr)) {
+        if (tonga_is_dpm_running(hwmgr)) {
                 /* If HW Virtualization is enabled, dpm_table_start will not have a valid value */
                 if (!data->dpm_table_start) {
                         return 1;
@@ -991,7 +994,7 @@ static int tonga_trim_voltage_table(struct pp_hwmgr *hwmgr,
 {
         uint32_t table_size, i, j;
         uint16_t vvalue;
-        bool bVoltageFound = 0;
+        bool bVoltageFound = false;
         pp_atomctrl_voltage_table *table;
 
         PP_ASSERT_WITH_CODE((NULL != voltage_table), "Voltage Table empty.", return -1;);
@@ -1007,11 +1010,11 @@ static int tonga_trim_voltage_table(struct pp_hwmgr *hwmgr,
 
         for (i = 0; i < voltage_table->count; i++) {
                 vvalue = voltage_table->entries[i].value;
-                bVoltageFound = 0;
+                bVoltageFound = false;
 
                 for (j = 0; j < table->count; j++) {
                         if (vvalue == table->entries[j].value) {
-                                bVoltageFound = 1;
+                                bVoltageFound = true;
                                 break;
                         }
                 }
@@ -1331,7 +1334,6 @@ static int tonga_populate_cac_tables(struct pp_hwmgr *hwmgr,
 {
         uint32_t count;
         uint8_t index;
-        int result = 0;
         tonga_hwmgr *data = (tonga_hwmgr *)(hwmgr->backend);
         struct phm_ppt_v1_information *pptable_info = (struct phm_ppt_v1_information *)(hwmgr->pptable);
         struct phm_ppt_v1_voltage_lookup_table *vddgfx_lookup_table = pptable_info->vddgfx_lookup_table;
@@ -1378,7 +1380,7 @@ static int tonga_populate_cac_tables(struct pp_hwmgr *hwmgr,
                 }
         }
 
-        return result;
+        return 0;
 }
 
 
@@ -2042,7 +2044,7 @@ static int tonga_populate_single_memory_level(
 
         if ((data->mclk_stutter_mode_threshold != 0) &&
             (memory_clock <= data->mclk_stutter_mode_threshold) &&
-            (data->is_uvd_enabled == 0)
+            (!data->is_uvd_enabled)
             && (PHM_READ_FIELD(hwmgr->device, DPG_PIPE_STUTTER_CONTROL, STUTTER_ENABLE) & 0x1)
             && (data->display_timing.num_existing_displays <= 2)
             && (data->display_timing.num_existing_displays != 0))
@@ -2705,7 +2707,7 @@ static int tonga_reset_single_dpm_table(
 
         dpm_table->count = count;
         for (i = 0; i < MAX_REGULAR_DPM_NUMBER; i++) {
-                dpm_table->dpm_levels[i].enabled = 0;
+                dpm_table->dpm_levels[i].enabled = false;
         }
 
         return 0;
@@ -2718,7 +2720,7 @@ static void tonga_setup_pcie_table_entry(
 {
         dpm_table->dpm_levels[index].value = pcie_gen;
         dpm_table->dpm_levels[index].param1 = pcie_lanes;
-        dpm_table->dpm_levels[index].enabled = 1;
+        dpm_table->dpm_levels[index].enabled = true;
 }
 
 static int tonga_setup_default_pcie_tables(struct pp_hwmgr *hwmgr)
@@ -2828,7 +2830,7 @@ static int tonga_setup_default_dpm_tables(struct pp_hwmgr *hwmgr)
                                 allowed_vdd_sclk_table->entries[i].clk) {
                         data->dpm_table.sclk_table.dpm_levels[data->dpm_table.sclk_table.count].value =
                                 allowed_vdd_sclk_table->entries[i].clk;
-                        data->dpm_table.sclk_table.dpm_levels[data->dpm_table.sclk_table.count].enabled = 1; /*(i==0) ? 1 : 0; to do */
+                        data->dpm_table.sclk_table.dpm_levels[data->dpm_table.sclk_table.count].enabled = true; /*(i==0) ? 1 : 0; to do */
                         data->dpm_table.sclk_table.count++;
                 }
         }
@@ -2842,7 +2844,7 @@ static int tonga_setup_default_dpm_tables(struct pp_hwmgr *hwmgr)
                         allowed_vdd_mclk_table->entries[i].clk) {
                         data->dpm_table.mclk_table.dpm_levels[data->dpm_table.mclk_table.count].value =
                                 allowed_vdd_mclk_table->entries[i].clk;
-                        data->dpm_table.mclk_table.dpm_levels[data->dpm_table.mclk_table.count].enabled = 1; /*(i==0) ? 1 : 0; */
+                        data->dpm_table.mclk_table.dpm_levels[data->dpm_table.mclk_table.count].enabled = true; /*(i==0) ? 1 : 0; */
                         data->dpm_table.mclk_table.count++;
                 }
         }
@@ -3026,8 +3028,8 @@ int tonga_init_smc_table(struct pp_hwmgr *hwmgr)
 
         reg_value = 0;
         if ((0 == reg_value) &&
-                (0 == atomctrl_get_pp_assign_pin(hwmgr,
-                        VDDC_VRHOT_GPIO_PINID, &gpio_pin_assignment))) {
+                (atomctrl_get_pp_assign_pin(hwmgr, VDDC_VRHOT_GPIO_PINID,
+                                                &gpio_pin_assignment))) {
                 table->VRHotGpio = gpio_pin_assignment.uc_gpio_pin_bit_shift;
                 phm_cap_set(hwmgr->platform_descriptor.platformCaps,
                         PHM_PlatformCaps_RegulatorHot);
@@ -3040,8 +3042,8 @@ int tonga_init_smc_table(struct pp_hwmgr *hwmgr)
         /* ACDC Switch GPIO */
         reg_value = 0;
         if ((0 == reg_value) &&
-                (0 == atomctrl_get_pp_assign_pin(hwmgr,
-                        PP_AC_DC_SWITCH_GPIO_PINID, &gpio_pin_assignment))) {
+                (atomctrl_get_pp_assign_pin(hwmgr, PP_AC_DC_SWITCH_GPIO_PINID,
+                                                &gpio_pin_assignment))) {
                 table->AcDcGpio = gpio_pin_assignment.uc_gpio_pin_bit_shift;
                 phm_cap_set(hwmgr->platform_descriptor.platformCaps,
                         PHM_PlatformCaps_AutomaticDCTransition);
@@ -3063,8 +3065,7 @@ int tonga_init_smc_table(struct pp_hwmgr *hwmgr)
         }
 
         reg_value = 0;
-        if ((0 == reg_value) &&
-                (0 == atomctrl_get_pp_assign_pin(hwmgr,
+        if ((0 == reg_value) && (atomctrl_get_pp_assign_pin(hwmgr,
                         THERMAL_INT_OUTPUT_GPIO_PINID, &gpio_pin_assignment))) {
                 phm_cap_set(hwmgr->platform_descriptor.platformCaps,
                         PHM_PlatformCaps_ThermalOutGPIO);
@@ -3135,7 +3136,7 @@ int tonga_upload_dpm_level_enable_mask(struct pp_hwmgr *hwmgr)
 
         if (0 == data->sclk_dpm_key_disabled) {
                 /* Checking if DPM is running.  If we discover hang because of this, we should skip this message.*/
-                if (0 != tonga_is_dpm_running(hwmgr))
+                if (tonga_is_dpm_running(hwmgr))
                         printk(KERN_ERR "[ powerplay ] Trying to set Enable Mask when DPM is disabled \n");
 
                 if (0 != data->dpm_level_enable_mask.sclk_dpm_enable_mask) {
@@ -3150,7 +3151,7 @@ int tonga_upload_dpm_level_enable_mask(struct pp_hwmgr *hwmgr)
 
         if (0 == data->mclk_dpm_key_disabled) {
                 /* Checking if DPM is running.  If we discover hang because of this, we should skip this message.*/
-                if (0 != tonga_is_dpm_running(hwmgr))
+                if (tonga_is_dpm_running(hwmgr))
                         printk(KERN_ERR "[ powerplay ] Trying to set Enable Mask when DPM is disabled \n");
 
                 if (0 != data->dpm_level_enable_mask.mclk_dpm_enable_mask) {
@@ -3261,7 +3262,7 @@ int tonga_initializa_dynamic_state_adjustment_rule_settings(struct pp_hwmgr *hwm
 
         /* initialize vddc_dep_on_dal_pwrl table */
         table_size = sizeof(uint32_t) + 4 * sizeof(struct phm_clock_voltage_dependency_record);
-        table_clk_vlt = (struct phm_clock_voltage_dependency_table *)kzalloc(table_size, GFP_KERNEL);
+        table_clk_vlt = kzalloc(table_size, GFP_KERNEL);
 
         if (NULL == table_clk_vlt) {
                 printk(KERN_ERR "[ powerplay ] Can not allocate space for vddc_dep_on_dal_pwrl! \n");
@@ -3336,9 +3337,9 @@ int tonga_unforce_dpm_levels(struct pp_hwmgr *hwmgr)
         tonga_hwmgr *data = (tonga_hwmgr *)(hwmgr->backend);
         int result = 1;
 
-        PP_ASSERT_WITH_CODE (0 == tonga_is_dpm_running(hwmgr),
-                "Trying to Unforce DPM when DPM is disabled. Returning without sending SMC message.",
-                                                        return result);
+        PP_ASSERT_WITH_CODE (!tonga_is_dpm_running(hwmgr),
+                             "Trying to Unforce DPM when DPM is disabled. Returning without sending SMC message.",
+                             return result);
 
         if (0 == data->pcie_dpm_key_disabled) {
                 PP_ASSERT_WITH_CODE((0 == smum_send_msg_to_smc(
@@ -3742,7 +3743,7 @@ uint8_t tonga_get_memory_modile_index(struct pp_hwmgr *hwmgr)
 
 bool tonga_check_s0_mc_reg_index(uint16_t inReg, uint16_t *outReg)
 {
-        bool result = 1;
+        bool result = true;
 
         switch (inReg) {
         case  mmMC_SEQ_RAS_TIMING:
@@ -3826,7 +3827,7 @@ bool tonga_check_s0_mc_reg_index(uint16_t inReg, uint16_t *outReg)
                 break;
 
         default:
-                result = 0;
+                result = false;
                 break;
         }
 
@@ -4422,13 +4423,6 @@ int tonga_reset_asic_tasks(struct pp_hwmgr *hwmgr)
 
 int tonga_hwmgr_backend_fini(struct pp_hwmgr *hwmgr)
 {
-        struct tonga_hwmgr *data = (struct tonga_hwmgr *)(hwmgr->backend);
-
-        if (data->soft_pp_table) {
-                kfree(data->soft_pp_table);
-                data->soft_pp_table = NULL;
-        }
-
         return phm_hwmgr_backend_fini(hwmgr);
 }
 
@@ -4442,7 +4436,7 @@ int tonga_hwmgr_backend_init(struct pp_hwmgr *hwmgr)
 {
         int result = 0;
         SMU72_Discrete_DpmTable  *table = NULL;
-        tonga_hwmgr *data = (struct tonga_hwmgr *)(hwmgr->backend);
+        tonga_hwmgr *data;
         pp_atomctrl_gpio_pin_assignment gpio_pin_assignment;
         struct phm_ppt_v1_information *pptable_info = (struct phm_ppt_v1_information *)(hwmgr->pptable);
         phw_tonga_ulv_parm *ulv;
@@ -4451,7 +4445,13 @@ int tonga_hwmgr_backend_init(struct pp_hwmgr *hwmgr)
         PP_ASSERT_WITH_CODE((NULL != hwmgr),
                 "Invalid Parameter!", return -1;);
 
-        data->dll_defaule_on = 0;
+        data = kzalloc(sizeof(struct tonga_hwmgr), GFP_KERNEL);
+        if (data == NULL)
+                return -ENOMEM;
+
+        hwmgr->backend = data;
+
+        data->dll_defaule_on = false;
         data->sram_end = SMC_RAM_END;
 
         data->activity_target[0] = PPTONGA_TARGETACTIVITY_DFLT;
@@ -4557,13 +4557,13 @@ int tonga_hwmgr_backend_init(struct pp_hwmgr *hwmgr)
 
         /* ULV Support*/
         ulv = &(data->ulv);
-        ulv->ulv_supported = 0;
+        ulv->ulv_supported = false;
 
         /* Initalize Dynamic State Adjustment Rule Settings*/
         result = tonga_initializa_dynamic_state_adjustment_rule_settings(hwmgr);
         if (result)
                 printk(KERN_ERR "[ powerplay ] tonga_initializa_dynamic_state_adjustment_rule_settings failed!\n");
-        data->uvd_enabled = 0;
+        data->uvd_enabled = false;
 
         table = &(data->smc_state_table);
 
@@ -4571,7 +4571,7 @@ int tonga_hwmgr_backend_init(struct pp_hwmgr *hwmgr)
         * if ucGPIO_ID=VDDC_PCC_GPIO_PINID in GPIO_LUTable,
         * Peak Current Control feature is enabled and we should program PCC HW register
         */
-        if (0 == atomctrl_get_pp_assign_pin(hwmgr, VDDC_PCC_GPIO_PINID, &gpio_pin_assignment)) {
+        if (atomctrl_get_pp_assign_pin(hwmgr, VDDC_PCC_GPIO_PINID, &gpio_pin_assignment)) {
                 uint32_t temp_reg = cgs_read_ind_register(hwmgr->device,
                                                                                 CGS_IND_REG__SMC, ixCNB_PWRMGT_CNTL);
 
@@ -4610,7 +4610,7 @@ int tonga_hwmgr_backend_init(struct pp_hwmgr *hwmgr)
         phm_cap_set(hwmgr->platform_descriptor.platformCaps,
                 PHM_PlatformCaps_SMU7);
 
-        data->vddc_phase_shed_control = 0;
+        data->vddc_phase_shed_control = false;
 
         phm_cap_unset(hwmgr->platform_descriptor.platformCaps,
                       PHM_PlatformCaps_UVDPowerGating);
@@ -4629,7 +4629,7 @@ int tonga_hwmgr_backend_init(struct pp_hwmgr *hwmgr)
         }
 
         if (0 == result) {
-                data->is_tlu_enabled = 0;
+                data->is_tlu_enabled = false;
                 hwmgr->platform_descriptor.hardwareActivityPerformanceLevels =
                         TONGA_MAX_HARDWARE_POWERLEVELS;
                 hwmgr->platform_descriptor.hardwarePerformanceLevels = 2;
@@ -4639,7 +4639,7 @@ int tonga_hwmgr_backend_init(struct pp_hwmgr *hwmgr)
                 sys_info.info_id = CGS_SYSTEM_INFO_PCIE_GEN_INFO;
                 result = cgs_query_system_info(hwmgr->device, &sys_info);
                 if (result)
-                        data->pcie_gen_cap = 0x30007;
+                        data->pcie_gen_cap = AMDGPU_DEFAULT_PCIE_GEN_MASK;
                 else
                         data->pcie_gen_cap = (uint32_t)sys_info.value;
                 if (data->pcie_gen_cap & CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3)
@@ -4648,7 +4648,7 @@ int tonga_hwmgr_backend_init(struct pp_hwmgr *hwmgr)
                 sys_info.info_id = CGS_SYSTEM_INFO_PCIE_MLW;
                 result = cgs_query_system_info(hwmgr->device, &sys_info);
                 if (result)
-                        data->pcie_lane_cap = 0x2f0000;
+                        data->pcie_lane_cap = AMDGPU_DEFAULT_PCIE_MLW_MASK;
                 else
                         data->pcie_lane_cap = (uint32_t)sys_info.value;
         } else {
@@ -5310,9 +5310,8 @@ static int tonga_freeze_sclk_mclk_dpm(struct pp_hwmgr *hwmgr)
         if ((0 == data->sclk_dpm_key_disabled) &&
                 (data->need_update_smu7_dpm_table &
                 (DPMTABLE_OD_UPDATE_SCLK + DPMTABLE_UPDATE_SCLK))) {
-                PP_ASSERT_WITH_CODE(
-                        0 == tonga_is_dpm_running(hwmgr),
-                        "Trying to freeze SCLK DPM when DPM is disabled",
+                PP_ASSERT_WITH_CODE(!tonga_is_dpm_running(hwmgr),
+                                    "Trying to freeze SCLK DPM when DPM is disabled",
                         );
                 PP_ASSERT_WITH_CODE(
                         0 == smum_send_msg_to_smc(hwmgr->smumgr,
@@ -5324,8 +5323,8 @@ static int tonga_freeze_sclk_mclk_dpm(struct pp_hwmgr *hwmgr)
         if ((0 == data->mclk_dpm_key_disabled) &&
                 (data->need_update_smu7_dpm_table &
                  DPMTABLE_OD_UPDATE_MCLK)) {
-                PP_ASSERT_WITH_CODE(0 == tonga_is_dpm_running(hwmgr),
-                        "Trying to freeze MCLK DPM when DPM is disabled",
+                PP_ASSERT_WITH_CODE(!tonga_is_dpm_running(hwmgr),
+                                    "Trying to freeze MCLK DPM when DPM is disabled",
                         );
                 PP_ASSERT_WITH_CODE(
                         0 == smum_send_msg_to_smc(hwmgr->smumgr,
@@ -5460,7 +5459,6 @@ static	int tonga_trim_single_dpm_states(struct pp_hwmgr *hwmgr,
 
 static int tonga_trim_dpm_states(struct pp_hwmgr *hwmgr, const struct tonga_power_state *hw_state)
 {
-        int result = 0;
         struct tonga_hwmgr *data = (struct tonga_hwmgr *)(hwmgr->backend);
         uint32_t high_limit_count;
 
@@ -5480,7 +5478,7 @@ static int tonga_trim_dpm_states(struct pp_hwmgr *hwmgr, const struct tonga_powe
                                                 hw_state->performance_levels[0].memory_clock,
                                                 hw_state->performance_levels[high_limit_count].memory_clock);
 
-        return result;
+        return 0;
 }
 
 static int tonga_generate_dpm_level_enable_mask(struct pp_hwmgr *hwmgr, const void *input)
@@ -5627,8 +5625,8 @@ static int tonga_unfreeze_sclk_mclk_dpm(struct pp_hwmgr *hwmgr)
                 (data->need_update_smu7_dpm_table &
                 (DPMTABLE_OD_UPDATE_SCLK + DPMTABLE_UPDATE_SCLK))) {
 
-                PP_ASSERT_WITH_CODE(0 == tonga_is_dpm_running(hwmgr),
-                        "Trying to Unfreeze SCLK DPM when DPM is disabled",
+                PP_ASSERT_WITH_CODE(!tonga_is_dpm_running(hwmgr),
+                                    "Trying to Unfreeze SCLK DPM when DPM is disabled",
                         );
                 PP_ASSERT_WITH_CODE(
                          0 == smum_send_msg_to_smc(hwmgr->smumgr,
@@ -5640,9 +5638,8 @@ static int tonga_unfreeze_sclk_mclk_dpm(struct pp_hwmgr *hwmgr)
         if ((0 == data->mclk_dpm_key_disabled) &&
                 (data->need_update_smu7_dpm_table & DPMTABLE_OD_UPDATE_MCLK)) {
 
-                PP_ASSERT_WITH_CODE(
-                                0 == tonga_is_dpm_running(hwmgr),
-                                "Trying to Unfreeze MCLK DPM when DPM is disabled",
+                PP_ASSERT_WITH_CODE(!tonga_is_dpm_running(hwmgr),
+                                    "Trying to Unfreeze MCLK DPM when DPM is disabled",
                                 );
                 PP_ASSERT_WITH_CODE(
                          0 == smum_send_msg_to_smc(hwmgr->smumgr,
@@ -6031,42 +6028,6 @@ static int tonga_get_fan_control_mode(struct pp_hwmgr *hwmgr)
                                 CG_FDO_CTRL2, FDO_PWM_MODE);
 }
 
-static int tonga_get_pp_table(struct pp_hwmgr *hwmgr, char **table)
-{
-        struct tonga_hwmgr *data = (struct tonga_hwmgr *)(hwmgr->backend);
-
-        if (!data->soft_pp_table) {
-                data->soft_pp_table = kmemdup(hwmgr->soft_pp_table,
-                                              hwmgr->soft_pp_table_size,
-                                              GFP_KERNEL);
-                if (!data->soft_pp_table)
-                        return -ENOMEM;
-        }
-
-        *table = (char *)&data->soft_pp_table;
-
-        return hwmgr->soft_pp_table_size;
-}
-
-static int tonga_set_pp_table(struct pp_hwmgr *hwmgr, const char *buf, size_t size)
-{
-        struct tonga_hwmgr *data = (struct tonga_hwmgr *)(hwmgr->backend);
-
-        if (!data->soft_pp_table) {
-                data->soft_pp_table = kzalloc(hwmgr->soft_pp_table_size, GFP_KERNEL);
-                if (!data->soft_pp_table)
-                        return -ENOMEM;
-        }
-
-        memcpy(data->soft_pp_table, buf, size);
-
-        hwmgr->soft_pp_table = data->soft_pp_table;
-
-        /* TODO: re-init powerplay to implement modified pptable */
-
-        return 0;
-}
-
 static int tonga_force_clock_level(struct pp_hwmgr *hwmgr,
                 enum pp_clock_type type, uint32_t mask)
 {
@@ -6174,11 +6135,96 @@ static int tonga_print_clock_levels(struct pp_hwmgr *hwmgr,
         return size;
 }
 
+static int tonga_get_sclk_od(struct pp_hwmgr *hwmgr)
+{
+        struct tonga_hwmgr *data = (struct tonga_hwmgr *)(hwmgr->backend);
+        struct tonga_single_dpm_table *sclk_table = &(data->dpm_table.sclk_table);
+        struct tonga_single_dpm_table *golden_sclk_table =
+                        &(data->golden_dpm_table.sclk_table);
+        int value;
+
+        value = (sclk_table->dpm_levels[sclk_table->count - 1].value -
+                        golden_sclk_table->dpm_levels[golden_sclk_table->count - 1].value) *
+                        100 /
+                        golden_sclk_table->dpm_levels[golden_sclk_table->count - 1].value;
+
+        return value;
+}
+
+static int tonga_set_sclk_od(struct pp_hwmgr *hwmgr, uint32_t value)
+{
+        struct tonga_hwmgr *data = (struct tonga_hwmgr *)(hwmgr->backend);
+        struct tonga_single_dpm_table *golden_sclk_table =
+                        &(data->golden_dpm_table.sclk_table);
+        struct pp_power_state  *ps;
+        struct tonga_power_state  *tonga_ps;
+
+        if (value > 20)
+                value = 20;
+
+        ps = hwmgr->request_ps;
+
+        if (ps == NULL)
+                return -EINVAL;
+
+        tonga_ps = cast_phw_tonga_power_state(&ps->hardware);
+
+        tonga_ps->performance_levels[tonga_ps->performance_level_count - 1].engine_clock =
+                        golden_sclk_table->dpm_levels[golden_sclk_table->count - 1].value *
+                        value / 100 +
+                        golden_sclk_table->dpm_levels[golden_sclk_table->count - 1].value;
+
+        return 0;
+}
+
+static int tonga_get_mclk_od(struct pp_hwmgr *hwmgr)
+{
+        struct tonga_hwmgr *data = (struct tonga_hwmgr *)(hwmgr->backend);
+        struct tonga_single_dpm_table *mclk_table = &(data->dpm_table.mclk_table);
+        struct tonga_single_dpm_table *golden_mclk_table =
+                        &(data->golden_dpm_table.mclk_table);
+        int value;
+
+        value = (mclk_table->dpm_levels[mclk_table->count - 1].value -
+                        golden_mclk_table->dpm_levels[golden_mclk_table->count - 1].value) *
+                        100 /
+                        golden_mclk_table->dpm_levels[golden_mclk_table->count - 1].value;
+
+        return value;
+}
+
+static int tonga_set_mclk_od(struct pp_hwmgr *hwmgr, uint32_t value)
+{
+        struct tonga_hwmgr *data = (struct tonga_hwmgr *)(hwmgr->backend);
+        struct tonga_single_dpm_table *golden_mclk_table =
+                        &(data->golden_dpm_table.mclk_table);
+        struct pp_power_state  *ps;
+        struct tonga_power_state  *tonga_ps;
+
+        if (value > 20)
+                value = 20;
+
+        ps = hwmgr->request_ps;
+
+        if (ps == NULL)
+                return -EINVAL;
+
+        tonga_ps = cast_phw_tonga_power_state(&ps->hardware);
+
+        tonga_ps->performance_levels[tonga_ps->performance_level_count - 1].memory_clock =
+                        golden_mclk_table->dpm_levels[golden_mclk_table->count - 1].value *
+                        value / 100 +
+                        golden_mclk_table->dpm_levels[golden_mclk_table->count - 1].value;
+
+        return 0;
+}
+
 static const struct pp_hwmgr_func tonga_hwmgr_funcs = {
         .backend_init = &tonga_hwmgr_backend_init,
         .backend_fini = &tonga_hwmgr_backend_fini,
         .asic_setup = &tonga_setup_asic_task,
         .dynamic_state_management_enable = &tonga_enable_dpm_tasks,
+        .dynamic_state_management_disable = &tonga_disable_dpm_tasks,
         .apply_state_adjust_rules = tonga_apply_state_adjust_rules,
         .force_dpm_level = &tonga_force_dpm_level,
         .power_state_set = tonga_set_power_state_tasks,
@@ -6212,22 +6258,16 @@ static const struct pp_hwmgr_func tonga_hwmgr_funcs = {
         .check_states_equal = tonga_check_states_equal,
         .set_fan_control_mode = tonga_set_fan_control_mode,
         .get_fan_control_mode = tonga_get_fan_control_mode,
-        .get_pp_table = tonga_get_pp_table,
-        .set_pp_table = tonga_set_pp_table,
         .force_clock_level = tonga_force_clock_level,
         .print_clock_levels = tonga_print_clock_levels,
+        .get_sclk_od = tonga_get_sclk_od,
+        .set_sclk_od = tonga_set_sclk_od,
+        .get_mclk_od = tonga_get_mclk_od,
+        .set_mclk_od = tonga_set_mclk_od,
 };
 
 int tonga_hwmgr_init(struct pp_hwmgr *hwmgr)
 {
-        tonga_hwmgr  *data;
-
-        data = kzalloc (sizeof(tonga_hwmgr), GFP_KERNEL);
-        if (data == NULL)
-                return -ENOMEM;
-        memset(data, 0x00, sizeof(tonga_hwmgr));
-
-        hwmgr->backend = data;
         hwmgr->hwmgr_func = &tonga_hwmgr_funcs;
         hwmgr->pptable_func = &tonga_pptable_funcs;
         pp_tonga_thermal_initialize(hwmgr);
diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/tonga_hwmgr.h b/drivers/gpu/drm/amd/powerplay/hwmgr/tonga_hwmgr.h
index 573cd39fe78d..3961884bfa9b 100644
--- a/drivers/gpu/drm/amd/powerplay/hwmgr/tonga_hwmgr.h
+++ b/drivers/gpu/drm/amd/powerplay/hwmgr/tonga_hwmgr.h
@@ -352,9 +352,6 @@ struct tonga_hwmgr {
         bool                           samu_power_gated; /* 1: gated, 0:not gated */
         bool                           acp_power_gated;  /* 1: gated, 0:not gated */
         bool                           pg_acp_init;
-
-        /* soft pptable for re-uploading into smu */
-        void *soft_pp_table;
 };
 
 typedef struct tonga_hwmgr tonga_hwmgr;
diff --git a/drivers/gpu/drm/amd/powerplay/hwmgr/tonga_processpptables.c b/drivers/gpu/drm/amd/powerplay/hwmgr/tonga_processpptables.c
index dccc859f638c..cfb647f76cbe 100644
--- a/drivers/gpu/drm/amd/powerplay/hwmgr/tonga_processpptables.c
+++ b/drivers/gpu/drm/amd/powerplay/hwmgr/tonga_processpptables.c
@@ -167,8 +167,7 @@ static int get_vddc_lookup_table(
         table_size = sizeof(uint32_t) +
                 sizeof(phm_ppt_v1_voltage_lookup_record) * max_levels;
 
-        table = (phm_ppt_v1_voltage_lookup_table *)
-                kzalloc(table_size, GFP_KERNEL);
+        table = kzalloc(table_size, GFP_KERNEL);
 
         if (NULL == table)
                 return -ENOMEM;
@@ -327,7 +326,7 @@ static int get_valid_clk(
         table_size = sizeof(uint32_t) +
                 sizeof(uint32_t) * clk_volt_pp_table->count;
 
-        table = (struct phm_clock_array *)kzalloc(table_size, GFP_KERNEL);
+        table = kzalloc(table_size, GFP_KERNEL);
 
         if (NULL == table)
                 return -ENOMEM;
@@ -377,8 +376,7 @@ static int get_mclk_voltage_dependency_table(
         table_size = sizeof(uint32_t) + sizeof(phm_ppt_v1_clock_voltage_dependency_record)
                 * mclk_dep_table->ucNumEntries;
 
-        mclk_table = (phm_ppt_v1_clock_voltage_dependency_table *)
-                kzalloc(table_size, GFP_KERNEL);
+        mclk_table = kzalloc(table_size, GFP_KERNEL);
 
         if (NULL == mclk_table)
                 return -ENOMEM;
@@ -424,8 +422,7 @@ static int get_sclk_voltage_dependency_table(
                 table_size = sizeof(uint32_t) + sizeof(phm_ppt_v1_clock_voltage_dependency_record)
                         * tonga_table->ucNumEntries;
 
-                sclk_table = (phm_ppt_v1_clock_voltage_dependency_table *)
-                        kzalloc(table_size, GFP_KERNEL);
+                sclk_table = kzalloc(table_size, GFP_KERNEL);
 
                 if (NULL == sclk_table)
                         return -ENOMEM;
@@ -456,8 +453,7 @@ static int get_sclk_voltage_dependency_table(
                 table_size = sizeof(uint32_t) + sizeof(phm_ppt_v1_clock_voltage_dependency_record)
                         * polaris_table->ucNumEntries;
 
-                sclk_table = (phm_ppt_v1_clock_voltage_dependency_table *)
-                        kzalloc(table_size, GFP_KERNEL);
+                sclk_table = kzalloc(table_size, GFP_KERNEL);
 
                 if (NULL == sclk_table)
                         return -ENOMEM;
@@ -504,7 +500,7 @@ static int get_pcie_table(
                 table_size = sizeof(uint32_t) +
                         sizeof(phm_ppt_v1_pcie_record) * atom_pcie_table->ucNumEntries;
 
-                pcie_table = (phm_ppt_v1_pcie_table *)kzalloc(table_size, GFP_KERNEL);
+                pcie_table = kzalloc(table_size, GFP_KERNEL);
 
                 if (pcie_table == NULL)
                         return -ENOMEM;
@@ -541,7 +537,7 @@ static int get_pcie_table(
                 table_size = sizeof(uint32_t) +
                         sizeof(phm_ppt_v1_pcie_record) * atom_pcie_table->ucNumEntries;
 
-                pcie_table = (phm_ppt_v1_pcie_table *)kzalloc(table_size, GFP_KERNEL);
+                pcie_table = kzalloc(table_size, GFP_KERNEL);
 
                 if (pcie_table == NULL)
                         return -ENOMEM;
@@ -695,8 +691,7 @@ static int get_mm_clock_voltage_table(
         table_size = sizeof(uint32_t) +
                 sizeof(phm_ppt_v1_mm_clock_voltage_dependency_record)
                 * mm_dependency_table->ucNumEntries;
-        mm_table = (phm_ppt_v1_mm_clock_voltage_dependency_table *)
-                kzalloc(table_size, GFP_KERNEL);
+        mm_table = kzalloc(table_size, GFP_KERNEL);
 
         if (NULL == mm_table)
                 return -ENOMEM;
@@ -1073,13 +1068,9 @@ int tonga_pp_tables_initialize(struct pp_hwmgr *hwmgr)
 
 int tonga_pp_tables_uninitialize(struct pp_hwmgr *hwmgr)
 {
-        int result = 0;
         struct phm_ppt_v1_information *pp_table_information =
                 (struct phm_ppt_v1_information *)(hwmgr->pptable);
 
-        if (NULL != hwmgr->soft_pp_table)
-                hwmgr->soft_pp_table = NULL;
-
         kfree(pp_table_information->vdd_dep_on_sclk);
         pp_table_information->vdd_dep_on_sclk = NULL;
 
@@ -1116,7 +1107,7 @@ int tonga_pp_tables_uninitialize(struct pp_hwmgr *hwmgr)
         kfree(hwmgr->pptable);
         hwmgr->pptable = NULL;
 
-        return result;
+        return 0;
 }
 
 const struct pp_table_func tonga_pptable_funcs = {
diff --git a/drivers/gpu/drm/amd/powerplay/inc/amd_powerplay.h b/drivers/gpu/drm/amd/powerplay/inc/amd_powerplay.h
index 50b367d44307..b764c8c05ec8 100644
--- a/drivers/gpu/drm/amd/powerplay/inc/amd_powerplay.h
+++ b/drivers/gpu/drm/amd/powerplay/inc/amd_powerplay.h
@@ -132,6 +132,7 @@ struct amd_pp_init {
         uint32_t chip_family;
         uint32_t chip_id;
         uint32_t rev_id;
+        bool powercontainment_enabled;
 };
 enum amd_pp_display_config_type{
         AMD_PP_DisplayConfigType_None = 0,
@@ -342,6 +343,10 @@ struct amd_powerplay_funcs {
         int (*set_pp_table)(void *handle, const char *buf, size_t size);
         int (*force_clock_level)(void *handle, enum pp_clock_type type, uint32_t mask);
         int (*print_clock_levels)(void *handle, enum pp_clock_type type, char *buf);
+        int (*get_sclk_od)(void *handle);
+        int (*set_sclk_od)(void *handle, uint32_t value);
+        int (*get_mclk_od)(void *handle);
+        int (*set_mclk_od)(void *handle, uint32_t value);
 };
 
 struct amd_powerplay {
@@ -355,6 +360,8 @@ int amd_powerplay_init(struct amd_pp_init *pp_init,
 
 int amd_powerplay_fini(void *handle);
 
+int amd_powerplay_reset(void *handle);
+
 int amd_powerplay_display_configuration_change(void *handle,
                 const struct amd_pp_display_configuration *input);
 
diff --git a/drivers/gpu/drm/amd/powerplay/inc/hardwaremanager.h b/drivers/gpu/drm/amd/powerplay/inc/hardwaremanager.h
index 56f712c7d07a..962cb5385951 100644
--- a/drivers/gpu/drm/amd/powerplay/inc/hardwaremanager.h
+++ b/drivers/gpu/drm/amd/powerplay/inc/hardwaremanager.h
@@ -340,6 +340,7 @@ extern int phm_powergate_vce(struct pp_hwmgr *hwmgr, bool gate);
 extern int phm_powerdown_uvd(struct pp_hwmgr *hwmgr);
 extern int phm_setup_asic(struct pp_hwmgr *hwmgr);
 extern int phm_enable_dynamic_state_management(struct pp_hwmgr *hwmgr);
+extern int phm_disable_dynamic_state_management(struct pp_hwmgr *hwmgr);
 extern void phm_init_dynamic_caps(struct pp_hwmgr *hwmgr);
 extern bool phm_is_hw_access_blocked(struct pp_hwmgr *hwmgr);
 extern int phm_block_hw_access(struct pp_hwmgr *hwmgr, bool block);
diff --git a/drivers/gpu/drm/amd/powerplay/inc/hwmgr.h b/drivers/gpu/drm/amd/powerplay/inc/hwmgr.h
index 77e8e33d5870..bf0d2accf7bf 100644
--- a/drivers/gpu/drm/amd/powerplay/inc/hwmgr.h
+++ b/drivers/gpu/drm/amd/powerplay/inc/hwmgr.h
@@ -278,6 +278,8 @@ struct pp_hwmgr_func {
 
         int (*dynamic_state_management_enable)(
                                                 struct pp_hwmgr *hw_mgr);
+        int (*dynamic_state_management_disable)(
+                                                struct pp_hwmgr *hw_mgr);
 
         int (*patch_boot_state)(struct pp_hwmgr *hwmgr,
                                      struct pp_hw_power_state *hw_ps);
@@ -333,11 +335,13 @@ struct pp_hwmgr_func {
         int (*get_clock_by_type)(struct pp_hwmgr *hwmgr, enum amd_pp_clock_type type, struct amd_pp_clocks *clocks);
         int (*get_max_high_clocks)(struct pp_hwmgr *hwmgr, struct amd_pp_simple_clock_info *clocks);
         int (*power_off_asic)(struct pp_hwmgr *hwmgr);
-        int (*get_pp_table)(struct pp_hwmgr *hwmgr, char **table);
-        int (*set_pp_table)(struct pp_hwmgr *hwmgr, const char *buf, size_t size);
         int (*force_clock_level)(struct pp_hwmgr *hwmgr, enum pp_clock_type type, uint32_t mask);
         int (*print_clock_levels)(struct pp_hwmgr *hwmgr, enum pp_clock_type type, char *buf);
         int (*enable_per_cu_power_gating)(struct pp_hwmgr *hwmgr, bool enable);
+        int (*get_sclk_od)(struct pp_hwmgr *hwmgr);
+        int (*set_sclk_od)(struct pp_hwmgr *hwmgr, uint32_t value);
+        int (*get_mclk_od)(struct pp_hwmgr *hwmgr);
+        int (*set_mclk_od)(struct pp_hwmgr *hwmgr, uint32_t value);
 };
 
 struct pp_table_func {
@@ -580,6 +584,7 @@ struct pp_hwmgr {
         struct pp_smumgr *smumgr;
         const void *soft_pp_table;
         uint32_t soft_pp_table_size;
+        void *hardcode_pp_table;
         bool need_pp_table_upload;
         enum amd_dpm_forced_level dpm_level;
         bool block_hw_access;
@@ -609,6 +614,7 @@ struct pp_hwmgr {
         uint32_t num_ps;
         struct pp_thermal_controller_info thermal_controller;
         bool fan_ctrl_is_in_default_mode;
+        bool powercontainment_enabled;
         uint32_t fan_ctrl_default_mode;
         uint32_t tmin;
         struct phm_microcode_version_info microcode_version_info;
diff --git a/drivers/gpu/drm/amd/powerplay/inc/smumgr.h b/drivers/gpu/drm/amd/powerplay/inc/smumgr.h
index fc9e3d1dd409..3c235f0177cd 100644
--- a/drivers/gpu/drm/amd/powerplay/inc/smumgr.h
+++ b/drivers/gpu/drm/amd/powerplay/inc/smumgr.h
@@ -131,6 +131,12 @@ extern int smu_free_memory(void *device, void *handle);
         smum_wait_on_indirect_register(smumgr,                          \
                                 mm##port##_INDEX, index, value, mask)
 
+#define SMUM_WAIT_INDIRECT_REGISTER(smumgr, port, reg, value, mask)    \
+            SMUM_WAIT_INDIRECT_REGISTER_GIVEN_INDEX(smumgr, port, ix##reg, value, mask)
+
+#define SMUM_WAIT_INDIRECT_FIELD(smumgr, port, reg, field, fieldval)                          \
+            SMUM_WAIT_INDIRECT_REGISTER(smumgr, port, reg, (fieldval) << SMUM_FIELD_SHIFT(reg, field), \
+                                    SMUM_FIELD_MASK(reg, field) )
 
 #define SMUM_WAIT_REGISTER_UNEQUAL_GIVEN_INDEX(smumgr,         \
                                                         index, value, mask) \
@@ -158,6 +164,10 @@ extern int smu_free_memory(void *device, void *handle);
                 (SMUM_FIELD_MASK(reg, field) & ((field_val) <<                 \
                         SMUM_FIELD_SHIFT(reg, field))))
 
+#define SMUM_READ_INDIRECT_FIELD(device, port, reg, field) \
+            SMUM_GET_FIELD(cgs_read_ind_register(device, port, ix##reg), \
+                           reg, field)
+
 #define SMUM_WAIT_VFPF_INDIRECT_REGISTER_GIVEN_INDEX(smumgr,            \
                                 port, index, value, mask)               \
         smum_wait_on_indirect_register(smumgr,                          \
@@ -191,6 +201,13 @@ extern int smu_free_memory(void *device, void *handle);
                         SMUM_SET_FIELD(cgs_read_ind_register(device, port, ix##reg), \
                         reg, field, fieldval))
 
+
+#define SMUM_WRITE_INDIRECT_FIELD(device, port, reg, field, fieldval)                   \
+                cgs_write_ind_register(device, port, ix##reg,                           \
+                        SMUM_SET_FIELD(cgs_read_ind_register(device, port, ix##reg),    \
+                                       reg, field, fieldval))
+
+
 #define SMUM_WAIT_VFPF_INDIRECT_FIELD(smumgr, port, reg, field, fieldval) \
         SMUM_WAIT_VFPF_INDIRECT_REGISTER(smumgr, port, reg,             \
                 (fieldval) << SMUM_FIELD_SHIFT(reg, field),             \
@@ -200,4 +217,16 @@ extern int smu_free_memory(void *device, void *handle);
         SMUM_WAIT_VFPF_INDIRECT_REGISTER_UNEQUAL(smumgr, port, reg,     \
                 (fieldval) << SMUM_FIELD_SHIFT(reg, field),             \
                 SMUM_FIELD_MASK(reg, field))
+
+#define SMUM_WAIT_INDIRECT_REGISTER_UNEQUAL_GIVEN_INDEX(smumgr, port, index, value, mask)    \
+        smum_wait_for_indirect_register_unequal(smumgr,                 \
+                mm##port##_INDEX, index, value, mask)
+
+#define SMUM_WAIT_INDIRECT_REGISTER_UNEQUAL(smumgr, port, reg, value, mask)    \
+            SMUM_WAIT_INDIRECT_REGISTER_UNEQUAL_GIVEN_INDEX(smumgr, port, ix##reg, value, mask)
+
+#define SMUM_WAIT_INDIRECT_FIELD_UNEQUAL(smumgr, port, reg, field, fieldval)                          \
+            SMUM_WAIT_INDIRECT_REGISTER_UNEQUAL(smumgr, port, reg, (fieldval) << SMUM_FIELD_SHIFT(reg, field), \
+                                    SMUM_FIELD_MASK(reg, field) )
+
 #endif
diff --git a/drivers/gpu/drm/amd/powerplay/smumgr/smumgr.c b/drivers/gpu/drm/amd/powerplay/smumgr/smumgr.c
index 0728c1e3d97a..7723473e51a0 100644
--- a/drivers/gpu/drm/amd/powerplay/smumgr/smumgr.c
+++ b/drivers/gpu/drm/amd/powerplay/smumgr/smumgr.c
@@ -23,6 +23,7 @@
 #include <linux/types.h>
 #include <linux/kernel.h>
 #include <linux/slab.h>
+#include <drm/amdgpu_drm.h>
 #include "pp_instance.h"
 #include "smumgr.h"
 #include "cgs_common.h"
@@ -52,10 +53,10 @@ int smum_init(struct amd_pp_init *pp_init, struct pp_instance *handle)
         handle->smu_mgr = smumgr;
 
         switch (smumgr->chip_family) {
-        case AMD_FAMILY_CZ:
+        case AMDGPU_FAMILY_CZ:
                 cz_smum_init(smumgr);
                 break;
-        case AMD_FAMILY_VI:
+        case AMDGPU_FAMILY_VI:
                 switch (smumgr->chip_id) {
                 case CHIP_TONGA:
                         tonga_smum_init(smumgr);
diff --git a/drivers/gpu/drm/amd/powerplay/smumgr/tonga_smumgr.c b/drivers/gpu/drm/amd/powerplay/smumgr/tonga_smumgr.c
index b22722eabafc..f42c536b3af1 100644
--- a/drivers/gpu/drm/amd/powerplay/smumgr/tonga_smumgr.c
+++ b/drivers/gpu/drm/amd/powerplay/smumgr/tonga_smumgr.c
@@ -479,7 +479,6 @@ static int tonga_request_smu_reload_fw(struct pp_smumgr *smumgr)
         struct tonga_smumgr *tonga_smu =
                 (struct tonga_smumgr *)(smumgr->backend);
         uint16_t fw_to_load;
-        int result = 0;
         struct SMU_DRAMData_TOC *toc;
         /**
          * First time this gets called during SmuMgr init,
@@ -563,7 +562,7 @@ static int tonga_request_smu_reload_fw(struct pp_smumgr *smumgr)
                 smumgr, PPSMC_MSG_LoadUcodes, fw_to_load),
                 "Fail to Request SMU Load uCode", return 0);
 
-        return result;
+        return 0;
 }
 
 static int tonga_request_smu_load_specific_fw(struct pp_smumgr *smumgr,
diff --git a/drivers/gpu/drm/amd/scheduler/gpu_sched_trace.h b/drivers/gpu/drm/amd/scheduler/gpu_sched_trace.h
index c89dc777768f..b961a1c6caf3 100644
--- a/drivers/gpu/drm/amd/scheduler/gpu_sched_trace.h
+++ b/drivers/gpu/drm/amd/scheduler/gpu_sched_trace.h
@@ -26,7 +26,7 @@ TRACE_EVENT(amd_sched_job,
             TP_fast_assign(
                            __entry->entity = sched_job->s_entity;
                            __entry->sched_job = sched_job;
-                           __entry->fence = &sched_job->s_fence->base;
+                           __entry->fence = &sched_job->s_fence->finished;
                            __entry->name = sched_job->sched->name;
                            __entry->job_count = kfifo_len(
                                    &sched_job->s_entity->job_queue) / sizeof(sched_job);
@@ -46,7 +46,7 @@ TRACE_EVENT(amd_sched_process_job,
                     ),
 
             TP_fast_assign(
-                    __entry->fence = &fence->base;
+                    __entry->fence = &fence->finished;
                     ),
             TP_printk("fence=%p signaled", __entry->fence)
 );
diff --git a/drivers/gpu/drm/amd/scheduler/gpu_scheduler.c b/drivers/gpu/drm/amd/scheduler/gpu_scheduler.c
index c16248cee779..ef312bb75fda 100644
--- a/drivers/gpu/drm/amd/scheduler/gpu_scheduler.c
+++ b/drivers/gpu/drm/amd/scheduler/gpu_scheduler.c
@@ -32,6 +32,7 @@
 
 static bool amd_sched_entity_is_ready(struct amd_sched_entity *entity);
 static void amd_sched_wakeup(struct amd_gpu_scheduler *sched);
+static void amd_sched_process_job(struct fence *f, struct fence_cb *cb);
 
 struct kmem_cache *sched_fence_slab;
 atomic_t sched_fence_slab_ref = ATOMIC_INIT(0);
@@ -140,7 +141,7 @@ int amd_sched_entity_init(struct amd_gpu_scheduler *sched,
                 return r;
 
         atomic_set(&entity->fence_seq, 0);
-        entity->fence_context = fence_context_alloc(1);
+        entity->fence_context = fence_context_alloc(2);
 
         return 0;
 }
@@ -251,17 +252,21 @@ static bool amd_sched_entity_add_dependency_cb(struct amd_sched_entity *entity)
 
         s_fence = to_amd_sched_fence(fence);
         if (s_fence && s_fence->sched == sched) {
-                /* Fence is from the same scheduler */
-                if (test_bit(AMD_SCHED_FENCE_SCHEDULED_BIT, &fence->flags)) {
-                        /* Ignore it when it is already scheduled */
-                        fence_put(entity->dependency);
-                        return false;
-                }
 
-                /* Wait for fence to be scheduled */
-                entity->cb.func = amd_sched_entity_clear_dep;
-                list_add_tail(&entity->cb.node, &s_fence->scheduled_cb);
-                return true;
+                /*
+                 * Fence is from the same scheduler, only need to wait for
+                 * it to be scheduled
+                 */
+                fence = fence_get(&s_fence->scheduled);
+                fence_put(entity->dependency);
+                entity->dependency = fence;
+                if (!fence_add_callback(fence, &entity->cb,
+                                        amd_sched_entity_clear_dep))
+                        return true;
+
+                /* Ignore it when it is already scheduled */
+                fence_put(fence);
+                return false;
         }
 
         if (!fence_add_callback(entity->dependency, &entity->cb,
@@ -319,46 +324,114 @@ static bool amd_sched_entity_in(struct amd_sched_job *sched_job)
         return added;
 }
 
-static void amd_sched_free_job(struct fence *f, struct fence_cb *cb) {
-        struct amd_sched_job *job = container_of(cb, struct amd_sched_job, cb_free_job);
-        schedule_work(&job->work_free_job);
-}
-
 /* job_finish is called after hw fence signaled, and
  * the job had already been deleted from ring_mirror_list
  */
-void amd_sched_job_finish(struct amd_sched_job *s_job)
+static void amd_sched_job_finish(struct work_struct *work)
 {
-        struct amd_sched_job *next;
+        struct amd_sched_job *s_job = container_of(work, struct amd_sched_job,
+                                                   finish_work);
         struct amd_gpu_scheduler *sched = s_job->sched;
 
+        /* remove job from ring_mirror_list */
+        spin_lock(&sched->job_list_lock);
+        list_del_init(&s_job->node);
         if (sched->timeout != MAX_SCHEDULE_TIMEOUT) {
-                if (cancel_delayed_work(&s_job->work_tdr))
-                        amd_sched_job_put(s_job);
+                struct amd_sched_job *next;
+
+                spin_unlock(&sched->job_list_lock);
+                cancel_delayed_work_sync(&s_job->work_tdr);
+                spin_lock(&sched->job_list_lock);
 
                 /* queue TDR for next job */
                 next = list_first_entry_or_null(&sched->ring_mirror_list,
                                                 struct amd_sched_job, node);
 
-                if (next) {
-                        INIT_DELAYED_WORK(&next->work_tdr, s_job->timeout_callback);
-                        amd_sched_job_get(next);
+                if (next)
                         schedule_delayed_work(&next->work_tdr, sched->timeout);
-                }
         }
+        spin_unlock(&sched->job_list_lock);
+        sched->ops->free_job(s_job);
 }
 
-void amd_sched_job_begin(struct amd_sched_job *s_job)
+static void amd_sched_job_finish_cb(struct fence *f, struct fence_cb *cb)
+{
+        struct amd_sched_job *job = container_of(cb, struct amd_sched_job,
+                                                 finish_cb);
+        schedule_work(&job->finish_work);
+}
+
+static void amd_sched_job_begin(struct amd_sched_job *s_job)
 {
         struct amd_gpu_scheduler *sched = s_job->sched;
 
+        spin_lock(&sched->job_list_lock);
+        list_add_tail(&s_job->node, &sched->ring_mirror_list);
         if (sched->timeout != MAX_SCHEDULE_TIMEOUT &&
-                list_first_entry_or_null(&sched->ring_mirror_list, struct amd_sched_job, node) == s_job)
-        {
-                INIT_DELAYED_WORK(&s_job->work_tdr, s_job->timeout_callback);
-                amd_sched_job_get(s_job);
+            list_first_entry_or_null(&sched->ring_mirror_list,
+                                     struct amd_sched_job, node) == s_job)
+                schedule_delayed_work(&s_job->work_tdr, sched->timeout);
+        spin_unlock(&sched->job_list_lock);
+}
+
+static void amd_sched_job_timedout(struct work_struct *work)
+{
+        struct amd_sched_job *job = container_of(work, struct amd_sched_job,
+                                                 work_tdr.work);
+
+        job->sched->ops->timedout_job(job);
+}
+
+void amd_sched_hw_job_reset(struct amd_gpu_scheduler *sched)
+{
+        struct amd_sched_job *s_job;
+
+        spin_lock(&sched->job_list_lock);
+        list_for_each_entry_reverse(s_job, &sched->ring_mirror_list, node) {
+                if (fence_remove_callback(s_job->s_fence->parent, &s_job->s_fence->cb)) {
+                        fence_put(s_job->s_fence->parent);
+                        s_job->s_fence->parent = NULL;
+                }
+        }
+        atomic_set(&sched->hw_rq_count, 0);
+        spin_unlock(&sched->job_list_lock);
+}
+
+void amd_sched_job_recovery(struct amd_gpu_scheduler *sched)
+{
+        struct amd_sched_job *s_job, *tmp;
+        int r;
+
+        spin_lock(&sched->job_list_lock);
+        s_job = list_first_entry_or_null(&sched->ring_mirror_list,
+                                         struct amd_sched_job, node);
+        if (s_job)
                 schedule_delayed_work(&s_job->work_tdr, sched->timeout);
+
+        list_for_each_entry_safe(s_job, tmp, &sched->ring_mirror_list, node) {
+                struct amd_sched_fence *s_fence = s_job->s_fence;
+                struct fence *fence;
+
+                spin_unlock(&sched->job_list_lock);
+                fence = sched->ops->run_job(s_job);
+                atomic_inc(&sched->hw_rq_count);
+                if (fence) {
+                        s_fence->parent = fence_get(fence);
+                        r = fence_add_callback(fence, &s_fence->cb,
+                                               amd_sched_process_job);
+                        if (r == -ENOENT)
+                                amd_sched_process_job(fence, &s_fence->cb);
+                        else if (r)
+                                DRM_ERROR("fence add callback failed (%d)\n",
+                                          r);
+                        fence_put(fence);
+                } else {
+                        DRM_ERROR("Failed to run job!\n");
+                        amd_sched_process_job(NULL, &s_fence->cb);
+                }
+                spin_lock(&sched->job_list_lock);
         }
+        spin_unlock(&sched->job_list_lock);
 }
 
 /**
@@ -372,36 +445,29 @@ void amd_sched_entity_push_job(struct amd_sched_job *sched_job)
 {
         struct amd_sched_entity *entity = sched_job->s_entity;
 
-        sched_job->use_sched = 1;
-        fence_add_callback(&sched_job->s_fence->base,
-                                        &sched_job->cb_free_job, amd_sched_free_job);
         trace_amd_sched_job(sched_job);
+        fence_add_callback(&sched_job->s_fence->finished, &sched_job->finish_cb,
+                           amd_sched_job_finish_cb);
         wait_event(entity->sched->job_scheduled,
                    amd_sched_entity_in(sched_job));
 }
 
 /* init a sched_job with basic field */
 int amd_sched_job_init(struct amd_sched_job *job,
-                                                struct amd_gpu_scheduler *sched,
-                                                struct amd_sched_entity *entity,
-                                                void (*timeout_cb)(struct work_struct *work),
-                                                void (*free_cb)(struct kref *refcount),
-                                                void *owner, struct fence **fence)
+                       struct amd_gpu_scheduler *sched,
+                       struct amd_sched_entity *entity,
+                       void *owner)
 {
-        INIT_LIST_HEAD(&job->node);
-        kref_init(&job->refcount);
         job->sched = sched;
         job->s_entity = entity;
         job->s_fence = amd_sched_fence_create(entity, owner);
         if (!job->s_fence)
                 return -ENOMEM;
 
-        job->s_fence->s_job = job;
-        job->timeout_callback = timeout_cb;
-        job->free_callback = free_cb;
+        INIT_WORK(&job->finish_work, amd_sched_job_finish);
+        INIT_LIST_HEAD(&job->node);
+        INIT_DELAYED_WORK(&job->work_tdr, amd_sched_job_timedout);
 
-        if (fence)
-                *fence = &job->s_fence->base;
         return 0;
 }
 
@@ -450,23 +516,25 @@ static void amd_sched_process_job(struct fence *f, struct fence_cb *cb)
         struct amd_sched_fence *s_fence =
                 container_of(cb, struct amd_sched_fence, cb);
         struct amd_gpu_scheduler *sched = s_fence->sched;
-        unsigned long flags;
 
         atomic_dec(&sched->hw_rq_count);
-
-        /* remove job from ring_mirror_list */
-        spin_lock_irqsave(&sched->job_list_lock, flags);
-        list_del_init(&s_fence->s_job->node);
-        sched->ops->finish_job(s_fence->s_job);
-        spin_unlock_irqrestore(&sched->job_list_lock, flags);
-
-        amd_sched_fence_signal(s_fence);
+        amd_sched_fence_finished(s_fence);
 
         trace_amd_sched_process_job(s_fence);
-        fence_put(&s_fence->base);
+        fence_put(&s_fence->finished);
         wake_up_interruptible(&sched->wake_up_worker);
 }
 
+static bool amd_sched_blocked(struct amd_gpu_scheduler *sched)
+{
+        if (kthread_should_park()) {
+                kthread_parkme();
+                return true;
+        }
+
+        return false;
+}
+
 static int amd_sched_main(void *param)
 {
         struct sched_param sparam = {.sched_priority = 1};
@@ -476,14 +544,15 @@ static int amd_sched_main(void *param)
         sched_setscheduler(current, SCHED_FIFO, &sparam);
 
         while (!kthread_should_stop()) {
-                struct amd_sched_entity *entity;
+                struct amd_sched_entity *entity = NULL;
                 struct amd_sched_fence *s_fence;
                 struct amd_sched_job *sched_job;
                 struct fence *fence;
 
                 wait_event_interruptible(sched->wake_up_worker,
-                        (entity = amd_sched_select_entity(sched)) ||
-                        kthread_should_stop());
+                                         (!amd_sched_blocked(sched) &&
+                                          (entity = amd_sched_select_entity(sched))) ||
+                                         kthread_should_stop());
 
                 if (!entity)
                         continue;
@@ -495,16 +564,19 @@ static int amd_sched_main(void *param)
                 s_fence = sched_job->s_fence;
 
                 atomic_inc(&sched->hw_rq_count);
-                amd_sched_job_pre_schedule(sched, sched_job);
+                amd_sched_job_begin(sched_job);
+
                 fence = sched->ops->run_job(sched_job);
                 amd_sched_fence_scheduled(s_fence);
                 if (fence) {
+                        s_fence->parent = fence_get(fence);
                         r = fence_add_callback(fence, &s_fence->cb,
                                                amd_sched_process_job);
                         if (r == -ENOENT)
                                 amd_sched_process_job(fence, &s_fence->cb);
                         else if (r)
-                                DRM_ERROR("fence add callback failed (%d)\n", r);
+                                DRM_ERROR("fence add callback failed (%d)\n",
+                                          r);
                         fence_put(fence);
                 } else {
                         DRM_ERROR("Failed to run job!\n");
diff --git a/drivers/gpu/drm/amd/scheduler/gpu_scheduler.h b/drivers/gpu/drm/amd/scheduler/gpu_scheduler.h
index 070095a9433c..7cbbbfb502ef 100644
--- a/drivers/gpu/drm/amd/scheduler/gpu_scheduler.h
+++ b/drivers/gpu/drm/amd/scheduler/gpu_scheduler.h
@@ -27,8 +27,6 @@
 #include <linux/kfifo.h>
 #include <linux/fence.h>
 
-#define AMD_SCHED_FENCE_SCHEDULED_BIT   FENCE_FLAG_USER_BITS
-
 struct amd_gpu_scheduler;
 struct amd_sched_rq;
 
@@ -68,36 +66,34 @@ struct amd_sched_rq {
 };
 
 struct amd_sched_fence {
-        struct fence                    base;
+        struct fence                    scheduled;
+        struct fence                    finished;
         struct fence_cb                 cb;
-        struct list_head                scheduled_cb;
+        struct fence                    *parent;
         struct amd_gpu_scheduler        *sched;
         spinlock_t                      lock;
         void                            *owner;
-        struct amd_sched_job    *s_job;
 };
 
 struct amd_sched_job {
-        struct kref refcount;
         struct amd_gpu_scheduler        *sched;
         struct amd_sched_entity         *s_entity;
         struct amd_sched_fence          *s_fence;
-        bool    use_sched;      /* true if the job goes to scheduler */
-        struct fence_cb                cb_free_job;
-        struct work_struct             work_free_job;
-        struct list_head                           node;
-        struct delayed_work work_tdr;
-        void (*timeout_callback) (struct work_struct *work);
-        void (*free_callback)(struct kref *refcount);
+        struct fence_cb                 finish_cb;
+        struct work_struct              finish_work;
+        struct list_head                node;
+        struct delayed_work             work_tdr;
 };
 
-extern const struct fence_ops amd_sched_fence_ops;
+extern const struct fence_ops amd_sched_fence_ops_scheduled;
+extern const struct fence_ops amd_sched_fence_ops_finished;
 static inline struct amd_sched_fence *to_amd_sched_fence(struct fence *f)
 {
-        struct amd_sched_fence *__f = container_of(f, struct amd_sched_fence, base);
+        if (f->ops == &amd_sched_fence_ops_scheduled)
+                return container_of(f, struct amd_sched_fence, scheduled);
 
-        if (__f->base.ops == &amd_sched_fence_ops)
-                return __f;
+        if (f->ops == &amd_sched_fence_ops_finished)
+                return container_of(f, struct amd_sched_fence, finished);
 
         return NULL;
 }
@@ -109,8 +105,8 @@ static inline struct amd_sched_fence *to_amd_sched_fence(struct fence *f)
 struct amd_sched_backend_ops {
         struct fence *(*dependency)(struct amd_sched_job *sched_job);
         struct fence *(*run_job)(struct amd_sched_job *sched_job);
-        void (*begin_job)(struct amd_sched_job *sched_job);
-        void (*finish_job)(struct amd_sched_job *sched_job);
+        void (*timedout_job)(struct amd_sched_job *sched_job);
+        void (*free_job)(struct amd_sched_job *sched_job);
 };
 
 enum amd_sched_priority {
@@ -152,25 +148,11 @@ void amd_sched_entity_push_job(struct amd_sched_job *sched_job);
 struct amd_sched_fence *amd_sched_fence_create(
         struct amd_sched_entity *s_entity, void *owner);
 void amd_sched_fence_scheduled(struct amd_sched_fence *fence);
-void amd_sched_fence_signal(struct amd_sched_fence *fence);
+void amd_sched_fence_finished(struct amd_sched_fence *fence);
 int amd_sched_job_init(struct amd_sched_job *job,
-                                        struct amd_gpu_scheduler *sched,
-                                        struct amd_sched_entity *entity,
-                                        void (*timeout_cb)(struct work_struct *work),
-                                        void (*free_cb)(struct kref* refcount),
-                                        void *owner, struct fence **fence);
-void amd_sched_job_pre_schedule(struct amd_gpu_scheduler *sched ,
-                                                                struct amd_sched_job *s_job);
-void amd_sched_job_finish(struct amd_sched_job *s_job);
-void amd_sched_job_begin(struct amd_sched_job *s_job);
-static inline void amd_sched_job_get(struct amd_sched_job *job) {
-        if (job)
-                kref_get(&job->refcount);
-}
-
-static inline void amd_sched_job_put(struct amd_sched_job *job) {
-        if (job)
-                kref_put(&job->refcount, job->free_callback);
-}
-
+                       struct amd_gpu_scheduler *sched,
+                       struct amd_sched_entity *entity,
+                       void *owner);
+void amd_sched_hw_job_reset(struct amd_gpu_scheduler *sched);
+void amd_sched_job_recovery(struct amd_gpu_scheduler *sched);
 #endif
diff --git a/drivers/gpu/drm/amd/scheduler/sched_fence.c b/drivers/gpu/drm/amd/scheduler/sched_fence.c
index 2a732c490375..6b63beaf7574 100644
--- a/drivers/gpu/drm/amd/scheduler/sched_fence.c
+++ b/drivers/gpu/drm/amd/scheduler/sched_fence.c
@@ -27,7 +27,8 @@
 #include <drm/drmP.h>
 #include "gpu_scheduler.h"
 
-struct amd_sched_fence *amd_sched_fence_create(struct amd_sched_entity *s_entity, void *owner)
+struct amd_sched_fence *amd_sched_fence_create(struct amd_sched_entity *entity,
+                                               void *owner)
 {
         struct amd_sched_fence *fence = NULL;
         unsigned seq;
@@ -36,46 +37,37 @@ struct amd_sched_fence *amd_sched_fence_create(struct amd_sched_entity *s_entity
         if (fence == NULL)
                 return NULL;
 
-        INIT_LIST_HEAD(&fence->scheduled_cb);
         fence->owner = owner;
-        fence->sched = s_entity->sched;
+        fence->sched = entity->sched;
         spin_lock_init(&fence->lock);
 
-        seq = atomic_inc_return(&s_entity->fence_seq);
-        fence_init(&fence->base, &amd_sched_fence_ops, &fence->lock,
-                   s_entity->fence_context, seq);
+        seq = atomic_inc_return(&entity->fence_seq);
+        fence_init(&fence->scheduled, &amd_sched_fence_ops_scheduled,
+                   &fence->lock, entity->fence_context, seq);
+        fence_init(&fence->finished, &amd_sched_fence_ops_finished,
+                   &fence->lock, entity->fence_context + 1, seq);
 
         return fence;
 }
 
-void amd_sched_fence_signal(struct amd_sched_fence *fence)
+void amd_sched_fence_scheduled(struct amd_sched_fence *fence)
 {
-        int ret = fence_signal(&fence->base);
+        int ret = fence_signal(&fence->scheduled);
+
         if (!ret)
-                FENCE_TRACE(&fence->base, "signaled from irq context\n");
+                FENCE_TRACE(&fence->scheduled, "signaled from irq context\n");
         else
-                FENCE_TRACE(&fence->base, "was already signaled\n");
-}
-
-void amd_sched_job_pre_schedule(struct amd_gpu_scheduler *sched ,
-                                struct amd_sched_job *s_job)
-{
-        unsigned long flags;
-        spin_lock_irqsave(&sched->job_list_lock, flags);
-        list_add_tail(&s_job->node, &sched->ring_mirror_list);
-        sched->ops->begin_job(s_job);
-        spin_unlock_irqrestore(&sched->job_list_lock, flags);
+                FENCE_TRACE(&fence->scheduled, "was already signaled\n");
 }
 
-void amd_sched_fence_scheduled(struct amd_sched_fence *s_fence)
+void amd_sched_fence_finished(struct amd_sched_fence *fence)
 {
-        struct fence_cb *cur, *tmp;
+        int ret = fence_signal(&fence->finished);
 
-        set_bit(AMD_SCHED_FENCE_SCHEDULED_BIT, &s_fence->base.flags);
-        list_for_each_entry_safe(cur, tmp, &s_fence->scheduled_cb, node) {
-                list_del_init(&cur->node);
-                cur->func(&s_fence->base, cur);
-        }
+        if (!ret)
+                FENCE_TRACE(&fence->finished, "signaled from irq context\n");
+        else
+                FENCE_TRACE(&fence->finished, "was already signaled\n");
 }
 
 static const char *amd_sched_fence_get_driver_name(struct fence *fence)
@@ -105,6 +97,8 @@ static void amd_sched_fence_free(struct rcu_head *rcu)
 {
         struct fence *f = container_of(rcu, struct fence, rcu);
         struct amd_sched_fence *fence = to_amd_sched_fence(f);
+
+        fence_put(fence->parent);
         kmem_cache_free(sched_fence_slab, fence);
 }
 
@@ -116,16 +110,41 @@ static void amd_sched_fence_free(struct rcu_head *rcu)
  * This function is called when the reference count becomes zero.
  * It just RCU schedules freeing up the fence.
  */
-static void amd_sched_fence_release(struct fence *f)
+static void amd_sched_fence_release_scheduled(struct fence *f)
 {
-        call_rcu(&f->rcu, amd_sched_fence_free);
+        struct amd_sched_fence *fence = to_amd_sched_fence(f);
+
+        call_rcu(&fence->finished.rcu, amd_sched_fence_free);
 }
 
-const struct fence_ops amd_sched_fence_ops = {
+/**
+ * amd_sched_fence_release_scheduled - drop extra reference
+ *
+ * @f: fence
+ *
+ * Drop the extra reference from the scheduled fence to the base fence.
+ */
+static void amd_sched_fence_release_finished(struct fence *f)
+{
+        struct amd_sched_fence *fence = to_amd_sched_fence(f);
+
+        fence_put(&fence->scheduled);
+}
+
+const struct fence_ops amd_sched_fence_ops_scheduled = {
+        .get_driver_name = amd_sched_fence_get_driver_name,
+        .get_timeline_name = amd_sched_fence_get_timeline_name,
+        .enable_signaling = amd_sched_fence_enable_signaling,
+        .signaled = NULL,
+        .wait = fence_default_wait,
+        .release = amd_sched_fence_release_scheduled,
+};
+
+const struct fence_ops amd_sched_fence_ops_finished = {
         .get_driver_name = amd_sched_fence_get_driver_name,
         .get_timeline_name = amd_sched_fence_get_timeline_name,
         .enable_signaling = amd_sched_fence_enable_signaling,
         .signaled = NULL,
         .wait = fence_default_wait,
-        .release = amd_sched_fence_release,
+        .release = amd_sched_fence_release_finished,
 };
diff --git a/drivers/gpu/drm/arc/Kconfig b/drivers/gpu/drm/arc/Kconfig
index f9a13b658fea..f47d88ba4fa5 100644
--- a/drivers/gpu/drm/arc/Kconfig
+++ b/drivers/gpu/drm/arc/Kconfig
@@ -2,7 +2,6 @@ config DRM_ARCPGU
         tristate "ARC PGU"
         depends on DRM && OF
         select DRM_KMS_CMA_HELPER
-        select DRM_KMS_FB_HELPER
         select DRM_KMS_HELPER
         help
           Choose this option if you have an ARC PGU controller.
diff --git a/drivers/gpu/drm/arc/Makefile b/drivers/gpu/drm/arc/Makefile
index d48fda70f857..73de56a0139a 100644
--- a/drivers/gpu/drm/arc/Makefile
+++ b/drivers/gpu/drm/arc/Makefile
@@ -1,2 +1,2 @@
-arcpgu-y := arcpgu_crtc.o arcpgu_hdmi.o arcpgu_drv.o
+arcpgu-y := arcpgu_crtc.o arcpgu_hdmi.o arcpgu_sim.o arcpgu_drv.o
 obj-$(CONFIG_DRM_ARCPGU) += arcpgu.o
diff --git a/drivers/gpu/drm/arc/arcpgu.h b/drivers/gpu/drm/arc/arcpgu.h
index 86574b698a78..e8fcf3ab1d9a 100644
--- a/drivers/gpu/drm/arc/arcpgu.h
+++ b/drivers/gpu/drm/arc/arcpgu.h
@@ -22,7 +22,6 @@ struct arcpgu_drm_private {
         struct clk              *clk;
         struct drm_fbdev_cma    *fbdev;
         struct drm_framebuffer  *fb;
-        struct list_head        event_list;
         struct drm_crtc         crtc;
         struct drm_plane        *plane;
 };
@@ -43,6 +42,7 @@ static inline u32 arc_pgu_read(struct arcpgu_drm_private *arcpgu,
 
 int arc_pgu_setup_crtc(struct drm_device *dev);
 int arcpgu_drm_hdmi_init(struct drm_device *drm, struct device_node *np);
+int arcpgu_drm_sim_init(struct drm_device *drm, struct device_node *np);
 struct drm_fbdev_cma *arcpgu_fbdev_cma_init(struct drm_device *dev,
         unsigned int preferred_bpp, unsigned int num_crtc,
         unsigned int max_conn_count);
diff --git a/drivers/gpu/drm/arc/arcpgu_crtc.c b/drivers/gpu/drm/arc/arcpgu_crtc.c
index 92f8beff8e60..ee0a61c2861b 100644
--- a/drivers/gpu/drm/arc/arcpgu_crtc.c
+++ b/drivers/gpu/drm/arc/arcpgu_crtc.c
@@ -145,20 +145,14 @@ static int arc_pgu_crtc_atomic_check(struct drm_crtc *crtc,
 static void arc_pgu_crtc_atomic_begin(struct drm_crtc *crtc,
                                       struct drm_crtc_state *state)
 {
-        struct arcpgu_drm_private *arcpgu = crtc_to_arcpgu_priv(crtc);
-        unsigned long flags;
-
-        if (crtc->state->event) {
-                struct drm_pending_vblank_event *event = crtc->state->event;
+        struct drm_pending_vblank_event *event = crtc->state->event;
 
+        if (event) {
                 crtc->state->event = NULL;
-                event->pipe = drm_crtc_index(crtc);
-
-                WARN_ON(drm_crtc_vblank_get(crtc) != 0);
 
-                spin_lock_irqsave(&crtc->dev->event_lock, flags);
-                list_add_tail(&event->base.link, &arcpgu->event_list);
-                spin_unlock_irqrestore(&crtc->dev->event_lock, flags);
+                spin_lock_irq(&crtc->dev->event_lock);
+                drm_crtc_send_vblank_event(crtc, event);
+                spin_unlock_irq(&crtc->dev->event_lock);
         }
 }
 
diff --git a/drivers/gpu/drm/arc/arcpgu_drv.c b/drivers/gpu/drm/arc/arcpgu_drv.c
index 76e187a5bde0..6d4ff34737cb 100644
--- a/drivers/gpu/drm/arc/arcpgu_drv.c
+++ b/drivers/gpu/drm/arc/arcpgu_drv.c
@@ -28,21 +28,14 @@ static void arcpgu_fb_output_poll_changed(struct drm_device *dev)
 {
         struct arcpgu_drm_private *arcpgu = dev->dev_private;
 
-        if (arcpgu->fbdev)
-                drm_fbdev_cma_hotplug_event(arcpgu->fbdev);
-}
-
-static int arcpgu_atomic_commit(struct drm_device *dev,
-                                    struct drm_atomic_state *state, bool async)
-{
-        return drm_atomic_helper_commit(dev, state, false);
+        drm_fbdev_cma_hotplug_event(arcpgu->fbdev);
 }
 
 static struct drm_mode_config_funcs arcpgu_drm_modecfg_funcs = {
         .fb_create  = drm_fb_cma_create,
         .output_poll_changed = arcpgu_fb_output_poll_changed,
         .atomic_check = drm_atomic_helper_check,
-        .atomic_commit = arcpgu_atomic_commit,
+        .atomic_commit = drm_atomic_helper_commit,
 };
 
 static void arcpgu_setup_mode_config(struct drm_device *drm)
@@ -55,7 +48,7 @@ static void arcpgu_setup_mode_config(struct drm_device *drm)
         drm->mode_config.funcs = &arcpgu_drm_modecfg_funcs;
 }
 
-int arcpgu_gem_mmap(struct file *filp, struct vm_area_struct *vma)
+static int arcpgu_gem_mmap(struct file *filp, struct vm_area_struct *vma)
 {
         int ret;
 
@@ -81,22 +74,6 @@ static const struct file_operations arcpgu_drm_ops = {
         .mmap = arcpgu_gem_mmap,
 };
 
-static void arcpgu_preclose(struct drm_device *drm, struct drm_file *file)
-{
-        struct arcpgu_drm_private *arcpgu = drm->dev_private;
-        struct drm_pending_vblank_event *e, *t;
-        unsigned long flags;
-
-        spin_lock_irqsave(&drm->event_lock, flags);
-        list_for_each_entry_safe(e, t, &arcpgu->event_list, base.link) {
-                if (e->base.file_priv != file)
-                        continue;
-                list_del(&e->base.link);
-                e->base.destroy(&e->base);
-        }
-        spin_unlock_irqrestore(&drm->event_lock, flags);
-}
-
 static void arcpgu_lastclose(struct drm_device *drm)
 {
         struct arcpgu_drm_private *arcpgu = drm->dev_private;
@@ -122,16 +99,12 @@ static int arcpgu_load(struct drm_device *drm)
         if (IS_ERR(arcpgu->clk))
                 return PTR_ERR(arcpgu->clk);
 
-        INIT_LIST_HEAD(&arcpgu->event_list);
-
         arcpgu_setup_mode_config(drm);
 
         res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
         arcpgu->regs = devm_ioremap_resource(&pdev->dev, res);
-        if (IS_ERR(arcpgu->regs)) {
-                dev_err(drm->dev, "Could not remap IO mem\n");
+        if (IS_ERR(arcpgu->regs))
                 return PTR_ERR(arcpgu->regs);
-        }
 
         dev_info(drm->dev, "arc_pgu ID: 0x%x\n",
                  arc_pgu_read(arcpgu, ARCPGU_REG_ID));
@@ -149,15 +122,17 @@ static int arcpgu_load(struct drm_device *drm)
 
         /* find the encoder node and initialize it */
         encoder_node = of_parse_phandle(drm->dev->of_node, "encoder-slave", 0);
-        if (!encoder_node) {
-                dev_err(drm->dev, "failed to get an encoder slave node\n");
-                return -ENODEV;
+        if (encoder_node) {
+                ret = arcpgu_drm_hdmi_init(drm, encoder_node);
+                of_node_put(encoder_node);
+                if (ret < 0)
+                        return ret;
+        } else {
+                ret = arcpgu_drm_sim_init(drm, NULL);
+                if (ret < 0)
+                        return ret;
         }
 
-        ret = arcpgu_drm_hdmi_init(drm, encoder_node);
-        if (ret < 0)
-                return ret;
-
         drm_mode_config_reset(drm);
         drm_kms_helper_poll_init(drm);
 
@@ -174,7 +149,7 @@ static int arcpgu_load(struct drm_device *drm)
         return 0;
 }
 
-int arcpgu_unload(struct drm_device *drm)
+static int arcpgu_unload(struct drm_device *drm)
 {
         struct arcpgu_drm_private *arcpgu = drm->dev_private;
 
@@ -192,7 +167,6 @@ int arcpgu_unload(struct drm_device *drm)
 static struct drm_driver arcpgu_drm_driver = {
         .driver_features = DRIVER_MODESET | DRIVER_GEM | DRIVER_PRIME |
                            DRIVER_ATOMIC,
-        .preclose = arcpgu_preclose,
         .lastclose = arcpgu_lastclose,
         .name = "drm-arcpgu",
         .desc = "ARC PGU Controller",
@@ -207,7 +181,7 @@ static struct drm_driver arcpgu_drm_driver = {
         .get_vblank_counter = drm_vblank_no_hw_counter,
         .prime_handle_to_fd = drm_gem_prime_handle_to_fd,
         .prime_fd_to_handle = drm_gem_prime_fd_to_handle,
-        .gem_free_object = drm_gem_cma_free_object,
+        .gem_free_object_unlocked = drm_gem_cma_free_object,
         .gem_vm_ops = &drm_gem_cma_vm_ops,
         .gem_prime_export = drm_gem_prime_export,
         .gem_prime_import = drm_gem_prime_import,
@@ -235,15 +209,8 @@ static int arcpgu_probe(struct platform_device *pdev)
         if (ret)
                 goto err_unload;
 
-        ret = drm_connector_register_all(drm);
-        if (ret)
-                goto err_unregister;
-
         return 0;
 
-err_unregister:
-        drm_dev_unregister(drm);
-
 err_unload:
         arcpgu_unload(drm);
 
@@ -257,7 +224,6 @@ static int arcpgu_remove(struct platform_device *pdev)
 {
         struct drm_device *drm = platform_get_drvdata(pdev);
 
-        drm_connector_unregister_all(drm);
         drm_dev_unregister(drm);
         arcpgu_unload(drm);
         drm_dev_unref(drm);
diff --git a/drivers/gpu/drm/arc/arcpgu_hdmi.c b/drivers/gpu/drm/arc/arcpgu_hdmi.c
index 08b6baeb320d..b7a8b2ac4055 100644
--- a/drivers/gpu/drm/arc/arcpgu_hdmi.c
+++ b/drivers/gpu/drm/arc/arcpgu_hdmi.c
@@ -46,23 +46,6 @@ static int arcpgu_drm_connector_get_modes(struct drm_connector *connector)
         return sfuncs->get_modes(&slave->base, connector);
 }
 
-struct drm_encoder *
-arcpgu_drm_connector_best_encoder(struct drm_connector *connector)
-{
-        struct drm_encoder_slave *slave;
-        struct arcpgu_drm_connector *con =
-                container_of(connector, struct arcpgu_drm_connector, connector);
-
-        slave = con->encoder_slave;
-        if (slave == NULL) {
-                dev_err(connector->dev->dev,
-                        "connector_best_encoder: cannot find slave encoder for connector\n");
-                return NULL;
-        }
-
-        return &slave->base;
-}
-
 static enum drm_connector_status
 arcpgu_drm_connector_detect(struct drm_connector *connector, bool force)
 {
@@ -97,7 +80,6 @@ static void arcpgu_drm_connector_destroy(struct drm_connector *connector)
 static const struct drm_connector_helper_funcs
 arcpgu_drm_connector_helper_funcs = {
         .get_modes = arcpgu_drm_connector_get_modes,
-        .best_encoder = arcpgu_drm_connector_best_encoder,
 };
 
 static const struct drm_connector_funcs arcpgu_drm_connector_funcs = {
diff --git a/drivers/gpu/drm/arc/arcpgu_sim.c b/drivers/gpu/drm/arc/arcpgu_sim.c
new file mode 100644
index 000000000000..2bf06d71556a
--- /dev/null
+++ b/drivers/gpu/drm/arc/arcpgu_sim.c
@@ -0,0 +1,128 @@
+/*
+ * ARC PGU DRM driver.
+ *
+ * Copyright (C) 2016 Synopsys, Inc. (www.synopsys.com)
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ */
+
+#include <drm/drm_crtc_helper.h>
+#include <drm/drm_encoder_slave.h>
+#include <drm/drm_atomic_helper.h>
+
+#include "arcpgu.h"
+
+#define XRES_DEF        640
+#define YRES_DEF        480
+
+#define XRES_MAX        8192
+#define YRES_MAX        8192
+
+
+struct arcpgu_drm_connector {
+        struct drm_connector connector;
+        struct drm_encoder_slave *encoder_slave;
+};
+
+static int arcpgu_drm_connector_get_modes(struct drm_connector *connector)
+{
+        int count;
+
+        count = drm_add_modes_noedid(connector, XRES_MAX, YRES_MAX);
+        drm_set_preferred_mode(connector, XRES_DEF, YRES_DEF);
+        return count;
+}
+
+static enum drm_connector_status
+arcpgu_drm_connector_detect(struct drm_connector *connector, bool force)
+{
+        return connector_status_connected;
+}
+
+static void arcpgu_drm_connector_destroy(struct drm_connector *connector)
+{
+        drm_connector_unregister(connector);
+        drm_connector_cleanup(connector);
+}
+
+static const struct drm_connector_helper_funcs
+arcpgu_drm_connector_helper_funcs = {
+        .get_modes = arcpgu_drm_connector_get_modes,
+};
+
+static const struct drm_connector_funcs arcpgu_drm_connector_funcs = {
+        .dpms = drm_helper_connector_dpms,
+        .reset = drm_atomic_helper_connector_reset,
+        .detect = arcpgu_drm_connector_detect,
+        .fill_modes = drm_helper_probe_single_connector_modes,
+        .destroy = arcpgu_drm_connector_destroy,
+        .atomic_duplicate_state = drm_atomic_helper_connector_duplicate_state,
+        .atomic_destroy_state = drm_atomic_helper_connector_destroy_state,
+};
+
+static struct drm_encoder_funcs arcpgu_drm_encoder_funcs = {
+        .destroy = drm_encoder_cleanup,
+};
+
+int arcpgu_drm_sim_init(struct drm_device *drm, struct device_node *np)
+{
+        struct arcpgu_drm_connector *arcpgu_connector;
+        struct drm_encoder_slave *encoder;
+        struct drm_connector *connector;
+        int ret;
+
+        encoder = devm_kzalloc(drm->dev, sizeof(*encoder), GFP_KERNEL);
+        if (encoder == NULL)
+                return -ENOMEM;
+
+        encoder->base.possible_crtcs = 1;
+        encoder->base.possible_clones = 0;
+
+        ret = drm_encoder_init(drm, &encoder->base, &arcpgu_drm_encoder_funcs,
+                               DRM_MODE_ENCODER_VIRTUAL, NULL);
+        if (ret)
+                return ret;
+
+        arcpgu_connector = devm_kzalloc(drm->dev, sizeof(*arcpgu_connector),
+                                        GFP_KERNEL);
+        if (!arcpgu_connector) {
+                ret = -ENOMEM;
+                goto error_encoder_cleanup;
+        }
+
+        connector = &arcpgu_connector->connector;
+        drm_connector_helper_add(connector, &arcpgu_drm_connector_helper_funcs);
+
+        ret = drm_connector_init(drm, connector, &arcpgu_drm_connector_funcs,
+                        DRM_MODE_CONNECTOR_VIRTUAL);
+        if (ret < 0) {
+                dev_err(drm->dev, "failed to initialize drm connector\n");
+                goto error_encoder_cleanup;
+        }
+
+        ret = drm_mode_connector_attach_encoder(connector, &encoder->base);
+        if (ret < 0) {
+                dev_err(drm->dev, "could not attach connector to encoder\n");
+                drm_connector_unregister(connector);
+                goto error_connector_cleanup;
+        }
+
+        arcpgu_connector->encoder_slave = encoder;
+
+        return 0;
+
+error_connector_cleanup:
+        drm_connector_cleanup(connector);
+
+error_encoder_cleanup:
+        drm_encoder_cleanup(&encoder->base);
+        return ret;
+}
diff --git a/drivers/gpu/drm/arm/Kconfig b/drivers/gpu/drm/arm/Kconfig
index eaed454e043c..9a18e1bd57b4 100644
--- a/drivers/gpu/drm/arm/Kconfig
+++ b/drivers/gpu/drm/arm/Kconfig
@@ -9,7 +9,6 @@ config DRM_HDLCD
         depends on COMMON_CLK
         select DRM_ARM
         select DRM_KMS_HELPER
-        select DRM_KMS_FB_HELPER
         select DRM_KMS_CMA_HELPER
         help
           Choose this option if you have an ARM High Definition Colour LCD
@@ -25,3 +24,19 @@ config DRM_HDLCD_SHOW_UNDERRUN
           Enable this option to show in red colour the pixels that the
           HDLCD device did not fetch from framebuffer due to underrun
           conditions.
+
+config DRM_MALI_DISPLAY
+        tristate "ARM Mali Display Processor"
+        depends on DRM && OF && (ARM || ARM64)
+        depends on COMMON_CLK
+        select DRM_ARM
+        select DRM_KMS_HELPER
+        select DRM_KMS_CMA_HELPER
+        select DRM_GEM_CMA_HELPER
+        select VIDEOMODE_HELPERS
+        help
+          Choose this option if you want to compile the ARM Mali Display
+          Processor driver. It supports the DP500, DP550 and DP650 variants
+          of the hardware.
+
+          If compiled as a module it will be called mali-dp.
diff --git a/drivers/gpu/drm/arm/Makefile b/drivers/gpu/drm/arm/Makefile
index 89dcb7bab93a..bb8b158ff90d 100644
--- a/drivers/gpu/drm/arm/Makefile
+++ b/drivers/gpu/drm/arm/Makefile
@@ -1,2 +1,4 @@
 hdlcd-y := hdlcd_drv.o hdlcd_crtc.o
 obj-$(CONFIG_DRM_HDLCD) += hdlcd.o
+mali-dp-y := malidp_drv.o malidp_hw.o malidp_planes.o malidp_crtc.o
+obj-$(CONFIG_DRM_MALI_DISPLAY)  += mali-dp.o
diff --git a/drivers/gpu/drm/arm/hdlcd_crtc.c b/drivers/gpu/drm/arm/hdlcd_crtc.c
index 0813c2f06931..48019ae22ddb 100644
--- a/drivers/gpu/drm/arm/hdlcd_crtc.c
+++ b/drivers/gpu/drm/arm/hdlcd_crtc.c
@@ -196,30 +196,11 @@ static void hdlcd_crtc_atomic_begin(struct drm_crtc *crtc,
         }
 }
 
-static void hdlcd_crtc_atomic_flush(struct drm_crtc *crtc,
-                                    struct drm_crtc_state *state)
-{
-}
-
-static bool hdlcd_crtc_mode_fixup(struct drm_crtc *crtc,
-                        const struct drm_display_mode *mode,
-                        struct drm_display_mode *adjusted_mode)
-{
-        return true;
-}
-
 static const struct drm_crtc_helper_funcs hdlcd_crtc_helper_funcs = {
-        .mode_fixup     = hdlcd_crtc_mode_fixup,
-        .mode_set       = drm_helper_crtc_mode_set,
-        .mode_set_base  = drm_helper_crtc_mode_set_base,
-        .mode_set_nofb  = hdlcd_crtc_mode_set_nofb,
         .enable         = hdlcd_crtc_enable,
         .disable        = hdlcd_crtc_disable,
-        .prepare        = hdlcd_crtc_disable,
-        .commit         = hdlcd_crtc_enable,
         .atomic_check   = hdlcd_crtc_atomic_check,
         .atomic_begin   = hdlcd_crtc_atomic_begin,
-        .atomic_flush   = hdlcd_crtc_atomic_flush,
 };
 
 static int hdlcd_plane_atomic_check(struct drm_plane *plane,
diff --git a/drivers/gpu/drm/arm/hdlcd_drv.c b/drivers/gpu/drm/arm/hdlcd_drv.c
index a6ca36f0096f..d83b46a30327 100644
--- a/drivers/gpu/drm/arm/hdlcd_drv.c
+++ b/drivers/gpu/drm/arm/hdlcd_drv.c
@@ -102,21 +102,14 @@ static void hdlcd_fb_output_poll_changed(struct drm_device *drm)
 {
         struct hdlcd_drm_private *hdlcd = drm->dev_private;
 
-        if (hdlcd->fbdev)
-                drm_fbdev_cma_hotplug_event(hdlcd->fbdev);
-}
-
-static int hdlcd_atomic_commit(struct drm_device *dev,
-                               struct drm_atomic_state *state, bool nonblock)
-{
-        return drm_atomic_helper_commit(dev, state, false);
+        drm_fbdev_cma_hotplug_event(hdlcd->fbdev);
 }
 
 static const struct drm_mode_config_funcs hdlcd_mode_config_funcs = {
         .fb_create = drm_fb_cma_create,
         .output_poll_changed = hdlcd_fb_output_poll_changed,
         .atomic_check = drm_atomic_helper_check,
-        .atomic_commit = hdlcd_atomic_commit,
+        .atomic_commit = drm_atomic_helper_commit,
 };
 
 static void hdlcd_setup_mode_config(struct drm_device *drm)
@@ -296,7 +289,7 @@ static struct drm_driver hdlcd_driver = {
         .get_vblank_counter = drm_vblank_no_hw_counter,
         .enable_vblank = hdlcd_enable_vblank,
         .disable_vblank = hdlcd_disable_vblank,
-        .gem_free_object = drm_gem_cma_free_object,
+        .gem_free_object_unlocked = drm_gem_cma_free_object,
         .gem_vm_ops = &drm_gem_cma_vm_ops,
         .dumb_create = drm_gem_cma_dumb_create,
         .dumb_map_offset = drm_gem_cma_dumb_map_offset,
diff --git a/drivers/gpu/drm/arm/malidp_crtc.c b/drivers/gpu/drm/arm/malidp_crtc.c
new file mode 100644
index 000000000000..08e6a71f5d05
--- /dev/null
+++ b/drivers/gpu/drm/arm/malidp_crtc.c
@@ -0,0 +1,216 @@
+/*
+ * (C) COPYRIGHT 2016 ARM Limited. All rights reserved.
+ * Author: Liviu Dudau <Liviu.Dudau@arm.com>
+ *
+ * This program is free software and is provided to you under the terms of the
+ * GNU General Public License version 2 as published by the Free Software
+ * Foundation, and any use by you of this program is subject to the terms
+ * of such GNU licence.
+ *
+ * ARM Mali DP500/DP550/DP650 driver (crtc operations)
+ */
+
+#include <drm/drmP.h>
+#include <drm/drm_atomic.h>
+#include <drm/drm_atomic_helper.h>
+#include <drm/drm_crtc.h>
+#include <drm/drm_crtc_helper.h>
+#include <linux/clk.h>
+#include <video/videomode.h>
+
+#include "malidp_drv.h"
+#include "malidp_hw.h"
+
+static bool malidp_crtc_mode_fixup(struct drm_crtc *crtc,
+                                   const struct drm_display_mode *mode,
+                                   struct drm_display_mode *adjusted_mode)
+{
+        struct malidp_drm *malidp = crtc_to_malidp_device(crtc);
+        struct malidp_hw_device *hwdev = malidp->dev;
+
+        /*
+         * check that the hardware can drive the required clock rate,
+         * but skip the check if the clock is meant to be disabled (req_rate = 0)
+         */
+        long rate, req_rate = mode->crtc_clock * 1000;
+
+        if (req_rate) {
+                rate = clk_round_rate(hwdev->mclk, req_rate);
+                if (rate < req_rate) {
+                        DRM_DEBUG_DRIVER("mclk clock unable to reach %d kHz\n",
+                                         mode->crtc_clock);
+                        return false;
+                }
+
+                rate = clk_round_rate(hwdev->pxlclk, req_rate);
+                if (rate != req_rate) {
+                        DRM_DEBUG_DRIVER("pxlclk doesn't support %ld Hz\n",
+                                         req_rate);
+                        return false;
+                }
+        }
+
+        return true;
+}
+
+static void malidp_crtc_enable(struct drm_crtc *crtc)
+{
+        struct malidp_drm *malidp = crtc_to_malidp_device(crtc);
+        struct malidp_hw_device *hwdev = malidp->dev;
+        struct videomode vm;
+
+        drm_display_mode_to_videomode(&crtc->state->adjusted_mode, &vm);
+
+        clk_prepare_enable(hwdev->pxlclk);
+
+        /* mclk needs to be set to the same or higher rate than pxlclk */
+        clk_set_rate(hwdev->mclk, crtc->state->adjusted_mode.crtc_clock * 1000);
+        clk_set_rate(hwdev->pxlclk, crtc->state->adjusted_mode.crtc_clock * 1000);
+
+        hwdev->modeset(hwdev, &vm);
+        hwdev->leave_config_mode(hwdev);
+        drm_crtc_vblank_on(crtc);
+}
+
+static void malidp_crtc_disable(struct drm_crtc *crtc)
+{
+        struct malidp_drm *malidp = crtc_to_malidp_device(crtc);
+        struct malidp_hw_device *hwdev = malidp->dev;
+
+        drm_crtc_vblank_off(crtc);
+        hwdev->enter_config_mode(hwdev);
+        clk_disable_unprepare(hwdev->pxlclk);
+}
+
+static int malidp_crtc_atomic_check(struct drm_crtc *crtc,
+                                    struct drm_crtc_state *state)
+{
+        struct malidp_drm *malidp = crtc_to_malidp_device(crtc);
+        struct malidp_hw_device *hwdev = malidp->dev;
+        struct drm_plane *plane;
+        const struct drm_plane_state *pstate;
+        u32 rot_mem_free, rot_mem_usable;
+        int rotated_planes = 0;
+
+        /*
+         * check if there is enough rotation memory available for planes
+         * that need 90° and 270° rotation. Each plane has set its required
+         * memory size in the ->plane_check() callback, here we only make
+         * sure that the sums are less that the total usable memory.
+         *
+         * The rotation memory allocation algorithm (for each plane):
+         *  a. If no more rotated planes exist, all remaining rotate
+         *     memory in the bank is available for use by the plane.
+         *  b. If other rotated planes exist, and plane's layer ID is
+         *     DE_VIDEO1, it can use all the memory from first bank if
+         *     secondary rotation memory bank is available, otherwise it can
+         *     use up to half the bank's memory.
+         *  c. If other rotated planes exist, and plane's layer ID is not
+         *     DE_VIDEO1, it can use half of the available memory
+         *
+         * Note: this algorithm assumes that the order in which the planes are
+         * checked always has DE_VIDEO1 plane first in the list if it is
+         * rotated. Because that is how we create the planes in the first
+         * place, under current DRM version things work, but if ever the order
+         * in which drm_atomic_crtc_state_for_each_plane() iterates over planes
+         * changes, we need to pre-sort the planes before validation.
+         */
+
+        /* first count the number of rotated planes */
+        drm_atomic_crtc_state_for_each_plane_state(plane, pstate, state) {
+                if (pstate->rotation & MALIDP_ROTATED_MASK)
+                        rotated_planes++;
+        }
+
+        rot_mem_free = hwdev->rotation_memory[0];
+        /*
+         * if we have more than 1 plane using rotation memory, use the second
+         * block of rotation memory as well
+         */
+        if (rotated_planes > 1)
+                rot_mem_free += hwdev->rotation_memory[1];
+
+        /* now validate the rotation memory requirements */
+        drm_atomic_crtc_state_for_each_plane_state(plane, pstate, state) {
+                struct malidp_plane *mp = to_malidp_plane(plane);
+                struct malidp_plane_state *ms = to_malidp_plane_state(pstate);
+
+                if (pstate->rotation & MALIDP_ROTATED_MASK) {
+                        /* process current plane */
+                        rotated_planes--;
+
+                        if (!rotated_planes) {
+                                /* no more rotated planes, we can use what's left */
+                                rot_mem_usable = rot_mem_free;
+                        } else {
+                                if ((mp->layer->id != DE_VIDEO1) ||
+                                    (hwdev->rotation_memory[1] == 0))
+                                        rot_mem_usable = rot_mem_free / 2;
+                                else
+                                        rot_mem_usable = hwdev->rotation_memory[0];
+                        }
+
+                        rot_mem_free -= rot_mem_usable;
+
+                        if (ms->rotmem_size > rot_mem_usable)
+                                return -EINVAL;
+                }
+        }
+
+        return 0;
+}
+
+static const struct drm_crtc_helper_funcs malidp_crtc_helper_funcs = {
+        .mode_fixup = malidp_crtc_mode_fixup,
+        .enable = malidp_crtc_enable,
+        .disable = malidp_crtc_disable,
+        .atomic_check = malidp_crtc_atomic_check,
+};
+
+static const struct drm_crtc_funcs malidp_crtc_funcs = {
+        .destroy = drm_crtc_cleanup,
+        .set_config = drm_atomic_helper_set_config,
+        .page_flip = drm_atomic_helper_page_flip,
+        .reset = drm_atomic_helper_crtc_reset,
+        .atomic_duplicate_state = drm_atomic_helper_crtc_duplicate_state,
+        .atomic_destroy_state = drm_atomic_helper_crtc_destroy_state,
+};
+
+int malidp_crtc_init(struct drm_device *drm)
+{
+        struct malidp_drm *malidp = drm->dev_private;
+        struct drm_plane *primary = NULL, *plane;
+        int ret;
+
+        ret = malidp_de_planes_init(drm);
+        if (ret < 0) {
+                DRM_ERROR("Failed to initialise planes\n");
+                return ret;
+        }
+
+        drm_for_each_plane(plane, drm) {
+                if (plane->type == DRM_PLANE_TYPE_PRIMARY) {
+                        primary = plane;
+                        break;
+                }
+        }
+
+        if (!primary) {
+                DRM_ERROR("no primary plane found\n");
+                ret = -EINVAL;
+                goto crtc_cleanup_planes;
+        }
+
+        ret = drm_crtc_init_with_planes(drm, &malidp->crtc, primary, NULL,
+                                        &malidp_crtc_funcs, NULL);
+
+        if (!ret) {
+                drm_crtc_helper_add(&malidp->crtc, &malidp_crtc_helper_funcs);
+                return 0;
+        }
+
+crtc_cleanup_planes:
+        malidp_de_planes_destroy(drm);
+
+        return ret;
+}
diff --git a/drivers/gpu/drm/arm/malidp_drv.c b/drivers/gpu/drm/arm/malidp_drv.c
new file mode 100644
index 000000000000..82171d223f2d
--- /dev/null
+++ b/drivers/gpu/drm/arm/malidp_drv.c
@@ -0,0 +1,519 @@
+/*
+ * (C) COPYRIGHT 2016 ARM Limited. All rights reserved.
+ * Author: Liviu Dudau <Liviu.Dudau@arm.com>
+ *
+ * This program is free software and is provided to you under the terms of the
+ * GNU General Public License version 2 as published by the Free Software
+ * Foundation, and any use by you of this program is subject to the terms
+ * of such GNU licence.
+ *
+ * ARM Mali DP500/DP550/DP650 KMS/DRM driver
+ */
+
+#include <linux/module.h>
+#include <linux/clk.h>
+#include <linux/component.h>
+#include <linux/of_device.h>
+#include <linux/of_graph.h>
+#include <linux/of_reserved_mem.h>
+
+#include <drm/drmP.h>
+#include <drm/drm_atomic.h>
+#include <drm/drm_atomic_helper.h>
+#include <drm/drm_crtc.h>
+#include <drm/drm_crtc_helper.h>
+#include <drm/drm_fb_helper.h>
+#include <drm/drm_fb_cma_helper.h>
+#include <drm/drm_gem_cma_helper.h>
+#include <drm/drm_of.h>
+
+#include "malidp_drv.h"
+#include "malidp_regs.h"
+#include "malidp_hw.h"
+
+#define MALIDP_CONF_VALID_TIMEOUT       250
+
+/*
+ * set the "config valid" bit and wait until the hardware acts on it
+ */
+static int malidp_set_and_wait_config_valid(struct drm_device *drm)
+{
+        struct malidp_drm *malidp = drm->dev_private;
+        struct malidp_hw_device *hwdev = malidp->dev;
+        int ret;
+
+        hwdev->set_config_valid(hwdev);
+        /* don't wait for config_valid flag if we are in config mode */
+        if (hwdev->in_config_mode(hwdev))
+                return 0;
+
+        ret = wait_event_interruptible_timeout(malidp->wq,
+                        atomic_read(&malidp->config_valid) == 1,
+                        msecs_to_jiffies(MALIDP_CONF_VALID_TIMEOUT));
+
+        return (ret > 0) ? 0 : -ETIMEDOUT;
+}
+
+static void malidp_output_poll_changed(struct drm_device *drm)
+{
+        struct malidp_drm *malidp = drm->dev_private;
+
+        drm_fbdev_cma_hotplug_event(malidp->fbdev);
+}
+
+static void malidp_atomic_commit_hw_done(struct drm_atomic_state *state)
+{
+        struct drm_pending_vblank_event *event;
+        struct drm_device *drm = state->dev;
+        struct malidp_drm *malidp = drm->dev_private;
+        int ret = malidp_set_and_wait_config_valid(drm);
+
+        if (ret)
+                DRM_DEBUG_DRIVER("timed out waiting for updated configuration\n");
+
+        event = malidp->crtc.state->event;
+        if (event) {
+                malidp->crtc.state->event = NULL;
+
+                spin_lock_irq(&drm->event_lock);
+                if (drm_crtc_vblank_get(&malidp->crtc) == 0)
+                        drm_crtc_arm_vblank_event(&malidp->crtc, event);
+                else
+                        drm_crtc_send_vblank_event(&malidp->crtc, event);
+                spin_unlock_irq(&drm->event_lock);
+        }
+        drm_atomic_helper_commit_hw_done(state);
+}
+
+static void malidp_atomic_commit_tail(struct drm_atomic_state *state)
+{
+        struct drm_device *drm = state->dev;
+
+        drm_atomic_helper_commit_modeset_disables(drm, state);
+        drm_atomic_helper_commit_modeset_enables(drm, state);
+        drm_atomic_helper_commit_planes(drm, state, true);
+
+        malidp_atomic_commit_hw_done(state);
+
+        drm_atomic_helper_wait_for_vblanks(drm, state);
+
+        drm_atomic_helper_cleanup_planes(drm, state);
+}
+
+static struct drm_mode_config_helper_funcs malidp_mode_config_helpers = {
+        .atomic_commit_tail = malidp_atomic_commit_tail,
+};
+
+static const struct drm_mode_config_funcs malidp_mode_config_funcs = {
+        .fb_create = drm_fb_cma_create,
+        .output_poll_changed = malidp_output_poll_changed,
+        .atomic_check = drm_atomic_helper_check,
+        .atomic_commit = drm_atomic_helper_commit,
+};
+
+static int malidp_enable_vblank(struct drm_device *drm, unsigned int crtc)
+{
+        struct malidp_drm *malidp = drm->dev_private;
+        struct malidp_hw_device *hwdev = malidp->dev;
+
+        malidp_hw_enable_irq(hwdev, MALIDP_DE_BLOCK,
+                             hwdev->map.de_irq_map.vsync_irq);
+        return 0;
+}
+
+static void malidp_disable_vblank(struct drm_device *drm, unsigned int pipe)
+{
+        struct malidp_drm *malidp = drm->dev_private;
+        struct malidp_hw_device *hwdev = malidp->dev;
+
+        malidp_hw_disable_irq(hwdev, MALIDP_DE_BLOCK,
+                              hwdev->map.de_irq_map.vsync_irq);
+}
+
+static int malidp_init(struct drm_device *drm)
+{
+        int ret;
+        struct malidp_drm *malidp = drm->dev_private;
+        struct malidp_hw_device *hwdev = malidp->dev;
+
+        drm_mode_config_init(drm);
+
+        drm->mode_config.min_width = hwdev->min_line_size;
+        drm->mode_config.min_height = hwdev->min_line_size;
+        drm->mode_config.max_width = hwdev->max_line_size;
+        drm->mode_config.max_height = hwdev->max_line_size;
+        drm->mode_config.funcs = &malidp_mode_config_funcs;
+        drm->mode_config.helper_private = &malidp_mode_config_helpers;
+
+        ret = malidp_crtc_init(drm);
+        if (ret) {
+                drm_mode_config_cleanup(drm);
+                return ret;
+        }
+
+        return 0;
+}
+
+static int malidp_irq_init(struct platform_device *pdev)
+{
+        int irq_de, irq_se, ret = 0;
+        struct drm_device *drm = dev_get_drvdata(&pdev->dev);
+
+        /* fetch the interrupts from DT */
+        irq_de = platform_get_irq_byname(pdev, "DE");
+        if (irq_de < 0) {
+                DRM_ERROR("no 'DE' IRQ specified!\n");
+                return irq_de;
+        }
+        irq_se = platform_get_irq_byname(pdev, "SE");
+        if (irq_se < 0) {
+                DRM_ERROR("no 'SE' IRQ specified!\n");
+                return irq_se;
+        }
+
+        ret = malidp_de_irq_init(drm, irq_de);
+        if (ret)
+                return ret;
+
+        ret = malidp_se_irq_init(drm, irq_se);
+        if (ret) {
+                malidp_de_irq_fini(drm);
+                return ret;
+        }
+
+        return 0;
+}
+
+static void malidp_lastclose(struct drm_device *drm)
+{
+        struct malidp_drm *malidp = drm->dev_private;
+
+        drm_fbdev_cma_restore_mode(malidp->fbdev);
+}
+
+static const struct file_operations fops = {
+        .owner = THIS_MODULE,
+        .open = drm_open,
+        .release = drm_release,
+        .unlocked_ioctl = drm_ioctl,
+#ifdef CONFIG_COMPAT
+        .compat_ioctl = drm_compat_ioctl,
+#endif
+        .poll = drm_poll,
+        .read = drm_read,
+        .llseek = noop_llseek,
+        .mmap = drm_gem_cma_mmap,
+};
+
+static struct drm_driver malidp_driver = {
+        .driver_features = DRIVER_GEM | DRIVER_MODESET | DRIVER_ATOMIC |
+                           DRIVER_PRIME,
+        .lastclose = malidp_lastclose,
+        .get_vblank_counter = drm_vblank_no_hw_counter,
+        .enable_vblank = malidp_enable_vblank,
+        .disable_vblank = malidp_disable_vblank,
+        .gem_free_object_unlocked = drm_gem_cma_free_object,
+        .gem_vm_ops = &drm_gem_cma_vm_ops,
+        .dumb_create = drm_gem_cma_dumb_create,
+        .dumb_map_offset = drm_gem_cma_dumb_map_offset,
+        .dumb_destroy = drm_gem_dumb_destroy,
+        .prime_handle_to_fd = drm_gem_prime_handle_to_fd,
+        .prime_fd_to_handle = drm_gem_prime_fd_to_handle,
+        .gem_prime_export = drm_gem_prime_export,
+        .gem_prime_import = drm_gem_prime_import,
+        .gem_prime_get_sg_table = drm_gem_cma_prime_get_sg_table,
+        .gem_prime_import_sg_table = drm_gem_cma_prime_import_sg_table,
+        .gem_prime_vmap = drm_gem_cma_prime_vmap,
+        .gem_prime_vunmap = drm_gem_cma_prime_vunmap,
+        .gem_prime_mmap = drm_gem_cma_prime_mmap,
+        .fops = &fops,
+        .name = "mali-dp",
+        .desc = "ARM Mali Display Processor driver",
+        .date = "20160106",
+        .major = 1,
+        .minor = 0,
+};
+
+static const struct of_device_id  malidp_drm_of_match[] = {
+        {
+                .compatible = "arm,mali-dp500",
+                .data = &malidp_device[MALIDP_500]
+        },
+        {
+                .compatible = "arm,mali-dp550",
+                .data = &malidp_device[MALIDP_550]
+        },
+        {
+                .compatible = "arm,mali-dp650",
+                .data = &malidp_device[MALIDP_650]
+        },
+        {},
+};
+MODULE_DEVICE_TABLE(of, malidp_drm_of_match);
+
+#define MAX_OUTPUT_CHANNELS     3
+
+static int malidp_bind(struct device *dev)
+{
+        struct resource *res;
+        struct drm_device *drm;
+        struct device_node *ep;
+        struct malidp_drm *malidp;
+        struct malidp_hw_device *hwdev;
+        struct platform_device *pdev = to_platform_device(dev);
+        /* number of lines for the R, G and B output */
+        u8 output_width[MAX_OUTPUT_CHANNELS];
+        int ret = 0, i;
+        u32 version, out_depth = 0;
+
+        malidp = devm_kzalloc(dev, sizeof(*malidp), GFP_KERNEL);
+        if (!malidp)
+                return -ENOMEM;
+
+        hwdev = devm_kzalloc(dev, sizeof(*hwdev), GFP_KERNEL);
+        if (!hwdev)
+                return -ENOMEM;
+
+        /*
+         * copy the associated data from malidp_drm_of_match to avoid
+         * having to keep a reference to the OF node after binding
+         */
+        memcpy(hwdev, of_device_get_match_data(dev), sizeof(*hwdev));
+        malidp->dev = hwdev;
+
+        INIT_LIST_HEAD(&malidp->event_list);
+
+        res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+        hwdev->regs = devm_ioremap_resource(dev, res);
+        if (IS_ERR(hwdev->regs))
+                return PTR_ERR(hwdev->regs);
+
+        hwdev->pclk = devm_clk_get(dev, "pclk");
+        if (IS_ERR(hwdev->pclk))
+                return PTR_ERR(hwdev->pclk);
+
+        hwdev->aclk = devm_clk_get(dev, "aclk");
+        if (IS_ERR(hwdev->aclk))
+                return PTR_ERR(hwdev->aclk);
+
+        hwdev->mclk = devm_clk_get(dev, "mclk");
+        if (IS_ERR(hwdev->mclk))
+                return PTR_ERR(hwdev->mclk);
+
+        hwdev->pxlclk = devm_clk_get(dev, "pxlclk");
+        if (IS_ERR(hwdev->pxlclk))
+                return PTR_ERR(hwdev->pxlclk);
+
+        /* Get the optional framebuffer memory resource */
+        ret = of_reserved_mem_device_init(dev);
+        if (ret && ret != -ENODEV)
+                return ret;
+
+        drm = drm_dev_alloc(&malidp_driver, dev);
+        if (!drm) {
+                ret = -ENOMEM;
+                goto alloc_fail;
+        }
+
+        /* Enable APB clock in order to get access to the registers */
+        clk_prepare_enable(hwdev->pclk);
+        /*
+         * Enable AXI clock and main clock so that prefetch can start once
+         * the registers are set
+         */
+        clk_prepare_enable(hwdev->aclk);
+        clk_prepare_enable(hwdev->mclk);
+
+        ret = hwdev->query_hw(hwdev);
+        if (ret) {
+                DRM_ERROR("Invalid HW configuration\n");
+                goto query_hw_fail;
+        }
+
+        version = malidp_hw_read(hwdev, hwdev->map.dc_base + MALIDP_DE_CORE_ID);
+        DRM_INFO("found ARM Mali-DP%3x version r%dp%d\n", version >> 16,
+                 (version >> 12) & 0xf, (version >> 8) & 0xf);
+
+        /* set the number of lines used for output of RGB data */
+        ret = of_property_read_u8_array(dev->of_node,
+                                        "arm,malidp-output-port-lines",
+                                        output_width, MAX_OUTPUT_CHANNELS);
+        if (ret)
+                goto query_hw_fail;
+
+        for (i = 0; i < MAX_OUTPUT_CHANNELS; i++)
+                out_depth = (out_depth << 8) | (output_width[i] & 0xf);
+        malidp_hw_write(hwdev, out_depth, hwdev->map.out_depth_base);
+
+        drm->dev_private = malidp;
+        dev_set_drvdata(dev, drm);
+        atomic_set(&malidp->config_valid, 0);
+        init_waitqueue_head(&malidp->wq);
+
+        ret = malidp_init(drm);
+        if (ret < 0)
+                goto init_fail;
+
+        ret = drm_dev_register(drm, 0);
+        if (ret)
+                goto register_fail;
+
+        /* Set the CRTC's port so that the encoder component can find it */
+        ep = of_graph_get_next_endpoint(dev->of_node, NULL);
+        if (!ep) {
+                ret = -EINVAL;
+                goto port_fail;
+        }
+        malidp->crtc.port = of_get_next_parent(ep);
+
+        ret = component_bind_all(dev, drm);
+        if (ret) {
+                DRM_ERROR("Failed to bind all components\n");
+                goto bind_fail;
+        }
+
+        ret = malidp_irq_init(pdev);
+        if (ret < 0)
+                goto irq_init_fail;
+
+        ret = drm_vblank_init(drm, drm->mode_config.num_crtc);
+        if (ret < 0) {
+                DRM_ERROR("failed to initialise vblank\n");
+                goto vblank_fail;
+        }
+
+        drm_mode_config_reset(drm);
+
+        malidp->fbdev = drm_fbdev_cma_init(drm, 32, drm->mode_config.num_crtc,
+                                           drm->mode_config.num_connector);
+
+        if (IS_ERR(malidp->fbdev)) {
+                ret = PTR_ERR(malidp->fbdev);
+                malidp->fbdev = NULL;
+                goto fbdev_fail;
+        }
+
+        drm_kms_helper_poll_init(drm);
+        return 0;
+
+fbdev_fail:
+        drm_vblank_cleanup(drm);
+vblank_fail:
+        malidp_se_irq_fini(drm);
+        malidp_de_irq_fini(drm);
+irq_init_fail:
+        component_unbind_all(dev, drm);
+bind_fail:
+        of_node_put(malidp->crtc.port);
+        malidp->crtc.port = NULL;
+port_fail:
+        drm_dev_unregister(drm);
+register_fail:
+        malidp_de_planes_destroy(drm);
+        drm_mode_config_cleanup(drm);
+init_fail:
+        drm->dev_private = NULL;
+        dev_set_drvdata(dev, NULL);
+query_hw_fail:
+        clk_disable_unprepare(hwdev->mclk);
+        clk_disable_unprepare(hwdev->aclk);
+        clk_disable_unprepare(hwdev->pclk);
+        drm_dev_unref(drm);
+alloc_fail:
+        of_reserved_mem_device_release(dev);
+
+        return ret;
+}
+
+static void malidp_unbind(struct device *dev)
+{
+        struct drm_device *drm = dev_get_drvdata(dev);
+        struct malidp_drm *malidp = drm->dev_private;
+        struct malidp_hw_device *hwdev = malidp->dev;
+
+        if (malidp->fbdev) {
+                drm_fbdev_cma_fini(malidp->fbdev);
+                malidp->fbdev = NULL;
+        }
+        drm_kms_helper_poll_fini(drm);
+        malidp_se_irq_fini(drm);
+        malidp_de_irq_fini(drm);
+        drm_vblank_cleanup(drm);
+        component_unbind_all(dev, drm);
+        of_node_put(malidp->crtc.port);
+        malidp->crtc.port = NULL;
+        drm_dev_unregister(drm);
+        malidp_de_planes_destroy(drm);
+        drm_mode_config_cleanup(drm);
+        drm->dev_private = NULL;
+        dev_set_drvdata(dev, NULL);
+        clk_disable_unprepare(hwdev->mclk);
+        clk_disable_unprepare(hwdev->aclk);
+        clk_disable_unprepare(hwdev->pclk);
+        drm_dev_unref(drm);
+        of_reserved_mem_device_release(dev);
+}
+
+static const struct component_master_ops malidp_master_ops = {
+        .bind = malidp_bind,
+        .unbind = malidp_unbind,
+};
+
+static int malidp_compare_dev(struct device *dev, void *data)
+{
+        struct device_node *np = data;
+
+        return dev->of_node == np;
+}
+
+static int malidp_platform_probe(struct platform_device *pdev)
+{
+        struct device_node *port, *ep;
+        struct component_match *match = NULL;
+
+        if (!pdev->dev.of_node)
+                return -ENODEV;
+
+        /* there is only one output port inside each device, find it */
+        ep = of_graph_get_next_endpoint(pdev->dev.of_node, NULL);
+        if (!ep)
+                return -ENODEV;
+
+        if (!of_device_is_available(ep)) {
+                of_node_put(ep);
+                return -ENODEV;
+        }
+
+        /* add the remote encoder port as component */
+        port = of_graph_get_remote_port_parent(ep);
+        of_node_put(ep);
+        if (!port || !of_device_is_available(port)) {
+                of_node_put(port);
+                return -EAGAIN;
+        }
+
+        component_match_add(&pdev->dev, &match, malidp_compare_dev, port);
+        return component_master_add_with_match(&pdev->dev, &malidp_master_ops,
+                                               match);
+}
+
+static int malidp_platform_remove(struct platform_device *pdev)
+{
+        component_master_del(&pdev->dev, &malidp_master_ops);
+        return 0;
+}
+
+static struct platform_driver malidp_platform_driver = {
+        .probe          = malidp_platform_probe,
+        .remove         = malidp_platform_remove,
+        .driver = {
+                .name = "mali-dp",
+                .of_match_table = malidp_drm_of_match,
+        },
+};
+
+module_platform_driver(malidp_platform_driver);
+
+MODULE_AUTHOR("Liviu Dudau <Liviu.Dudau@arm.com>");
+MODULE_DESCRIPTION("ARM Mali DP DRM driver");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/gpu/drm/arm/malidp_drv.h b/drivers/gpu/drm/arm/malidp_drv.h
new file mode 100644
index 000000000000..95558fde214b
--- /dev/null
+++ b/drivers/gpu/drm/arm/malidp_drv.h
@@ -0,0 +1,54 @@
+/*
+ * (C) COPYRIGHT 2016 ARM Limited. All rights reserved.
+ * Author: Liviu Dudau <Liviu.Dudau@arm.com>
+ *
+ * This program is free software and is provided to you under the terms of the
+ * GNU General Public License version 2 as published by the Free Software
+ * Foundation, and any use by you of this program is subject to the terms
+ * of such GNU licence.
+ *
+ * ARM Mali DP500/DP550/DP650 KMS/DRM driver structures
+ */
+
+#ifndef __MALIDP_DRV_H__
+#define __MALIDP_DRV_H__
+
+#include <linux/mutex.h>
+#include <linux/wait.h>
+#include "malidp_hw.h"
+
+struct malidp_drm {
+        struct malidp_hw_device *dev;
+        struct drm_fbdev_cma *fbdev;
+        struct list_head event_list;
+        struct drm_crtc crtc;
+        wait_queue_head_t wq;
+        atomic_t config_valid;
+};
+
+#define crtc_to_malidp_device(x) container_of(x, struct malidp_drm, crtc)
+
+struct malidp_plane {
+        struct drm_plane base;
+        struct malidp_hw_device *hwdev;
+        const struct malidp_layer *layer;
+};
+
+struct malidp_plane_state {
+        struct drm_plane_state base;
+
+        /* size of the required rotation memory if plane is rotated */
+        u32 rotmem_size;
+};
+
+#define to_malidp_plane(x) container_of(x, struct malidp_plane, base)
+#define to_malidp_plane_state(x) container_of(x, struct malidp_plane_state, base)
+
+int malidp_de_planes_init(struct drm_device *drm);
+void malidp_de_planes_destroy(struct drm_device *drm);
+int malidp_crtc_init(struct drm_device *drm);
+
+/* often used combination of rotational bits */
+#define MALIDP_ROTATED_MASK     (BIT(DRM_ROTATE_90) | BIT(DRM_ROTATE_270))
+
+#endif  /* __MALIDP_DRV_H__ */
diff --git a/drivers/gpu/drm/arm/malidp_hw.c b/drivers/gpu/drm/arm/malidp_hw.c
new file mode 100644
index 000000000000..a6132f1d58c1
--- /dev/null
+++ b/drivers/gpu/drm/arm/malidp_hw.c
@@ -0,0 +1,691 @@
+/*
+ * (C) COPYRIGHT 2016 ARM Limited. All rights reserved.
+ * Author: Liviu Dudau <Liviu.Dudau@arm.com>
+ *
+ * This program is free software and is provided to you under the terms of the
+ * GNU General Public License version 2 as published by the Free Software
+ * Foundation, and any use by you of this program is subject to the terms
+ * of such GNU licence.
+ *
+ * ARM Mali DP500/DP550/DP650 hardware manipulation routines. This is where
+ * the difference between various versions of the hardware is being dealt with
+ * in an attempt to provide to the rest of the driver code a unified view
+ */
+
+#include <linux/types.h>
+#include <linux/io.h>
+#include <drm/drmP.h>
+#include <video/videomode.h>
+#include <video/display_timing.h>
+
+#include "malidp_drv.h"
+#include "malidp_hw.h"
+
+static const struct malidp_input_format malidp500_de_formats[] = {
+        /*    fourcc,   layers supporting the format,     internal id  */
+        { DRM_FORMAT_ARGB2101010, DE_VIDEO1 | DE_GRAPHICS1 | DE_GRAPHICS2,  0 },
+        { DRM_FORMAT_ABGR2101010, DE_VIDEO1 | DE_GRAPHICS1 | DE_GRAPHICS2,  1 },
+        { DRM_FORMAT_ARGB8888, DE_VIDEO1 | DE_GRAPHICS1 | DE_GRAPHICS2,  2 },
+        { DRM_FORMAT_ABGR8888, DE_VIDEO1 | DE_GRAPHICS1 | DE_GRAPHICS2,  3 },
+        { DRM_FORMAT_XRGB8888, DE_VIDEO1 | DE_GRAPHICS1 | DE_GRAPHICS2,  4 },
+        { DRM_FORMAT_XBGR8888, DE_VIDEO1 | DE_GRAPHICS1 | DE_GRAPHICS2,  5 },
+        { DRM_FORMAT_RGB888, DE_VIDEO1 | DE_GRAPHICS1 | DE_GRAPHICS2,  6 },
+        { DRM_FORMAT_BGR888, DE_VIDEO1 | DE_GRAPHICS1 | DE_GRAPHICS2,  7 },
+        { DRM_FORMAT_RGBA5551, DE_VIDEO1 | DE_GRAPHICS1 | DE_GRAPHICS2,  8 },
+        { DRM_FORMAT_ABGR1555, DE_VIDEO1 | DE_GRAPHICS1 | DE_GRAPHICS2,  9 },
+        { DRM_FORMAT_RGB565, DE_VIDEO1 | DE_GRAPHICS1 | DE_GRAPHICS2, 10 },
+        { DRM_FORMAT_BGR565, DE_VIDEO1 | DE_GRAPHICS1 | DE_GRAPHICS2, 11 },
+        { DRM_FORMAT_UYVY, DE_VIDEO1, 12 },
+        { DRM_FORMAT_YUYV, DE_VIDEO1, 13 },
+        { DRM_FORMAT_NV12, DE_VIDEO1, 14 },
+        { DRM_FORMAT_YUV420, DE_VIDEO1, 15 },
+};
+
+#define MALIDP_ID(__group, __format) \
+        ((((__group) & 0x7) << 3) | ((__format) & 0x7))
+
+#define MALIDP_COMMON_FORMATS \
+        /*    fourcc,   layers supporting the format,      internal id   */ \
+        { DRM_FORMAT_ARGB2101010, DE_VIDEO1 | DE_GRAPHICS1 | DE_VIDEO2, MALIDP_ID(0, 0) }, \
+        { DRM_FORMAT_ABGR2101010, DE_VIDEO1 | DE_GRAPHICS1 | DE_VIDEO2, MALIDP_ID(0, 1) }, \
+        { DRM_FORMAT_RGBA1010102, DE_VIDEO1 | DE_GRAPHICS1 | DE_VIDEO2, MALIDP_ID(0, 2) }, \
+        { DRM_FORMAT_BGRA1010102, DE_VIDEO1 | DE_GRAPHICS1 | DE_VIDEO2, MALIDP_ID(0, 3) }, \
+        { DRM_FORMAT_ARGB8888, DE_VIDEO1 | DE_GRAPHICS1 | DE_VIDEO2 | DE_SMART, MALIDP_ID(1, 0) }, \
+        { DRM_FORMAT_ABGR8888, DE_VIDEO1 | DE_GRAPHICS1 | DE_VIDEO2 | DE_SMART, MALIDP_ID(1, 1) }, \
+        { DRM_FORMAT_RGBA8888, DE_VIDEO1 | DE_GRAPHICS1 | DE_VIDEO2 | DE_SMART, MALIDP_ID(1, 2) }, \
+        { DRM_FORMAT_BGRA8888, DE_VIDEO1 | DE_GRAPHICS1 | DE_VIDEO2 | DE_SMART, MALIDP_ID(1, 3) }, \
+        { DRM_FORMAT_XRGB8888, DE_VIDEO1 | DE_GRAPHICS1 | DE_VIDEO2 | DE_SMART, MALIDP_ID(2, 0) }, \
+        { DRM_FORMAT_XBGR8888, DE_VIDEO1 | DE_GRAPHICS1 | DE_VIDEO2 | DE_SMART, MALIDP_ID(2, 1) }, \
+        { DRM_FORMAT_RGBX8888, DE_VIDEO1 | DE_GRAPHICS1 | DE_VIDEO2 | DE_SMART, MALIDP_ID(2, 2) }, \
+        { DRM_FORMAT_BGRX8888, DE_VIDEO1 | DE_GRAPHICS1 | DE_VIDEO2 | DE_SMART, MALIDP_ID(2, 3) }, \
+        { DRM_FORMAT_RGB888, DE_VIDEO1 | DE_GRAPHICS1 | DE_VIDEO2, MALIDP_ID(3, 0) }, \
+        { DRM_FORMAT_BGR888, DE_VIDEO1 | DE_GRAPHICS1 | DE_VIDEO2, MALIDP_ID(3, 1) }, \
+        { DRM_FORMAT_RGBA5551, DE_VIDEO1 | DE_GRAPHICS1 | DE_VIDEO2, MALIDP_ID(4, 0) }, \
+        { DRM_FORMAT_ABGR1555, DE_VIDEO1 | DE_GRAPHICS1 | DE_VIDEO2, MALIDP_ID(4, 1) }, \
+        { DRM_FORMAT_RGB565, DE_VIDEO1 | DE_GRAPHICS1 | DE_VIDEO2, MALIDP_ID(4, 2) }, \
+        { DRM_FORMAT_BGR565, DE_VIDEO1 | DE_GRAPHICS1 | DE_VIDEO2, MALIDP_ID(4, 3) }, \
+        { DRM_FORMAT_YUYV, DE_VIDEO1 | DE_VIDEO2, MALIDP_ID(5, 2) },    \
+        { DRM_FORMAT_UYVY, DE_VIDEO1 | DE_VIDEO2, MALIDP_ID(5, 3) },    \
+        { DRM_FORMAT_NV12, DE_VIDEO1 | DE_VIDEO2, MALIDP_ID(5, 6) },    \
+        { DRM_FORMAT_YUV420, DE_VIDEO1 | DE_VIDEO2, MALIDP_ID(5, 7) }
+
+static const struct malidp_input_format malidp550_de_formats[] = {
+        MALIDP_COMMON_FORMATS,
+};
+
+static const struct malidp_layer malidp500_layers[] = {
+        { DE_VIDEO1, MALIDP500_DE_LV_BASE, MALIDP500_DE_LV_PTR_BASE },
+        { DE_GRAPHICS1, MALIDP500_DE_LG1_BASE, MALIDP500_DE_LG1_PTR_BASE },
+        { DE_GRAPHICS2, MALIDP500_DE_LG2_BASE, MALIDP500_DE_LG2_PTR_BASE },
+};
+
+static const struct malidp_layer malidp550_layers[] = {
+        { DE_VIDEO1, MALIDP550_DE_LV1_BASE, MALIDP550_DE_LV1_PTR_BASE },
+        { DE_GRAPHICS1, MALIDP550_DE_LG_BASE, MALIDP550_DE_LG_PTR_BASE },
+        { DE_VIDEO2, MALIDP550_DE_LV2_BASE, MALIDP550_DE_LV2_PTR_BASE },
+        { DE_SMART, MALIDP550_DE_LS_BASE, MALIDP550_DE_LS_PTR_BASE },
+};
+
+#define MALIDP_DE_DEFAULT_PREFETCH_START        5
+
+static int malidp500_query_hw(struct malidp_hw_device *hwdev)
+{
+        u32 conf = malidp_hw_read(hwdev, MALIDP500_CONFIG_ID);
+        /* bit 4 of the CONFIG_ID register holds the line size multiplier */
+        u8 ln_size_mult = conf & 0x10 ? 2 : 1;
+
+        hwdev->min_line_size = 2;
+        hwdev->max_line_size = SZ_2K * ln_size_mult;
+        hwdev->rotation_memory[0] = SZ_1K * 64 * ln_size_mult;
+        hwdev->rotation_memory[1] = 0; /* no second rotation memory bank */
+
+        return 0;
+}
+
+static void malidp500_enter_config_mode(struct malidp_hw_device *hwdev)
+{
+        u32 status, count = 100;
+
+        malidp_hw_setbits(hwdev, MALIDP500_DC_CONFIG_REQ, MALIDP500_DC_CONTROL);
+        while (count) {
+                status = malidp_hw_read(hwdev, hwdev->map.dc_base + MALIDP_REG_STATUS);
+                if ((status & MALIDP500_DC_CONFIG_REQ) == MALIDP500_DC_CONFIG_REQ)
+                        break;
+                /*
+                 * entering config mode can take as long as the rendering
+                 * of a full frame, hence the long sleep here
+                 */
+                usleep_range(1000, 10000);
+                count--;
+        }
+        WARN(count == 0, "timeout while entering config mode");
+}
+
+static void malidp500_leave_config_mode(struct malidp_hw_device *hwdev)
+{
+        u32 status, count = 100;
+
+        malidp_hw_clearbits(hwdev, MALIDP500_DC_CONFIG_REQ, MALIDP500_DC_CONTROL);
+        while (count) {
+                status = malidp_hw_read(hwdev, hwdev->map.dc_base + MALIDP_REG_STATUS);
+                if ((status & MALIDP500_DC_CONFIG_REQ) == 0)
+                        break;
+                usleep_range(100, 1000);
+                count--;
+        }
+        WARN(count == 0, "timeout while leaving config mode");
+}
+
+static bool malidp500_in_config_mode(struct malidp_hw_device *hwdev)
+{
+        u32 status;
+
+        status = malidp_hw_read(hwdev, hwdev->map.dc_base + MALIDP_REG_STATUS);
+        if ((status & MALIDP500_DC_CONFIG_REQ) == MALIDP500_DC_CONFIG_REQ)
+                return true;
+
+        return false;
+}
+
+static void malidp500_set_config_valid(struct malidp_hw_device *hwdev)
+{
+        malidp_hw_setbits(hwdev, MALIDP_CFG_VALID, MALIDP500_CONFIG_VALID);
+}
+
+static void malidp500_modeset(struct malidp_hw_device *hwdev, struct videomode *mode)
+{
+        u32 val = 0;
+
+        malidp_hw_clearbits(hwdev, MALIDP500_DC_CLEAR_MASK, MALIDP500_DC_CONTROL);
+        if (mode->flags & DISPLAY_FLAGS_HSYNC_HIGH)
+                val |= MALIDP500_HSYNCPOL;
+        if (mode->flags & DISPLAY_FLAGS_VSYNC_HIGH)
+                val |= MALIDP500_VSYNCPOL;
+        val |= MALIDP_DE_DEFAULT_PREFETCH_START;
+        malidp_hw_setbits(hwdev, val, MALIDP500_DC_CONTROL);
+
+        /*
+         * Mali-DP500 encodes the background color like this:
+         *    - red   @ MALIDP500_BGND_COLOR[12:0]
+         *    - green @ MALIDP500_BGND_COLOR[27:16]
+         *    - blue  @ (MALIDP500_BGND_COLOR + 4)[12:0]
+         */
+        val = ((MALIDP_BGND_COLOR_G & 0xfff) << 16) |
+              (MALIDP_BGND_COLOR_R & 0xfff);
+        malidp_hw_write(hwdev, val, MALIDP500_BGND_COLOR);
+        malidp_hw_write(hwdev, MALIDP_BGND_COLOR_B, MALIDP500_BGND_COLOR + 4);
+
+        val = MALIDP_DE_H_FRONTPORCH(mode->hfront_porch) |
+                MALIDP_DE_H_BACKPORCH(mode->hback_porch);
+        malidp_hw_write(hwdev, val, MALIDP500_TIMINGS_BASE + MALIDP_DE_H_TIMINGS);
+
+        val = MALIDP500_DE_V_FRONTPORCH(mode->vfront_porch) |
+                MALIDP_DE_V_BACKPORCH(mode->vback_porch);
+        malidp_hw_write(hwdev, val, MALIDP500_TIMINGS_BASE + MALIDP_DE_V_TIMINGS);
+
+        val = MALIDP_DE_H_SYNCWIDTH(mode->hsync_len) |
+                MALIDP_DE_V_SYNCWIDTH(mode->vsync_len);
+        malidp_hw_write(hwdev, val, MALIDP500_TIMINGS_BASE + MALIDP_DE_SYNC_WIDTH);
+
+        val = MALIDP_DE_H_ACTIVE(mode->hactive) | MALIDP_DE_V_ACTIVE(mode->vactive);
+        malidp_hw_write(hwdev, val, MALIDP500_TIMINGS_BASE + MALIDP_DE_HV_ACTIVE);
+
+        if (mode->flags & DISPLAY_FLAGS_INTERLACED)
+                malidp_hw_setbits(hwdev, MALIDP_DISP_FUNC_ILACED, MALIDP_DE_DISPLAY_FUNC);
+        else
+                malidp_hw_clearbits(hwdev, MALIDP_DISP_FUNC_ILACED, MALIDP_DE_DISPLAY_FUNC);
+}
+
+static int malidp500_rotmem_required(struct malidp_hw_device *hwdev, u16 w, u16 h, u32 fmt)
+{
+        unsigned int depth;
+        int bpp;
+
+        /* RGB888 or BGR888 can't be rotated */
+        if ((fmt == DRM_FORMAT_RGB888) || (fmt == DRM_FORMAT_BGR888))
+                return -EINVAL;
+
+        /*
+         * Each layer needs enough rotation memory to fit 8 lines
+         * worth of pixel data. Required size is then:
+         *    size = rotated_width * (bpp / 8) * 8;
+         */
+        drm_fb_get_bpp_depth(fmt, &depth, &bpp);
+
+        return w * bpp;
+}
+
+static int malidp550_query_hw(struct malidp_hw_device *hwdev)
+{
+        u32 conf = malidp_hw_read(hwdev, MALIDP550_CONFIG_ID);
+        u8 ln_size = (conf >> 4) & 0x3, rsize;
+
+        hwdev->min_line_size = 2;
+
+        switch (ln_size) {
+        case 0:
+                hwdev->max_line_size = SZ_2K;
+                /* two banks of 64KB for rotation memory */
+                rsize = 64;
+                break;
+        case 1:
+                hwdev->max_line_size = SZ_4K;
+                /* two banks of 128KB for rotation memory */
+                rsize = 128;
+                break;
+        case 2:
+                hwdev->max_line_size = 1280;
+                /* two banks of 40KB for rotation memory */
+                rsize = 40;
+                break;
+        case 3:
+                /* reserved value */
+                hwdev->max_line_size = 0;
+                return -EINVAL;
+        }
+
+        hwdev->rotation_memory[0] = hwdev->rotation_memory[1] = rsize * SZ_1K;
+        return 0;
+}
+
+static void malidp550_enter_config_mode(struct malidp_hw_device *hwdev)
+{
+        u32 status, count = 100;
+
+        malidp_hw_setbits(hwdev, MALIDP550_DC_CONFIG_REQ, MALIDP550_DC_CONTROL);
+        while (count) {
+                status = malidp_hw_read(hwdev, hwdev->map.dc_base + MALIDP_REG_STATUS);
+                if ((status & MALIDP550_DC_CONFIG_REQ) == MALIDP550_DC_CONFIG_REQ)
+                        break;
+                /*
+                 * entering config mode can take as long as the rendering
+                 * of a full frame, hence the long sleep here
+                 */
+                usleep_range(1000, 10000);
+                count--;
+        }
+        WARN(count == 0, "timeout while entering config mode");
+}
+
+static void malidp550_leave_config_mode(struct malidp_hw_device *hwdev)
+{
+        u32 status, count = 100;
+
+        malidp_hw_clearbits(hwdev, MALIDP550_DC_CONFIG_REQ, MALIDP550_DC_CONTROL);
+        while (count) {
+                status = malidp_hw_read(hwdev, hwdev->map.dc_base + MALIDP_REG_STATUS);
+                if ((status & MALIDP550_DC_CONFIG_REQ) == 0)
+                        break;
+                usleep_range(100, 1000);
+                count--;
+        }
+        WARN(count == 0, "timeout while leaving config mode");
+}
+
+static bool malidp550_in_config_mode(struct malidp_hw_device *hwdev)
+{
+        u32 status;
+
+        status = malidp_hw_read(hwdev, hwdev->map.dc_base + MALIDP_REG_STATUS);
+        if ((status & MALIDP550_DC_CONFIG_REQ) == MALIDP550_DC_CONFIG_REQ)
+                return true;
+
+        return false;
+}
+
+static void malidp550_set_config_valid(struct malidp_hw_device *hwdev)
+{
+        malidp_hw_setbits(hwdev, MALIDP_CFG_VALID, MALIDP550_CONFIG_VALID);
+}
+
+static void malidp550_modeset(struct malidp_hw_device *hwdev, struct videomode *mode)
+{
+        u32 val = MALIDP_DE_DEFAULT_PREFETCH_START;
+
+        malidp_hw_write(hwdev, val, MALIDP550_DE_CONTROL);
+        /*
+         * Mali-DP550 and Mali-DP650 encode the background color like this:
+         *   - red   @ MALIDP550_DE_BGND_COLOR[23:16]
+         *   - green @ MALIDP550_DE_BGND_COLOR[15:8]
+         *   - blue  @ MALIDP550_DE_BGND_COLOR[7:0]
+         *
+         * We need to truncate the least significant 4 bits from the default
+         * MALIDP_BGND_COLOR_x values
+         */
+        val = (((MALIDP_BGND_COLOR_R >> 4) & 0xff) << 16) |
+              (((MALIDP_BGND_COLOR_G >> 4) & 0xff) << 8) |
+              ((MALIDP_BGND_COLOR_B >> 4) & 0xff);
+        malidp_hw_write(hwdev, val, MALIDP550_DE_BGND_COLOR);
+
+        val = MALIDP_DE_H_FRONTPORCH(mode->hfront_porch) |
+                MALIDP_DE_H_BACKPORCH(mode->hback_porch);
+        malidp_hw_write(hwdev, val, MALIDP550_TIMINGS_BASE + MALIDP_DE_H_TIMINGS);
+
+        val = MALIDP550_DE_V_FRONTPORCH(mode->vfront_porch) |
+                MALIDP_DE_V_BACKPORCH(mode->vback_porch);
+        malidp_hw_write(hwdev, val, MALIDP550_TIMINGS_BASE + MALIDP_DE_V_TIMINGS);
+
+        val = MALIDP_DE_H_SYNCWIDTH(mode->hsync_len) |
+                MALIDP_DE_V_SYNCWIDTH(mode->vsync_len);
+        if (mode->flags & DISPLAY_FLAGS_HSYNC_HIGH)
+                val |= MALIDP550_HSYNCPOL;
+        if (mode->flags & DISPLAY_FLAGS_VSYNC_HIGH)
+                val |= MALIDP550_VSYNCPOL;
+        malidp_hw_write(hwdev, val, MALIDP550_TIMINGS_BASE + MALIDP_DE_SYNC_WIDTH);
+
+        val = MALIDP_DE_H_ACTIVE(mode->hactive) | MALIDP_DE_V_ACTIVE(mode->vactive);
+        malidp_hw_write(hwdev, val, MALIDP550_TIMINGS_BASE + MALIDP_DE_HV_ACTIVE);
+
+        if (mode->flags & DISPLAY_FLAGS_INTERLACED)
+                malidp_hw_setbits(hwdev, MALIDP_DISP_FUNC_ILACED, MALIDP_DE_DISPLAY_FUNC);
+        else
+                malidp_hw_clearbits(hwdev, MALIDP_DISP_FUNC_ILACED, MALIDP_DE_DISPLAY_FUNC);
+}
+
+static int malidp550_rotmem_required(struct malidp_hw_device *hwdev, u16 w, u16 h, u32 fmt)
+{
+        u32 bytes_per_col;
+
+        /* raw RGB888 or BGR888 can't be rotated */
+        if ((fmt == DRM_FORMAT_RGB888) || (fmt == DRM_FORMAT_BGR888))
+                return -EINVAL;
+
+        switch (fmt) {
+        /* 8 lines at 4 bytes per pixel */
+        case DRM_FORMAT_ARGB2101010:
+        case DRM_FORMAT_ABGR2101010:
+        case DRM_FORMAT_RGBA1010102:
+        case DRM_FORMAT_BGRA1010102:
+        case DRM_FORMAT_ARGB8888:
+        case DRM_FORMAT_ABGR8888:
+        case DRM_FORMAT_RGBA8888:
+        case DRM_FORMAT_BGRA8888:
+        case DRM_FORMAT_XRGB8888:
+        case DRM_FORMAT_XBGR8888:
+        case DRM_FORMAT_RGBX8888:
+        case DRM_FORMAT_BGRX8888:
+        case DRM_FORMAT_RGB888:
+        case DRM_FORMAT_BGR888:
+        /* 16 lines at 2 bytes per pixel */
+        case DRM_FORMAT_RGBA5551:
+        case DRM_FORMAT_ABGR1555:
+        case DRM_FORMAT_RGB565:
+        case DRM_FORMAT_BGR565:
+        case DRM_FORMAT_UYVY:
+        case DRM_FORMAT_YUYV:
+                bytes_per_col = 32;
+                break;
+        /* 16 lines at 1.5 bytes per pixel */
+        case DRM_FORMAT_NV12:
+        case DRM_FORMAT_YUV420:
+                bytes_per_col = 24;
+                break;
+        default:
+                return -EINVAL;
+        }
+
+        return w * bytes_per_col;
+}
+
+static int malidp650_query_hw(struct malidp_hw_device *hwdev)
+{
+        u32 conf = malidp_hw_read(hwdev, MALIDP550_CONFIG_ID);
+        u8 ln_size = (conf >> 4) & 0x3, rsize;
+
+        hwdev->min_line_size = 4;
+
+        switch (ln_size) {
+        case 0:
+        case 2:
+                /* reserved values */
+                hwdev->max_line_size = 0;
+                return -EINVAL;
+        case 1:
+                hwdev->max_line_size = SZ_4K;
+                /* two banks of 128KB for rotation memory */
+                rsize = 128;
+                break;
+        case 3:
+                hwdev->max_line_size = 2560;
+                /* two banks of 80KB for rotation memory */
+                rsize = 80;
+        }
+
+        hwdev->rotation_memory[0] = hwdev->rotation_memory[1] = rsize * SZ_1K;
+        return 0;
+}
+
+const struct malidp_hw_device malidp_device[MALIDP_MAX_DEVICES] = {
+        [MALIDP_500] = {
+                .map = {
+                        .se_base = MALIDP500_SE_BASE,
+                        .dc_base = MALIDP500_DC_BASE,
+                        .out_depth_base = MALIDP500_OUTPUT_DEPTH,
+                        .features = 0,  /* no CLEARIRQ register */
+                        .n_layers = ARRAY_SIZE(malidp500_layers),
+                        .layers = malidp500_layers,
+                        .de_irq_map = {
+                                .irq_mask = MALIDP_DE_IRQ_UNDERRUN |
+                                            MALIDP500_DE_IRQ_AXI_ERR |
+                                            MALIDP500_DE_IRQ_VSYNC |
+                                            MALIDP500_DE_IRQ_GLOBAL,
+                                .vsync_irq = MALIDP500_DE_IRQ_VSYNC,
+                        },
+                        .se_irq_map = {
+                                .irq_mask = MALIDP500_SE_IRQ_CONF_MODE,
+                                .vsync_irq = 0,
+                        },
+                        .dc_irq_map = {
+                                .irq_mask = MALIDP500_DE_IRQ_CONF_VALID,
+                                .vsync_irq = MALIDP500_DE_IRQ_CONF_VALID,
+                        },
+                        .input_formats = malidp500_de_formats,
+                        .n_input_formats = ARRAY_SIZE(malidp500_de_formats),
+                },
+                .query_hw = malidp500_query_hw,
+                .enter_config_mode = malidp500_enter_config_mode,
+                .leave_config_mode = malidp500_leave_config_mode,
+                .in_config_mode = malidp500_in_config_mode,
+                .set_config_valid = malidp500_set_config_valid,
+                .modeset = malidp500_modeset,
+                .rotmem_required = malidp500_rotmem_required,
+        },
+        [MALIDP_550] = {
+                .map = {
+                        .se_base = MALIDP550_SE_BASE,
+                        .dc_base = MALIDP550_DC_BASE,
+                        .out_depth_base = MALIDP550_DE_OUTPUT_DEPTH,
+                        .features = MALIDP_REGMAP_HAS_CLEARIRQ,
+                        .n_layers = ARRAY_SIZE(malidp550_layers),
+                        .layers = malidp550_layers,
+                        .de_irq_map = {
+                                .irq_mask = MALIDP_DE_IRQ_UNDERRUN |
+                                            MALIDP550_DE_IRQ_VSYNC,
+                                .vsync_irq = MALIDP550_DE_IRQ_VSYNC,
+                        },
+                        .se_irq_map = {
+                                .irq_mask = MALIDP550_SE_IRQ_EOW |
+                                            MALIDP550_SE_IRQ_AXI_ERR,
+                        },
+                        .dc_irq_map = {
+                                .irq_mask = MALIDP550_DC_IRQ_CONF_VALID,
+                                .vsync_irq = MALIDP550_DC_IRQ_CONF_VALID,
+                        },
+                        .input_formats = malidp550_de_formats,
+                        .n_input_formats = ARRAY_SIZE(malidp550_de_formats),
+                },
+                .query_hw = malidp550_query_hw,
+                .enter_config_mode = malidp550_enter_config_mode,
+                .leave_config_mode = malidp550_leave_config_mode,
+                .in_config_mode = malidp550_in_config_mode,
+                .set_config_valid = malidp550_set_config_valid,
+                .modeset = malidp550_modeset,
+                .rotmem_required = malidp550_rotmem_required,
+        },
+        [MALIDP_650] = {
+                .map = {
+                        .se_base = MALIDP550_SE_BASE,
+                        .dc_base = MALIDP550_DC_BASE,
+                        .out_depth_base = MALIDP550_DE_OUTPUT_DEPTH,
+                        .features = MALIDP_REGMAP_HAS_CLEARIRQ,
+                        .n_layers = ARRAY_SIZE(malidp550_layers),
+                        .layers = malidp550_layers,
+                        .de_irq_map = {
+                                .irq_mask = MALIDP_DE_IRQ_UNDERRUN |
+                                            MALIDP650_DE_IRQ_DRIFT |
+                                            MALIDP550_DE_IRQ_VSYNC,
+                                .vsync_irq = MALIDP550_DE_IRQ_VSYNC,
+                        },
+                        .se_irq_map = {
+                                .irq_mask = MALIDP550_SE_IRQ_EOW |
+                                            MALIDP550_SE_IRQ_AXI_ERR,
+                        },
+                        .dc_irq_map = {
+                                .irq_mask = MALIDP550_DC_IRQ_CONF_VALID,
+                                .vsync_irq = MALIDP550_DC_IRQ_CONF_VALID,
+                        },
+                        .input_formats = malidp550_de_formats,
+                        .n_input_formats = ARRAY_SIZE(malidp550_de_formats),
+                },
+                .query_hw = malidp650_query_hw,
+                .enter_config_mode = malidp550_enter_config_mode,
+                .leave_config_mode = malidp550_leave_config_mode,
+                .in_config_mode = malidp550_in_config_mode,
+                .set_config_valid = malidp550_set_config_valid,
+                .modeset = malidp550_modeset,
+                .rotmem_required = malidp550_rotmem_required,
+        },
+};
+
+u8 malidp_hw_get_format_id(const struct malidp_hw_regmap *map,
+                           u8 layer_id, u32 format)
+{
+        unsigned int i;
+
+        for (i = 0; i < map->n_input_formats; i++) {
+                if (((map->input_formats[i].layer & layer_id) == layer_id) &&
+                    (map->input_formats[i].format == format))
+                        return map->input_formats[i].id;
+        }
+
+        return MALIDP_INVALID_FORMAT_ID;
+}
+
+static void malidp_hw_clear_irq(struct malidp_hw_device *hwdev, u8 block, u32 irq)
+{
+        u32 base = malidp_get_block_base(hwdev, block);
+
+        if (hwdev->map.features & MALIDP_REGMAP_HAS_CLEARIRQ)
+                malidp_hw_write(hwdev, irq, base + MALIDP_REG_CLEARIRQ);
+        else
+                malidp_hw_write(hwdev, irq, base + MALIDP_REG_STATUS);
+}
+
+static irqreturn_t malidp_de_irq(int irq, void *arg)
+{
+        struct drm_device *drm = arg;
+        struct malidp_drm *malidp = drm->dev_private;
+        struct malidp_hw_device *hwdev;
+        const struct malidp_irq_map *de;
+        u32 status, mask, dc_status;
+        irqreturn_t ret = IRQ_NONE;
+
+        if (!drm->dev_private)
+                return IRQ_HANDLED;
+
+        hwdev = malidp->dev;
+        de = &hwdev->map.de_irq_map;
+
+        /* first handle the config valid IRQ */
+        dc_status = malidp_hw_read(hwdev, hwdev->map.dc_base + MALIDP_REG_STATUS);
+        if (dc_status & hwdev->map.dc_irq_map.vsync_irq) {
+                /* we have a page flip event */
+                atomic_set(&malidp->config_valid, 1);
+                malidp_hw_clear_irq(hwdev, MALIDP_DC_BLOCK, dc_status);
+                ret = IRQ_WAKE_THREAD;
+        }
+
+        status = malidp_hw_read(hwdev, MALIDP_REG_STATUS);
+        if (!(status & de->irq_mask))
+                return ret;
+
+        mask = malidp_hw_read(hwdev, MALIDP_REG_MASKIRQ);
+        status &= mask;
+        if (status & de->vsync_irq)
+                drm_crtc_handle_vblank(&malidp->crtc);
+
+        malidp_hw_clear_irq(hwdev, MALIDP_DE_BLOCK, status);
+
+        return (ret == IRQ_NONE) ? IRQ_HANDLED : ret;
+}
+
+static irqreturn_t malidp_de_irq_thread_handler(int irq, void *arg)
+{
+        struct drm_device *drm = arg;
+        struct malidp_drm *malidp = drm->dev_private;
+
+        wake_up(&malidp->wq);
+
+        return IRQ_HANDLED;
+}
+
+int malidp_de_irq_init(struct drm_device *drm, int irq)
+{
+        struct malidp_drm *malidp = drm->dev_private;
+        struct malidp_hw_device *hwdev = malidp->dev;
+        int ret;
+
+        /* ensure interrupts are disabled */
+        malidp_hw_disable_irq(hwdev, MALIDP_DE_BLOCK, 0xffffffff);
+        malidp_hw_clear_irq(hwdev, MALIDP_DE_BLOCK, 0xffffffff);
+        malidp_hw_disable_irq(hwdev, MALIDP_DC_BLOCK, 0xffffffff);
+        malidp_hw_clear_irq(hwdev, MALIDP_DC_BLOCK, 0xffffffff);
+
+        ret = devm_request_threaded_irq(drm->dev, irq, malidp_de_irq,
+                                        malidp_de_irq_thread_handler,
+                                        IRQF_SHARED, "malidp-de", drm);
+        if (ret < 0) {
+                DRM_ERROR("failed to install DE IRQ handler\n");
+                return ret;
+        }
+
+        /* first enable the DC block IRQs */
+        malidp_hw_enable_irq(hwdev, MALIDP_DC_BLOCK,
+                             hwdev->map.dc_irq_map.irq_mask);
+
+        /* now enable the DE block IRQs */
+        malidp_hw_enable_irq(hwdev, MALIDP_DE_BLOCK,
+                             hwdev->map.de_irq_map.irq_mask);
+
+        return 0;
+}
+
+void malidp_de_irq_fini(struct drm_device *drm)
+{
+        struct malidp_drm *malidp = drm->dev_private;
+        struct malidp_hw_device *hwdev = malidp->dev;
+
+        malidp_hw_disable_irq(hwdev, MALIDP_DE_BLOCK,
+                              hwdev->map.de_irq_map.irq_mask);
+        malidp_hw_disable_irq(hwdev, MALIDP_DC_BLOCK,
+                              hwdev->map.dc_irq_map.irq_mask);
+}
+
+static irqreturn_t malidp_se_irq(int irq, void *arg)
+{
+        struct drm_device *drm = arg;
+        struct malidp_drm *malidp = drm->dev_private;
+        struct malidp_hw_device *hwdev = malidp->dev;
+        u32 status, mask;
+
+        status = malidp_hw_read(hwdev, hwdev->map.se_base + MALIDP_REG_STATUS);
+        if (!(status & hwdev->map.se_irq_map.irq_mask))
+                return IRQ_NONE;
+
+        mask = malidp_hw_read(hwdev, hwdev->map.se_base + MALIDP_REG_MASKIRQ);
+        status = malidp_hw_read(hwdev, hwdev->map.se_base + MALIDP_REG_STATUS);
+        status &= mask;
+        /* ToDo: status decoding and firing up of VSYNC and page flip events */
+
+        malidp_hw_clear_irq(hwdev, MALIDP_SE_BLOCK, status);
+
+        return IRQ_HANDLED;
+}
+
+static irqreturn_t malidp_se_irq_thread_handler(int irq, void *arg)
+{
+        return IRQ_HANDLED;
+}
+
+int malidp_se_irq_init(struct drm_device *drm, int irq)
+{
+        struct malidp_drm *malidp = drm->dev_private;
+        struct malidp_hw_device *hwdev = malidp->dev;
+        int ret;
+
+        /* ensure interrupts are disabled */
+        malidp_hw_disable_irq(hwdev, MALIDP_SE_BLOCK, 0xffffffff);
+        malidp_hw_clear_irq(hwdev, MALIDP_SE_BLOCK, 0xffffffff);
+
+        ret = devm_request_threaded_irq(drm->dev, irq, malidp_se_irq,
+                                        malidp_se_irq_thread_handler,
+                                        IRQF_SHARED, "malidp-se", drm);
+        if (ret < 0) {
+                DRM_ERROR("failed to install SE IRQ handler\n");
+                return ret;
+        }
+
+        malidp_hw_enable_irq(hwdev, MALIDP_SE_BLOCK,
+                             hwdev->map.se_irq_map.irq_mask);
+
+        return 0;
+}
+
+void malidp_se_irq_fini(struct drm_device *drm)
+{
+        struct malidp_drm *malidp = drm->dev_private;
+        struct malidp_hw_device *hwdev = malidp->dev;
+
+        malidp_hw_disable_irq(hwdev, MALIDP_SE_BLOCK,
+                              hwdev->map.se_irq_map.irq_mask);
+}
diff --git a/drivers/gpu/drm/arm/malidp_hw.h b/drivers/gpu/drm/arm/malidp_hw.h
new file mode 100644
index 000000000000..141743e9f3a6
--- /dev/null
+++ b/drivers/gpu/drm/arm/malidp_hw.h
@@ -0,0 +1,241 @@
+/*
+ *
+ * (C) COPYRIGHT 2013-2016 ARM Limited. All rights reserved.
+ *
+ * This program is free software and is provided to you under the terms of the
+ * GNU General Public License version 2 as published by the Free Software
+ * Foundation, and any use by you of this program is subject to the terms
+ * of such GNU licence.
+ *
+ * ARM Mali DP hardware manipulation routines.
+ */
+
+#ifndef __MALIDP_HW_H__
+#define __MALIDP_HW_H__
+
+#include <linux/bitops.h>
+#include "malidp_regs.h"
+
+struct videomode;
+struct clk;
+
+/* Mali DP IP blocks */
+enum {
+        MALIDP_DE_BLOCK = 0,
+        MALIDP_SE_BLOCK,
+        MALIDP_DC_BLOCK
+};
+
+/* Mali DP layer IDs */
+enum {
+        DE_VIDEO1 = BIT(0),
+        DE_GRAPHICS1 = BIT(1),
+        DE_GRAPHICS2 = BIT(2), /* used only in DP500 */
+        DE_VIDEO2 = BIT(3),
+        DE_SMART = BIT(4),
+};
+
+struct malidp_input_format {
+        u32 format;             /* DRM fourcc */
+        u8 layer;               /* bitmask of layers supporting it */
+        u8 id;                  /* used internally */
+};
+
+#define MALIDP_INVALID_FORMAT_ID        0xff
+
+/*
+ * hide the differences between register maps
+ * by using a common structure to hold the
+ * base register offsets
+ */
+
+struct malidp_irq_map {
+        u32 irq_mask;           /* mask of IRQs that can be enabled in the block */
+        u32 vsync_irq;          /* IRQ bit used for signaling during VSYNC */
+};
+
+struct malidp_layer {
+        u16 id;                 /* layer ID */
+        u16 base;               /* address offset for the register bank */
+        u16 ptr;                /* address offset for the pointer register */
+};
+
+/* regmap features */
+#define MALIDP_REGMAP_HAS_CLEARIRQ      (1 << 0)
+
+struct malidp_hw_regmap {
+        /* address offset of the DE register bank */
+        /* is always 0x0000 */
+        /* address offset of the SE registers bank */
+        const u16 se_base;
+        /* address offset of the DC registers bank */
+        const u16 dc_base;
+
+        /* address offset for the output depth register */
+        const u16 out_depth_base;
+
+        /* bitmap with register map features */
+        const u8 features;
+
+        /* list of supported layers */
+        const u8 n_layers;
+        const struct malidp_layer *layers;
+
+        const struct malidp_irq_map de_irq_map;
+        const struct malidp_irq_map se_irq_map;
+        const struct malidp_irq_map dc_irq_map;
+
+        /* list of supported input formats for each layer */
+        const struct malidp_input_format *input_formats;
+        const u8 n_input_formats;
+};
+
+struct malidp_hw_device {
+        const struct malidp_hw_regmap map;
+        void __iomem *regs;
+
+        /* APB clock */
+        struct clk *pclk;
+        /* AXI clock */
+        struct clk *aclk;
+        /* main clock for display core */
+        struct clk *mclk;
+        /* pixel clock for display core */
+        struct clk *pxlclk;
+
+        /*
+         * Validate the driver instance against the hardware bits
+         */
+        int (*query_hw)(struct malidp_hw_device *hwdev);
+
+        /*
+         * Set the hardware into config mode, ready to accept mode changes
+         */
+        void (*enter_config_mode)(struct malidp_hw_device *hwdev);
+
+        /*
+         * Tell hardware to exit configuration mode
+         */
+        void (*leave_config_mode)(struct malidp_hw_device *hwdev);
+
+        /*
+         * Query if hardware is in configuration mode
+         */
+        bool (*in_config_mode)(struct malidp_hw_device *hwdev);
+
+        /*
+         * Set configuration valid flag for hardware parameters that can
+         * be changed outside the configuration mode. Hardware will use
+         * the new settings when config valid is set after the end of the
+         * current buffer scanout
+         */
+        void (*set_config_valid)(struct malidp_hw_device *hwdev);
+
+        /*
+         * Set a new mode in hardware. Requires the hardware to be in
+         * configuration mode before this function is called.
+         */
+        void (*modeset)(struct malidp_hw_device *hwdev, struct videomode *m);
+
+        /*
+         * Calculate the required rotation memory given the active area
+         * and the buffer format.
+         */
+        int (*rotmem_required)(struct malidp_hw_device *hwdev, u16 w, u16 h, u32 fmt);
+
+        u8 features;
+
+        u8 min_line_size;
+        u16 max_line_size;
+
+        /* size of memory used for rotating layers, up to two banks available */
+        u32 rotation_memory[2];
+};
+
+/* Supported variants of the hardware */
+enum {
+        MALIDP_500 = 0,
+        MALIDP_550,
+        MALIDP_650,
+        /* keep the next entry last */
+        MALIDP_MAX_DEVICES
+};
+
+extern const struct malidp_hw_device malidp_device[MALIDP_MAX_DEVICES];
+
+static inline u32 malidp_hw_read(struct malidp_hw_device *hwdev, u32 reg)
+{
+        return readl(hwdev->regs + reg);
+}
+
+static inline void malidp_hw_write(struct malidp_hw_device *hwdev,
+                                   u32 value, u32 reg)
+{
+        writel(value, hwdev->regs + reg);
+}
+
+static inline void malidp_hw_setbits(struct malidp_hw_device *hwdev,
+                                     u32 mask, u32 reg)
+{
+        u32 data = malidp_hw_read(hwdev, reg);
+
+        data |= mask;
+        malidp_hw_write(hwdev, data, reg);
+}
+
+static inline void malidp_hw_clearbits(struct malidp_hw_device *hwdev,
+                                       u32 mask, u32 reg)
+{
+        u32 data = malidp_hw_read(hwdev, reg);
+
+        data &= ~mask;
+        malidp_hw_write(hwdev, data, reg);
+}
+
+static inline u32 malidp_get_block_base(struct malidp_hw_device *hwdev,
+                                        u8 block)
+{
+        switch (block) {
+        case MALIDP_SE_BLOCK:
+                return hwdev->map.se_base;
+        case MALIDP_DC_BLOCK:
+                return hwdev->map.dc_base;
+        }
+
+        return 0;
+}
+
+static inline void malidp_hw_disable_irq(struct malidp_hw_device *hwdev,
+                                         u8 block, u32 irq)
+{
+        u32 base = malidp_get_block_base(hwdev, block);
+
+        malidp_hw_clearbits(hwdev, irq, base + MALIDP_REG_MASKIRQ);
+}
+
+static inline void malidp_hw_enable_irq(struct malidp_hw_device *hwdev,
+                                        u8 block, u32 irq)
+{
+        u32 base = malidp_get_block_base(hwdev, block);
+
+        malidp_hw_setbits(hwdev, irq, base + MALIDP_REG_MASKIRQ);
+}
+
+int malidp_de_irq_init(struct drm_device *drm, int irq);
+void malidp_de_irq_fini(struct drm_device *drm);
+int malidp_se_irq_init(struct drm_device *drm, int irq);
+void malidp_se_irq_fini(struct drm_device *drm);
+
+u8 malidp_hw_get_format_id(const struct malidp_hw_regmap *map,
+                           u8 layer_id, u32 format);
+
+/*
+ * background color components are defined as 12bits values,
+ * they will be shifted right when stored on hardware that
+ * supports only 8bits per channel
+ */
+#define MALIDP_BGND_COLOR_R             0x000
+#define MALIDP_BGND_COLOR_G             0x000
+#define MALIDP_BGND_COLOR_B             0x000
+
+#endif  /* __MALIDP_HW_H__ */
diff --git a/drivers/gpu/drm/arm/malidp_planes.c b/drivers/gpu/drm/arm/malidp_planes.c
new file mode 100644
index 000000000000..725098d6179a
--- /dev/null
+++ b/drivers/gpu/drm/arm/malidp_planes.c
@@ -0,0 +1,298 @@
+/*
+ * (C) COPYRIGHT 2016 ARM Limited. All rights reserved.
+ * Author: Liviu Dudau <Liviu.Dudau@arm.com>
+ *
+ * This program is free software and is provided to you under the terms of the
+ * GNU General Public License version 2 as published by the Free Software
+ * Foundation, and any use by you of this program is subject to the terms
+ * of such GNU licence.
+ *
+ * ARM Mali DP plane manipulation routines.
+ */
+
+#include <drm/drmP.h>
+#include <drm/drm_atomic_helper.h>
+#include <drm/drm_fb_cma_helper.h>
+#include <drm/drm_gem_cma_helper.h>
+#include <drm/drm_plane_helper.h>
+
+#include "malidp_hw.h"
+#include "malidp_drv.h"
+
+/* Layer specific register offsets */
+#define MALIDP_LAYER_FORMAT             0x000
+#define MALIDP_LAYER_CONTROL            0x004
+#define   LAYER_ENABLE                  (1 << 0)
+#define   LAYER_ROT_OFFSET              8
+#define   LAYER_H_FLIP                  (1 << 10)
+#define   LAYER_V_FLIP                  (1 << 11)
+#define   LAYER_ROT_MASK                (0xf << 8)
+#define MALIDP_LAYER_SIZE               0x00c
+#define   LAYER_H_VAL(x)                (((x) & 0x1fff) << 0)
+#define   LAYER_V_VAL(x)                (((x) & 0x1fff) << 16)
+#define MALIDP_LAYER_COMP_SIZE          0x010
+#define MALIDP_LAYER_OFFSET             0x014
+#define MALIDP_LAYER_STRIDE             0x018
+
+static void malidp_de_plane_destroy(struct drm_plane *plane)
+{
+        struct malidp_plane *mp = to_malidp_plane(plane);
+
+        if (mp->base.fb)
+                drm_framebuffer_unreference(mp->base.fb);
+
+        drm_plane_helper_disable(plane);
+        drm_plane_cleanup(plane);
+        devm_kfree(plane->dev->dev, mp);
+}
+
+struct drm_plane_state *malidp_duplicate_plane_state(struct drm_plane *plane)
+{
+        struct malidp_plane_state *state, *m_state;
+
+        if (!plane->state)
+                return NULL;
+
+        state = kmalloc(sizeof(*state), GFP_KERNEL);
+        if (state) {
+                m_state = to_malidp_plane_state(plane->state);
+                __drm_atomic_helper_plane_duplicate_state(plane, &state->base);
+                state->rotmem_size = m_state->rotmem_size;
+        }
+
+        return &state->base;
+}
+
+void malidp_destroy_plane_state(struct drm_plane *plane,
+                                struct drm_plane_state *state)
+{
+        struct malidp_plane_state *m_state = to_malidp_plane_state(state);
+
+        __drm_atomic_helper_plane_destroy_state(state);
+        kfree(m_state);
+}
+
+static const struct drm_plane_funcs malidp_de_plane_funcs = {
+        .update_plane = drm_atomic_helper_update_plane,
+        .disable_plane = drm_atomic_helper_disable_plane,
+        .destroy = malidp_de_plane_destroy,
+        .reset = drm_atomic_helper_plane_reset,
+        .atomic_duplicate_state = malidp_duplicate_plane_state,
+        .atomic_destroy_state = malidp_destroy_plane_state,
+};
+
+static int malidp_de_plane_check(struct drm_plane *plane,
+                                 struct drm_plane_state *state)
+{
+        struct malidp_plane *mp = to_malidp_plane(plane);
+        struct malidp_plane_state *ms = to_malidp_plane_state(state);
+        u8 format_id;
+        u32 src_w, src_h;
+
+        if (!state->crtc || !state->fb)
+                return 0;
+
+        format_id = malidp_hw_get_format_id(&mp->hwdev->map, mp->layer->id,
+                                            state->fb->pixel_format);
+        if (format_id == MALIDP_INVALID_FORMAT_ID)
+                return -EINVAL;
+
+        src_w = state->src_w >> 16;
+        src_h = state->src_h >> 16;
+
+        if ((state->crtc_w > mp->hwdev->max_line_size) ||
+            (state->crtc_h > mp->hwdev->max_line_size) ||
+            (state->crtc_w < mp->hwdev->min_line_size) ||
+            (state->crtc_h < mp->hwdev->min_line_size) ||
+            (state->crtc_w != src_w) || (state->crtc_h != src_h))
+                return -EINVAL;
+
+        /* packed RGB888 / BGR888 can't be rotated or flipped */
+        if (state->rotation != BIT(DRM_ROTATE_0) &&
+            (state->fb->pixel_format == DRM_FORMAT_RGB888 ||
+             state->fb->pixel_format == DRM_FORMAT_BGR888))
+                return -EINVAL;
+
+        ms->rotmem_size = 0;
+        if (state->rotation & MALIDP_ROTATED_MASK) {
+                int val;
+
+                val = mp->hwdev->rotmem_required(mp->hwdev, state->crtc_h,
+                                                 state->crtc_w,
+                                                 state->fb->pixel_format);
+                if (val < 0)
+                        return val;
+
+                ms->rotmem_size = val;
+        }
+
+        return 0;
+}
+
+static void malidp_de_plane_update(struct drm_plane *plane,
+                                   struct drm_plane_state *old_state)
+{
+        struct drm_gem_cma_object *obj;
+        struct malidp_plane *mp;
+        const struct malidp_hw_regmap *map;
+        u8 format_id;
+        u16 ptr;
+        u32 format, src_w, src_h, dest_w, dest_h, val = 0;
+        int num_planes, i;
+
+        mp = to_malidp_plane(plane);
+
+        map = &mp->hwdev->map;
+        format = plane->state->fb->pixel_format;
+        format_id = malidp_hw_get_format_id(map, mp->layer->id, format);
+        num_planes = drm_format_num_planes(format);
+
+        /* convert src values from Q16 fixed point to integer */
+        src_w = plane->state->src_w >> 16;
+        src_h = plane->state->src_h >> 16;
+        if (plane->state->rotation & MALIDP_ROTATED_MASK) {
+                dest_w = plane->state->crtc_h;
+                dest_h = plane->state->crtc_w;
+        } else {
+                dest_w = plane->state->crtc_w;
+                dest_h = plane->state->crtc_h;
+        }
+
+        malidp_hw_write(mp->hwdev, format_id, mp->layer->base);
+
+        for (i = 0; i < num_planes; i++) {
+                /* calculate the offset for the layer's plane registers */
+                ptr = mp->layer->ptr + (i << 4);
+
+                obj = drm_fb_cma_get_gem_obj(plane->state->fb, i);
+                malidp_hw_write(mp->hwdev, lower_32_bits(obj->paddr), ptr);
+                malidp_hw_write(mp->hwdev, upper_32_bits(obj->paddr), ptr + 4);
+                malidp_hw_write(mp->hwdev, plane->state->fb->pitches[i],
+                                mp->layer->base + MALIDP_LAYER_STRIDE);
+        }
+
+        malidp_hw_write(mp->hwdev, LAYER_H_VAL(src_w) | LAYER_V_VAL(src_h),
+                        mp->layer->base + MALIDP_LAYER_SIZE);
+
+        malidp_hw_write(mp->hwdev, LAYER_H_VAL(dest_w) | LAYER_V_VAL(dest_h),
+                        mp->layer->base + MALIDP_LAYER_COMP_SIZE);
+
+        malidp_hw_write(mp->hwdev, LAYER_H_VAL(plane->state->crtc_x) |
+                        LAYER_V_VAL(plane->state->crtc_y),
+                        mp->layer->base + MALIDP_LAYER_OFFSET);
+
+        /* first clear the rotation bits in the register */
+        malidp_hw_clearbits(mp->hwdev, LAYER_ROT_MASK,
+                            mp->layer->base + MALIDP_LAYER_CONTROL);
+
+        /* setup the rotation and axis flip bits */
+        if (plane->state->rotation & DRM_ROTATE_MASK)
+                val = ilog2(plane->state->rotation & DRM_ROTATE_MASK) << LAYER_ROT_OFFSET;
+        if (plane->state->rotation & BIT(DRM_REFLECT_X))
+                val |= LAYER_V_FLIP;
+        if (plane->state->rotation & BIT(DRM_REFLECT_Y))
+                val |= LAYER_H_FLIP;
+
+        /* set the 'enable layer' bit */
+        val |= LAYER_ENABLE;
+
+        malidp_hw_setbits(mp->hwdev, val,
+                          mp->layer->base + MALIDP_LAYER_CONTROL);
+}
+
+static void malidp_de_plane_disable(struct drm_plane *plane,
+                                    struct drm_plane_state *state)
+{
+        struct malidp_plane *mp = to_malidp_plane(plane);
+
+        malidp_hw_clearbits(mp->hwdev, LAYER_ENABLE,
+                            mp->layer->base + MALIDP_LAYER_CONTROL);
+}
+
+static const struct drm_plane_helper_funcs malidp_de_plane_helper_funcs = {
+        .atomic_check = malidp_de_plane_check,
+        .atomic_update = malidp_de_plane_update,
+        .atomic_disable = malidp_de_plane_disable,
+};
+
+int malidp_de_planes_init(struct drm_device *drm)
+{
+        struct malidp_drm *malidp = drm->dev_private;
+        const struct malidp_hw_regmap *map = &malidp->dev->map;
+        struct malidp_plane *plane = NULL;
+        enum drm_plane_type plane_type;
+        unsigned long crtcs = 1 << drm->mode_config.num_crtc;
+        u32 *formats;
+        int ret, i, j, n;
+
+        formats = kcalloc(map->n_input_formats, sizeof(*formats), GFP_KERNEL);
+        if (!formats) {
+                ret = -ENOMEM;
+                goto cleanup;
+        }
+
+        for (i = 0; i < map->n_layers; i++) {
+                u8 id = map->layers[i].id;
+
+                plane = kzalloc(sizeof(*plane), GFP_KERNEL);
+                if (!plane) {
+                        ret = -ENOMEM;
+                        goto cleanup;
+                }
+
+                /* build the list of DRM supported formats based on the map */
+                for (n = 0, j = 0;  j < map->n_input_formats; j++) {
+                        if ((map->input_formats[j].layer & id) == id)
+                                formats[n++] = map->input_formats[j].format;
+                }
+
+                plane_type = (i == 0) ? DRM_PLANE_TYPE_PRIMARY :
+                                        DRM_PLANE_TYPE_OVERLAY;
+                ret = drm_universal_plane_init(drm, &plane->base, crtcs,
+                                               &malidp_de_plane_funcs, formats,
+                                               n, plane_type, NULL);
+                if (ret < 0)
+                        goto cleanup;
+
+                if (!drm->mode_config.rotation_property) {
+                        unsigned long flags = BIT(DRM_ROTATE_0) |
+                                              BIT(DRM_ROTATE_90) |
+                                              BIT(DRM_ROTATE_180) |
+                                              BIT(DRM_ROTATE_270) |
+                                              BIT(DRM_REFLECT_X) |
+                                              BIT(DRM_REFLECT_Y);
+                        drm->mode_config.rotation_property =
+                                drm_mode_create_rotation_property(drm, flags);
+                }
+                /* SMART layer can't be rotated */
+                if (drm->mode_config.rotation_property && (id != DE_SMART))
+                        drm_object_attach_property(&plane->base.base,
+                                                   drm->mode_config.rotation_property,
+                                                   BIT(DRM_ROTATE_0));
+
+                drm_plane_helper_add(&plane->base,
+                                     &malidp_de_plane_helper_funcs);
+                plane->hwdev = malidp->dev;
+                plane->layer = &map->layers[i];
+        }
+
+        kfree(formats);
+
+        return 0;
+
+cleanup:
+        malidp_de_planes_destroy(drm);
+        kfree(formats);
+
+        return ret;
+}
+
+void malidp_de_planes_destroy(struct drm_device *drm)
+{
+        struct drm_plane *p, *pt;
+
+        list_for_each_entry_safe(p, pt, &drm->mode_config.plane_list, head) {
+                drm_plane_cleanup(p);
+                kfree(p);
+        }
+}
diff --git a/drivers/gpu/drm/arm/malidp_regs.h b/drivers/gpu/drm/arm/malidp_regs.h
new file mode 100644
index 000000000000..73fecb38f955
--- /dev/null
+++ b/drivers/gpu/drm/arm/malidp_regs.h
@@ -0,0 +1,172 @@
+/*
+ * (C) COPYRIGHT 2016 ARM Limited. All rights reserved.
+ * Author: Liviu Dudau <Liviu.Dudau@arm.com>
+ *
+ * This program is free software and is provided to you under the terms of the
+ * GNU General Public License version 2 as published by the Free Software
+ * Foundation, and any use by you of this program is subject to the terms
+ * of such GNU licence.
+ *
+ * ARM Mali DP500/DP550/DP650 registers definition.
+ */
+
+#ifndef __MALIDP_REGS_H__
+#define __MALIDP_REGS_H__
+
+/*
+ * abbreviations used:
+ *    - DC - display core (general settings)
+ *    - DE - display engine
+ *    - SE - scaling engine
+ */
+
+/* interrupt bit masks */
+#define MALIDP_DE_IRQ_UNDERRUN                  (1 << 0)
+
+#define MALIDP500_DE_IRQ_AXI_ERR                (1 << 4)
+#define MALIDP500_DE_IRQ_VSYNC                  (1 << 5)
+#define MALIDP500_DE_IRQ_PROG_LINE              (1 << 6)
+#define MALIDP500_DE_IRQ_SATURATION             (1 << 7)
+#define MALIDP500_DE_IRQ_CONF_VALID             (1 << 8)
+#define MALIDP500_DE_IRQ_CONF_MODE              (1 << 11)
+#define MALIDP500_DE_IRQ_CONF_ACTIVE            (1 << 17)
+#define MALIDP500_DE_IRQ_PM_ACTIVE              (1 << 18)
+#define MALIDP500_DE_IRQ_TESTMODE_ACTIVE        (1 << 19)
+#define MALIDP500_DE_IRQ_FORCE_BLNK_ACTIVE      (1 << 24)
+#define MALIDP500_DE_IRQ_AXI_BUSY               (1 << 28)
+#define MALIDP500_DE_IRQ_GLOBAL                 (1 << 31)
+#define MALIDP500_SE_IRQ_CONF_MODE              (1 << 0)
+#define MALIDP500_SE_IRQ_CONF_VALID             (1 << 4)
+#define MALIDP500_SE_IRQ_INIT_BUSY              (1 << 5)
+#define MALIDP500_SE_IRQ_AXI_ERROR              (1 << 8)
+#define MALIDP500_SE_IRQ_OVERRUN                (1 << 9)
+#define MALIDP500_SE_IRQ_PROG_LINE1             (1 << 12)
+#define MALIDP500_SE_IRQ_PROG_LINE2             (1 << 13)
+#define MALIDP500_SE_IRQ_CONF_ACTIVE            (1 << 17)
+#define MALIDP500_SE_IRQ_PM_ACTIVE              (1 << 18)
+#define MALIDP500_SE_IRQ_AXI_BUSY               (1 << 28)
+#define MALIDP500_SE_IRQ_GLOBAL                 (1 << 31)
+
+#define MALIDP550_DE_IRQ_SATURATION             (1 << 8)
+#define MALIDP550_DE_IRQ_VSYNC                  (1 << 12)
+#define MALIDP550_DE_IRQ_PROG_LINE              (1 << 13)
+#define MALIDP550_DE_IRQ_AXI_ERR                (1 << 16)
+#define MALIDP550_SE_IRQ_EOW                    (1 << 0)
+#define MALIDP550_SE_IRQ_AXI_ERR                (1 << 16)
+#define MALIDP550_DC_IRQ_CONF_VALID             (1 << 0)
+#define MALIDP550_DC_IRQ_CONF_MODE              (1 << 4)
+#define MALIDP550_DC_IRQ_CONF_ACTIVE            (1 << 16)
+#define MALIDP550_DC_IRQ_DE                     (1 << 20)
+#define MALIDP550_DC_IRQ_SE                     (1 << 24)
+
+#define MALIDP650_DE_IRQ_DRIFT                  (1 << 4)
+
+/* bit masks that are common between products */
+#define   MALIDP_CFG_VALID              (1 << 0)
+#define   MALIDP_DISP_FUNC_ILACED       (1 << 8)
+
+/* register offsets for IRQ management */
+#define MALIDP_REG_STATUS               0x00000
+#define MALIDP_REG_SETIRQ               0x00004
+#define MALIDP_REG_MASKIRQ              0x00008
+#define MALIDP_REG_CLEARIRQ             0x0000c
+
+/* register offsets */
+#define MALIDP_DE_CORE_ID               0x00018
+#define MALIDP_DE_DISPLAY_FUNC          0x00020
+
+/* these offsets are relative to MALIDP5x0_TIMINGS_BASE */
+#define MALIDP_DE_H_TIMINGS             0x0
+#define MALIDP_DE_V_TIMINGS             0x4
+#define MALIDP_DE_SYNC_WIDTH            0x8
+#define MALIDP_DE_HV_ACTIVE             0xc
+
+/* macros to set values into registers */
+#define MALIDP_DE_H_FRONTPORCH(x)       (((x) & 0xfff) << 0)
+#define MALIDP_DE_H_BACKPORCH(x)        (((x) & 0x3ff) << 16)
+#define MALIDP500_DE_V_FRONTPORCH(x)    (((x) & 0xff) << 0)
+#define MALIDP550_DE_V_FRONTPORCH(x)    (((x) & 0xfff) << 0)
+#define MALIDP_DE_V_BACKPORCH(x)        (((x) & 0xff) << 16)
+#define MALIDP_DE_H_SYNCWIDTH(x)        (((x) & 0x3ff) << 0)
+#define MALIDP_DE_V_SYNCWIDTH(x)        (((x) & 0xff) << 16)
+#define MALIDP_DE_H_ACTIVE(x)           (((x) & 0x1fff) << 0)
+#define MALIDP_DE_V_ACTIVE(x)           (((x) & 0x1fff) << 16)
+
+/* register offsets and bits specific to DP500 */
+#define MALIDP500_DC_BASE               0x00000
+#define MALIDP500_DC_CONTROL            0x0000c
+#define   MALIDP500_DC_CONFIG_REQ       (1 << 17)
+#define   MALIDP500_HSYNCPOL            (1 << 20)
+#define   MALIDP500_VSYNCPOL            (1 << 21)
+#define   MALIDP500_DC_CLEAR_MASK       0x300fff
+#define MALIDP500_DE_LINE_COUNTER       0x00010
+#define MALIDP500_DE_AXI_CONTROL        0x00014
+#define MALIDP500_DE_SECURE_CTRL        0x0001c
+#define MALIDP500_DE_CHROMA_KEY         0x00024
+#define MALIDP500_TIMINGS_BASE          0x00028
+
+#define MALIDP500_CONFIG_3D             0x00038
+#define MALIDP500_BGND_COLOR            0x0003c
+#define MALIDP500_OUTPUT_DEPTH          0x00044
+#define MALIDP500_YUV_RGB_COEF          0x00048
+#define MALIDP500_COLOR_ADJ_COEF        0x00078
+#define MALIDP500_COEF_TABLE_ADDR       0x000a8
+#define MALIDP500_COEF_TABLE_DATA       0x000ac
+#define MALIDP500_DE_LV_BASE            0x00100
+#define MALIDP500_DE_LV_PTR_BASE        0x00124
+#define MALIDP500_DE_LG1_BASE           0x00200
+#define MALIDP500_DE_LG1_PTR_BASE       0x0021c
+#define MALIDP500_DE_LG2_BASE           0x00300
+#define MALIDP500_DE_LG2_PTR_BASE       0x0031c
+#define MALIDP500_SE_BASE               0x00c00
+#define MALIDP500_SE_PTR_BASE           0x00e0c
+#define MALIDP500_DC_IRQ_BASE           0x00f00
+#define MALIDP500_CONFIG_VALID          0x00f00
+#define MALIDP500_CONFIG_ID             0x00fd4
+
+/* register offsets and bits specific to DP550/DP650 */
+#define MALIDP550_DE_CONTROL            0x00010
+#define MALIDP550_DE_LINE_COUNTER       0x00014
+#define MALIDP550_DE_AXI_CONTROL        0x00018
+#define MALIDP550_DE_QOS                0x0001c
+#define MALIDP550_TIMINGS_BASE          0x00030
+#define MALIDP550_HSYNCPOL              (1 << 12)
+#define MALIDP550_VSYNCPOL              (1 << 28)
+
+#define MALIDP550_DE_DISP_SIDEBAND      0x00040
+#define MALIDP550_DE_BGND_COLOR         0x00044
+#define MALIDP550_DE_OUTPUT_DEPTH       0x0004c
+#define MALIDP550_DE_COLOR_COEF         0x00050
+#define MALIDP550_DE_COEF_TABLE_ADDR    0x00080
+#define MALIDP550_DE_COEF_TABLE_DATA    0x00084
+#define MALIDP550_DE_LV1_BASE           0x00100
+#define MALIDP550_DE_LV1_PTR_BASE       0x00124
+#define MALIDP550_DE_LV2_BASE           0x00200
+#define MALIDP550_DE_LV2_PTR_BASE       0x00224
+#define MALIDP550_DE_LG_BASE            0x00300
+#define MALIDP550_DE_LG_PTR_BASE        0x0031c
+#define MALIDP550_DE_LS_BASE            0x00400
+#define MALIDP550_DE_LS_PTR_BASE        0x0042c
+#define MALIDP550_DE_PERF_BASE          0x00500
+#define MALIDP550_SE_BASE               0x08000
+#define MALIDP550_DC_BASE               0x0c000
+#define MALIDP550_DC_CONTROL            0x0c010
+#define   MALIDP550_DC_CONFIG_REQ       (1 << 16)
+#define MALIDP550_CONFIG_VALID          0x0c014
+#define MALIDP550_CONFIG_ID             0x0ffd4
+
+/*
+ * Starting with DP550 the register map blocks has been standardised to the
+ * following layout:
+ *
+ *   Offset            Block registers
+ *  0x00000            Display Engine
+ *  0x08000            Scaling Engine
+ *  0x0c000            Display Core
+ *  0x10000            Secure control
+ *
+ * The old DP500 IP mixes some DC with the DE registers, hence the need
+ * for a mapping structure.
+ */
+
+#endif /* __MALIDP_REGS_H__ */
diff --git a/drivers/gpu/drm/armada/Kconfig b/drivers/gpu/drm/armada/Kconfig
index eb773e9af313..15f3ecfb16f1 100644
--- a/drivers/gpu/drm/armada/Kconfig
+++ b/drivers/gpu/drm/armada/Kconfig
@@ -1,11 +1,7 @@
 config DRM_ARMADA
         tristate "DRM support for Marvell Armada SoCs"
         depends on DRM && HAVE_CLK && ARM
-        select FB_CFB_FILLRECT
-        select FB_CFB_COPYAREA
-        select FB_CFB_IMAGEBLIT
         select DRM_KMS_HELPER
-        select DRM_KMS_FB_HELPER
         help
           Support the "LCD" controllers found on the Marvell Armada 510
           devices.  There are two controllers on the device, each controller
diff --git a/drivers/gpu/drm/armada/armada_crtc.c b/drivers/gpu/drm/armada/armada_crtc.c
index 3130aa8bcdd0..2f58e9e2a59c 100644
--- a/drivers/gpu/drm/armada/armada_crtc.c
+++ b/drivers/gpu/drm/armada/armada_crtc.c
@@ -199,7 +199,7 @@ static void armada_drm_plane_work_run(struct armada_crtc *dcrtc,
         /* Handle any pending frame work. */
         if (work) {
                 work->fn(dcrtc, plane, work);
-                drm_vblank_put(dcrtc->crtc.dev, dcrtc->num);
+                drm_crtc_vblank_put(&dcrtc->crtc);
         }
 
         wake_up(&plane->frame_wait);
@@ -210,7 +210,7 @@ int armada_drm_plane_work_queue(struct armada_crtc *dcrtc,
 {
         int ret;
 
-        ret = drm_vblank_get(dcrtc->crtc.dev, dcrtc->num);
+        ret = drm_crtc_vblank_get(&dcrtc->crtc);
         if (ret) {
                 DRM_ERROR("failed to acquire vblank counter\n");
                 return ret;
@@ -218,7 +218,7 @@ int armada_drm_plane_work_queue(struct armada_crtc *dcrtc,
 
         ret = cmpxchg(&plane->work, NULL, work) ? -EBUSY : 0;
         if (ret)
-                drm_vblank_put(dcrtc->crtc.dev, dcrtc->num);
+                drm_crtc_vblank_put(&dcrtc->crtc);
 
         return ret;
 }
@@ -234,7 +234,7 @@ struct armada_plane_work *armada_drm_plane_work_cancel(
         struct armada_plane_work *work = xchg(&plane->work, NULL);
 
         if (work)
-                drm_vblank_put(dcrtc->crtc.dev, dcrtc->num);
+                drm_crtc_vblank_put(&dcrtc->crtc);
 
         return work;
 }
@@ -260,7 +260,7 @@ static void armada_drm_crtc_complete_frame_work(struct armada_crtc *dcrtc,
 
         if (fwork->event) {
                 spin_lock_irqsave(&dev->event_lock, flags);
-                drm_send_vblank_event(dev, dcrtc->num, fwork->event);
+                drm_crtc_send_vblank_event(&dcrtc->crtc, fwork->event);
                 spin_unlock_irqrestore(&dev->event_lock, flags);
         }
 
@@ -410,7 +410,7 @@ static void armada_drm_crtc_irq(struct armada_crtc *dcrtc, u32 stat)
                 DRM_ERROR("graphics underflow on crtc %u\n", dcrtc->num);
 
         if (stat & VSYNC_IRQ)
-                drm_handle_vblank(dcrtc->crtc.dev, dcrtc->num);
+                drm_crtc_handle_vblank(&dcrtc->crtc);
 
         spin_lock(&dcrtc->irq_lock);
         ovl_plane = dcrtc->plane;
@@ -592,9 +592,9 @@ static int armada_drm_crtc_mode_set(struct drm_crtc *crtc,
 
         if (interlaced ^ dcrtc->interlaced) {
                 if (adj->flags & DRM_MODE_FLAG_INTERLACE)
-                        drm_vblank_get(dcrtc->crtc.dev, dcrtc->num);
+                        drm_crtc_vblank_get(&dcrtc->crtc);
                 else
-                        drm_vblank_put(dcrtc->crtc.dev, dcrtc->num);
+                        drm_crtc_vblank_put(&dcrtc->crtc);
                 dcrtc->interlaced = interlaced;
         }
 
diff --git a/drivers/gpu/drm/armada/armada_drv.c b/drivers/gpu/drm/armada/armada_drv.c
index 439824a61aa5..f5ebdd681445 100644
--- a/drivers/gpu/drm/armada/armada_drv.c
+++ b/drivers/gpu/drm/armada/armada_drv.c
@@ -189,7 +189,6 @@ static struct drm_driver armada_drm_driver = {
         .load                   = armada_drm_load,
         .lastclose              = armada_drm_lastclose,
         .unload                 = armada_drm_unload,
-        .set_busid              = drm_platform_set_busid,
         .get_vblank_counter     = drm_vblank_no_hw_counter,
         .enable_vblank          = armada_drm_enable_vblank,
         .disable_vblank         = armada_drm_disable_vblank,
@@ -197,7 +196,7 @@ static struct drm_driver armada_drm_driver = {
         .debugfs_init           = armada_drm_debugfs_init,
         .debugfs_cleanup        = armada_drm_debugfs_cleanup,
 #endif
-        .gem_free_object        = armada_gem_free_object,
+        .gem_free_object_unlocked = armada_gem_free_object,
         .prime_handle_to_fd     = drm_gem_prime_handle_to_fd,
         .prime_fd_to_handle     = drm_gem_prime_fd_to_handle,
         .gem_prime_export       = armada_gem_prime_export,
diff --git a/drivers/gpu/drm/armada/armada_overlay.c b/drivers/gpu/drm/armada/armada_overlay.c
index 148e8a42b2c6..1ee707ef6b8d 100644
--- a/drivers/gpu/drm/armada/armada_overlay.c
+++ b/drivers/gpu/drm/armada/armada_overlay.c
@@ -121,6 +121,7 @@ armada_ovl_plane_update(struct drm_plane *plane, struct drm_crtc *crtc,
         int ret;
 
         ret = drm_plane_helper_check_update(plane, crtc, fb, &src, &dest, &clip,
+                                            BIT(DRM_ROTATE_0),
                                             0, INT_MAX, true, false, &visible);
         if (ret)
                 return ret;
diff --git a/drivers/gpu/drm/ast/Kconfig b/drivers/gpu/drm/ast/Kconfig
index 8a784c460c89..15f6ce7acb2a 100644
--- a/drivers/gpu/drm/ast/Kconfig
+++ b/drivers/gpu/drm/ast/Kconfig
@@ -2,11 +2,7 @@ config DRM_AST
         tristate "AST server chips"
         depends on DRM && PCI
         select DRM_TTM
-        select FB_SYS_COPYAREA
-        select FB_SYS_FILLRECT
-        select FB_SYS_IMAGEBLIT
         select DRM_KMS_HELPER
-        select DRM_KMS_FB_HELPER
         select DRM_TTM
         help
          Say yes for experimental AST GPU driver. Do not enable
diff --git a/drivers/gpu/drm/ast/ast_drv.c b/drivers/gpu/drm/ast/ast_drv.c
index fcd9c0714836..f54afd2113a9 100644
--- a/drivers/gpu/drm/ast/ast_drv.c
+++ b/drivers/gpu/drm/ast/ast_drv.c
@@ -209,7 +209,7 @@ static struct drm_driver driver = {
         .minor = DRIVER_MINOR,
         .patchlevel = DRIVER_PATCHLEVEL,
 
-        .gem_free_object = ast_gem_free_object,
+        .gem_free_object_unlocked = ast_gem_free_object,
         .dumb_create = ast_dumb_create,
         .dumb_map_offset = ast_dumb_mmap_offset,
         .dumb_destroy = drm_gem_dumb_destroy,
diff --git a/drivers/gpu/drm/ast/ast_fb.c b/drivers/gpu/drm/ast/ast_fb.c
index 5320f8c57884..c017a9330a18 100644
--- a/drivers/gpu/drm/ast/ast_fb.c
+++ b/drivers/gpu/drm/ast/ast_fb.c
@@ -167,12 +167,9 @@ static int astfb_create_object(struct ast_fbdev *afbdev,
                                struct drm_gem_object **gobj_p)
 {
         struct drm_device *dev = afbdev->helper.dev;
-        u32 bpp, depth;
         u32 size;
         struct drm_gem_object *gobj;
-
         int ret = 0;
-        drm_fb_get_bpp_depth(mode_cmd->pixel_format, &depth, &bpp);
 
         size = mode_cmd->pitches[0] * mode_cmd->height;
         ret = ast_gem_create(dev, size, true, &gobj);
diff --git a/drivers/gpu/drm/ast/ast_main.c b/drivers/gpu/drm/ast/ast_main.c
index 7bc3aa6dda8c..904beaa932d0 100644
--- a/drivers/gpu/drm/ast/ast_main.c
+++ b/drivers/gpu/drm/ast/ast_main.c
@@ -295,9 +295,8 @@ static int ast_get_dram_info(struct drm_device *dev)
 static void ast_user_framebuffer_destroy(struct drm_framebuffer *fb)
 {
         struct ast_framebuffer *ast_fb = to_ast_framebuffer(fb);
-        if (ast_fb->obj)
-                drm_gem_object_unreference_unlocked(ast_fb->obj);
 
+        drm_gem_object_unreference_unlocked(ast_fb->obj);
         drm_framebuffer_cleanup(fb);
         kfree(fb);
 }
diff --git a/drivers/gpu/drm/ast/ast_mode.c b/drivers/gpu/drm/ast/ast_mode.c
index c337922606e3..5957c3e659fe 100644
--- a/drivers/gpu/drm/ast/ast_mode.c
+++ b/drivers/gpu/drm/ast/ast_mode.c
@@ -624,19 +624,21 @@ static void ast_crtc_reset(struct drm_crtc *crtc)
 
 }
 
-static void ast_crtc_gamma_set(struct drm_crtc *crtc, u16 *red, u16 *green,
-                                 u16 *blue, uint32_t start, uint32_t size)
+static int ast_crtc_gamma_set(struct drm_crtc *crtc, u16 *red, u16 *green,
+                              u16 *blue, uint32_t size)
 {
         struct ast_crtc *ast_crtc = to_ast_crtc(crtc);
-        int end = (start + size > 256) ? 256 : start + size, i;
+        int i;
 
         /* userspace palettes are always correct as is */
-        for (i = start; i < end; i++) {
+        for (i = 0; i < size; i++) {
                 ast_crtc->lut_r[i] = red[i] >> 8;
                 ast_crtc->lut_g[i] = green[i] >> 8;
                 ast_crtc->lut_b[i] = blue[i] >> 8;
         }
         ast_crtc_load_lut(crtc);
+
+        return 0;
 }
 
 
diff --git a/drivers/gpu/drm/ast/ast_ttm.c b/drivers/gpu/drm/ast/ast_ttm.c
index 59f2f93b6f84..b29a41218fc9 100644
--- a/drivers/gpu/drm/ast/ast_ttm.c
+++ b/drivers/gpu/drm/ast/ast_ttm.c
@@ -186,17 +186,6 @@ static void ast_ttm_io_mem_free(struct ttm_bo_device *bdev, struct ttm_mem_reg *
 {
 }
 
-static int ast_bo_move(struct ttm_buffer_object *bo,
-                       bool evict, bool interruptible,
-                       bool no_wait_gpu,
-                       struct ttm_mem_reg *new_mem)
-{
-        int r;
-        r = ttm_bo_move_memcpy(bo, evict, no_wait_gpu, new_mem);
-        return r;
-}
-
-
 static void ast_ttm_backend_destroy(struct ttm_tt *tt)
 {
         ttm_tt_fini(tt);
@@ -241,7 +230,7 @@ struct ttm_bo_driver ast_bo_driver = {
         .ttm_tt_unpopulate = ast_ttm_tt_unpopulate,
         .init_mem_type = ast_bo_init_mem_type,
         .evict_flags = ast_bo_evict_flags,
-        .move = ast_bo_move,
+        .move = NULL,
         .verify_access = ast_bo_verify_access,
         .io_mem_reserve = &ast_ttm_io_mem_reserve,
         .io_mem_free = &ast_ttm_io_mem_free,
diff --git a/drivers/gpu/drm/atmel-hlcdc/Kconfig b/drivers/gpu/drm/atmel-hlcdc/Kconfig
index 99b4f0698a30..32bcc4bad06a 100644
--- a/drivers/gpu/drm/atmel-hlcdc/Kconfig
+++ b/drivers/gpu/drm/atmel-hlcdc/Kconfig
@@ -3,7 +3,6 @@ config DRM_ATMEL_HLCDC
         depends on DRM && OF && COMMON_CLK && MFD_ATMEL_HLCDC && ARM
         select DRM_GEM_CMA_HELPER
         select DRM_KMS_HELPER
-        select DRM_KMS_FB_HELPER
         select DRM_KMS_CMA_HELPER
         select DRM_PANEL
         help
diff --git a/drivers/gpu/drm/atmel-hlcdc/atmel_hlcdc_crtc.c b/drivers/gpu/drm/atmel-hlcdc/atmel_hlcdc_crtc.c
index bd12231ab0cd..a978381ef95b 100644
--- a/drivers/gpu/drm/atmel-hlcdc/atmel_hlcdc_crtc.c
+++ b/drivers/gpu/drm/atmel-hlcdc/atmel_hlcdc_crtc.c
@@ -374,8 +374,8 @@ static void atmel_hlcdc_crtc_finish_page_flip(struct atmel_hlcdc_crtc *crtc)
 
         spin_lock_irqsave(&dev->event_lock, flags);
         if (crtc->event) {
-                drm_send_vblank_event(dev, crtc->id, crtc->event);
-                drm_vblank_put(dev, crtc->id);
+                drm_crtc_send_vblank_event(&crtc->base, crtc->event);
+                drm_crtc_vblank_put(&crtc->base);
                 crtc->event = NULL;
         }
         spin_unlock_irqrestore(&dev->event_lock, flags);
@@ -383,7 +383,7 @@ static void atmel_hlcdc_crtc_finish_page_flip(struct atmel_hlcdc_crtc *crtc)
 
 void atmel_hlcdc_crtc_irq(struct drm_crtc *c)
 {
-        drm_handle_vblank(c->dev, 0);
+        drm_crtc_handle_vblank(c);
         atmel_hlcdc_crtc_finish_page_flip(drm_crtc_to_atmel_hlcdc_crtc(c));
 }
 
diff --git a/drivers/gpu/drm/atmel-hlcdc/atmel_hlcdc_dc.c b/drivers/gpu/drm/atmel-hlcdc/atmel_hlcdc_dc.c
index 8ded7645747e..d4a3d61b7b06 100644
--- a/drivers/gpu/drm/atmel-hlcdc/atmel_hlcdc_dc.c
+++ b/drivers/gpu/drm/atmel-hlcdc/atmel_hlcdc_dc.c
@@ -519,7 +519,7 @@ static int atmel_hlcdc_dc_atomic_commit(struct drm_device *dev,
         }
 
         /* Swap the state, this is the point of no return. */
-        drm_atomic_helper_swap_state(dev, state);
+        drm_atomic_helper_swap_state(state, true);
 
         if (async)
                 queue_work(dc->wq, &commit->work);
@@ -691,13 +691,6 @@ static void atmel_hlcdc_dc_unload(struct drm_device *dev)
         destroy_workqueue(dc->wq);
 }
 
-static void atmel_hlcdc_dc_connector_unplug_all(struct drm_device *dev)
-{
-        mutex_lock(&dev->mode_config.mutex);
-        drm_connector_unregister_all(dev);
-        mutex_unlock(&dev->mode_config.mutex);
-}
-
 static void atmel_hlcdc_dc_lastclose(struct drm_device *dev)
 {
         struct atmel_hlcdc_dc *dc = dev->dev_private;
@@ -776,7 +769,7 @@ static struct drm_driver atmel_hlcdc_dc_driver = {
         .get_vblank_counter = drm_vblank_no_hw_counter,
         .enable_vblank = atmel_hlcdc_dc_enable_vblank,
         .disable_vblank = atmel_hlcdc_dc_disable_vblank,
-        .gem_free_object = drm_gem_cma_free_object,
+        .gem_free_object_unlocked = drm_gem_cma_free_object,
         .gem_vm_ops = &drm_gem_cma_vm_ops,
         .prime_handle_to_fd = drm_gem_prime_handle_to_fd,
         .prime_fd_to_handle = drm_gem_prime_fd_to_handle,
@@ -815,15 +808,8 @@ static int atmel_hlcdc_dc_drm_probe(struct platform_device *pdev)
         if (ret)
                 goto err_unload;
 
-        ret = drm_connector_register_all(ddev);
-        if (ret)
-                goto err_unregister;
-
         return 0;
 
-err_unregister:
-        drm_dev_unregister(ddev);
-
 err_unload:
         atmel_hlcdc_dc_unload(ddev);
 
@@ -837,7 +823,6 @@ static int atmel_hlcdc_dc_drm_remove(struct platform_device *pdev)
 {
         struct drm_device *ddev = platform_get_drvdata(pdev);
 
-        atmel_hlcdc_dc_connector_unplug_all(ddev);
         drm_dev_unregister(ddev);
         atmel_hlcdc_dc_unload(ddev);
         drm_dev_unref(ddev);
diff --git a/drivers/gpu/drm/atmel-hlcdc/atmel_hlcdc_output.c b/drivers/gpu/drm/atmel-hlcdc/atmel_hlcdc_output.c
index 3d34fc4ca826..6119b5085501 100644
--- a/drivers/gpu/drm/atmel-hlcdc/atmel_hlcdc_output.c
+++ b/drivers/gpu/drm/atmel-hlcdc/atmel_hlcdc_output.c
@@ -113,21 +113,9 @@ static int atmel_hlcdc_rgb_mode_valid(struct drm_connector *connector,
         return atmel_hlcdc_dc_mode_valid(rgb->dc, mode);
 }
 
-
-
-static struct drm_encoder *
-atmel_hlcdc_rgb_best_encoder(struct drm_connector *connector)
-{
-        struct atmel_hlcdc_rgb_output *rgb =
-                        drm_connector_to_atmel_hlcdc_rgb_output(connector);
-
-        return &rgb->encoder;
-}
-
 static const struct drm_connector_helper_funcs atmel_hlcdc_panel_connector_helper_funcs = {
         .get_modes = atmel_hlcdc_panel_get_modes,
         .mode_valid = atmel_hlcdc_rgb_mode_valid,
-        .best_encoder = atmel_hlcdc_rgb_best_encoder,
 };
 
 static enum drm_connector_status
diff --git a/drivers/gpu/drm/bochs/Kconfig b/drivers/gpu/drm/bochs/Kconfig
index 5f8b0c2b9a44..f739763f47ce 100644
--- a/drivers/gpu/drm/bochs/Kconfig
+++ b/drivers/gpu/drm/bochs/Kconfig
@@ -2,10 +2,6 @@ config DRM_BOCHS
         tristate "DRM Support for bochs dispi vga interface (qemu stdvga)"
         depends on DRM && PCI
         select DRM_KMS_HELPER
-        select DRM_KMS_FB_HELPER
-        select FB_SYS_FILLRECT
-        select FB_SYS_COPYAREA
-        select FB_SYS_IMAGEBLIT
         select DRM_TTM
         help
           Choose this option for qemu.
diff --git a/drivers/gpu/drm/bochs/bochs_drv.c b/drivers/gpu/drm/bochs/bochs_drv.c
index b332b4d3b0e2..abace82de6ea 100644
--- a/drivers/gpu/drm/bochs/bochs_drv.c
+++ b/drivers/gpu/drm/bochs/bochs_drv.c
@@ -89,7 +89,7 @@ static struct drm_driver bochs_driver = {
         .date                   = "20130925",
         .major                  = 1,
         .minor                  = 0,
-        .gem_free_object        = bochs_gem_free_object,
+        .gem_free_object_unlocked = bochs_gem_free_object,
         .dumb_create            = bochs_dumb_create,
         .dumb_map_offset        = bochs_dumb_mmap_offset,
         .dumb_destroy           = drm_gem_dumb_destroy,
diff --git a/drivers/gpu/drm/bochs/bochs_mm.c b/drivers/gpu/drm/bochs/bochs_mm.c
index 6cf912c45e48..5c5638a777a1 100644
--- a/drivers/gpu/drm/bochs/bochs_mm.c
+++ b/drivers/gpu/drm/bochs/bochs_mm.c
@@ -165,15 +165,6 @@ static void bochs_ttm_io_mem_free(struct ttm_bo_device *bdev,
 {
 }
 
-static int bochs_bo_move(struct ttm_buffer_object *bo,
-                         bool evict, bool interruptible,
-                         bool no_wait_gpu,
-                         struct ttm_mem_reg *new_mem)
-{
-        return ttm_bo_move_memcpy(bo, evict, no_wait_gpu, new_mem);
-}
-
-
 static void bochs_ttm_backend_destroy(struct ttm_tt *tt)
 {
         ttm_tt_fini(tt);
@@ -208,7 +199,7 @@ struct ttm_bo_driver bochs_bo_driver = {
         .ttm_tt_unpopulate = ttm_pool_unpopulate,
         .init_mem_type = bochs_bo_init_mem_type,
         .evict_flags = bochs_bo_evict_flags,
-        .move = bochs_bo_move,
+        .move = NULL,
         .verify_access = bochs_bo_verify_access,
         .io_mem_reserve = &bochs_ttm_io_mem_reserve,
         .io_mem_free = &bochs_ttm_io_mem_free,
@@ -474,8 +465,8 @@ int bochs_dumb_mmap_offset(struct drm_file *file, struct drm_device *dev,
 static void bochs_user_framebuffer_destroy(struct drm_framebuffer *fb)
 {
         struct bochs_framebuffer *bochs_fb = to_bochs_framebuffer(fb);
-        if (bochs_fb->obj)
-                drm_gem_object_unreference_unlocked(bochs_fb->obj);
+
+        drm_gem_object_unreference_unlocked(bochs_fb->obj);
         drm_framebuffer_cleanup(fb);
         kfree(fb);
 }
diff --git a/drivers/gpu/drm/bridge/Kconfig b/drivers/gpu/drm/bridge/Kconfig
index 8f7423f18da5..b590e678052d 100644
--- a/drivers/gpu/drm/bridge/Kconfig
+++ b/drivers/gpu/drm/bridge/Kconfig
@@ -50,6 +50,25 @@ config DRM_PARADE_PS8622
         ---help---
           Parade eDP-LVDS bridge chip driver.
 
+config DRM_SII902X
+        tristate "Silicon Image sii902x RGB/HDMI bridge"
+        depends on OF
+        select DRM_KMS_HELPER
+        select REGMAP_I2C
+        ---help---
+          Silicon Image sii902x bridge chip driver.
+
+config DRM_TOSHIBA_TC358767
+        tristate "Toshiba TC358767 eDP bridge"
+        depends on OF
+        select DRM_KMS_HELPER
+        select REGMAP_I2C
+        select DRM_PANEL
+        ---help---
+          Toshiba TC358767 eDP bridge chip driver.
+
 source "drivers/gpu/drm/bridge/analogix/Kconfig"
 
+source "drivers/gpu/drm/bridge/adv7511/Kconfig"
+
 endmenu
diff --git a/drivers/gpu/drm/bridge/Makefile b/drivers/gpu/drm/bridge/Makefile
index 96b13b30e6ab..efdb07e878f5 100644
--- a/drivers/gpu/drm/bridge/Makefile
+++ b/drivers/gpu/drm/bridge/Makefile
@@ -5,4 +5,7 @@ obj-$(CONFIG_DRM_DW_HDMI) += dw-hdmi.o
 obj-$(CONFIG_DRM_DW_HDMI_AHB_AUDIO) += dw-hdmi-ahb-audio.o
 obj-$(CONFIG_DRM_NXP_PTN3460) += nxp-ptn3460.o
 obj-$(CONFIG_DRM_PARADE_PS8622) += parade-ps8622.o
+obj-$(CONFIG_DRM_SII902X) += sii902x.o
+obj-$(CONFIG_DRM_TOSHIBA_TC358767) += tc358767.o
 obj-$(CONFIG_DRM_ANALOGIX_DP) += analogix/
+obj-$(CONFIG_DRM_I2C_ADV7511) += adv7511/
diff --git a/drivers/gpu/drm/bridge/adv7511/Kconfig b/drivers/gpu/drm/bridge/adv7511/Kconfig
new file mode 100644
index 000000000000..d2b0499ab7d7
--- /dev/null
+++ b/drivers/gpu/drm/bridge/adv7511/Kconfig
@@ -0,0 +1,15 @@
+config DRM_I2C_ADV7511
+        tristate "AV7511 encoder"
+        depends on OF
+        select DRM_KMS_HELPER
+        select REGMAP_I2C
+        help
+          Support for the Analog Device ADV7511(W) and ADV7513 HDMI encoders.
+
+config DRM_I2C_ADV7533
+        bool "ADV7533 encoder"
+        depends on DRM_I2C_ADV7511
+        select DRM_MIPI_DSI
+        default y
+        help
+          Support for the Analog Devices ADV7533 DSI to HDMI encoder.
diff --git a/drivers/gpu/drm/bridge/adv7511/Makefile b/drivers/gpu/drm/bridge/adv7511/Makefile
new file mode 100644
index 000000000000..9019327fff4c
--- /dev/null
+++ b/drivers/gpu/drm/bridge/adv7511/Makefile
@@ -0,0 +1,3 @@
+adv7511-y := adv7511_drv.o
+adv7511-$(CONFIG_DRM_I2C_ADV7533) += adv7533.o
+obj-$(CONFIG_DRM_I2C_ADV7511) += adv7511.o
diff --git a/drivers/gpu/drm/i2c/adv7511.h b/drivers/gpu/drm/bridge/adv7511/adv7511.h
index 38515b30cedf..161c923d6162 100644
--- a/drivers/gpu/drm/i2c/adv7511.h
+++ b/drivers/gpu/drm/bridge/adv7511/adv7511.h
@@ -10,6 +10,11 @@
 #define __DRM_I2C_ADV7511_H__
 
 #include <linux/hdmi.h>
+#include <linux/i2c.h>
+#include <linux/regmap.h>
+
+#include <drm/drm_crtc_helper.h>
+#include <drm/drm_mipi_dsi.h>
 
 #define ADV7511_REG_CHIP_REVISION               0x00
 #define ADV7511_REG_N0                          0x01
@@ -286,4 +291,102 @@ struct adv7511_video_config {
         struct hdmi_avi_infoframe avi_infoframe;
 };
 
+enum adv7511_type {
+        ADV7511,
+        ADV7533,
+};
+
+struct adv7511 {
+        struct i2c_client *i2c_main;
+        struct i2c_client *i2c_edid;
+        struct i2c_client *i2c_cec;
+
+        struct regmap *regmap;
+        struct regmap *regmap_cec;
+        enum drm_connector_status status;
+        bool powered;
+
+        struct drm_display_mode curr_mode;
+
+        unsigned int f_tmds;
+
+        unsigned int current_edid_segment;
+        uint8_t edid_buf[256];
+        bool edid_read;
+
+        wait_queue_head_t wq;
+        struct drm_bridge bridge;
+        struct drm_connector connector;
+
+        bool embedded_sync;
+        enum adv7511_sync_polarity vsync_polarity;
+        enum adv7511_sync_polarity hsync_polarity;
+        bool rgb;
+
+        struct edid *edid;
+
+        struct gpio_desc *gpio_pd;
+
+        /* ADV7533 DSI RX related params */
+        struct device_node *host_node;
+        struct mipi_dsi_device *dsi;
+        u8 num_dsi_lanes;
+        bool use_timing_gen;
+
+        enum adv7511_type type;
+};
+
+#ifdef CONFIG_DRM_I2C_ADV7533
+void adv7533_dsi_power_on(struct adv7511 *adv);
+void adv7533_dsi_power_off(struct adv7511 *adv);
+void adv7533_mode_set(struct adv7511 *adv, struct drm_display_mode *mode);
+int adv7533_patch_registers(struct adv7511 *adv);
+void adv7533_uninit_cec(struct adv7511 *adv);
+int adv7533_init_cec(struct adv7511 *adv);
+int adv7533_attach_dsi(struct adv7511 *adv);
+void adv7533_detach_dsi(struct adv7511 *adv);
+int adv7533_parse_dt(struct device_node *np, struct adv7511 *adv);
+#else
+static inline void adv7533_dsi_power_on(struct adv7511 *adv)
+{
+}
+
+static inline void adv7533_dsi_power_off(struct adv7511 *adv)
+{
+}
+
+static inline void adv7533_mode_set(struct adv7511 *adv,
+                                    struct drm_display_mode *mode)
+{
+}
+
+static inline int adv7533_patch_registers(struct adv7511 *adv)
+{
+        return -ENODEV;
+}
+
+static inline void adv7533_uninit_cec(struct adv7511 *adv)
+{
+}
+
+static inline int adv7533_init_cec(struct adv7511 *adv)
+{
+        return -ENODEV;
+}
+
+static inline int adv7533_attach_dsi(struct adv7511 *adv)
+{
+        return -ENODEV;
+}
+
+static inline void adv7533_detach_dsi(struct adv7511 *adv)
+{
+}
+
+static inline int adv7533_parse_dt(struct device_node *np, struct adv7511 *adv)
+{
+        return -ENODEV;
+}
+#endif
+
 #endif /* __DRM_I2C_ADV7511_H__ */
diff --git a/drivers/gpu/drm/i2c/adv7511.c b/drivers/gpu/drm/bridge/adv7511/adv7511_drv.c
index a02112ba1c3d..ec8fb2ed3275 100644
--- a/drivers/gpu/drm/i2c/adv7511.c
+++ b/drivers/gpu/drm/bridge/adv7511/adv7511_drv.c
@@ -8,51 +8,17 @@
 
 #include <linux/device.h>
 #include <linux/gpio/consumer.h>
-#include <linux/i2c.h>
 #include <linux/module.h>
-#include <linux/regmap.h>
+#include <linux/of_device.h>
 #include <linux/slab.h>
 
 #include <drm/drmP.h>
-#include <drm/drm_crtc_helper.h>
+#include <drm/drm_atomic.h>
+#include <drm/drm_atomic_helper.h>
 #include <drm/drm_edid.h>
-#include <drm/drm_encoder_slave.h>
 
 #include "adv7511.h"
 
-struct adv7511 {
-        struct i2c_client *i2c_main;
-        struct i2c_client *i2c_edid;
-
-        struct regmap *regmap;
-        struct regmap *packet_memory_regmap;
-        enum drm_connector_status status;
-        bool powered;
-
-        unsigned int f_tmds;
-
-        unsigned int current_edid_segment;
-        uint8_t edid_buf[256];
-        bool edid_read;
-
-        wait_queue_head_t wq;
-        struct drm_encoder *encoder;
-
-        bool embedded_sync;
-        enum adv7511_sync_polarity vsync_polarity;
-        enum adv7511_sync_polarity hsync_polarity;
-        bool rgb;
-
-        struct edid *edid;
-
-        struct gpio_desc *gpio_pd;
-};
-
-static struct adv7511 *encoder_to_adv7511(struct drm_encoder *encoder)
-{
-        return to_encoder_slave(encoder)->slave_priv;
-}
-
 /* ADI recommended values for proper operation. */
 static const struct reg_sequence adv7511_fixed_registers[] = {
         { 0x98, 0x03 },
@@ -394,6 +360,9 @@ static void adv7511_power_on(struct adv7511 *adv7511)
          */
         regcache_sync(adv7511->regmap);
 
+        if (adv7511->type == ADV7533)
+                adv7533_dsi_power_on(adv7511);
+
         adv7511->powered = true;
 }
 
@@ -405,6 +374,9 @@ static void adv7511_power_off(struct adv7511 *adv7511)
                            ADV7511_POWER_POWER_DOWN);
         regcache_mark_dirty(adv7511->regmap);
 
+        if (adv7511->type == ADV7533)
+                adv7533_dsi_power_off(adv7511);
+
         adv7511->powered = false;
 }
 
@@ -430,7 +402,7 @@ static bool adv7511_hpd(struct adv7511 *adv7511)
         return false;
 }
 
-static int adv7511_irq_process(struct adv7511 *adv7511)
+static int adv7511_irq_process(struct adv7511 *adv7511, bool process_hpd)
 {
         unsigned int irq0, irq1;
         int ret;
@@ -446,8 +418,8 @@ static int adv7511_irq_process(struct adv7511 *adv7511)
         regmap_write(adv7511->regmap, ADV7511_REG_INT(0), irq0);
         regmap_write(adv7511->regmap, ADV7511_REG_INT(1), irq1);
 
-        if (irq0 & ADV7511_INT0_HPD && adv7511->encoder)
-                drm_helper_hpd_irq_event(adv7511->encoder->dev);
+        if (process_hpd && irq0 & ADV7511_INT0_HPD && adv7511->bridge.encoder)
+                drm_helper_hpd_irq_event(adv7511->connector.dev);
 
         if (irq0 & ADV7511_INT0_EDID_READY || irq1 & ADV7511_INT1_DDC_ERROR) {
                 adv7511->edid_read = true;
@@ -464,7 +436,7 @@ static irqreturn_t adv7511_irq_handler(int irq, void *devid)
         struct adv7511 *adv7511 = devid;
         int ret;
 
-        ret = adv7511_irq_process(adv7511);
+        ret = adv7511_irq_process(adv7511, true);
         return ret < 0 ? IRQ_NONE : IRQ_HANDLED;
 }
 
@@ -481,7 +453,7 @@ static int adv7511_wait_for_edid(struct adv7511 *adv7511, int timeout)
                                 adv7511->edid_read, msecs_to_jiffies(timeout));
         } else {
                 for (; timeout > 0; timeout -= 25) {
-                        ret = adv7511_irq_process(adv7511);
+                        ret = adv7511_irq_process(adv7511, false);
                         if (ret < 0)
                                 break;
 
@@ -563,13 +535,12 @@ static int adv7511_get_edid_block(void *data, u8 *buf, unsigned int block,
 }
 
 /* -----------------------------------------------------------------------------
- * Encoder operations
+ * ADV75xx helpers
  */
 
-static int adv7511_get_modes(struct drm_encoder *encoder,
+static int adv7511_get_modes(struct adv7511 *adv7511,
                              struct drm_connector *connector)
 {
-        struct adv7511 *adv7511 = encoder_to_adv7511(encoder);
         struct edid *edid;
         unsigned int count;
 
@@ -606,21 +577,9 @@ static int adv7511_get_modes(struct drm_encoder *encoder,
         return count;
 }
 
-static void adv7511_encoder_dpms(struct drm_encoder *encoder, int mode)
-{
-        struct adv7511 *adv7511 = encoder_to_adv7511(encoder);
-
-        if (mode == DRM_MODE_DPMS_ON)
-                adv7511_power_on(adv7511);
-        else
-                adv7511_power_off(adv7511);
-}
-
 static enum drm_connector_status
-adv7511_encoder_detect(struct drm_encoder *encoder,
-                       struct drm_connector *connector)
+adv7511_detect(struct adv7511 *adv7511, struct drm_connector *connector)
 {
-        struct adv7511 *adv7511 = encoder_to_adv7511(encoder);
         enum drm_connector_status status;
         unsigned int val;
         bool hpd;
@@ -644,7 +603,7 @@ adv7511_encoder_detect(struct drm_encoder *encoder,
         if (status == connector_status_connected && hpd && adv7511->powered) {
                 regcache_mark_dirty(adv7511->regmap);
                 adv7511_power_on(adv7511);
-                adv7511_get_modes(encoder, connector);
+                adv7511_get_modes(adv7511, connector);
                 if (adv7511->status == connector_status_connected)
                         status = connector_status_disconnected;
         } else {
@@ -658,8 +617,8 @@ adv7511_encoder_detect(struct drm_encoder *encoder,
         return status;
 }
 
-static int adv7511_encoder_mode_valid(struct drm_encoder *encoder,
-                                      struct drm_display_mode *mode)
+static int adv7511_mode_valid(struct adv7511 *adv7511,
+                              struct drm_display_mode *mode)
 {
         if (mode->clock > 165000)
                 return MODE_CLOCK_HIGH;
@@ -667,11 +626,10 @@ static int adv7511_encoder_mode_valid(struct drm_encoder *encoder,
         return MODE_OK;
 }
 
-static void adv7511_encoder_mode_set(struct drm_encoder *encoder,
-                                     struct drm_display_mode *mode,
-                                     struct drm_display_mode *adj_mode)
+static void adv7511_mode_set(struct adv7511 *adv7511,
+                             struct drm_display_mode *mode,
+                             struct drm_display_mode *adj_mode)
 {
-        struct adv7511 *adv7511 = encoder_to_adv7511(encoder);
         unsigned int low_refresh_rate;
         unsigned int hsync_polarity = 0;
         unsigned int vsync_polarity = 0;
@@ -754,6 +712,11 @@ static void adv7511_encoder_mode_set(struct drm_encoder *encoder,
         regmap_update_bits(adv7511->regmap, 0x17,
                 0x60, (vsync_polarity << 6) | (hsync_polarity << 5));
 
+        if (adv7511->type == ADV7533)
+                adv7533_mode_set(adv7511, adj_mode);
+
+        drm_mode_copy(&adv7511->curr_mode, adj_mode);
+
         /*
          * TODO Test first order 4:2:2 to 4:4:4 up conversion method, which is
          * supposed to give better results.
@@ -762,12 +725,114 @@ static void adv7511_encoder_mode_set(struct drm_encoder *encoder,
         adv7511->f_tmds = mode->clock;
 }
 
-static const struct drm_encoder_slave_funcs adv7511_encoder_funcs = {
-        .dpms = adv7511_encoder_dpms,
-        .mode_valid = adv7511_encoder_mode_valid,
-        .mode_set = adv7511_encoder_mode_set,
-        .detect = adv7511_encoder_detect,
-        .get_modes = adv7511_get_modes,
+/* Connector funcs */
+static struct adv7511 *connector_to_adv7511(struct drm_connector *connector)
+{
+        return container_of(connector, struct adv7511, connector);
+}
+
+static int adv7511_connector_get_modes(struct drm_connector *connector)
+{
+        struct adv7511 *adv = connector_to_adv7511(connector);
+
+        return adv7511_get_modes(adv, connector);
+}
+
+static enum drm_mode_status
+adv7511_connector_mode_valid(struct drm_connector *connector,
+                             struct drm_display_mode *mode)
+{
+        struct adv7511 *adv = connector_to_adv7511(connector);
+
+        return adv7511_mode_valid(adv, mode);
+}
+
+static struct drm_connector_helper_funcs adv7511_connector_helper_funcs = {
+        .get_modes = adv7511_connector_get_modes,
+        .mode_valid = adv7511_connector_mode_valid,
+};
+
+static enum drm_connector_status
+adv7511_connector_detect(struct drm_connector *connector, bool force)
+{
+        struct adv7511 *adv = connector_to_adv7511(connector);
+
+        return adv7511_detect(adv, connector);
+}
+
+static struct drm_connector_funcs adv7511_connector_funcs = {
+        .dpms = drm_atomic_helper_connector_dpms,
+        .fill_modes = drm_helper_probe_single_connector_modes,
+        .detect = adv7511_connector_detect,
+        .destroy = drm_connector_cleanup,
+        .reset = drm_atomic_helper_connector_reset,
+        .atomic_duplicate_state = drm_atomic_helper_connector_duplicate_state,
+        .atomic_destroy_state = drm_atomic_helper_connector_destroy_state,
+};
+
+/* Bridge funcs */
+static struct adv7511 *bridge_to_adv7511(struct drm_bridge *bridge)
+{
+        return container_of(bridge, struct adv7511, bridge);
+}
+
+static void adv7511_bridge_enable(struct drm_bridge *bridge)
+{
+        struct adv7511 *adv = bridge_to_adv7511(bridge);
+
+        adv7511_power_on(adv);
+}
+
+static void adv7511_bridge_disable(struct drm_bridge *bridge)
+{
+        struct adv7511 *adv = bridge_to_adv7511(bridge);
+
+        adv7511_power_off(adv);
+}
+
+static void adv7511_bridge_mode_set(struct drm_bridge *bridge,
+                                    struct drm_display_mode *mode,
+                                    struct drm_display_mode *adj_mode)
+{
+        struct adv7511 *adv = bridge_to_adv7511(bridge);
+
+        adv7511_mode_set(adv, mode, adj_mode);
+}
+
+static int adv7511_bridge_attach(struct drm_bridge *bridge)
+{
+        struct adv7511 *adv = bridge_to_adv7511(bridge);
+        int ret;
+
+        if (!bridge->encoder) {
+                DRM_ERROR("Parent encoder object not found");
+                return -ENODEV;
+        }
+
+        adv->connector.polled = DRM_CONNECTOR_POLL_HPD;
+
+        ret = drm_connector_init(bridge->dev, &adv->connector,
+                                 &adv7511_connector_funcs,
+                                 DRM_MODE_CONNECTOR_HDMIA);
+        if (ret) {
+                DRM_ERROR("Failed to initialize connector with drm\n");
+                return ret;
+        }
+        drm_connector_helper_add(&adv->connector,
+                                 &adv7511_connector_helper_funcs);
+        drm_mode_connector_attach_encoder(&adv->connector, bridge->encoder);
+
+        if (adv->type == ADV7533)
+                ret = adv7533_attach_dsi(adv);
+
+        return ret;
+}
+
+static struct drm_bridge_funcs adv7511_bridge_funcs = {
+        .enable = adv7511_bridge_enable,
+        .disable = adv7511_bridge_disable,
+        .mode_set = adv7511_bridge_mode_set,
+        .attach = adv7511_bridge_attach,
 };
 
 /* -----------------------------------------------------------------------------
@@ -780,8 +845,6 @@ static int adv7511_parse_dt(struct device_node *np,
         const char *str;
         int ret;
 
-        memset(config, 0, sizeof(*config));
-
         of_property_read_u32(np, "adi,input-depth", &config->input_color_depth);
         if (config->input_color_depth != 8 && config->input_color_depth != 10 &&
             config->input_color_depth != 12)
@@ -881,7 +944,17 @@ static int adv7511_probe(struct i2c_client *i2c, const struct i2c_device_id *id)
         adv7511->powered = false;
         adv7511->status = connector_status_disconnected;
 
-        ret = adv7511_parse_dt(dev->of_node, &link_config);
+        if (dev->of_node)
+                adv7511->type = (enum adv7511_type)of_device_get_match_data(dev);
+        else
+                adv7511->type = id->driver_data;
+
+        memset(&link_config, 0, sizeof(link_config));
+
+        if (adv7511->type == ADV7511)
+                ret = adv7511_parse_dt(dev->of_node, &link_config);
+        else
+                ret = adv7533_parse_dt(dev->of_node, adv7511);
         if (ret)
                 return ret;
 
@@ -907,8 +980,12 @@ static int adv7511_probe(struct i2c_client *i2c, const struct i2c_device_id *id)
                 return ret;
         dev_dbg(dev, "Rev. %d\n", val);
 
-        ret = regmap_register_patch(adv7511->regmap, adv7511_fixed_registers,
-                                    ARRAY_SIZE(adv7511_fixed_registers));
+        if (adv7511->type == ADV7511)
+                ret = regmap_register_patch(adv7511->regmap,
+                                            adv7511_fixed_registers,
+                                            ARRAY_SIZE(adv7511_fixed_registers));
+        else
+                ret = adv7533_patch_registers(adv7511);
         if (ret)
                 return ret;
 
@@ -923,6 +1000,12 @@ static int adv7511_probe(struct i2c_client *i2c, const struct i2c_device_id *id)
         if (!adv7511->i2c_edid)
                 return -ENOMEM;
 
+        if (adv7511->type == ADV7533) {
+                ret = adv7533_init_cec(adv7511);
+                if (ret)
+                        goto err_i2c_unregister_edid;
+        }
+
         if (i2c->irq) {
                 init_waitqueue_head(&adv7511->wq);
 
@@ -931,7 +1014,7 @@ static int adv7511_probe(struct i2c_client *i2c, const struct i2c_device_id *id)
                                                 IRQF_ONESHOT, dev_name(dev),
                                                 adv7511);
                 if (ret)
-                        goto err_i2c_unregister_device;
+                        goto err_unregister_cec;
         }
 
         /* CEC is unused for now */
@@ -942,11 +1025,23 @@ static int adv7511_probe(struct i2c_client *i2c, const struct i2c_device_id *id)
 
         i2c_set_clientdata(i2c, adv7511);
 
-        adv7511_set_link_config(adv7511, &link_config);
+        if (adv7511->type == ADV7511)
+                adv7511_set_link_config(adv7511, &link_config);
+
+        adv7511->bridge.funcs = &adv7511_bridge_funcs;
+        adv7511->bridge.of_node = dev->of_node;
+
+        ret = drm_bridge_add(&adv7511->bridge);
+        if (ret) {
+                dev_err(dev, "failed to add adv7511 bridge\n");
+                goto err_unregister_cec;
+        }
 
         return 0;
 
-err_i2c_unregister_device:
+err_unregister_cec:
+        adv7533_uninit_cec(adv7511);
+err_i2c_unregister_edid:
         i2c_unregister_device(adv7511->i2c_edid);
 
         return ret;
@@ -956,66 +1051,71 @@ static int adv7511_remove(struct i2c_client *i2c)
 {
         struct adv7511 *adv7511 = i2c_get_clientdata(i2c);
 
-        i2c_unregister_device(adv7511->i2c_edid);
-
-        kfree(adv7511->edid);
-
-        return 0;
-}
-
-static int adv7511_encoder_init(struct i2c_client *i2c, struct drm_device *dev,
-                                struct drm_encoder_slave *encoder)
-{
+        if (adv7511->type == ADV7533) {
+                adv7533_detach_dsi(adv7511);
+                adv7533_uninit_cec(adv7511);
+        }
 
-        struct adv7511 *adv7511 = i2c_get_clientdata(i2c);
+        drm_bridge_remove(&adv7511->bridge);
 
-        encoder->slave_priv = adv7511;
-        encoder->slave_funcs = &adv7511_encoder_funcs;
+        i2c_unregister_device(adv7511->i2c_edid);
 
-        adv7511->encoder = &encoder->base;
+        kfree(adv7511->edid);
 
         return 0;
 }
 
 static const struct i2c_device_id adv7511_i2c_ids[] = {
-        { "adv7511", 0 },
-        { "adv7511w", 0 },
-        { "adv7513", 0 },
+        { "adv7511", ADV7511 },
+        { "adv7511w", ADV7511 },
+        { "adv7513", ADV7511 },
+#ifdef CONFIG_DRM_I2C_ADV7533
+        { "adv7533", ADV7533 },
+#endif
         { }
 };
 MODULE_DEVICE_TABLE(i2c, adv7511_i2c_ids);
 
 static const struct of_device_id adv7511_of_ids[] = {
-        { .compatible = "adi,adv7511", },
-        { .compatible = "adi,adv7511w", },
-        { .compatible = "adi,adv7513", },
+        { .compatible = "adi,adv7511", .data = (void *)ADV7511 },
+        { .compatible = "adi,adv7511w", .data = (void *)ADV7511 },
+        { .compatible = "adi,adv7513", .data = (void *)ADV7511 },
+#ifdef CONFIG_DRM_I2C_ADV7533
+        { .compatible = "adi,adv7533", .data = (void *)ADV7533 },
+#endif
         { }
 };
 MODULE_DEVICE_TABLE(of, adv7511_of_ids);
 
-static struct drm_i2c_encoder_driver adv7511_driver = {
-        .i2c_driver = {
-                .driver = {
-                        .name = "adv7511",
-                        .of_match_table = adv7511_of_ids,
-                },
-                .id_table = adv7511_i2c_ids,
-                .probe = adv7511_probe,
-                .remove = adv7511_remove,
-        },
+static struct mipi_dsi_driver adv7533_dsi_driver = {
+        .driver.name = "adv7533",
+};
 
-        .encoder_init = adv7511_encoder_init,
+static struct i2c_driver adv7511_driver = {
+        .driver = {
+                .name = "adv7511",
+                .of_match_table = adv7511_of_ids,
+        },
+        .id_table = adv7511_i2c_ids,
+        .probe = adv7511_probe,
+        .remove = adv7511_remove,
 };
 
 static int __init adv7511_init(void)
 {
-        return drm_i2c_encoder_register(THIS_MODULE, &adv7511_driver);
+        if (IS_ENABLED(CONFIG_DRM_MIPI_DSI))
+                mipi_dsi_driver_register(&adv7533_dsi_driver);
+
+        return i2c_add_driver(&adv7511_driver);
 }
 module_init(adv7511_init);
 
 static void __exit adv7511_exit(void)
 {
-        drm_i2c_encoder_unregister(&adv7511_driver);
+        i2c_del_driver(&adv7511_driver);
+
+        if (IS_ENABLED(CONFIG_DRM_MIPI_DSI))
+                mipi_dsi_driver_unregister(&adv7533_dsi_driver);
 }
 module_exit(adv7511_exit);
 
diff --git a/drivers/gpu/drm/bridge/adv7511/adv7533.c b/drivers/gpu/drm/bridge/adv7511/adv7533.c
new file mode 100644
index 000000000000..5eebd15899b1
--- /dev/null
+++ b/drivers/gpu/drm/bridge/adv7511/adv7533.c
@@ -0,0 +1,265 @@
+/*
+ * Copyright (c) 2016, The Linux Foundation. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 and
+ * only version 2 as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+
+#include <linux/of_graph.h>
+
+#include "adv7511.h"
+
+static const struct reg_sequence adv7533_fixed_registers[] = {
+        { 0x16, 0x20 },
+        { 0x9a, 0xe0 },
+        { 0xba, 0x70 },
+        { 0xde, 0x82 },
+        { 0xe4, 0x40 },
+        { 0xe5, 0x80 },
+};
+
+static const struct reg_sequence adv7533_cec_fixed_registers[] = {
+        { 0x15, 0xd0 },
+        { 0x17, 0xd0 },
+        { 0x24, 0x20 },
+        { 0x57, 0x11 },
+};
+
+static const struct regmap_config adv7533_cec_regmap_config = {
+        .reg_bits = 8,
+        .val_bits = 8,
+
+        .max_register = 0xff,
+        .cache_type = REGCACHE_RBTREE,
+};
+
+static void adv7511_dsi_config_timing_gen(struct adv7511 *adv)
+{
+        struct mipi_dsi_device *dsi = adv->dsi;
+        struct drm_display_mode *mode = &adv->curr_mode;
+        unsigned int hsw, hfp, hbp, vsw, vfp, vbp;
+        u8 clock_div_by_lanes[] = { 6, 4, 3 };  /* 2, 3, 4 lanes */
+
+        hsw = mode->hsync_end - mode->hsync_start;
+        hfp = mode->hsync_start - mode->hdisplay;
+        hbp = mode->htotal - mode->hsync_end;
+        vsw = mode->vsync_end - mode->vsync_start;
+        vfp = mode->vsync_start - mode->vdisplay;
+        vbp = mode->vtotal - mode->vsync_end;
+
+        /* set pixel clock divider mode */
+        regmap_write(adv->regmap_cec, 0x16,
+                     clock_div_by_lanes[dsi->lanes - 2] << 3);
+
+        /* horizontal porch params */
+        regmap_write(adv->regmap_cec, 0x28, mode->htotal >> 4);
+        regmap_write(adv->regmap_cec, 0x29, (mode->htotal << 4) & 0xff);
+        regmap_write(adv->regmap_cec, 0x2a, hsw >> 4);
+        regmap_write(adv->regmap_cec, 0x2b, (hsw << 4) & 0xff);
+        regmap_write(adv->regmap_cec, 0x2c, hfp >> 4);
+        regmap_write(adv->regmap_cec, 0x2d, (hfp << 4) & 0xff);
+        regmap_write(adv->regmap_cec, 0x2e, hbp >> 4);
+        regmap_write(adv->regmap_cec, 0x2f, (hbp << 4) & 0xff);
+
+        /* vertical porch params */
+        regmap_write(adv->regmap_cec, 0x30, mode->vtotal >> 4);
+        regmap_write(adv->regmap_cec, 0x31, (mode->vtotal << 4) & 0xff);
+        regmap_write(adv->regmap_cec, 0x32, vsw >> 4);
+        regmap_write(adv->regmap_cec, 0x33, (vsw << 4) & 0xff);
+        regmap_write(adv->regmap_cec, 0x34, vfp >> 4);
+        regmap_write(adv->regmap_cec, 0x35, (vfp << 4) & 0xff);
+        regmap_write(adv->regmap_cec, 0x36, vbp >> 4);
+        regmap_write(adv->regmap_cec, 0x37, (vbp << 4) & 0xff);
+}
+
+void adv7533_dsi_power_on(struct adv7511 *adv)
+{
+        struct mipi_dsi_device *dsi = adv->dsi;
+
+        if (adv->use_timing_gen)
+                adv7511_dsi_config_timing_gen(adv);
+
+        /* set number of dsi lanes */
+        regmap_write(adv->regmap_cec, 0x1c, dsi->lanes << 4);
+
+        if (adv->use_timing_gen) {
+                /* reset internal timing generator */
+                regmap_write(adv->regmap_cec, 0x27, 0xcb);
+                regmap_write(adv->regmap_cec, 0x27, 0x8b);
+                regmap_write(adv->regmap_cec, 0x27, 0xcb);
+        } else {
+                /* disable internal timing generator */
+                regmap_write(adv->regmap_cec, 0x27, 0x0b);
+        }
+
+        /* enable hdmi */
+        regmap_write(adv->regmap_cec, 0x03, 0x89);
+        /* disable test mode */
+        regmap_write(adv->regmap_cec, 0x55, 0x00);
+
+        regmap_register_patch(adv->regmap_cec, adv7533_cec_fixed_registers,
+                              ARRAY_SIZE(adv7533_cec_fixed_registers));
+}
+
+void adv7533_dsi_power_off(struct adv7511 *adv)
+{
+        /* disable hdmi */
+        regmap_write(adv->regmap_cec, 0x03, 0x0b);
+        /* disable internal timing generator */
+        regmap_write(adv->regmap_cec, 0x27, 0x0b);
+}
+
+void adv7533_mode_set(struct adv7511 *adv, struct drm_display_mode *mode)
+{
+        struct mipi_dsi_device *dsi = adv->dsi;
+        int lanes, ret;
+
+        if (adv->num_dsi_lanes != 4)
+                return;
+
+        if (mode->clock > 80000)
+                lanes = 4;
+        else
+                lanes = 3;
+
+        if (lanes != dsi->lanes) {
+                mipi_dsi_detach(dsi);
+                dsi->lanes = lanes;
+                ret = mipi_dsi_attach(dsi);
+                if (ret)
+                        dev_err(&dsi->dev, "failed to change host lanes\n");
+        }
+}
+
+int adv7533_patch_registers(struct adv7511 *adv)
+{
+        return regmap_register_patch(adv->regmap,
+                                     adv7533_fixed_registers,
+                                     ARRAY_SIZE(adv7533_fixed_registers));
+}
+
+void adv7533_uninit_cec(struct adv7511 *adv)
+{
+        i2c_unregister_device(adv->i2c_cec);
+}
+
+static const int cec_i2c_addr = 0x78;
+
+int adv7533_init_cec(struct adv7511 *adv)
+{
+        int ret;
+
+        adv->i2c_cec = i2c_new_dummy(adv->i2c_main->adapter, cec_i2c_addr >> 1);
+        if (!adv->i2c_cec)
+                return -ENOMEM;
+
+        adv->regmap_cec = devm_regmap_init_i2c(adv->i2c_cec,
+                                        &adv7533_cec_regmap_config);
+        if (IS_ERR(adv->regmap_cec)) {
+                ret = PTR_ERR(adv->regmap_cec);
+                goto err;
+        }
+
+        ret = regmap_register_patch(adv->regmap_cec,
+                                    adv7533_cec_fixed_registers,
+                                    ARRAY_SIZE(adv7533_cec_fixed_registers));
+        if (ret)
+                goto err;
+
+        return 0;
+err:
+        adv7533_uninit_cec(adv);
+        return ret;
+}
+
+int adv7533_attach_dsi(struct adv7511 *adv)
+{
+        struct device *dev = &adv->i2c_main->dev;
+        struct mipi_dsi_host *host;
+        struct mipi_dsi_device *dsi;
+        int ret = 0;
+        const struct mipi_dsi_device_info info = { .type = "adv7533",
+                                                   .channel = 0,
+                                                   .node = NULL,
+                                                 };
+
+        host = of_find_mipi_dsi_host_by_node(adv->host_node);
+        if (!host) {
+                dev_err(dev, "failed to find dsi host\n");
+                return -EPROBE_DEFER;
+        }
+
+        dsi = mipi_dsi_device_register_full(host, &info);
+        if (IS_ERR(dsi)) {
+                dev_err(dev, "failed to create dsi device\n");
+                ret = PTR_ERR(dsi);
+                goto err_dsi_device;
+        }
+
+        adv->dsi = dsi;
+
+        dsi->lanes = adv->num_dsi_lanes;
+        dsi->format = MIPI_DSI_FMT_RGB888;
+        dsi->mode_flags = MIPI_DSI_MODE_VIDEO | MIPI_DSI_MODE_VIDEO_SYNC_PULSE |
+                          MIPI_DSI_MODE_EOT_PACKET | MIPI_DSI_MODE_VIDEO_HSE;
+
+        ret = mipi_dsi_attach(dsi);
+        if (ret < 0) {
+                dev_err(dev, "failed to attach dsi to host\n");
+                goto err_dsi_attach;
+        }
+
+        return 0;
+
+err_dsi_attach:
+        mipi_dsi_device_unregister(dsi);
+err_dsi_device:
+        return ret;
+}
+
+void adv7533_detach_dsi(struct adv7511 *adv)
+{
+        mipi_dsi_detach(adv->dsi);
+        mipi_dsi_device_unregister(adv->dsi);
+}
+
+int adv7533_parse_dt(struct device_node *np, struct adv7511 *adv)
+{
+        u32 num_lanes;
+        struct device_node *endpoint;
+
+        of_property_read_u32(np, "adi,dsi-lanes", &num_lanes);
+
+        if (num_lanes < 1 || num_lanes > 4)
+                return -EINVAL;
+
+        adv->num_dsi_lanes = num_lanes;
+
+        endpoint = of_graph_get_next_endpoint(np, NULL);
+        if (!endpoint)
+                return -ENODEV;
+
+        adv->host_node = of_graph_get_remote_port_parent(endpoint);
+        if (!adv->host_node) {
+                of_node_put(endpoint);
+                return -ENODEV;
+        }
+
+        of_node_put(endpoint);
+        of_node_put(adv->host_node);
+
+        adv->use_timing_gen = !of_property_read_bool(np,
+                                                "adi,disable-timing-generator");
+
+        /* TODO: Check if these need to be parsed by DT or not */
+        adv->rgb = true;
+        adv->embedded_sync = false;
+
+        return 0;
+}
diff --git a/drivers/gpu/drm/bridge/analogix-anx78xx.c b/drivers/gpu/drm/bridge/analogix-anx78xx.c
index d087b054c360..f9f03bcba0af 100644
--- a/drivers/gpu/drm/bridge/analogix-anx78xx.c
+++ b/drivers/gpu/drm/bridge/analogix-anx78xx.c
@@ -986,16 +986,8 @@ unlock:
         return num_modes;
 }
 
-static struct drm_encoder *anx78xx_best_encoder(struct drm_connector *connector)
-{
-        struct anx78xx *anx78xx = connector_to_anx78xx(connector);
-
-        return anx78xx->bridge.encoder;
-}
-
 static const struct drm_connector_helper_funcs anx78xx_connector_helper_funcs = {
         .get_modes = anx78xx_get_modes,
-        .best_encoder = anx78xx_best_encoder,
 };
 
 static enum drm_connector_status anx78xx_detect(struct drm_connector *connector,
diff --git a/drivers/gpu/drm/bridge/analogix/analogix_dp_core.c b/drivers/gpu/drm/bridge/analogix/analogix_dp_core.c
index 7699597070a1..32715daf73cb 100644
--- a/drivers/gpu/drm/bridge/analogix/analogix_dp_core.c
+++ b/drivers/gpu/drm/bridge/analogix/analogix_dp_core.c
@@ -938,7 +938,7 @@ int analogix_dp_get_modes(struct drm_connector *connector)
                 num_modes += drm_panel_get_modes(dp->plat_data->panel);
 
         if (dp->plat_data->get_modes)
-                num_modes += dp->plat_data->get_modes(dp->plat_data);
+                num_modes += dp->plat_data->get_modes(dp->plat_data, connector);
 
         return num_modes;
 }
@@ -1208,6 +1208,7 @@ static int analogix_dp_dt_parse_pdata(struct analogix_dp_device *dp)
 
         switch (dp->plat_data->dev_type) {
         case RK3288_DP:
+        case RK3399_EDP:
                 /*
                  * Like Rk3288 DisplayPort TRM indicate that "Main link
                  * containing 4 physical lanes of 2.7/1.62 Gbps/lane".
diff --git a/drivers/gpu/drm/bridge/analogix/analogix_dp_core.h b/drivers/gpu/drm/bridge/analogix/analogix_dp_core.h
index f09275d40f70..b45638043ec4 100644
--- a/drivers/gpu/drm/bridge/analogix/analogix_dp_core.h
+++ b/drivers/gpu/drm/bridge/analogix/analogix_dp_core.h
@@ -127,10 +127,10 @@ enum analog_power_block {
 };
 
 enum dp_irq_type {
-        DP_IRQ_TYPE_HP_CABLE_IN,
-        DP_IRQ_TYPE_HP_CABLE_OUT,
-        DP_IRQ_TYPE_HP_CHANGE,
-        DP_IRQ_TYPE_UNKNOWN,
+        DP_IRQ_TYPE_HP_CABLE_IN  = BIT(0),
+        DP_IRQ_TYPE_HP_CABLE_OUT = BIT(1),
+        DP_IRQ_TYPE_HP_CHANGE    = BIT(2),
+        DP_IRQ_TYPE_UNKNOWN      = BIT(3),
 };
 
 struct video_info {
diff --git a/drivers/gpu/drm/bridge/analogix/analogix_dp_reg.c b/drivers/gpu/drm/bridge/analogix/analogix_dp_reg.c
index 49205ef02be3..48030f0cf497 100644
--- a/drivers/gpu/drm/bridge/analogix/analogix_dp_reg.c
+++ b/drivers/gpu/drm/bridge/analogix/analogix_dp_reg.c
@@ -74,8 +74,12 @@ void analogix_dp_init_analog_param(struct analogix_dp_device *dp)
         reg = SEL_24M | TX_DVDD_BIT_1_0625V;
         writel(reg, dp->reg_base + ANALOGIX_DP_ANALOG_CTL_2);
 
-        if (dp->plat_data && (dp->plat_data->dev_type == RK3288_DP)) {
-                writel(REF_CLK_24M, dp->reg_base + ANALOGIX_DP_PLL_REG_1);
+        if (dp->plat_data && is_rockchip(dp->plat_data->dev_type)) {
+                reg = REF_CLK_24M;
+                if (dp->plat_data->dev_type == RK3288_DP)
+                        reg ^= REF_CLK_MASK;
+
+                writel(reg, dp->reg_base + ANALOGIX_DP_PLL_REG_1);
                 writel(0x95, dp->reg_base + ANALOGIX_DP_PLL_REG_2);
                 writel(0x40, dp->reg_base + ANALOGIX_DP_PLL_REG_3);
                 writel(0x58, dp->reg_base + ANALOGIX_DP_PLL_REG_4);
@@ -244,7 +248,7 @@ void analogix_dp_set_analog_power_down(struct analogix_dp_device *dp,
         u32 reg;
         u32 phy_pd_addr = ANALOGIX_DP_PHY_PD;
 
-        if (dp->plat_data && (dp->plat_data->dev_type == RK3288_DP))
+        if (dp->plat_data && is_rockchip(dp->plat_data->dev_type))
                 phy_pd_addr = ANALOGIX_DP_PD;
 
         switch (block) {
@@ -448,7 +452,7 @@ void analogix_dp_init_aux(struct analogix_dp_device *dp)
         analogix_dp_reset_aux(dp);
 
         /* Disable AUX transaction H/W retry */
-        if (dp->plat_data && (dp->plat_data->dev_type == RK3288_DP))
+        if (dp->plat_data && is_rockchip(dp->plat_data->dev_type))
                 reg = AUX_BIT_PERIOD_EXPECTED_DELAY(0) |
                       AUX_HW_RETRY_COUNT_SEL(3) |
                       AUX_HW_RETRY_INTERVAL_600_MICROSECONDS;
diff --git a/drivers/gpu/drm/bridge/analogix/analogix_dp_reg.h b/drivers/gpu/drm/bridge/analogix/analogix_dp_reg.h
index 337912b0aeab..cdcc6c5add5e 100644
--- a/drivers/gpu/drm/bridge/analogix/analogix_dp_reg.h
+++ b/drivers/gpu/drm/bridge/analogix/analogix_dp_reg.h
@@ -163,8 +163,9 @@
 #define HSYNC_POLARITY_CFG                      (0x1 << 0)
 
 /* ANALOGIX_DP_PLL_REG_1 */
-#define REF_CLK_24M                             (0x1 << 1)
-#define REF_CLK_27M                             (0x0 << 1)
+#define REF_CLK_24M                             (0x1 << 0)
+#define REF_CLK_27M                             (0x0 << 0)
+#define REF_CLK_MASK                            (0x1 << 0)
 
 /* ANALOGIX_DP_LANE_MAP */
 #define LANE3_MAP_LOGIC_LANE_0                  (0x0 << 6)
diff --git a/drivers/gpu/drm/bridge/dw-hdmi.c b/drivers/gpu/drm/bridge/dw-hdmi.c
index c9d941283d30..77ab47341658 100644
--- a/drivers/gpu/drm/bridge/dw-hdmi.c
+++ b/drivers/gpu/drm/bridge/dw-hdmi.c
@@ -1476,15 +1476,6 @@ dw_hdmi_connector_mode_valid(struct drm_connector *connector,
         return mode_status;
 }
 
-static struct drm_encoder *dw_hdmi_connector_best_encoder(struct drm_connector
-                                                           *connector)
-{
-        struct dw_hdmi *hdmi = container_of(connector, struct dw_hdmi,
-                                             connector);
-
-        return hdmi->encoder;
-}
-
 static void dw_hdmi_connector_destroy(struct drm_connector *connector)
 {
         drm_connector_unregister(connector);
@@ -1504,14 +1495,6 @@ static void dw_hdmi_connector_force(struct drm_connector *connector)
 }
 
 static const struct drm_connector_funcs dw_hdmi_connector_funcs = {
-        .dpms = drm_helper_connector_dpms,
-        .fill_modes = drm_helper_probe_single_connector_modes,
-        .detect = dw_hdmi_connector_detect,
-        .destroy = dw_hdmi_connector_destroy,
-        .force = dw_hdmi_connector_force,
-};
-
-static const struct drm_connector_funcs dw_hdmi_atomic_connector_funcs = {
         .dpms = drm_atomic_helper_connector_dpms,
         .fill_modes = drm_helper_probe_single_connector_modes,
         .detect = dw_hdmi_connector_detect,
@@ -1525,7 +1508,7 @@ static const struct drm_connector_funcs dw_hdmi_atomic_connector_funcs = {
 static const struct drm_connector_helper_funcs dw_hdmi_connector_helper_funcs = {
         .get_modes = dw_hdmi_connector_get_modes,
         .mode_valid = dw_hdmi_connector_mode_valid,
-        .best_encoder = dw_hdmi_connector_best_encoder,
+        .best_encoder = drm_atomic_helper_best_encoder,
 };
 
 static const struct drm_bridge_funcs dw_hdmi_bridge_funcs = {
@@ -1643,14 +1626,9 @@ static int dw_hdmi_register(struct drm_device *drm, struct dw_hdmi *hdmi)
         drm_connector_helper_add(&hdmi->connector,
                                  &dw_hdmi_connector_helper_funcs);
 
-        if (drm_core_check_feature(drm, DRIVER_ATOMIC))
-                drm_connector_init(drm, &hdmi->connector,
-                                   &dw_hdmi_atomic_connector_funcs,
-                                   DRM_MODE_CONNECTOR_HDMIA);
-        else
-                drm_connector_init(drm, &hdmi->connector,
-                                   &dw_hdmi_connector_funcs,
-                                   DRM_MODE_CONNECTOR_HDMIA);
+        drm_connector_init(drm, &hdmi->connector,
+                           &dw_hdmi_connector_funcs,
+                           DRM_MODE_CONNECTOR_HDMIA);
 
         drm_mode_connector_attach_encoder(&hdmi->connector, encoder);
 
diff --git a/drivers/gpu/drm/bridge/nxp-ptn3460.c b/drivers/gpu/drm/bridge/nxp-ptn3460.c
index 7ecd59f70b8e..93f3dacf9e27 100644
--- a/drivers/gpu/drm/bridge/nxp-ptn3460.c
+++ b/drivers/gpu/drm/bridge/nxp-ptn3460.c
@@ -235,16 +235,8 @@ out:
         return num_modes;
 }
 
-static struct drm_encoder *ptn3460_best_encoder(struct drm_connector *connector)
-{
-        struct ptn3460_bridge *ptn_bridge = connector_to_ptn3460(connector);
-
-        return ptn_bridge->bridge.encoder;
-}
-
 static const struct drm_connector_helper_funcs ptn3460_connector_helper_funcs = {
         .get_modes = ptn3460_get_modes,
-        .best_encoder = ptn3460_best_encoder,
 };
 
 static enum drm_connector_status ptn3460_detect(struct drm_connector *connector,
diff --git a/drivers/gpu/drm/bridge/parade-ps8622.c b/drivers/gpu/drm/bridge/parade-ps8622.c
index be881e9fef8f..583b8ce614e3 100644
--- a/drivers/gpu/drm/bridge/parade-ps8622.c
+++ b/drivers/gpu/drm/bridge/parade-ps8622.c
@@ -474,18 +474,8 @@ static int ps8622_get_modes(struct drm_connector *connector)
         return drm_panel_get_modes(ps8622->panel);
 }
 
-static struct drm_encoder *ps8622_best_encoder(struct drm_connector *connector)
-{
-        struct ps8622_bridge *ps8622;
-
-        ps8622 = connector_to_ps8622(connector);
-
-        return ps8622->bridge.encoder;
-}
-
 static const struct drm_connector_helper_funcs ps8622_connector_helper_funcs = {
         .get_modes = ps8622_get_modes,
-        .best_encoder = ps8622_best_encoder,
 };
 
 static enum drm_connector_status ps8622_detect(struct drm_connector *connector,
@@ -646,9 +636,7 @@ static int ps8622_remove(struct i2c_client *client)
 {
         struct ps8622_bridge *ps8622 = i2c_get_clientdata(client);
 
-        if (ps8622->bl)
-                backlight_device_unregister(ps8622->bl);
-
+        backlight_device_unregister(ps8622->bl);
         drm_bridge_remove(&ps8622->bridge);
 
         return 0;
diff --git a/drivers/gpu/drm/bridge/sii902x.c b/drivers/gpu/drm/bridge/sii902x.c
new file mode 100644
index 000000000000..9126d0306ab5
--- /dev/null
+++ b/drivers/gpu/drm/bridge/sii902x.c
@@ -0,0 +1,467 @@
+/*
+ * Copyright (C) 2016 Atmel
+ *                    Bo Shen <voice.shen@atmel.com>
+ *
+ * Authors:           Bo Shen <voice.shen@atmel.com>
+ *                    Boris Brezillon <boris.brezillon@free-electrons.com>
+ *                    Wu, Songjun <Songjun.Wu@atmel.com>
+ *
+ *
+ * Copyright (C) 2010-2011 Freescale Semiconductor, Inc. All Rights Reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+
+#include <linux/gpio/consumer.h>
+#include <linux/i2c.h>
+#include <linux/module.h>
+#include <linux/regmap.h>
+
+#include <drm/drmP.h>
+#include <drm/drm_atomic_helper.h>
+#include <drm/drm_crtc_helper.h>
+#include <drm/drm_edid.h>
+
+#define SII902X_TPI_VIDEO_DATA                  0x0
+
+#define SII902X_TPI_PIXEL_REPETITION            0x8
+#define SII902X_TPI_AVI_PIXEL_REP_BUS_24BIT     BIT(5)
+#define SII902X_TPI_AVI_PIXEL_REP_RISING_EDGE   BIT(4)
+#define SII902X_TPI_AVI_PIXEL_REP_4X            3
+#define SII902X_TPI_AVI_PIXEL_REP_2X            1
+#define SII902X_TPI_AVI_PIXEL_REP_NONE          0
+#define SII902X_TPI_CLK_RATIO_HALF              (0 << 6)
+#define SII902X_TPI_CLK_RATIO_1X                (1 << 6)
+#define SII902X_TPI_CLK_RATIO_2X                (2 << 6)
+#define SII902X_TPI_CLK_RATIO_4X                (3 << 6)
+
+#define SII902X_TPI_AVI_IN_FORMAT               0x9
+#define SII902X_TPI_AVI_INPUT_BITMODE_12BIT     BIT(7)
+#define SII902X_TPI_AVI_INPUT_DITHER            BIT(6)
+#define SII902X_TPI_AVI_INPUT_RANGE_LIMITED     (2 << 2)
+#define SII902X_TPI_AVI_INPUT_RANGE_FULL        (1 << 2)
+#define SII902X_TPI_AVI_INPUT_RANGE_AUTO        (0 << 2)
+#define SII902X_TPI_AVI_INPUT_COLORSPACE_BLACK  (3 << 0)
+#define SII902X_TPI_AVI_INPUT_COLORSPACE_YUV422 (2 << 0)
+#define SII902X_TPI_AVI_INPUT_COLORSPACE_YUV444 (1 << 0)
+#define SII902X_TPI_AVI_INPUT_COLORSPACE_RGB    (0 << 0)
+
+#define SII902X_TPI_AVI_INFOFRAME               0x0c
+
+#define SII902X_SYS_CTRL_DATA                   0x1a
+#define SII902X_SYS_CTRL_PWR_DWN                BIT(4)
+#define SII902X_SYS_CTRL_AV_MUTE                BIT(3)
+#define SII902X_SYS_CTRL_DDC_BUS_REQ            BIT(2)
+#define SII902X_SYS_CTRL_DDC_BUS_GRTD           BIT(1)
+#define SII902X_SYS_CTRL_OUTPUT_MODE            BIT(0)
+#define SII902X_SYS_CTRL_OUTPUT_HDMI            1
+#define SII902X_SYS_CTRL_OUTPUT_DVI             0
+
+#define SII902X_REG_CHIPID(n)                   (0x1b + (n))
+
+#define SII902X_PWR_STATE_CTRL                  0x1e
+#define SII902X_AVI_POWER_STATE_MSK             GENMASK(1, 0)
+#define SII902X_AVI_POWER_STATE_D(l)            ((l) & SII902X_AVI_POWER_STATE_MSK)
+
+#define SII902X_INT_ENABLE                      0x3c
+#define SII902X_INT_STATUS                      0x3d
+#define SII902X_HOTPLUG_EVENT                   BIT(0)
+#define SII902X_PLUGGED_STATUS                  BIT(2)
+
+#define SII902X_REG_TPI_RQB                     0xc7
+
+#define SII902X_I2C_BUS_ACQUISITION_TIMEOUT_MS  500
+
+struct sii902x {
+        struct i2c_client *i2c;
+        struct regmap *regmap;
+        struct drm_bridge bridge;
+        struct drm_connector connector;
+        struct gpio_desc *reset_gpio;
+};
+
+static inline struct sii902x *bridge_to_sii902x(struct drm_bridge *bridge)
+{
+        return container_of(bridge, struct sii902x, bridge);
+}
+
+static inline struct sii902x *connector_to_sii902x(struct drm_connector *con)
+{
+        return container_of(con, struct sii902x, connector);
+}
+
+static void sii902x_reset(struct sii902x *sii902x)
+{
+        if (!sii902x->reset_gpio)
+                return;
+
+        gpiod_set_value(sii902x->reset_gpio, 1);
+
+        /* The datasheet says treset-min = 100us. Make it 150us to be sure. */
+        usleep_range(150, 200);
+
+        gpiod_set_value(sii902x->reset_gpio, 0);
+}
+
+static enum drm_connector_status
+sii902x_connector_detect(struct drm_connector *connector, bool force)
+{
+        struct sii902x *sii902x = connector_to_sii902x(connector);
+        unsigned int status;
+
+        regmap_read(sii902x->regmap, SII902X_INT_STATUS, &status);
+
+        return (status & SII902X_PLUGGED_STATUS) ?
+               connector_status_connected : connector_status_disconnected;
+}
+
+static const struct drm_connector_funcs sii902x_connector_funcs = {
+        .dpms = drm_atomic_helper_connector_dpms,
+        .detect = sii902x_connector_detect,
+        .fill_modes = drm_helper_probe_single_connector_modes,
+        .destroy = drm_connector_cleanup,
+        .reset = drm_atomic_helper_connector_reset,
+        .atomic_duplicate_state = drm_atomic_helper_connector_duplicate_state,
+        .atomic_destroy_state = drm_atomic_helper_connector_destroy_state,
+};
+
+static int sii902x_get_modes(struct drm_connector *connector)
+{
+        struct sii902x *sii902x = connector_to_sii902x(connector);
+        struct regmap *regmap = sii902x->regmap;
+        u32 bus_format = MEDIA_BUS_FMT_RGB888_1X24;
+        unsigned long timeout;
+        unsigned int status;
+        struct edid *edid;
+        int num = 0;
+        int ret;
+
+        ret = regmap_update_bits(regmap, SII902X_SYS_CTRL_DATA,
+                                 SII902X_SYS_CTRL_DDC_BUS_REQ,
+                                 SII902X_SYS_CTRL_DDC_BUS_REQ);
+        if (ret)
+                return ret;
+
+        timeout = jiffies +
+                  msecs_to_jiffies(SII902X_I2C_BUS_ACQUISITION_TIMEOUT_MS);
+        do {
+                ret = regmap_read(regmap, SII902X_SYS_CTRL_DATA, &status);
+                if (ret)
+                        return ret;
+        } while (!(status & SII902X_SYS_CTRL_DDC_BUS_GRTD) &&
+                 time_before(jiffies, timeout));
+
+        if (!(status & SII902X_SYS_CTRL_DDC_BUS_GRTD)) {
+                dev_err(&sii902x->i2c->dev, "failed to acquire the i2c bus");
+                return -ETIMEDOUT;
+        }
+
+        ret = regmap_write(regmap, SII902X_SYS_CTRL_DATA, status);
+        if (ret)
+                return ret;
+
+        edid = drm_get_edid(connector, sii902x->i2c->adapter);
+        drm_mode_connector_update_edid_property(connector, edid);
+        if (edid) {
+                num = drm_add_edid_modes(connector, edid);
+                kfree(edid);
+        }
+
+        ret = drm_display_info_set_bus_formats(&connector->display_info,
+                                               &bus_format, 1);
+        if (ret)
+                return ret;
+
+        ret = regmap_read(regmap, SII902X_SYS_CTRL_DATA, &status);
+        if (ret)
+                return ret;
+
+        ret = regmap_update_bits(regmap, SII902X_SYS_CTRL_DATA,
+                                 SII902X_SYS_CTRL_DDC_BUS_REQ |
+                                 SII902X_SYS_CTRL_DDC_BUS_GRTD, 0);
+        if (ret)
+                return ret;
+
+        timeout = jiffies +
+                  msecs_to_jiffies(SII902X_I2C_BUS_ACQUISITION_TIMEOUT_MS);
+        do {
+                ret = regmap_read(regmap, SII902X_SYS_CTRL_DATA, &status);
+                if (ret)
+                        return ret;
+        } while (status & (SII902X_SYS_CTRL_DDC_BUS_REQ |
+                           SII902X_SYS_CTRL_DDC_BUS_GRTD) &&
+                 time_before(jiffies, timeout));
+
+        if (status & (SII902X_SYS_CTRL_DDC_BUS_REQ |
+                      SII902X_SYS_CTRL_DDC_BUS_GRTD)) {
+                dev_err(&sii902x->i2c->dev, "failed to release the i2c bus");
+                return -ETIMEDOUT;
+        }
+
+        return num;
+}
+
+static enum drm_mode_status sii902x_mode_valid(struct drm_connector *connector,
+                                               struct drm_display_mode *mode)
+{
+        /* TODO: check mode */
+
+        return MODE_OK;
+}
+
+static const struct drm_connector_helper_funcs sii902x_connector_helper_funcs = {
+        .get_modes = sii902x_get_modes,
+        .mode_valid = sii902x_mode_valid,
+};
+
+static void sii902x_bridge_disable(struct drm_bridge *bridge)
+{
+        struct sii902x *sii902x = bridge_to_sii902x(bridge);
+
+        regmap_update_bits(sii902x->regmap, SII902X_SYS_CTRL_DATA,
+                           SII902X_SYS_CTRL_PWR_DWN,
+                           SII902X_SYS_CTRL_PWR_DWN);
+}
+
+static void sii902x_bridge_enable(struct drm_bridge *bridge)
+{
+        struct sii902x *sii902x = bridge_to_sii902x(bridge);
+
+        regmap_update_bits(sii902x->regmap, SII902X_PWR_STATE_CTRL,
+                           SII902X_AVI_POWER_STATE_MSK,
+                           SII902X_AVI_POWER_STATE_D(0));
+        regmap_update_bits(sii902x->regmap, SII902X_SYS_CTRL_DATA,
+                           SII902X_SYS_CTRL_PWR_DWN, 0);
+}
+
+static void sii902x_bridge_mode_set(struct drm_bridge *bridge,
+                                    struct drm_display_mode *mode,
+                                    struct drm_display_mode *adj)
+{
+        struct sii902x *sii902x = bridge_to_sii902x(bridge);
+        struct regmap *regmap = sii902x->regmap;
+        u8 buf[HDMI_INFOFRAME_SIZE(AVI)];
+        struct hdmi_avi_infoframe frame;
+        int ret;
+
+        buf[0] = adj->clock;
+        buf[1] = adj->clock >> 8;
+        buf[2] = adj->vrefresh;
+        buf[3] = 0x00;
+        buf[4] = adj->hdisplay;
+        buf[5] = adj->hdisplay >> 8;
+        buf[6] = adj->vdisplay;
+        buf[7] = adj->vdisplay >> 8;
+        buf[8] = SII902X_TPI_CLK_RATIO_1X | SII902X_TPI_AVI_PIXEL_REP_NONE |
+                 SII902X_TPI_AVI_PIXEL_REP_BUS_24BIT;
+        buf[9] = SII902X_TPI_AVI_INPUT_RANGE_AUTO |
+                 SII902X_TPI_AVI_INPUT_COLORSPACE_RGB;
+
+        ret = regmap_bulk_write(regmap, SII902X_TPI_VIDEO_DATA, buf, 10);
+        if (ret)
+                return;
+
+        ret = drm_hdmi_avi_infoframe_from_display_mode(&frame, adj);
+        if (ret < 0) {
+                DRM_ERROR("couldn't fill AVI infoframe\n");
+                return;
+        }
+
+        ret = hdmi_avi_infoframe_pack(&frame, buf, sizeof(buf));
+        if (ret < 0) {
+                DRM_ERROR("failed to pack AVI infoframe: %d\n", ret);
+                return;
+        }
+
+        /* Do not send the infoframe header, but keep the CRC field. */
+        regmap_bulk_write(regmap, SII902X_TPI_AVI_INFOFRAME,
+                          buf + HDMI_INFOFRAME_HEADER_SIZE - 1,
+                          HDMI_AVI_INFOFRAME_SIZE + 1);
+}
+
+static int sii902x_bridge_attach(struct drm_bridge *bridge)
+{
+        struct sii902x *sii902x = bridge_to_sii902x(bridge);
+        struct drm_device *drm = bridge->dev;
+        int ret;
+
+        drm_connector_helper_add(&sii902x->connector,
+                                 &sii902x_connector_helper_funcs);
+
+        if (!drm_core_check_feature(drm, DRIVER_ATOMIC)) {
+                dev_err(&sii902x->i2c->dev,
+                        "sii902x driver is only compatible with DRM devices supporting atomic updates");
+                return -ENOTSUPP;
+        }
+
+        ret = drm_connector_init(drm, &sii902x->connector,
+                                 &sii902x_connector_funcs,
+                                 DRM_MODE_CONNECTOR_HDMIA);
+        if (ret)
+                return ret;
+
+        if (sii902x->i2c->irq > 0)
+                sii902x->connector.polled = DRM_CONNECTOR_POLL_HPD;
+        else
+                sii902x->connector.polled = DRM_CONNECTOR_POLL_CONNECT;
+
+        drm_mode_connector_attach_encoder(&sii902x->connector, bridge->encoder);
+
+        return 0;
+}
+
+static const struct drm_bridge_funcs sii902x_bridge_funcs = {
+        .attach = sii902x_bridge_attach,
+        .mode_set = sii902x_bridge_mode_set,
+        .disable = sii902x_bridge_disable,
+        .enable = sii902x_bridge_enable,
+};
+
+static const struct regmap_range sii902x_volatile_ranges[] = {
+        { .range_min = 0, .range_max = 0xff },
+};
+
+static const struct regmap_access_table sii902x_volatile_table = {
+        .yes_ranges = sii902x_volatile_ranges,
+        .n_yes_ranges = ARRAY_SIZE(sii902x_volatile_ranges),
+};
+
+static const struct regmap_config sii902x_regmap_config = {
+        .reg_bits = 8,
+        .val_bits = 8,
+        .volatile_table = &sii902x_volatile_table,
+        .cache_type = REGCACHE_NONE,
+};
+
+static irqreturn_t sii902x_interrupt(int irq, void *data)
+{
+        struct sii902x *sii902x = data;
+        unsigned int status = 0;
+
+        regmap_read(sii902x->regmap, SII902X_INT_STATUS, &status);
+        regmap_write(sii902x->regmap, SII902X_INT_STATUS, status);
+
+        if ((status & SII902X_HOTPLUG_EVENT) && sii902x->bridge.dev)
+                drm_helper_hpd_irq_event(sii902x->bridge.dev);
+
+        return IRQ_HANDLED;
+}
+
+static int sii902x_probe(struct i2c_client *client,
+                         const struct i2c_device_id *id)
+{
+        struct device *dev = &client->dev;
+        unsigned int status = 0;
+        struct sii902x *sii902x;
+        u8 chipid[4];
+        int ret;
+
+        sii902x = devm_kzalloc(dev, sizeof(*sii902x), GFP_KERNEL);
+        if (!sii902x)
+                return -ENOMEM;
+
+        sii902x->i2c = client;
+        sii902x->regmap = devm_regmap_init_i2c(client, &sii902x_regmap_config);
+        if (IS_ERR(sii902x->regmap))
+                return PTR_ERR(sii902x->regmap);
+
+        sii902x->reset_gpio = devm_gpiod_get_optional(dev, "reset",
+                                                      GPIOD_OUT_LOW);
+        if (IS_ERR(sii902x->reset_gpio)) {
+                dev_err(dev, "Failed to retrieve/request reset gpio: %ld\n",
+                        PTR_ERR(sii902x->reset_gpio));
+                return PTR_ERR(sii902x->reset_gpio);
+        }
+
+        sii902x_reset(sii902x);
+
+        ret = regmap_write(sii902x->regmap, SII902X_REG_TPI_RQB, 0x0);
+        if (ret)
+                return ret;
+
+        ret = regmap_bulk_read(sii902x->regmap, SII902X_REG_CHIPID(0),
+                               &chipid, 4);
+        if (ret) {
+                dev_err(dev, "regmap_read failed %d\n", ret);
+                return ret;
+        }
+
+        if (chipid[0] != 0xb0) {
+                dev_err(dev, "Invalid chipid: %02x (expecting 0xb0)\n",
+                        chipid[0]);
+                return -EINVAL;
+        }
+
+        /* Clear all pending interrupts */
+        regmap_read(sii902x->regmap, SII902X_INT_STATUS, &status);
+        regmap_write(sii902x->regmap, SII902X_INT_STATUS, status);
+
+        if (client->irq > 0) {
+                regmap_write(sii902x->regmap, SII902X_INT_ENABLE,
+                             SII902X_HOTPLUG_EVENT);
+
+                ret = devm_request_threaded_irq(dev, client->irq, NULL,
+                                                sii902x_interrupt,
+                                                IRQF_ONESHOT, dev_name(dev),
+                                                sii902x);
+                if (ret)
+                        return ret;
+        }
+
+        sii902x->bridge.funcs = &sii902x_bridge_funcs;
+        sii902x->bridge.of_node = dev->of_node;
+        ret = drm_bridge_add(&sii902x->bridge);
+        if (ret) {
+                dev_err(dev, "Failed to add drm_bridge\n");
+                return ret;
+        }
+
+        i2c_set_clientdata(client, sii902x);
+
+        return 0;
+}
+
+static int sii902x_remove(struct i2c_client *client)
+
+{
+        struct sii902x *sii902x = i2c_get_clientdata(client);
+
+        drm_bridge_remove(&sii902x->bridge);
+
+        return 0;
+}
+
+static const struct of_device_id sii902x_dt_ids[] = {
+        { .compatible = "sil,sii9022", },
+        { }
+};
+MODULE_DEVICE_TABLE(of, sii902x_dt_ids);
+
+static const struct i2c_device_id sii902x_i2c_ids[] = {
+        { "sii9022", 0 },
+        { },
+};
+MODULE_DEVICE_TABLE(i2c, sii902x_i2c_ids);
+
+static struct i2c_driver sii902x_driver = {
+        .probe = sii902x_probe,
+        .remove = sii902x_remove,
+        .driver = {
+                .name = "sii902x",
+                .of_match_table = sii902x_dt_ids,
+        },
+        .id_table = sii902x_i2c_ids,
+};
+module_i2c_driver(sii902x_driver);
+
+MODULE_AUTHOR("Boris Brezillon <boris.brezillon@free-electrons.com>");
+MODULE_DESCRIPTION("SII902x RGB -> HDMI bridges");
+MODULE_LICENSE("GPL");
diff --git a/drivers/gpu/drm/bridge/tc358767.c b/drivers/gpu/drm/bridge/tc358767.c
new file mode 100644
index 000000000000..a09825d8c94a
--- /dev/null
+++ b/drivers/gpu/drm/bridge/tc358767.c
@@ -0,0 +1,1413 @@
+/*
+ * tc358767 eDP bridge driver
+ *
+ * Copyright (C) 2016 CogentEmbedded Inc
+ * Author: Andrey Gusakov <andrey.gusakov@cogentembedded.com>
+ *
+ * Copyright (C) 2016 Pengutronix, Philipp Zabel <p.zabel@pengutronix.de>
+ *
+ * Initially based on: drivers/gpu/drm/i2c/tda998x_drv.c
+ *
+ * Copyright (C) 2012 Texas Instruments
+ * Author: Rob Clark <robdclark@gmail.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+
+#include <linux/clk.h>
+#include <linux/device.h>
+#include <linux/gpio/consumer.h>
+#include <linux/i2c.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/regmap.h>
+#include <linux/slab.h>
+
+#include <drm/drm_atomic_helper.h>
+#include <drm/drm_crtc_helper.h>
+#include <drm/drm_dp_helper.h>
+#include <drm/drm_edid.h>
+#include <drm/drm_of.h>
+#include <drm/drm_panel.h>
+
+/* Registers */
+
+/* Display Parallel Interface */
+#define DPIPXLFMT               0x0440
+#define VS_POL_ACTIVE_LOW               (1 << 10)
+#define HS_POL_ACTIVE_LOW               (1 << 9)
+#define DE_POL_ACTIVE_HIGH              (0 << 8)
+#define SUB_CFG_TYPE_CONFIG1            (0 << 2) /* LSB aligned */
+#define SUB_CFG_TYPE_CONFIG2            (1 << 2) /* Loosely Packed */
+#define SUB_CFG_TYPE_CONFIG3            (2 << 2) /* LSB aligned 8-bit */
+#define DPI_BPP_RGB888                  (0 << 0)
+#define DPI_BPP_RGB666                  (1 << 0)
+#define DPI_BPP_RGB565                  (2 << 0)
+
+/* Video Path */
+#define VPCTRL0                 0x0450
+#define OPXLFMT_RGB666                  (0 << 8)
+#define OPXLFMT_RGB888                  (1 << 8)
+#define FRMSYNC_DISABLED                (0 << 4) /* Video Timing Gen Disabled */
+#define FRMSYNC_ENABLED                 (1 << 4) /* Video Timing Gen Enabled */
+#define MSF_DISABLED                    (0 << 0) /* Magic Square FRC disabled */
+#define MSF_ENABLED                     (1 << 0) /* Magic Square FRC enabled */
+#define HTIM01                  0x0454
+#define HTIM02                  0x0458
+#define VTIM01                  0x045c
+#define VTIM02                  0x0460
+#define VFUEN0                  0x0464
+#define VFUEN                           BIT(0)   /* Video Frame Timing Upload */
+
+/* System */
+#define TC_IDREG                0x0500
+#define SYSCTRL                 0x0510
+#define DP0_AUDSRC_NO_INPUT             (0 << 3)
+#define DP0_AUDSRC_I2S_RX               (1 << 3)
+#define DP0_VIDSRC_NO_INPUT             (0 << 0)
+#define DP0_VIDSRC_DSI_RX               (1 << 0)
+#define DP0_VIDSRC_DPI_RX               (2 << 0)
+#define DP0_VIDSRC_COLOR_BAR            (3 << 0)
+
+/* Control */
+#define DP0CTL                  0x0600
+#define VID_MN_GEN                      BIT(6)   /* Auto-generate M/N values */
+#define EF_EN                           BIT(5)   /* Enable Enhanced Framing */
+#define VID_EN                          BIT(1)   /* Video transmission enable */
+#define DP_EN                           BIT(0)   /* Enable DPTX function */
+
+/* Clocks */
+#define DP0_VIDMNGEN0           0x0610
+#define DP0_VIDMNGEN1           0x0614
+#define DP0_VMNGENSTATUS        0x0618
+
+/* Main Channel */
+#define DP0_SECSAMPLE           0x0640
+#define DP0_VIDSYNCDELAY        0x0644
+#define DP0_TOTALVAL            0x0648
+#define DP0_STARTVAL            0x064c
+#define DP0_ACTIVEVAL           0x0650
+#define DP0_SYNCVAL             0x0654
+#define DP0_MISC                0x0658
+#define TU_SIZE_RECOMMENDED             (0x3f << 16) /* LSCLK cycles per TU */
+#define BPC_6                           (0 << 5)
+#define BPC_8                           (1 << 5)
+
+/* AUX channel */
+#define DP0_AUXCFG0             0x0660
+#define DP0_AUXCFG1             0x0664
+#define AUX_RX_FILTER_EN                BIT(16)
+
+#define DP0_AUXADDR             0x0668
+#define DP0_AUXWDATA(i)         (0x066c + (i) * 4)
+#define DP0_AUXRDATA(i)         (0x067c + (i) * 4)
+#define DP0_AUXSTATUS           0x068c
+#define AUX_STATUS_MASK                 0xf0
+#define AUX_STATUS_SHIFT                4
+#define AUX_TIMEOUT                     BIT(1)
+#define AUX_BUSY                        BIT(0)
+#define DP0_AUXI2CADR           0x0698
+
+/* Link Training */
+#define DP0_SRCCTRL             0x06a0
+#define DP0_SRCCTRL_SCRMBLDIS           BIT(13)
+#define DP0_SRCCTRL_EN810B              BIT(12)
+#define DP0_SRCCTRL_NOTP                (0 << 8)
+#define DP0_SRCCTRL_TP1                 (1 << 8)
+#define DP0_SRCCTRL_TP2                 (2 << 8)
+#define DP0_SRCCTRL_LANESKEW            BIT(7)
+#define DP0_SRCCTRL_SSCG                BIT(3)
+#define DP0_SRCCTRL_LANES_1             (0 << 2)
+#define DP0_SRCCTRL_LANES_2             (1 << 2)
+#define DP0_SRCCTRL_BW27                (1 << 1)
+#define DP0_SRCCTRL_BW162               (0 << 1)
+#define DP0_SRCCTRL_AUTOCORRECT         BIT(0)
+#define DP0_LTSTAT              0x06d0
+#define LT_LOOPDONE                     BIT(13)
+#define LT_STATUS_MASK                  (0x1f << 8)
+#define LT_CHANNEL1_EQ_BITS             (DP_CHANNEL_EQ_BITS << 4)
+#define LT_INTERLANE_ALIGN_DONE         BIT(3)
+#define LT_CHANNEL0_EQ_BITS             (DP_CHANNEL_EQ_BITS)
+#define DP0_SNKLTCHGREQ         0x06d4
+#define DP0_LTLOOPCTRL          0x06d8
+#define DP0_SNKLTCTRL           0x06e4
+
+/* PHY */
+#define DP_PHY_CTRL             0x0800
+#define DP_PHY_RST                      BIT(28)  /* DP PHY Global Soft Reset */
+#define BGREN                           BIT(25)  /* AUX PHY BGR Enable */
+#define PWR_SW_EN                       BIT(24)  /* PHY Power Switch Enable */
+#define PHY_M1_RST                      BIT(12)  /* Reset PHY1 Main Channel */
+#define PHY_RDY                         BIT(16)  /* PHY Main Channels Ready */
+#define PHY_M0_RST                      BIT(8)   /* Reset PHY0 Main Channel */
+#define PHY_A0_EN                       BIT(1)   /* PHY Aux Channel0 Enable */
+#define PHY_M0_EN                       BIT(0)   /* PHY Main Channel0 Enable */
+
+/* PLL */
+#define DP0_PLLCTRL             0x0900
+#define DP1_PLLCTRL             0x0904  /* not defined in DS */
+#define PXL_PLLCTRL             0x0908
+#define PLLUPDATE                       BIT(2)
+#define PLLBYP                          BIT(1)
+#define PLLEN                           BIT(0)
+#define PXL_PLLPARAM            0x0914
+#define IN_SEL_REFCLK                   (0 << 14)
+#define SYS_PLLPARAM            0x0918
+#define REF_FREQ_38M4                   (0 << 8) /* 38.4 MHz */
+#define REF_FREQ_19M2                   (1 << 8) /* 19.2 MHz */
+#define REF_FREQ_26M                    (2 << 8) /* 26 MHz */
+#define REF_FREQ_13M                    (3 << 8) /* 13 MHz */
+#define SYSCLK_SEL_LSCLK                (0 << 4)
+#define LSCLK_DIV_1                     (0 << 0)
+#define LSCLK_DIV_2                     (1 << 0)
+
+/* Test & Debug */
+#define TSTCTL                  0x0a00
+#define PLL_DBG                 0x0a04
+
+static bool tc_test_pattern;
+module_param_named(test, tc_test_pattern, bool, 0644);
+
+struct tc_edp_link {
+        struct drm_dp_link      base;
+        u8                      assr;
+        int                     scrambler_dis;
+        int                     spread;
+        int                     coding8b10b;
+        u8                      swing;
+        u8                      preemp;
+};
+
+struct tc_data {
+        struct device           *dev;
+        struct regmap           *regmap;
+        struct drm_dp_aux       aux;
+
+        struct drm_bridge       bridge;
+        struct drm_connector    connector;
+        struct drm_panel        *panel;
+
+        /* link settings */
+        struct tc_edp_link      link;
+
+        /* display edid */
+        struct edid             *edid;
+        /* current mode */
+        struct drm_display_mode *mode;
+
+        u32                     rev;
+        u8                      assr;
+
+        struct gpio_desc        *sd_gpio;
+        struct gpio_desc        *reset_gpio;
+        struct clk              *refclk;
+};
+
+static inline struct tc_data *aux_to_tc(struct drm_dp_aux *a)
+{
+        return container_of(a, struct tc_data, aux);
+}
+
+static inline struct tc_data *bridge_to_tc(struct drm_bridge *b)
+{
+        return container_of(b, struct tc_data, bridge);
+}
+
+static inline struct tc_data *connector_to_tc(struct drm_connector *c)
+{
+        return container_of(c, struct tc_data, connector);
+}
+
+/* Simple macros to avoid repeated error checks */
+#define tc_write(reg, var)                                      \
+        do {                                                    \
+                ret = regmap_write(tc->regmap, reg, var);       \
+                if (ret)                                        \
+                        goto err;                               \
+        } while (0)
+#define tc_read(reg, var)                                       \
+        do {                                                    \
+                ret = regmap_read(tc->regmap, reg, var);        \
+                if (ret)                                        \
+                        goto err;                               \
+        } while (0)
+
+static inline int tc_poll_timeout(struct regmap *map, unsigned int addr,
+                                  unsigned int cond_mask,
+                                  unsigned int cond_value,
+                                  unsigned long sleep_us, u64 timeout_us)
+{
+        ktime_t timeout = ktime_add_us(ktime_get(), timeout_us);
+        unsigned int val;
+        int ret;
+
+        for (;;) {
+                ret = regmap_read(map, addr, &val);
+                if (ret)
+                        break;
+                if ((val & cond_mask) == cond_value)
+                        break;
+                if (timeout_us && ktime_compare(ktime_get(), timeout) > 0) {
+                        ret = regmap_read(map, addr, &val);
+                        break;
+                }
+                if (sleep_us)
+                        usleep_range((sleep_us >> 2) + 1, sleep_us);
+        }
+        return ret ?: (((val & cond_mask) == cond_value) ? 0 : -ETIMEDOUT);
+}
+
+static int tc_aux_wait_busy(struct tc_data *tc, unsigned int timeout_ms)
+{
+        return tc_poll_timeout(tc->regmap, DP0_AUXSTATUS, AUX_BUSY, 0,
+                               1000, 1000 * timeout_ms);
+}
+
+static int tc_aux_get_status(struct tc_data *tc, u8 *reply)
+{
+        int ret;
+        u32 value;
+
+        ret = regmap_read(tc->regmap, DP0_AUXSTATUS, &value);
+        if (ret < 0)
+                return ret;
+        if (value & AUX_BUSY) {
+                if (value & AUX_TIMEOUT) {
+                        dev_err(tc->dev, "i2c access timeout!\n");
+                        return -ETIMEDOUT;
+                }
+                return -EBUSY;
+        }
+
+        *reply = (value & AUX_STATUS_MASK) >> AUX_STATUS_SHIFT;
+        return 0;
+}
+
+static ssize_t tc_aux_transfer(struct drm_dp_aux *aux,
+                               struct drm_dp_aux_msg *msg)
+{
+        struct tc_data *tc = aux_to_tc(aux);
+        size_t size = min_t(size_t, 8, msg->size);
+        u8 request = msg->request & ~DP_AUX_I2C_MOT;
+        u8 *buf = msg->buffer;
+        u32 tmp = 0;
+        int i = 0;
+        int ret;
+
+        if (size == 0)
+                return 0;
+
+        ret = tc_aux_wait_busy(tc, 100);
+        if (ret)
+                goto err;
+
+        if (request == DP_AUX_I2C_WRITE || request == DP_AUX_NATIVE_WRITE) {
+                /* Store data */
+                while (i < size) {
+                        if (request == DP_AUX_NATIVE_WRITE)
+                                tmp = tmp | (buf[i] << (8 * (i & 0x3)));
+                        else
+                                tmp = (tmp << 8) | buf[i];
+                        i++;
+                        if (((i % 4) == 0) || (i == size)) {
+                                tc_write(DP0_AUXWDATA(i >> 2), tmp);
+                                tmp = 0;
+                        }
+                }
+        } else if (request != DP_AUX_I2C_READ &&
+                   request != DP_AUX_NATIVE_READ) {
+                return -EINVAL;
+        }
+
+        /* Store address */
+        tc_write(DP0_AUXADDR, msg->address);
+        /* Start transfer */
+        tc_write(DP0_AUXCFG0, ((size - 1) << 8) | request);
+
+        ret = tc_aux_wait_busy(tc, 100);
+        if (ret)
+                goto err;
+
+        ret = tc_aux_get_status(tc, &msg->reply);
+        if (ret)
+                goto err;
+
+        if (request == DP_AUX_I2C_READ || request == DP_AUX_NATIVE_READ) {
+                /* Read data */
+                while (i < size) {
+                        if ((i % 4) == 0)
+                                tc_read(DP0_AUXRDATA(i >> 2), &tmp);
+                        buf[i] = tmp & 0xff;
+                        tmp = tmp >> 8;
+                        i++;
+                }
+        }
+
+        return size;
+err:
+        return ret;
+}
+
+static const char * const training_pattern1_errors[] = {
+        "No errors",
+        "Aux write error",
+        "Aux read error",
+        "Max voltage reached error",
+        "Loop counter expired error",
+        "res", "res", "res"
+};
+
+static const char * const training_pattern2_errors[] = {
+        "No errors",
+        "Aux write error",
+        "Aux read error",
+        "Clock recovery failed error",
+        "Loop counter expired error",
+        "res", "res", "res"
+};
+
+static u32 tc_srcctrl(struct tc_data *tc)
+{
+        /*
+         * No training pattern, skew lane 1 data by two LSCLK cycles with
+         * respect to lane 0 data, AutoCorrect Mode = 0
+         */
+        u32 reg = DP0_SRCCTRL_NOTP | DP0_SRCCTRL_LANESKEW;
+
+        if (tc->link.scrambler_dis)
+                reg |= DP0_SRCCTRL_SCRMBLDIS;   /* Scrambler Disabled */
+        if (tc->link.coding8b10b)
+                /* Enable 8/10B Encoder (TxData[19:16] not used) */
+                reg |= DP0_SRCCTRL_EN810B;
+        if (tc->link.spread)
+                reg |= DP0_SRCCTRL_SSCG;        /* Spread Spectrum Enable */
+        if (tc->link.base.num_lanes == 2)
+                reg |= DP0_SRCCTRL_LANES_2;     /* Two Main Channel Lanes */
+        if (tc->link.base.rate != 162000)
+                reg |= DP0_SRCCTRL_BW27;        /* 2.7 Gbps link */
+        return reg;
+}
+
+static void tc_wait_pll_lock(struct tc_data *tc)
+{
+        /* Wait for PLL to lock: up to 2.09 ms, depending on refclk */
+        usleep_range(3000, 6000);
+}
+
+static int tc_pxl_pll_en(struct tc_data *tc, u32 refclk, u32 pixelclock)
+{
+        int ret;
+        int i_pre, best_pre = 1;
+        int i_post, best_post = 1;
+        int div, best_div = 1;
+        int mul, best_mul = 1;
+        int delta, best_delta;
+        int ext_div[] = {1, 2, 3, 5, 7};
+        int best_pixelclock = 0;
+        int vco_hi = 0;
+
+        dev_dbg(tc->dev, "PLL: requested %d pixelclock, ref %d\n", pixelclock,
+                refclk);
+        best_delta = pixelclock;
+        /* Loop over all possible ext_divs, skipping invalid configurations */
+        for (i_pre = 0; i_pre < ARRAY_SIZE(ext_div); i_pre++) {
+                /*
+                 * refclk / ext_pre_div should be in the 1 to 200 MHz range.
+                 * We don't allow any refclk > 200 MHz, only check lower bounds.
+                 */
+                if (refclk / ext_div[i_pre] < 1000000)
+                        continue;
+                for (i_post = 0; i_post < ARRAY_SIZE(ext_div); i_post++) {
+                        for (div = 1; div <= 16; div++) {
+                                u32 clk;
+                                u64 tmp;
+
+                                tmp = pixelclock * ext_div[i_pre] *
+                                      ext_div[i_post] * div;
+                                do_div(tmp, refclk);
+                                mul = tmp;
+
+                                /* Check limits */
+                                if ((mul < 1) || (mul > 128))
+                                        continue;
+
+                                clk = (refclk / ext_div[i_pre] / div) * mul;
+                                /*
+                                 * refclk * mul / (ext_pre_div * pre_div)
+                                 * should be in the 150 to 650 MHz range
+                                 */
+                                if ((clk > 650000000) || (clk < 150000000))
+                                        continue;
+
+                                clk = clk / ext_div[i_post];
+                                delta = clk - pixelclock;
+
+                                if (abs(delta) < abs(best_delta)) {
+                                        best_pre = i_pre;
+                                        best_post = i_post;
+                                        best_div = div;
+                                        best_mul = mul;
+                                        best_delta = delta;
+                                        best_pixelclock = clk;
+                                }
+                        }
+                }
+        }
+        if (best_pixelclock == 0) {
+                dev_err(tc->dev, "Failed to calc clock for %d pixelclock\n",
+                        pixelclock);
+                return -EINVAL;
+        }
+
+        dev_dbg(tc->dev, "PLL: got %d, delta %d\n", best_pixelclock,
+                best_delta);
+        dev_dbg(tc->dev, "PLL: %d / %d / %d * %d / %d\n", refclk,
+                ext_div[best_pre], best_div, best_mul, ext_div[best_post]);
+
+        /* if VCO >= 300 MHz */
+        if (refclk / ext_div[best_pre] / best_div * best_mul >= 300000000)
+                vco_hi = 1;
+        /* see DS */
+        if (best_div == 16)
+                best_div = 0;
+        if (best_mul == 128)
+                best_mul = 0;
+
+        /* Power up PLL and switch to bypass */
+        tc_write(PXL_PLLCTRL, PLLBYP | PLLEN);
+
+        tc_write(PXL_PLLPARAM,
+                 (vco_hi << 24) |               /* For PLL VCO >= 300 MHz = 1 */
+                 (ext_div[best_pre] << 20) |    /* External Pre-divider */
+                 (ext_div[best_post] << 16) |   /* External Post-divider */
+                 IN_SEL_REFCLK |                /* Use RefClk as PLL input */
+                 (best_div << 8) |              /* Divider for PLL RefClk */
+                 (best_mul << 0));              /* Multiplier for PLL */
+
+        /* Force PLL parameter update and disable bypass */
+        tc_write(PXL_PLLCTRL, PLLUPDATE | PLLEN);
+
+        tc_wait_pll_lock(tc);
+
+        return 0;
+err:
+        return ret;
+}
+
+static int tc_pxl_pll_dis(struct tc_data *tc)
+{
+        /* Enable PLL bypass, power down PLL */
+        return regmap_write(tc->regmap, PXL_PLLCTRL, PLLBYP);
+}
+
+static int tc_stream_clock_calc(struct tc_data *tc)
+{
+        int ret;
+        /*
+         * If the Stream clock and Link Symbol clock are
+         * asynchronous with each other, the value of M changes over
+         * time. This way of generating link clock and stream
+         * clock is called Asynchronous Clock mode. The value M
+         * must change while the value N stays constant. The
+         * value of N in this Asynchronous Clock mode must be set
+         * to 2^15 or 32,768.
+         *
+         * LSCLK = 1/10 of high speed link clock
+         *
+         * f_STRMCLK = M/N * f_LSCLK
+         * M/N = f_STRMCLK / f_LSCLK
+         *
+         */
+        tc_write(DP0_VIDMNGEN1, 32768);
+
+        return 0;
+err:
+        return ret;
+}
+
+static int tc_aux_link_setup(struct tc_data *tc)
+{
+        unsigned long rate;
+        u32 value;
+        int ret;
+
+        rate = clk_get_rate(tc->refclk);
+        switch (rate) {
+        case 38400000:
+                value = REF_FREQ_38M4;
+                break;
+        case 26000000:
+                value = REF_FREQ_26M;
+                break;
+        case 19200000:
+                value = REF_FREQ_19M2;
+                break;
+        case 13000000:
+                value = REF_FREQ_13M;
+                break;
+        default:
+                dev_err(tc->dev, "Invalid refclk rate: %lu Hz\n", rate);
+                return -EINVAL;
+        }
+
+        /* Setup DP-PHY / PLL */
+        value |= SYSCLK_SEL_LSCLK | LSCLK_DIV_2;
+        tc_write(SYS_PLLPARAM, value);
+
+        tc_write(DP_PHY_CTRL, BGREN | PWR_SW_EN | BIT(2) | PHY_A0_EN);
+
+        /*
+         * Initially PLLs are in bypass. Force PLL parameter update,
+         * disable PLL bypass, enable PLL
+         */
+        tc_write(DP0_PLLCTRL, PLLUPDATE | PLLEN);
+        tc_wait_pll_lock(tc);
+
+        tc_write(DP1_PLLCTRL, PLLUPDATE | PLLEN);
+        tc_wait_pll_lock(tc);
+
+        ret = tc_poll_timeout(tc->regmap, DP_PHY_CTRL, PHY_RDY, PHY_RDY, 1,
+                              1000);
+        if (ret == -ETIMEDOUT) {
+                dev_err(tc->dev, "Timeout waiting for PHY to become ready");
+                return ret;
+        } else if (ret)
+                goto err;
+
+        /* Setup AUX link */
+        tc_write(DP0_AUXCFG1, AUX_RX_FILTER_EN |
+                 (0x06 << 8) |  /* Aux Bit Period Calculator Threshold */
+                 (0x3f << 0));  /* Aux Response Timeout Timer */
+
+        return 0;
+err:
+        dev_err(tc->dev, "tc_aux_link_setup failed: %d\n", ret);
+        return ret;
+}
+
+static int tc_get_display_props(struct tc_data *tc)
+{
+        int ret;
+        /* temp buffer */
+        u8 tmp[8];
+
+        /* Read DP Rx Link Capability */
+        ret = drm_dp_link_probe(&tc->aux, &tc->link.base);
+        if (ret < 0)
+                goto err_dpcd_read;
+        if ((tc->link.base.rate != 162000) && (tc->link.base.rate != 270000))
+                goto err_dpcd_inval;
+
+        ret = drm_dp_dpcd_readb(&tc->aux, DP_MAX_DOWNSPREAD, tmp);
+        if (ret < 0)
+                goto err_dpcd_read;
+        tc->link.spread = tmp[0] & BIT(0); /* 0.5% down spread */
+
+        ret = drm_dp_dpcd_readb(&tc->aux, DP_MAIN_LINK_CHANNEL_CODING, tmp);
+        if (ret < 0)
+                goto err_dpcd_read;
+        tc->link.coding8b10b = tmp[0] & BIT(0);
+        tc->link.scrambler_dis = 0;
+        /* read assr */
+        ret = drm_dp_dpcd_readb(&tc->aux, DP_EDP_CONFIGURATION_SET, tmp);
+        if (ret < 0)
+                goto err_dpcd_read;
+        tc->link.assr = tmp[0] & DP_ALTERNATE_SCRAMBLER_RESET_ENABLE;
+
+        dev_dbg(tc->dev, "DPCD rev: %d.%d, rate: %s, lanes: %d, framing: %s\n",
+                tc->link.base.revision >> 4, tc->link.base.revision & 0x0f,
+                (tc->link.base.rate == 162000) ? "1.62Gbps" : "2.7Gbps",
+                tc->link.base.num_lanes,
+                (tc->link.base.capabilities & DP_LINK_CAP_ENHANCED_FRAMING) ?
+                "enhanced" : "non-enhanced");
+        dev_dbg(tc->dev, "ANSI 8B/10B: %d\n", tc->link.coding8b10b);
+        dev_dbg(tc->dev, "Display ASSR: %d, TC358767 ASSR: %d\n",
+                tc->link.assr, tc->assr);
+
+        return 0;
+
+err_dpcd_read:
+        dev_err(tc->dev, "failed to read DPCD: %d\n", ret);
+        return ret;
+err_dpcd_inval:
+        dev_err(tc->dev, "invalid DPCD\n");
+        return -EINVAL;
+}
+
+static int tc_set_video_mode(struct tc_data *tc, struct drm_display_mode *mode)
+{
+        int ret;
+        int vid_sync_dly;
+        int max_tu_symbol;
+
+        int left_margin = mode->htotal - mode->hsync_end;
+        int right_margin = mode->hsync_start - mode->hdisplay;
+        int hsync_len = mode->hsync_end - mode->hsync_start;
+        int upper_margin = mode->vtotal - mode->vsync_end;
+        int lower_margin = mode->vsync_start - mode->vdisplay;
+        int vsync_len = mode->vsync_end - mode->vsync_start;
+
+        dev_dbg(tc->dev, "set mode %dx%d\n",
+                mode->hdisplay, mode->vdisplay);
+        dev_dbg(tc->dev, "H margin %d,%d sync %d\n",
+                left_margin, right_margin, hsync_len);
+        dev_dbg(tc->dev, "V margin %d,%d sync %d\n",
+                upper_margin, lower_margin, vsync_len);
+        dev_dbg(tc->dev, "total: %dx%d\n", mode->htotal, mode->vtotal);
+
+
+        /* LCD Ctl Frame Size */
+        tc_write(VPCTRL0, (0x40 << 20) /* VSDELAY */ |
+                 OPXLFMT_RGB888 | FRMSYNC_DISABLED | MSF_DISABLED);
+        tc_write(HTIM01, (left_margin << 16) |          /* H back porch */
+                         (hsync_len << 0));             /* Hsync */
+        tc_write(HTIM02, (right_margin << 16) |         /* H front porch */
+                         (mode->hdisplay << 0));        /* width */
+        tc_write(VTIM01, (upper_margin << 16) |         /* V back porch */
+                         (vsync_len << 0));             /* Vsync */
+        tc_write(VTIM02, (lower_margin << 16) |         /* V front porch */
+                         (mode->vdisplay << 0));        /* height */
+        tc_write(VFUEN0, VFUEN);                /* update settings */
+
+        /* Test pattern settings */
+        tc_write(TSTCTL,
+                 (120 << 24) |  /* Red Color component value */
+                 (20 << 16) |   /* Green Color component value */
+                 (99 << 8) |    /* Blue Color component value */
+                 (1 << 4) |     /* Enable I2C Filter */
+                 (2 << 0) |     /* Color bar Mode */
+                 0);
+
+        /* DP Main Stream Attributes */
+        vid_sync_dly = hsync_len + left_margin + mode->hdisplay;
+        tc_write(DP0_VIDSYNCDELAY,
+                 (0x003e << 16) |       /* thresh_dly */
+                 (vid_sync_dly << 0));
+
+        tc_write(DP0_TOTALVAL, (mode->vtotal << 16) | (mode->htotal));
+
+        tc_write(DP0_STARTVAL,
+                 ((upper_margin + vsync_len) << 16) |
+                 ((left_margin + hsync_len) << 0));
+
+        tc_write(DP0_ACTIVEVAL, (mode->vdisplay << 16) | (mode->hdisplay));
+
+        tc_write(DP0_SYNCVAL, (vsync_len << 16) | (hsync_len << 0));
+
+        tc_write(DPIPXLFMT, VS_POL_ACTIVE_LOW | HS_POL_ACTIVE_LOW |
+                 DE_POL_ACTIVE_HIGH | SUB_CFG_TYPE_CONFIG1 | DPI_BPP_RGB888);
+
+        /*
+         * Recommended maximum number of symbols transferred in a transfer unit:
+         * DIV_ROUND_UP((input active video bandwidth in bytes) * tu_size,
+         *              (output active video bandwidth in bytes))
+         * Must be less than tu_size.
+         */
+        max_tu_symbol = TU_SIZE_RECOMMENDED - 1;
+        tc_write(DP0_MISC, (max_tu_symbol << 23) | TU_SIZE_RECOMMENDED | BPC_8);
+
+        return 0;
+err:
+        return ret;
+}
+
+static int tc_link_training(struct tc_data *tc, int pattern)
+{
+        const char * const *errors;
+        u32 srcctrl = tc_srcctrl(tc) | DP0_SRCCTRL_SCRMBLDIS |
+                      DP0_SRCCTRL_AUTOCORRECT;
+        int timeout;
+        int retry;
+        u32 value;
+        int ret;
+
+        if (pattern == DP_TRAINING_PATTERN_1) {
+                srcctrl |= DP0_SRCCTRL_TP1;
+                errors = training_pattern1_errors;
+        } else {
+                srcctrl |= DP0_SRCCTRL_TP2;
+                errors = training_pattern2_errors;
+        }
+
+        /* Set DPCD 0x102 for Training Part 1 or 2 */
+        tc_write(DP0_SNKLTCTRL, DP_LINK_SCRAMBLING_DISABLE | pattern);
+
+        tc_write(DP0_LTLOOPCTRL,
+                 (0x0f << 28) | /* Defer Iteration Count */
+                 (0x0f << 24) | /* Loop Iteration Count */
+                 (0x0d << 0));  /* Loop Timer Delay */
+
+        retry = 5;
+        do {
+                /* Set DP0 Training Pattern */
+                tc_write(DP0_SRCCTRL, srcctrl);
+
+                /* Enable DP0 to start Link Training */
+                tc_write(DP0CTL, DP_EN);
+
+                /* wait */
+                timeout = 1000;
+                do {
+                        tc_read(DP0_LTSTAT, &value);
+                        udelay(1);
+                } while ((!(value & LT_LOOPDONE)) && (--timeout));
+                if (timeout == 0) {
+                        dev_err(tc->dev, "Link training timeout!\n");
+                } else {
+                        int pattern = (value >> 11) & 0x3;
+                        int error = (value >> 8) & 0x7;
+
+                        dev_dbg(tc->dev,
+                                "Link training phase %d done after %d uS: %s\n",
+                                pattern, 1000 - timeout, errors[error]);
+                        if (pattern == DP_TRAINING_PATTERN_1 && error == 0)
+                                break;
+                        if (pattern == DP_TRAINING_PATTERN_2) {
+                                value &= LT_CHANNEL1_EQ_BITS |
+                                         LT_INTERLANE_ALIGN_DONE |
+                                         LT_CHANNEL0_EQ_BITS;
+                                /* in case of two lanes */
+                                if ((tc->link.base.num_lanes == 2) &&
+                                    (value == (LT_CHANNEL1_EQ_BITS |
+                                               LT_INTERLANE_ALIGN_DONE |
+                                               LT_CHANNEL0_EQ_BITS)))
+                                        break;
+                                /* in case of one line */
+                                if ((tc->link.base.num_lanes == 1) &&
+                                    (value == (LT_INTERLANE_ALIGN_DONE |
+                                               LT_CHANNEL0_EQ_BITS)))
+                                        break;
+                        }
+                }
+                /* restart */
+                tc_write(DP0CTL, 0);
+                usleep_range(10, 20);
+        } while (--retry);
+        if (retry == 0) {
+                dev_err(tc->dev, "Failed to finish training phase %d\n",
+                        pattern);
+        }
+
+        return 0;
+err:
+        return ret;
+}
+
+static int tc_main_link_setup(struct tc_data *tc)
+{
+        struct drm_dp_aux *aux = &tc->aux;
+        struct device *dev = tc->dev;
+        unsigned int rate;
+        u32 dp_phy_ctrl;
+        int timeout;
+        bool aligned;
+        bool ready;
+        u32 value;
+        int ret;
+        u8 tmp[8];
+
+        /* display mode should be set at this point */
+        if (!tc->mode)
+                return -EINVAL;
+
+        /* from excel file - DP0_SrcCtrl */
+        tc_write(DP0_SRCCTRL, DP0_SRCCTRL_SCRMBLDIS | DP0_SRCCTRL_EN810B |
+                 DP0_SRCCTRL_LANESKEW | DP0_SRCCTRL_LANES_2 |
+                 DP0_SRCCTRL_BW27 | DP0_SRCCTRL_AUTOCORRECT);
+        /* from excel file - DP1_SrcCtrl */
+        tc_write(0x07a0, 0x00003083);
+
+        rate = clk_get_rate(tc->refclk);
+        switch (rate) {
+        case 38400000:
+                value = REF_FREQ_38M4;
+                break;
+        case 26000000:
+                value = REF_FREQ_26M;
+                break;
+        case 19200000:
+                value = REF_FREQ_19M2;
+                break;
+        case 13000000:
+                value = REF_FREQ_13M;
+                break;
+        default:
+                return -EINVAL;
+        }
+        value |= SYSCLK_SEL_LSCLK | LSCLK_DIV_2;
+        tc_write(SYS_PLLPARAM, value);
+        /* Setup Main Link */
+        dp_phy_ctrl = BGREN | PWR_SW_EN | BIT(2) | PHY_A0_EN |  PHY_M0_EN;
+        tc_write(DP_PHY_CTRL, dp_phy_ctrl);
+        msleep(100);
+
+        /* PLL setup */
+        tc_write(DP0_PLLCTRL, PLLUPDATE | PLLEN);
+        tc_wait_pll_lock(tc);
+
+        tc_write(DP1_PLLCTRL, PLLUPDATE | PLLEN);
+        tc_wait_pll_lock(tc);
+
+        /* PXL PLL setup */
+        if (tc_test_pattern) {
+                ret = tc_pxl_pll_en(tc, clk_get_rate(tc->refclk),
+                                    1000 * tc->mode->clock);
+                if (ret)
+                        goto err;
+        }
+
+        /* Reset/Enable Main Links */
+        dp_phy_ctrl |= DP_PHY_RST | PHY_M1_RST | PHY_M0_RST;
+        tc_write(DP_PHY_CTRL, dp_phy_ctrl);
+        usleep_range(100, 200);
+        dp_phy_ctrl &= ~(DP_PHY_RST | PHY_M1_RST | PHY_M0_RST);
+        tc_write(DP_PHY_CTRL, dp_phy_ctrl);
+
+        timeout = 1000;
+        do {
+                tc_read(DP_PHY_CTRL, &value);
+                udelay(1);
+        } while ((!(value & PHY_RDY)) && (--timeout));
+
+        if (timeout == 0) {
+                dev_err(dev, "timeout waiting for phy become ready");
+                return -ETIMEDOUT;
+        }
+
+        /* Set misc: 8 bits per color */
+        ret = regmap_update_bits(tc->regmap, DP0_MISC, BPC_8, BPC_8);
+        if (ret)
+                goto err;
+
+        /*
+         * ASSR mode
+         * on TC358767 side ASSR configured through strap pin
+         * seems there is no way to change this setting from SW
+         *
+         * check is tc configured for same mode
+         */
+        if (tc->assr != tc->link.assr) {
+                dev_dbg(dev, "Trying to set display to ASSR: %d\n",
+                        tc->assr);
+                /* try to set ASSR on display side */
+                tmp[0] = tc->assr;
+                ret = drm_dp_dpcd_writeb(aux, DP_EDP_CONFIGURATION_SET, tmp[0]);
+                if (ret < 0)
+                        goto err_dpcd_read;
+                /* read back */
+                ret = drm_dp_dpcd_readb(aux, DP_EDP_CONFIGURATION_SET, tmp);
+                if (ret < 0)
+                        goto err_dpcd_read;
+
+                if (tmp[0] != tc->assr) {
+                        dev_warn(dev, "Failed to switch display ASSR to %d, falling back to unscrambled mode\n",
+                                 tc->assr);
+                        /* trying with disabled scrambler */
+                        tc->link.scrambler_dis = 1;
+                }
+        }
+
+        /* Setup Link & DPRx Config for Training */
+        ret = drm_dp_link_configure(aux, &tc->link.base);
+        if (ret < 0)
+                goto err_dpcd_write;
+
+        /* DOWNSPREAD_CTRL */
+        tmp[0] = tc->link.spread ? DP_SPREAD_AMP_0_5 : 0x00;
+        /* MAIN_LINK_CHANNEL_CODING_SET */
+        tmp[1] =  tc->link.coding8b10b ? DP_SET_ANSI_8B10B : 0x00;
+        ret = drm_dp_dpcd_write(aux, DP_DOWNSPREAD_CTRL, tmp, 2);
+        if (ret < 0)
+                goto err_dpcd_write;
+
+        ret = tc_link_training(tc, DP_TRAINING_PATTERN_1);
+        if (ret)
+                goto err;
+
+        ret = tc_link_training(tc, DP_TRAINING_PATTERN_2);
+        if (ret)
+                goto err;
+
+        /* Clear DPCD 0x102 */
+        /* Note: Can Not use DP0_SNKLTCTRL (0x06E4) short cut */
+        tmp[0] = tc->link.scrambler_dis ? DP_LINK_SCRAMBLING_DISABLE : 0x00;
+        ret = drm_dp_dpcd_writeb(aux, DP_TRAINING_PATTERN_SET, tmp[0]);
+        if (ret < 0)
+                goto err_dpcd_write;
+
+        /* Clear Training Pattern, set AutoCorrect Mode = 1 */
+        tc_write(DP0_SRCCTRL, tc_srcctrl(tc) | DP0_SRCCTRL_AUTOCORRECT);
+
+        /* Wait */
+        timeout = 100;
+        do {
+                udelay(1);
+                /* Read DPCD 0x202-0x207 */
+                ret = drm_dp_dpcd_read_link_status(aux, tmp + 2);
+                if (ret < 0)
+                        goto err_dpcd_read;
+                ready = (tmp[2] == ((DP_CHANNEL_EQ_BITS << 4) | /* Lane1 */
+                                     DP_CHANNEL_EQ_BITS));      /* Lane0 */
+                aligned = tmp[4] & DP_INTERLANE_ALIGN_DONE;
+        } while ((--timeout) && !(ready && aligned));
+
+        if (timeout == 0) {
+                /* Read DPCD 0x200-0x201 */
+                ret = drm_dp_dpcd_read(aux, DP_SINK_COUNT, tmp, 2);
+                if (ret < 0)
+                        goto err_dpcd_read;
+                dev_info(dev, "0x0200 SINK_COUNT: 0x%02x\n", tmp[0]);
+                dev_info(dev, "0x0201 DEVICE_SERVICE_IRQ_VECTOR: 0x%02x\n",
+                         tmp[1]);
+                dev_info(dev, "0x0202 LANE0_1_STATUS: 0x%02x\n", tmp[2]);
+                dev_info(dev, "0x0204 LANE_ALIGN_STATUS_UPDATED: 0x%02x\n",
+                         tmp[4]);
+                dev_info(dev, "0x0205 SINK_STATUS: 0x%02x\n", tmp[5]);
+                dev_info(dev, "0x0206 ADJUST_REQUEST_LANE0_1: 0x%02x\n",
+                         tmp[6]);
+
+                if (!ready)
+                        dev_err(dev, "Lane0/1 not ready\n");
+                if (!aligned)
+                        dev_err(dev, "Lane0/1 not aligned\n");
+                return -EAGAIN;
+        }
+
+        ret = tc_set_video_mode(tc, tc->mode);
+        if (ret)
+                goto err;
+
+        /* Set M/N */
+        ret = tc_stream_clock_calc(tc);
+        if (ret)
+                goto err;
+
+        return 0;
+err_dpcd_read:
+        dev_err(tc->dev, "Failed to read DPCD: %d\n", ret);
+        return ret;
+err_dpcd_write:
+        dev_err(tc->dev, "Failed to write DPCD: %d\n", ret);
+err:
+        return ret;
+}
+
+static int tc_main_link_stream(struct tc_data *tc, int state)
+{
+        int ret;
+        u32 value;
+
+        dev_dbg(tc->dev, "stream: %d\n", state);
+
+        if (state) {
+                value = VID_MN_GEN | DP_EN;
+                if (tc->link.base.capabilities & DP_LINK_CAP_ENHANCED_FRAMING)
+                        value |= EF_EN;
+                tc_write(DP0CTL, value);
+                /*
+                 * VID_EN assertion should be delayed by at least N * LSCLK
+                 * cycles from the time VID_MN_GEN is enabled in order to
+                 * generate stable values for VID_M. LSCLK is 270 MHz or
+                 * 162 MHz, VID_N is set to 32768 in  tc_stream_clock_calc(),
+                 * so a delay of at least 203 us should suffice.
+                 */
+                usleep_range(500, 1000);
+                value |= VID_EN;
+                tc_write(DP0CTL, value);
+                /* Set input interface */
+                value = DP0_AUDSRC_NO_INPUT;
+                if (tc_test_pattern)
+                        value |= DP0_VIDSRC_COLOR_BAR;
+                else
+                        value |= DP0_VIDSRC_DPI_RX;
+                tc_write(SYSCTRL, value);
+        } else {
+                tc_write(DP0CTL, 0);
+        }
+
+        return 0;
+err:
+        return ret;
+}
+
+static enum drm_connector_status
+tc_connector_detect(struct drm_connector *connector, bool force)
+{
+        return connector_status_connected;
+}
+
+static void tc_bridge_pre_enable(struct drm_bridge *bridge)
+{
+        struct tc_data *tc = bridge_to_tc(bridge);
+
+        drm_panel_prepare(tc->panel);
+}
+
+static void tc_bridge_enable(struct drm_bridge *bridge)
+{
+        struct tc_data *tc = bridge_to_tc(bridge);
+        int ret;
+
+        ret = tc_main_link_setup(tc);
+        if (ret < 0) {
+                dev_err(tc->dev, "main link setup error: %d\n", ret);
+                return;
+        }
+
+        ret = tc_main_link_stream(tc, 1);
+        if (ret < 0) {
+                dev_err(tc->dev, "main link stream start error: %d\n", ret);
+                return;
+        }
+
+        drm_panel_enable(tc->panel);
+}
+
+static void tc_bridge_disable(struct drm_bridge *bridge)
+{
+        struct tc_data *tc = bridge_to_tc(bridge);
+        int ret;
+
+        drm_panel_disable(tc->panel);
+
+        ret = tc_main_link_stream(tc, 0);
+        if (ret < 0)
+                dev_err(tc->dev, "main link stream stop error: %d\n", ret);
+}
+
+static void tc_bridge_post_disable(struct drm_bridge *bridge)
+{
+        struct tc_data *tc = bridge_to_tc(bridge);
+
+        drm_panel_unprepare(tc->panel);
+}
+
+static bool tc_bridge_mode_fixup(struct drm_bridge *bridge,
+                                 const struct drm_display_mode *mode,
+                                 struct drm_display_mode *adj)
+{
+        /* Fixup sync polarities, both hsync and vsync are active low */
+        adj->flags = mode->flags;
+        adj->flags |= (DRM_MODE_FLAG_NHSYNC | DRM_MODE_FLAG_NVSYNC);
+        adj->flags &= ~(DRM_MODE_FLAG_PHSYNC | DRM_MODE_FLAG_PVSYNC);
+
+        return true;
+}
+
+static int tc_connector_mode_valid(struct drm_connector *connector,
+                                   struct drm_display_mode *mode)
+{
+        /* Accept any mode */
+        return MODE_OK;
+}
+
+static void tc_bridge_mode_set(struct drm_bridge *bridge,
+                               struct drm_display_mode *mode,
+                               struct drm_display_mode *adj)
+{
+        struct tc_data *tc = bridge_to_tc(bridge);
+
+        tc->mode = mode;
+}
+
+static int tc_connector_get_modes(struct drm_connector *connector)
+{
+        struct tc_data *tc = connector_to_tc(connector);
+        struct edid *edid;
+        unsigned int count;
+
+        if (tc->panel && tc->panel->funcs && tc->panel->funcs->get_modes) {
+                count = tc->panel->funcs->get_modes(tc->panel);
+                if (count > 0)
+                        return count;
+        }
+
+        edid = drm_get_edid(connector, &tc->aux.ddc);
+
+        kfree(tc->edid);
+        tc->edid = edid;
+        if (!edid)
+                return 0;
+
+        drm_mode_connector_update_edid_property(connector, edid);
+        count = drm_add_edid_modes(connector, edid);
+
+        return count;
+}
+
+static void tc_connector_set_polling(struct tc_data *tc,
+                                     struct drm_connector *connector)
+{
+        /* TODO: add support for HPD */
+        connector->polled = DRM_CONNECTOR_POLL_CONNECT |
+                            DRM_CONNECTOR_POLL_DISCONNECT;
+}
+
+static struct drm_encoder *
+tc_connector_best_encoder(struct drm_connector *connector)
+{
+        struct tc_data *tc = connector_to_tc(connector);
+
+        return tc->bridge.encoder;
+}
+
+static const struct drm_connector_helper_funcs tc_connector_helper_funcs = {
+        .get_modes = tc_connector_get_modes,
+        .mode_valid = tc_connector_mode_valid,
+        .best_encoder = tc_connector_best_encoder,
+};
+
+static void tc_connector_destroy(struct drm_connector *connector)
+{
+        drm_connector_unregister(connector);
+        drm_connector_cleanup(connector);
+}
+
+static const struct drm_connector_funcs tc_connector_funcs = {
+        .dpms = drm_atomic_helper_connector_dpms,
+        .fill_modes = drm_helper_probe_single_connector_modes,
+        .detect = tc_connector_detect,
+        .destroy = tc_connector_destroy,
+        .reset = drm_atomic_helper_connector_reset,
+        .atomic_duplicate_state = drm_atomic_helper_connector_duplicate_state,
+        .atomic_destroy_state = drm_atomic_helper_connector_destroy_state,
+};
+
+static int tc_bridge_attach(struct drm_bridge *bridge)
+{
+        u32 bus_format = MEDIA_BUS_FMT_RGB888_1X24;
+        struct tc_data *tc = bridge_to_tc(bridge);
+        struct drm_device *drm = bridge->dev;
+        int ret;
+
+        /* Create eDP connector */
+        drm_connector_helper_add(&tc->connector, &tc_connector_helper_funcs);
+        ret = drm_connector_init(drm, &tc->connector, &tc_connector_funcs,
+                                 DRM_MODE_CONNECTOR_eDP);
+        if (ret)
+                return ret;
+
+        if (tc->panel)
+                drm_panel_attach(tc->panel, &tc->connector);
+
+        drm_display_info_set_bus_formats(&tc->connector.display_info,
+                                         &bus_format, 1);
+        drm_mode_connector_attach_encoder(&tc->connector, tc->bridge.encoder);
+
+        return 0;
+}
+
+static const struct drm_bridge_funcs tc_bridge_funcs = {
+        .attach = tc_bridge_attach,
+        .mode_set = tc_bridge_mode_set,
+        .pre_enable = tc_bridge_pre_enable,
+        .enable = tc_bridge_enable,
+        .disable = tc_bridge_disable,
+        .post_disable = tc_bridge_post_disable,
+        .mode_fixup = tc_bridge_mode_fixup,
+};
+
+static bool tc_readable_reg(struct device *dev, unsigned int reg)
+{
+        return reg != SYSCTRL;
+}
+
+static const struct regmap_range tc_volatile_ranges[] = {
+        regmap_reg_range(DP0_AUXWDATA(0), DP0_AUXSTATUS),
+        regmap_reg_range(DP0_LTSTAT, DP0_SNKLTCHGREQ),
+        regmap_reg_range(DP_PHY_CTRL, DP_PHY_CTRL),
+        regmap_reg_range(DP0_PLLCTRL, PXL_PLLCTRL),
+        regmap_reg_range(VFUEN0, VFUEN0),
+};
+
+static const struct regmap_access_table tc_volatile_table = {
+        .yes_ranges = tc_volatile_ranges,
+        .n_yes_ranges = ARRAY_SIZE(tc_volatile_ranges),
+};
+
+static bool tc_writeable_reg(struct device *dev, unsigned int reg)
+{
+        return (reg != TC_IDREG) &&
+               (reg != DP0_LTSTAT) &&
+               (reg != DP0_SNKLTCHGREQ);
+}
+
+static const struct regmap_config tc_regmap_config = {
+        .name = "tc358767",
+        .reg_bits = 16,
+        .val_bits = 32,
+        .reg_stride = 4,
+        .max_register = PLL_DBG,
+        .cache_type = REGCACHE_RBTREE,
+        .readable_reg = tc_readable_reg,
+        .volatile_table = &tc_volatile_table,
+        .writeable_reg = tc_writeable_reg,
+        .reg_format_endian = REGMAP_ENDIAN_BIG,
+        .val_format_endian = REGMAP_ENDIAN_LITTLE,
+};
+
+static int tc_probe(struct i2c_client *client, const struct i2c_device_id *id)
+{
+        struct device *dev = &client->dev;
+        struct device_node *ep;
+        struct tc_data *tc;
+        int ret;
+
+        tc = devm_kzalloc(dev, sizeof(*tc), GFP_KERNEL);
+        if (!tc)
+                return -ENOMEM;
+
+        tc->dev = dev;
+
+        /* port@2 is the output port */
+        ep = of_graph_get_endpoint_by_regs(dev->of_node, 2, -1);
+        if (ep) {
+                struct device_node *remote;
+
+                remote = of_graph_get_remote_port_parent(ep);
+                if (!remote) {
+                        dev_warn(dev, "endpoint %s not connected\n",
+                                 ep->full_name);
+                        of_node_put(ep);
+                        return -ENODEV;
+                }
+                of_node_put(ep);
+                tc->panel = of_drm_find_panel(remote);
+                if (tc->panel) {
+                        dev_dbg(dev, "found panel %s\n", remote->full_name);
+                } else {
+                        dev_dbg(dev, "waiting for panel %s\n",
+                                remote->full_name);
+                        of_node_put(remote);
+                        return -EPROBE_DEFER;
+                }
+                of_node_put(remote);
+        }
+
+        /* Shut down GPIO is optional */
+        tc->sd_gpio = devm_gpiod_get_optional(dev, "shutdown", GPIOD_OUT_HIGH);
+        if (IS_ERR(tc->sd_gpio))
+                return PTR_ERR(tc->sd_gpio);
+
+        if (tc->sd_gpio) {
+                gpiod_set_value_cansleep(tc->sd_gpio, 0);
+                usleep_range(5000, 10000);
+        }
+
+        /* Reset GPIO is optional */
+        tc->reset_gpio = devm_gpiod_get_optional(dev, "reset", GPIOD_OUT_LOW);
+        if (IS_ERR(tc->reset_gpio))
+                return PTR_ERR(tc->reset_gpio);
+
+        if (tc->reset_gpio) {
+                gpiod_set_value_cansleep(tc->reset_gpio, 1);
+                usleep_range(5000, 10000);
+        }
+
+        tc->refclk = devm_clk_get(dev, "ref");
+        if (IS_ERR(tc->refclk)) {
+                ret = PTR_ERR(tc->refclk);
+                dev_err(dev, "Failed to get refclk: %d\n", ret);
+                return ret;
+        }
+
+        tc->regmap = devm_regmap_init_i2c(client, &tc_regmap_config);
+        if (IS_ERR(tc->regmap)) {
+                ret = PTR_ERR(tc->regmap);
+                dev_err(dev, "Failed to initialize regmap: %d\n", ret);
+                return ret;
+        }
+
+        ret = regmap_read(tc->regmap, TC_IDREG, &tc->rev);
+        if (ret) {
+                dev_err(tc->dev, "can not read device ID: %d\n", ret);
+                return ret;
+        }
+
+        if ((tc->rev != 0x6601) && (tc->rev != 0x6603)) {
+                dev_err(tc->dev, "invalid device ID: 0x%08x\n", tc->rev);
+                return -EINVAL;
+        }
+
+        tc->assr = (tc->rev == 0x6601); /* Enable ASSR for eDP panels */
+
+        ret = tc_aux_link_setup(tc);
+        if (ret)
+                return ret;
+
+        /* Register DP AUX channel */
+        tc->aux.name = "TC358767 AUX i2c adapter";
+        tc->aux.dev = tc->dev;
+        tc->aux.transfer = tc_aux_transfer;
+        ret = drm_dp_aux_register(&tc->aux);
+        if (ret)
+                return ret;
+
+        ret = tc_get_display_props(tc);
+        if (ret)
+                goto err_unregister_aux;
+
+        tc_connector_set_polling(tc, &tc->connector);
+
+        tc->bridge.funcs = &tc_bridge_funcs;
+        tc->bridge.of_node = dev->of_node;
+        ret = drm_bridge_add(&tc->bridge);
+        if (ret) {
+                dev_err(dev, "Failed to add drm_bridge: %d\n", ret);
+                goto err_unregister_aux;
+        }
+
+        i2c_set_clientdata(client, tc);
+
+        return 0;
+err_unregister_aux:
+        drm_dp_aux_unregister(&tc->aux);
+        return ret;
+}
+
+static int tc_remove(struct i2c_client *client)
+{
+        struct tc_data *tc = i2c_get_clientdata(client);
+
+        drm_bridge_remove(&tc->bridge);
+        drm_dp_aux_unregister(&tc->aux);
+
+        tc_pxl_pll_dis(tc);
+
+        return 0;
+}
+
+static const struct i2c_device_id tc358767_i2c_ids[] = {
+        { "tc358767", 0 },
+        { }
+};
+MODULE_DEVICE_TABLE(i2c, tc358767_i2c_ids);
+
+static const struct of_device_id tc358767_of_ids[] = {
+        { .compatible = "toshiba,tc358767", },
+        { }
+};
+MODULE_DEVICE_TABLE(of, tc358767_of_ids);
+
+static struct i2c_driver tc358767_driver = {
+        .driver = {
+                .name = "tc358767",
+                .of_match_table = tc358767_of_ids,
+        },
+        .id_table = tc358767_i2c_ids,
+        .probe = tc_probe,
+        .remove = tc_remove,
+};
+module_i2c_driver(tc358767_driver);
+
+MODULE_AUTHOR("Andrey Gusakov <andrey.gusakov@cogentembedded.com>");
+MODULE_DESCRIPTION("tc358767 eDP encoder driver");
+MODULE_LICENSE("GPL");
diff --git a/drivers/gpu/drm/cirrus/Kconfig b/drivers/gpu/drm/cirrus/Kconfig
index 9864559e5fb9..04b3c161dfae 100644
--- a/drivers/gpu/drm/cirrus/Kconfig
+++ b/drivers/gpu/drm/cirrus/Kconfig
@@ -1,11 +1,7 @@
 config DRM_CIRRUS_QEMU
         tristate "Cirrus driver for QEMU emulated device"
         depends on DRM && PCI
-        select FB_SYS_FILLRECT
-        select FB_SYS_COPYAREA
-        select FB_SYS_IMAGEBLIT
         select DRM_KMS_HELPER
-        select DRM_KMS_FB_HELPER
         select DRM_TTM
         help
          This is a KMS driver for emulated cirrus device in qemu.
diff --git a/drivers/gpu/drm/cirrus/cirrus_drv.c b/drivers/gpu/drm/cirrus/cirrus_drv.c
index dc83f69da6f1..b05f7eae32ce 100644
--- a/drivers/gpu/drm/cirrus/cirrus_drv.c
+++ b/drivers/gpu/drm/cirrus/cirrus_drv.c
@@ -142,7 +142,7 @@ static struct drm_driver driver = {
         .major = DRIVER_MAJOR,
         .minor = DRIVER_MINOR,
         .patchlevel = DRIVER_PATCHLEVEL,
-        .gem_free_object = cirrus_gem_free_object,
+        .gem_free_object_unlocked = cirrus_gem_free_object,
         .dumb_create = cirrus_dumb_create,
         .dumb_map_offset = cirrus_dumb_mmap_offset,
         .dumb_destroy = drm_gem_dumb_destroy,
diff --git a/drivers/gpu/drm/cirrus/cirrus_main.c b/drivers/gpu/drm/cirrus/cirrus_main.c
index 32d32c5b7b17..80446e2d3ab6 100644
--- a/drivers/gpu/drm/cirrus/cirrus_main.c
+++ b/drivers/gpu/drm/cirrus/cirrus_main.c
@@ -17,8 +17,8 @@
 static void cirrus_user_framebuffer_destroy(struct drm_framebuffer *fb)
 {
         struct cirrus_framebuffer *cirrus_fb = to_cirrus_framebuffer(fb);
-        if (cirrus_fb->obj)
-                drm_gem_object_unreference_unlocked(cirrus_fb->obj);
+
+        drm_gem_object_unreference_unlocked(cirrus_fb->obj);
         drm_framebuffer_cleanup(fb);
         kfree(fb);
 }
diff --git a/drivers/gpu/drm/cirrus/cirrus_mode.c b/drivers/gpu/drm/cirrus/cirrus_mode.c
index d3d8d7bfcc57..17c915d9a03e 100644
--- a/drivers/gpu/drm/cirrus/cirrus_mode.c
+++ b/drivers/gpu/drm/cirrus/cirrus_mode.c
@@ -325,21 +325,20 @@ static void cirrus_crtc_commit(struct drm_crtc *crtc)
  * use this for 8-bit mode so can't perform smooth fades on deeper modes,
  * but it's a requirement that we provide the function
  */
-static void cirrus_crtc_gamma_set(struct drm_crtc *crtc, u16 *red, u16 *green,
-                                  u16 *blue, uint32_t start, uint32_t size)
+static int cirrus_crtc_gamma_set(struct drm_crtc *crtc, u16 *red, u16 *green,
+                                 u16 *blue, uint32_t size)
 {
         struct cirrus_crtc *cirrus_crtc = to_cirrus_crtc(crtc);
         int i;
 
-        if (size != CIRRUS_LUT_SIZE)
-                return;
-
-        for (i = 0; i < CIRRUS_LUT_SIZE; i++) {
+        for (i = 0; i < size; i++) {
                 cirrus_crtc->lut_r[i] = red[i];
                 cirrus_crtc->lut_g[i] = green[i];
                 cirrus_crtc->lut_b[i] = blue[i];
         }
         cirrus_crtc_load_lut(crtc);
+
+        return 0;
 }
 
 /* Simple cleanup function */
diff --git a/drivers/gpu/drm/cirrus/cirrus_ttm.c b/drivers/gpu/drm/cirrus/cirrus_ttm.c
index 6768b7b1af32..1cc9ee607128 100644
--- a/drivers/gpu/drm/cirrus/cirrus_ttm.c
+++ b/drivers/gpu/drm/cirrus/cirrus_ttm.c
@@ -186,17 +186,6 @@ static void cirrus_ttm_io_mem_free(struct ttm_bo_device *bdev, struct ttm_mem_re
 {
 }
 
-static int cirrus_bo_move(struct ttm_buffer_object *bo,
-                       bool evict, bool interruptible,
-                       bool no_wait_gpu,
-                       struct ttm_mem_reg *new_mem)
-{
-        int r;
-        r = ttm_bo_move_memcpy(bo, evict, no_wait_gpu, new_mem);
-        return r;
-}
-
-
 static void cirrus_ttm_backend_destroy(struct ttm_tt *tt)
 {
         ttm_tt_fini(tt);
@@ -241,7 +230,7 @@ struct ttm_bo_driver cirrus_bo_driver = {
         .ttm_tt_unpopulate = cirrus_ttm_tt_unpopulate,
         .init_mem_type = cirrus_bo_init_mem_type,
         .evict_flags = cirrus_bo_evict_flags,
-        .move = cirrus_bo_move,
+        .move = NULL,
         .verify_access = cirrus_bo_verify_access,
         .io_mem_reserve = &cirrus_ttm_io_mem_reserve,
         .io_mem_free = &cirrus_ttm_io_mem_free,
diff --git a/drivers/gpu/drm/drm_atomic.c b/drivers/gpu/drm/drm_atomic.c
index 9bb99e274d23..8d2f111fa113 100644
--- a/drivers/gpu/drm/drm_atomic.c
+++ b/drivers/gpu/drm/drm_atomic.c
@@ -33,6 +33,20 @@
 
 #include "drm_crtc_internal.h"
 
+static void crtc_commit_free(struct kref *kref)
+{
+        struct drm_crtc_commit *commit =
+                container_of(kref, struct drm_crtc_commit, ref);
+
+        kfree(commit);
+}
+
+void drm_crtc_commit_put(struct drm_crtc_commit *commit)
+{
+        kref_put(&commit->ref, crtc_commit_free);
+}
+EXPORT_SYMBOL(drm_crtc_commit_put);
+
 /**
  * drm_atomic_state_default_release -
  * release memory initialized by drm_atomic_state_init
@@ -44,11 +58,8 @@
 void drm_atomic_state_default_release(struct drm_atomic_state *state)
 {
         kfree(state->connectors);
-        kfree(state->connector_states);
         kfree(state->crtcs);
-        kfree(state->crtc_states);
         kfree(state->planes);
-        kfree(state->plane_states);
 }
 EXPORT_SYMBOL(drm_atomic_state_default_release);
 
@@ -72,18 +83,10 @@ drm_atomic_state_init(struct drm_device *dev, struct drm_atomic_state *state)
                                sizeof(*state->crtcs), GFP_KERNEL);
         if (!state->crtcs)
                 goto fail;
-        state->crtc_states = kcalloc(dev->mode_config.num_crtc,
-                                     sizeof(*state->crtc_states), GFP_KERNEL);
-        if (!state->crtc_states)
-                goto fail;
         state->planes = kcalloc(dev->mode_config.num_total_plane,
                                 sizeof(*state->planes), GFP_KERNEL);
         if (!state->planes)
                 goto fail;
-        state->plane_states = kcalloc(dev->mode_config.num_total_plane,
-                                      sizeof(*state->plane_states), GFP_KERNEL);
-        if (!state->plane_states)
-                goto fail;
 
         state->dev = dev;
 
@@ -139,40 +142,48 @@ void drm_atomic_state_default_clear(struct drm_atomic_state *state)
         DRM_DEBUG_ATOMIC("Clearing atomic state %p\n", state);
 
         for (i = 0; i < state->num_connector; i++) {
-                struct drm_connector *connector = state->connectors[i];
+                struct drm_connector *connector = state->connectors[i].ptr;
 
                 if (!connector)
                         continue;
 
                 connector->funcs->atomic_destroy_state(connector,
-                                                       state->connector_states[i]);
-                state->connectors[i] = NULL;
-                state->connector_states[i] = NULL;
+                                                       state->connectors[i].state);
+                state->connectors[i].ptr = NULL;
+                state->connectors[i].state = NULL;
                 drm_connector_unreference(connector);
         }
 
         for (i = 0; i < config->num_crtc; i++) {
-                struct drm_crtc *crtc = state->crtcs[i];
+                struct drm_crtc *crtc = state->crtcs[i].ptr;
 
                 if (!crtc)
                         continue;
 
                 crtc->funcs->atomic_destroy_state(crtc,
-                                                  state->crtc_states[i]);
-                state->crtcs[i] = NULL;
-                state->crtc_states[i] = NULL;
+                                                  state->crtcs[i].state);
+
+                if (state->crtcs[i].commit) {
+                        kfree(state->crtcs[i].commit->event);
+                        state->crtcs[i].commit->event = NULL;
+                        drm_crtc_commit_put(state->crtcs[i].commit);
+                }
+
+                state->crtcs[i].commit = NULL;
+                state->crtcs[i].ptr = NULL;
+                state->crtcs[i].state = NULL;
         }
 
         for (i = 0; i < config->num_total_plane; i++) {
-                struct drm_plane *plane = state->planes[i];
+                struct drm_plane *plane = state->planes[i].ptr;
 
                 if (!plane)
                         continue;
 
                 plane->funcs->atomic_destroy_state(plane,
-                                                   state->plane_states[i]);
-                state->planes[i] = NULL;
-                state->plane_states[i] = NULL;
+                                                   state->planes[i].state);
+                state->planes[i].ptr = NULL;
+                state->planes[i].state = NULL;
         }
 }
 EXPORT_SYMBOL(drm_atomic_state_default_clear);
@@ -270,8 +281,8 @@ drm_atomic_get_crtc_state(struct drm_atomic_state *state,
         if (!crtc_state)
                 return ERR_PTR(-ENOMEM);
 
-        state->crtc_states[index] = crtc_state;
-        state->crtcs[index] = crtc;
+        state->crtcs[index].state = crtc_state;
+        state->crtcs[index].ptr = crtc;
         crtc_state->state = state;
 
         DRM_DEBUG_ATOMIC("Added [CRTC:%d:%s] %p state to %p\n",
@@ -393,8 +404,7 @@ drm_atomic_replace_property_blob(struct drm_property_blob **blob,
         if (old_blob == new_blob)
                 return;
 
-        if (old_blob)
-                drm_property_unreference_blob(old_blob);
+        drm_property_unreference_blob(old_blob);
         if (new_blob)
                 drm_property_reference_blob(new_blob);
         *blob = new_blob;
@@ -632,8 +642,8 @@ drm_atomic_get_plane_state(struct drm_atomic_state *state,
         if (!plane_state)
                 return ERR_PTR(-ENOMEM);
 
-        state->plane_states[index] = plane_state;
-        state->planes[index] = plane;
+        state->planes[index].state = plane_state;
+        state->planes[index].ptr = plane;
         plane_state->state = state;
 
         DRM_DEBUG_ATOMIC("Added [PLANE:%d:%s] %p state to %p\n",
@@ -897,8 +907,7 @@ drm_atomic_get_connector_state(struct drm_atomic_state *state,
         index = drm_connector_index(connector);
 
         if (index >= state->num_connector) {
-                struct drm_connector **c;
-                struct drm_connector_state **cs;
+                struct __drm_connnectors_state *c;
                 int alloc = max(index + 1, config->num_connector);
 
                 c = krealloc(state->connectors, alloc * sizeof(*state->connectors), GFP_KERNEL);
@@ -909,26 +918,19 @@ drm_atomic_get_connector_state(struct drm_atomic_state *state,
                 memset(&state->connectors[state->num_connector], 0,
                        sizeof(*state->connectors) * (alloc - state->num_connector));
 
-                cs = krealloc(state->connector_states, alloc * sizeof(*state->connector_states), GFP_KERNEL);
-                if (!cs)
-                        return ERR_PTR(-ENOMEM);
-
-                state->connector_states = cs;
-                memset(&state->connector_states[state->num_connector], 0,
-                       sizeof(*state->connector_states) * (alloc - state->num_connector));
                 state->num_connector = alloc;
         }
 
-        if (state->connector_states[index])
-                return state->connector_states[index];
+        if (state->connectors[index].state)
+                return state->connectors[index].state;
 
         connector_state = connector->funcs->atomic_duplicate_state(connector);
         if (!connector_state)
                 return ERR_PTR(-ENOMEM);
 
         drm_connector_reference(connector);
-        state->connector_states[index] = connector_state;
-        state->connectors[index] = connector;
+        state->connectors[index].state = connector_state;
+        state->connectors[index].ptr = connector;
         connector_state->state = state;
 
         DRM_DEBUG_ATOMIC("Added [CONNECTOR:%d] %p state to %p\n",
@@ -1457,7 +1459,8 @@ EXPORT_SYMBOL(drm_atomic_nonblocking_commit);
  */
 
 static struct drm_pending_vblank_event *create_vblank_event(
-                struct drm_device *dev, struct drm_file *file_priv, uint64_t user_data)
+                struct drm_device *dev, struct drm_file *file_priv,
+                struct fence *fence, uint64_t user_data)
 {
         struct drm_pending_vblank_event *e = NULL;
         int ret;
@@ -1470,12 +1473,17 @@ static struct drm_pending_vblank_event *create_vblank_event(
         e->event.base.length = sizeof(e->event);
         e->event.user_data = user_data;
 
-        ret = drm_event_reserve_init(dev, file_priv, &e->base, &e->event.base);
-        if (ret) {
-                kfree(e);
-                return NULL;
+        if (file_priv) {
+                ret = drm_event_reserve_init(dev, file_priv, &e->base,
+                                             &e->event.base);
+                if (ret) {
+                        kfree(e);
+                        return NULL;
+                }
         }
 
+        e->base.fence = fence;
+
         return e;
 }
 
@@ -1715,7 +1723,8 @@ retry:
                 for_each_crtc_in_state(state, crtc, crtc_state, i) {
                         struct drm_pending_vblank_event *e;
 
-                        e = create_vblank_event(dev, file_priv, arg->user_data);
+                        e = create_vblank_event(dev, file_priv, NULL,
+                                                arg->user_data);
                         if (!e) {
                                 ret = -ENOMEM;
                                 goto out;
diff --git a/drivers/gpu/drm/drm_atomic_helper.c b/drivers/gpu/drm/drm_atomic_helper.c
index ddfa0d120e39..de7fddce3cef 100644
--- a/drivers/gpu/drm/drm_atomic_helper.c
+++ b/drivers/gpu/drm/drm_atomic_helper.c
@@ -110,8 +110,10 @@ static int handle_conflicting_encoders(struct drm_atomic_state *state,
 
                 if (funcs->atomic_best_encoder)
                         new_encoder = funcs->atomic_best_encoder(connector, conn_state);
-                else
+                else if (funcs->best_encoder)
                         new_encoder = funcs->best_encoder(connector);
+                else
+                        new_encoder = drm_atomic_helper_best_encoder(connector);
 
                 if (new_encoder) {
                         if (encoder_mask & (1 << drm_encoder_index(new_encoder))) {
@@ -298,8 +300,10 @@ update_connector_routing(struct drm_atomic_state *state,
         if (funcs->atomic_best_encoder)
                 new_encoder = funcs->atomic_best_encoder(connector,
                                                          connector_state);
-        else
+        else if (funcs->best_encoder)
                 new_encoder = funcs->best_encoder(connector);
+        else
+                new_encoder = drm_atomic_helper_best_encoder(connector);
 
         if (!new_encoder) {
                 DRM_DEBUG_ATOMIC("No suitable encoder found for [CONNECTOR:%d:%s]\n",
@@ -414,6 +418,9 @@ mode_fixup(struct drm_atomic_state *state)
         for_each_crtc_in_state(state, crtc, crtc_state, i) {
                 const struct drm_crtc_helper_funcs *funcs;
 
+                if (!crtc_state->enable)
+                        continue;
+
                 if (!crtc_state->mode_changed &&
                     !crtc_state->connectors_changed)
                         continue;
@@ -458,7 +465,7 @@ mode_fixup(struct drm_atomic_state *state)
  * times for the same update, e.g. when the ->atomic_check functions depend upon
  * the adjusted dotclock for fifo space allocation and watermark computation.
  *
- * RETURNS
+ * RETURNS:
  * Zero for success or -errno
  */
 int
@@ -572,7 +579,7 @@ EXPORT_SYMBOL(drm_atomic_helper_check_modeset);
  * It also sets crtc_state->planes_changed to indicate that a crtc has
  * updated planes.
  *
- * RETURNS
+ * RETURNS:
  * Zero for success or -errno
  */
 int
@@ -611,7 +618,7 @@ drm_atomic_helper_check_planes(struct drm_device *dev,
                 if (!funcs || !funcs->atomic_check)
                         continue;
 
-                ret = funcs->atomic_check(crtc, state->crtc_states[i]);
+                ret = funcs->atomic_check(crtc, crtc_state);
                 if (ret) {
                         DRM_DEBUG_ATOMIC("[CRTC:%d:%s] atomic driver check failed\n",
                                          crtc->base.id, crtc->name);
@@ -640,7 +647,7 @@ EXPORT_SYMBOL(drm_atomic_helper_check_planes);
  * ->atomic_check functions depend upon an updated adjusted_mode.clock to
  * e.g. properly compute watermarks.
  *
- * RETURNS
+ * RETURNS:
  * Zero for success or -errno
  */
 int drm_atomic_helper_check(struct drm_device *dev,
@@ -1113,22 +1120,17 @@ drm_atomic_helper_wait_for_vblanks(struct drm_device *dev,
 EXPORT_SYMBOL(drm_atomic_helper_wait_for_vblanks);
 
 /**
- * drm_atomic_helper_commit - commit validated state object
- * @dev: DRM device
- * @state: the driver state object
- * @nonblocking: whether nonblocking behavior is requested.
+ * drm_atomic_helper_commit_tail - commit atomic update to hardware
+ * @state: new modeset state to be committed
  *
- * This function commits a with drm_atomic_helper_check() pre-validated state
- * object. This can still fail when e.g. the framebuffer reservation fails. For
- * now this doesn't implement nonblocking commits.
+ * This is the default implemenation for the ->atomic_commit_tail() hook of the
+ * &drm_mode_config_helper_funcs vtable.
  *
- * Note that right now this function does not support nonblocking commits, hence
- * driver writers must implement their own version for now. Also note that the
- * default ordering of how the various stages are called is to match the legacy
- * modeset helper library closest. One peculiarity of that is that it doesn't
- * mesh well with runtime PM at all.
+ * Note that the default ordering of how the various stages are called is to
+ * match the legacy modeset helper library closest. One peculiarity of that is
+ * that it doesn't mesh well with runtime PM at all.
  *
- * For drivers supporting runtime PM the recommended sequence is
+ * For drivers supporting runtime PM the recommended sequence is instead ::
  *
  *     drm_atomic_helper_commit_modeset_disables(dev, state);
  *
@@ -1136,9 +1138,75 @@ EXPORT_SYMBOL(drm_atomic_helper_wait_for_vblanks);
  *
  *     drm_atomic_helper_commit_planes(dev, state, true);
  *
- * See the kerneldoc entries for these three functions for more details.
+ * for committing the atomic update to hardware.  See the kerneldoc entries for
+ * these three functions for more details.
+ */
+void drm_atomic_helper_commit_tail(struct drm_atomic_state *state)
+{
+        struct drm_device *dev = state->dev;
+
+        drm_atomic_helper_commit_modeset_disables(dev, state);
+
+        drm_atomic_helper_commit_planes(dev, state, false);
+
+        drm_atomic_helper_commit_modeset_enables(dev, state);
+
+        drm_atomic_helper_commit_hw_done(state);
+
+        drm_atomic_helper_wait_for_vblanks(dev, state);
+
+        drm_atomic_helper_cleanup_planes(dev, state);
+}
+EXPORT_SYMBOL(drm_atomic_helper_commit_tail);
+
+static void commit_tail(struct drm_atomic_state *state)
+{
+        struct drm_device *dev = state->dev;
+        struct drm_mode_config_helper_funcs *funcs;
+
+        funcs = dev->mode_config.helper_private;
+
+        drm_atomic_helper_wait_for_fences(dev, state);
+
+        drm_atomic_helper_wait_for_dependencies(state);
+
+        if (funcs && funcs->atomic_commit_tail)
+                funcs->atomic_commit_tail(state);
+        else
+                drm_atomic_helper_commit_tail(state);
+
+        drm_atomic_helper_commit_cleanup_done(state);
+
+        drm_atomic_state_free(state);
+}
+
+static void commit_work(struct work_struct *work)
+{
+        struct drm_atomic_state *state = container_of(work,
+                                                      struct drm_atomic_state,
+                                                      commit_work);
+        commit_tail(state);
+}
+
+/**
+ * drm_atomic_helper_commit - commit validated state object
+ * @dev: DRM device
+ * @state: the driver state object
+ * @nonblock: whether nonblocking behavior is requested.
+ *
+ * This function commits a with drm_atomic_helper_check() pre-validated state
+ * object. This can still fail when e.g. the framebuffer reservation fails. This
+ * function implements nonblocking commits, using
+ * drm_atomic_helper_setup_commit() and related functions.
+ *
+ * Note that right now this function does not support nonblocking commits, hence
+ * driver writers must implement their own version for now.
+ *
+ * Committing the actual hardware state is done through the
+ * ->atomic_commit_tail() callback of the &drm_mode_config_helper_funcs vtable,
+ * or it's default implementation drm_atomic_helper_commit_tail().
  *
- * RETURNS
+ * RETURNS:
  * Zero for success or -errno.
  */
 int drm_atomic_helper_commit(struct drm_device *dev,
@@ -1147,8 +1215,11 @@ int drm_atomic_helper_commit(struct drm_device *dev,
 {
         int ret;
 
-        if (nonblock)
-                return -EBUSY;
+        ret = drm_atomic_helper_setup_commit(state, nonblock);
+        if (ret)
+                return ret;
+
+        INIT_WORK(&state->commit_work, commit_work);
 
         ret = drm_atomic_helper_prepare_planes(dev, state);
         if (ret)
@@ -1160,7 +1231,7 @@ int drm_atomic_helper_commit(struct drm_device *dev,
          * the software side now.
          */
 
-        drm_atomic_helper_swap_state(dev, state);
+        drm_atomic_helper_swap_state(state, true);
 
         /*
          * Everything below can be run asynchronously without the need to grab
@@ -1176,21 +1247,16 @@ int drm_atomic_helper_commit(struct drm_device *dev,
          * update. Which is important since compositors need to figure out the
          * composition of the next frame right after having submitted the
          * current layout.
+         *
+         * NOTE: Commit work has multiple phases, first hardware commit, then
+         * cleanup. We want them to overlap, hence need system_unbound_wq to
+         * make sure work items don't artifically stall on each another.
          */
 
-        drm_atomic_helper_wait_for_fences(dev, state);
-
-        drm_atomic_helper_commit_modeset_disables(dev, state);
-
-        drm_atomic_helper_commit_planes(dev, state, false);
-
-        drm_atomic_helper_commit_modeset_enables(dev, state);
-
-        drm_atomic_helper_wait_for_vblanks(dev, state);
-
-        drm_atomic_helper_cleanup_planes(dev, state);
-
-        drm_atomic_state_free(state);
+        if (nonblock)
+                queue_work(system_unbound_wq, &state->commit_work);
+        else
+                commit_tail(state);
 
         return 0;
 }
@@ -1199,12 +1265,7 @@ EXPORT_SYMBOL(drm_atomic_helper_commit);
 /**
  * DOC: implementing nonblocking commit
  *
- * For now the atomic helpers don't support nonblocking commit directly. If
- * there is real need it could be added though, using the dma-buf fence
- * infrastructure for generic synchronization with outstanding rendering.
- *
- * For now drivers have to implement nonblocking commit themselves, with the
- * following sequence being the recommended one:
+ * Nonblocking atomic commits have to be implemented in the following sequence:
  *
  * 1. Run drm_atomic_helper_prepare_planes() first. This is the only function
  * which commit needs to call which can fail, so we want to run it first and
@@ -1216,10 +1277,14 @@ EXPORT_SYMBOL(drm_atomic_helper_commit);
  * cancelled updates. Note that it is important to ensure that the framebuffer
  * cleanup is still done when cancelling.
  *
- * For sufficient parallelism it is recommended to have a work item per crtc
- * (for updates which don't touch global state) and a global one. Then we only
- * need to synchronize with the crtc work items for changed crtcs and the global
- * work item, which allows nice concurrent updates on disjoint sets of crtcs.
+ * Asynchronous workers need to have sufficient parallelism to be able to run
+ * different atomic commits on different CRTCs in parallel. The simplest way to
+ * achive this is by running them on the &system_unbound_wq work queue. Note
+ * that drivers are not required to split up atomic commits and run an
+ * individual commit in parallel - userspace is supposed to do that if it cares.
+ * But it might be beneficial to do that for modesets, since those necessarily
+ * must be done as one global operation, and enabling or disabling a CRTC can
+ * take a long time. But even that is not required.
  *
  * 3. The software state is updated synchronously with
  * drm_atomic_helper_swap_state(). Doing this under the protection of all modeset
@@ -1232,8 +1297,310 @@ EXPORT_SYMBOL(drm_atomic_helper_commit);
  * commit helpers: a) pre-plane commit b) plane commit c) post-plane commit and
  * then cleaning up the framebuffers after the old framebuffer is no longer
  * being displayed.
+ *
+ * The above scheme is implemented in the atomic helper libraries in
+ * drm_atomic_helper_commit() using a bunch of helper functions. See
+ * drm_atomic_helper_setup_commit() for a starting point.
  */
 
+static int stall_checks(struct drm_crtc *crtc, bool nonblock)
+{
+        struct drm_crtc_commit *commit, *stall_commit = NULL;
+        bool completed = true;
+        int i;
+        long ret = 0;
+
+        spin_lock(&crtc->commit_lock);
+        i = 0;
+        list_for_each_entry(commit, &crtc->commit_list, commit_entry) {
+                if (i == 0) {
+                        completed = try_wait_for_completion(&commit->flip_done);
+                        /* Userspace is not allowed to get ahead of the previous
+                         * commit with nonblocking ones. */
+                        if (!completed && nonblock) {
+                                spin_unlock(&crtc->commit_lock);
+                                return -EBUSY;
+                        }
+                } else if (i == 1) {
+                        stall_commit = commit;
+                        drm_crtc_commit_get(stall_commit);
+                        break;
+                }
+
+                i++;
+        }
+        spin_unlock(&crtc->commit_lock);
+
+        if (!stall_commit)
+                return 0;
+
+        /* We don't want to let commits get ahead of cleanup work too much,
+         * stalling on 2nd previous commit means triple-buffer won't ever stall.
+         */
+        ret = wait_for_completion_interruptible_timeout(&stall_commit->cleanup_done,
+                                                        10*HZ);
+        if (ret == 0)
+                DRM_ERROR("[CRTC:%d:%s] cleanup_done timed out\n",
+                          crtc->base.id, crtc->name);
+
+        drm_crtc_commit_put(stall_commit);
+
+        return ret < 0 ? ret : 0;
+}
+
+/**
+ * drm_atomic_helper_setup_commit - setup possibly nonblocking commit
+ * @state: new modeset state to be committed
+ * @nonblock: whether nonblocking behavior is requested.
+ *
+ * This function prepares @state to be used by the atomic helper's support for
+ * nonblocking commits. Drivers using the nonblocking commit infrastructure
+ * should always call this function from their ->atomic_commit hook.
+ *
+ * To be able to use this support drivers need to use a few more helper
+ * functions. drm_atomic_helper_wait_for_dependencies() must be called before
+ * actually committing the hardware state, and for nonblocking commits this call
+ * must be placed in the async worker. See also drm_atomic_helper_swap_state()
+ * and it's stall parameter, for when a driver's commit hooks look at the
+ * ->state pointers of struct &drm_crtc, &drm_plane or &drm_connector directly.
+ *
+ * Completion of the hardware commit step must be signalled using
+ * drm_atomic_helper_commit_hw_done(). After this step the driver is not allowed
+ * to read or change any permanent software or hardware modeset state. The only
+ * exception is state protected by other means than &drm_modeset_lock locks.
+ * Only the free standing @state with pointers to the old state structures can
+ * be inspected, e.g. to clean up old buffers using
+ * drm_atomic_helper_cleanup_planes().
+ *
+ * At the very end, before cleaning up @state drivers must call
+ * drm_atomic_helper_commit_cleanup_done().
+ *
+ * This is all implemented by in drm_atomic_helper_commit(), giving drivers a
+ * complete and esay-to-use default implementation of the atomic_commit() hook.
+ *
+ * The tracking of asynchronously executed and still pending commits is done
+ * using the core structure &drm_crtc_commit.
+ *
+ * By default there's no need to clean up resources allocated by this function
+ * explicitly: drm_atomic_state_default_clear() will take care of that
+ * automatically.
+ *
+ * Returns:
+ *
+ * 0 on success. -EBUSY when userspace schedules nonblocking commits too fast,
+ * -ENOMEM on allocation failures and -EINTR when a signal is pending.
+ */
+int drm_atomic_helper_setup_commit(struct drm_atomic_state *state,
+                                   bool nonblock)
+{
+        struct drm_crtc *crtc;
+        struct drm_crtc_state *crtc_state;
+        struct drm_crtc_commit *commit;
+        int i, ret;
+
+        for_each_crtc_in_state(state, crtc, crtc_state, i) {
+                commit = kzalloc(sizeof(*commit), GFP_KERNEL);
+                if (!commit)
+                        return -ENOMEM;
+
+                init_completion(&commit->flip_done);
+                init_completion(&commit->hw_done);
+                init_completion(&commit->cleanup_done);
+                INIT_LIST_HEAD(&commit->commit_entry);
+                kref_init(&commit->ref);
+                commit->crtc = crtc;
+
+                state->crtcs[i].commit = commit;
+
+                ret = stall_checks(crtc, nonblock);
+                if (ret)
+                        return ret;
+
+                /* Drivers only send out events when at least either current or
+                 * new CRTC state is active. Complete right away if everything
+                 * stays off. */
+                if (!crtc->state->active && !crtc_state->active) {
+                        complete_all(&commit->flip_done);
+                        continue;
+                }
+
+                /* Legacy cursor updates are fully unsynced. */
+                if (state->legacy_cursor_update) {
+                        complete_all(&commit->flip_done);
+                        continue;
+                }
+
+                if (!crtc_state->event) {
+                        commit->event = kzalloc(sizeof(*commit->event),
+                                                GFP_KERNEL);
+                        if (!commit->event)
+                                return -ENOMEM;
+
+                        crtc_state->event = commit->event;
+                }
+
+                crtc_state->event->base.completion = &commit->flip_done;
+        }
+
+        return 0;
+}
+EXPORT_SYMBOL(drm_atomic_helper_setup_commit);
+
+
+static struct drm_crtc_commit *preceeding_commit(struct drm_crtc *crtc)
+{
+        struct drm_crtc_commit *commit;
+        int i = 0;
+
+        list_for_each_entry(commit, &crtc->commit_list, commit_entry) {
+                /* skip the first entry, that's the current commit */
+                if (i == 1)
+                        return commit;
+                i++;
+        }
+
+        return NULL;
+}
+
+/**
+ * drm_atomic_helper_wait_for_dependencies - wait for required preceeding commits
+ * @state: new modeset state to be committed
+ *
+ * This function waits for all preceeding commits that touch the same CRTC as
+ * @state to both be committed to the hardware (as signalled by
+ * drm_atomic_helper_commit_hw_done) and executed by the hardware (as signalled
+ * by calling drm_crtc_vblank_send_event on the event member of
+ * &drm_crtc_state).
+ *
+ * This is part of the atomic helper support for nonblocking commits, see
+ * drm_atomic_helper_setup_commit() for an overview.
+ */
+void drm_atomic_helper_wait_for_dependencies(struct drm_atomic_state *state)
+{
+        struct drm_crtc *crtc;
+        struct drm_crtc_state *crtc_state;
+        struct drm_crtc_commit *commit;
+        int i;
+        long ret;
+
+        for_each_crtc_in_state(state, crtc, crtc_state, i) {
+                spin_lock(&crtc->commit_lock);
+                commit = preceeding_commit(crtc);
+                if (commit)
+                        drm_crtc_commit_get(commit);
+                spin_unlock(&crtc->commit_lock);
+
+                if (!commit)
+                        continue;
+
+                ret = wait_for_completion_timeout(&commit->hw_done,
+                                                  10*HZ);
+                if (ret == 0)
+                        DRM_ERROR("[CRTC:%d:%s] hw_done timed out\n",
+                                  crtc->base.id, crtc->name);
+
+                /* Currently no support for overwriting flips, hence
+                 * stall for previous one to execute completely. */
+                ret = wait_for_completion_timeout(&commit->flip_done,
+                                                  10*HZ);
+                if (ret == 0)
+                        DRM_ERROR("[CRTC:%d:%s] flip_done timed out\n",
+                                  crtc->base.id, crtc->name);
+
+                drm_crtc_commit_put(commit);
+        }
+}
+EXPORT_SYMBOL(drm_atomic_helper_wait_for_dependencies);
+
+/**
+ * drm_atomic_helper_commit_hw_done - setup possible nonblocking commit
+ * @state: new modeset state to be committed
+ *
+ * This function is used to signal completion of the hardware commit step. After
+ * this step the driver is not allowed to read or change any permanent software
+ * or hardware modeset state. The only exception is state protected by other
+ * means than &drm_modeset_lock locks.
+ *
+ * Drivers should try to postpone any expensive or delayed cleanup work after
+ * this function is called.
+ *
+ * This is part of the atomic helper support for nonblocking commits, see
+ * drm_atomic_helper_setup_commit() for an overview.
+ */
+void drm_atomic_helper_commit_hw_done(struct drm_atomic_state *state)
+{
+        struct drm_crtc *crtc;
+        struct drm_crtc_state *crtc_state;
+        struct drm_crtc_commit *commit;
+        int i;
+
+        for_each_crtc_in_state(state, crtc, crtc_state, i) {
+                commit = state->crtcs[i].commit;
+                if (!commit)
+                        continue;
+
+                /* backend must have consumed any event by now */
+                WARN_ON(crtc->state->event);
+                spin_lock(&crtc->commit_lock);
+                complete_all(&commit->hw_done);
+                spin_unlock(&crtc->commit_lock);
+        }
+}
+EXPORT_SYMBOL(drm_atomic_helper_commit_hw_done);
+
+/**
+ * drm_atomic_helper_commit_cleanup_done - signal completion of commit
+ * @state: new modeset state to be committed
+ *
+ * This signals completion of the atomic update @state, including any cleanup
+ * work. If used, it must be called right before calling
+ * drm_atomic_state_free().
+ *
+ * This is part of the atomic helper support for nonblocking commits, see
+ * drm_atomic_helper_setup_commit() for an overview.
+ */
+void drm_atomic_helper_commit_cleanup_done(struct drm_atomic_state *state)
+{
+        struct drm_crtc *crtc;
+        struct drm_crtc_state *crtc_state;
+        struct drm_crtc_commit *commit;
+        int i;
+        long ret;
+
+        for_each_crtc_in_state(state, crtc, crtc_state, i) {
+                commit = state->crtcs[i].commit;
+                if (WARN_ON(!commit))
+                        continue;
+
+                spin_lock(&crtc->commit_lock);
+                complete_all(&commit->cleanup_done);
+                WARN_ON(!try_wait_for_completion(&commit->hw_done));
+
+                /* commit_list borrows our reference, need to remove before we
+                 * clean up our drm_atomic_state. But only after it actually
+                 * completed, otherwise subsequent commits won't stall properly. */
+                if (try_wait_for_completion(&commit->flip_done))
+                        goto del_commit;
+
+                spin_unlock(&crtc->commit_lock);
+
+                /* We must wait for the vblank event to signal our completion
+                 * before releasing our reference, since the vblank work does
+                 * not hold a reference of its own. */
+                ret = wait_for_completion_timeout(&commit->flip_done,
+                                                  10*HZ);
+                if (ret == 0)
+                        DRM_ERROR("[CRTC:%d:%s] flip_done timed out\n",
+                                  crtc->base.id, crtc->name);
+
+                spin_lock(&crtc->commit_lock);
+del_commit:
+                list_del(&commit->commit_entry);
+                spin_unlock(&crtc->commit_lock);
+        }
+}
+EXPORT_SYMBOL(drm_atomic_helper_commit_cleanup_done);
+
 /**
  * drm_atomic_helper_prepare_planes - prepare plane resources before commit
  * @dev: DRM device
@@ -1249,16 +1616,12 @@ EXPORT_SYMBOL(drm_atomic_helper_commit);
 int drm_atomic_helper_prepare_planes(struct drm_device *dev,
                                      struct drm_atomic_state *state)
 {
-        int nplanes = dev->mode_config.num_total_plane;
-        int ret, i;
+        struct drm_plane *plane;
+        struct drm_plane_state *plane_state;
+        int ret, i, j;
 
-        for (i = 0; i < nplanes; i++) {
+        for_each_plane_in_state(state, plane, plane_state, i) {
                 const struct drm_plane_helper_funcs *funcs;
-                struct drm_plane *plane = state->planes[i];
-                struct drm_plane_state *plane_state = state->plane_states[i];
-
-                if (!plane)
-                        continue;
 
                 funcs = plane->helper_private;
 
@@ -1272,12 +1635,10 @@ int drm_atomic_helper_prepare_planes(struct drm_device *dev,
         return 0;
 
 fail:
-        for (i--; i >= 0; i--) {
+        for_each_plane_in_state(state, plane, plane_state, j) {
                 const struct drm_plane_helper_funcs *funcs;
-                struct drm_plane *plane = state->planes[i];
-                struct drm_plane_state *plane_state = state->plane_states[i];
 
-                if (!plane)
+                if (j >= i)
                         continue;
 
                 funcs = plane->helper_private;
@@ -1537,8 +1898,8 @@ EXPORT_SYMBOL(drm_atomic_helper_cleanup_planes);
 
 /**
  * drm_atomic_helper_swap_state - store atomic state into current sw state
- * @dev: DRM device
  * @state: atomic state
+ * @stall: stall for proceeding commits
  *
  * This function stores the atomic state into the current state pointers in all
  * driver objects. It should be called after all failing steps have been done
@@ -1559,42 +1920,70 @@ EXPORT_SYMBOL(drm_atomic_helper_cleanup_planes);
  *
  * 5. Call drm_atomic_helper_cleanup_planes() with @state, which since step 3
  * contains the old state. Also do any other cleanup required with that state.
+ *
+ * @stall must be set when nonblocking commits for this driver directly access
+ * the ->state pointer of &drm_plane, &drm_crtc or &drm_connector. With the
+ * current atomic helpers this is almost always the case, since the helpers
+ * don't pass the right state structures to the callbacks.
  */
-void drm_atomic_helper_swap_state(struct drm_device *dev,
-                                  struct drm_atomic_state *state)
+void drm_atomic_helper_swap_state(struct drm_atomic_state *state,
+                                  bool stall)
 {
         int i;
+        long ret;
+        struct drm_connector *connector;
+        struct drm_connector_state *conn_state;
+        struct drm_crtc *crtc;
+        struct drm_crtc_state *crtc_state;
+        struct drm_plane *plane;
+        struct drm_plane_state *plane_state;
+        struct drm_crtc_commit *commit;
+
+        if (stall) {
+                for_each_crtc_in_state(state, crtc, crtc_state, i) {
+                        spin_lock(&crtc->commit_lock);
+                        commit = list_first_entry_or_null(&crtc->commit_list,
+                                        struct drm_crtc_commit, commit_entry);
+                        if (commit)
+                                drm_crtc_commit_get(commit);
+                        spin_unlock(&crtc->commit_lock);
+
+                        if (!commit)
+                                continue;
 
-        for (i = 0; i < state->num_connector; i++) {
-                struct drm_connector *connector = state->connectors[i];
-
-                if (!connector)
-                        continue;
+                        ret = wait_for_completion_timeout(&commit->hw_done,
+                                                          10*HZ);
+                        if (ret == 0)
+                                DRM_ERROR("[CRTC:%d:%s] hw_done timed out\n",
+                                          crtc->base.id, crtc->name);
+                        drm_crtc_commit_put(commit);
+                }
+        }
 
+        for_each_connector_in_state(state, connector, conn_state, i) {
                 connector->state->state = state;
-                swap(state->connector_states[i], connector->state);
+                swap(state->connectors[i].state, connector->state);
                 connector->state->state = NULL;
         }
 
-        for (i = 0; i < dev->mode_config.num_crtc; i++) {
-                struct drm_crtc *crtc = state->crtcs[i];
-
-                if (!crtc)
-                        continue;
-
+        for_each_crtc_in_state(state, crtc, crtc_state, i) {
                 crtc->state->state = state;
-                swap(state->crtc_states[i], crtc->state);
+                swap(state->crtcs[i].state, crtc->state);
                 crtc->state->state = NULL;
-        }
 
-        for (i = 0; i < dev->mode_config.num_total_plane; i++) {
-                struct drm_plane *plane = state->planes[i];
+                if (state->crtcs[i].commit) {
+                        spin_lock(&crtc->commit_lock);
+                        list_add(&state->crtcs[i].commit->commit_entry,
+                                 &crtc->commit_list);
+                        spin_unlock(&crtc->commit_lock);
 
-                if (!plane)
-                        continue;
+                        state->crtcs[i].commit->event = NULL;
+                }
+        }
 
+        for_each_plane_in_state(state, plane, plane_state, i) {
                 plane->state->state = state;
-                swap(state->plane_states[i], plane->state);
+                swap(state->planes[i].state, plane->state);
                 plane->state->state = NULL;
         }
 }
@@ -2409,7 +2798,7 @@ EXPORT_SYMBOL(drm_atomic_helper_page_flip);
  * This is the main helper function provided by the atomic helper framework for
  * implementing the legacy DPMS connector interface. It computes the new desired
  * ->active state for the corresponding CRTC (if the connector is enabled) and
- *  updates it.
+ * updates it.
  *
  * Returns:
  * Returns 0 on success, negative errno numbers on failure.
@@ -2930,16 +3319,15 @@ EXPORT_SYMBOL(drm_atomic_helper_connector_destroy_state);
  * @red: red correction table
  * @green: green correction table
  * @blue: green correction table
- * @start:
  * @size: size of the tables
  *
  * Implements support for legacy gamma correction table for drivers
  * that support color management through the DEGAMMA_LUT/GAMMA_LUT
  * properties.
  */
-void drm_atomic_helper_legacy_gamma_set(struct drm_crtc *crtc,
-                                        u16 *red, u16 *green, u16 *blue,
-                                        uint32_t start, uint32_t size)
+int drm_atomic_helper_legacy_gamma_set(struct drm_crtc *crtc,
+                                       u16 *red, u16 *green, u16 *blue,
+                                       uint32_t size)
 {
         struct drm_device *dev = crtc->dev;
         struct drm_mode_config *config = &dev->mode_config;
@@ -2951,7 +3339,7 @@ void drm_atomic_helper_legacy_gamma_set(struct drm_crtc *crtc,
 
         state = drm_atomic_state_alloc(crtc->dev);
         if (!state)
-                return;
+                return -ENOMEM;
 
         blob = drm_property_create_blob(dev,
                                         sizeof(struct drm_color_lut) * size,
@@ -3002,7 +3390,7 @@ retry:
 
         drm_property_unreference_blob(blob);
 
-        return;
+        return 0;
 fail:
         if (ret == -EDEADLK)
                 goto backoff;
@@ -3010,7 +3398,7 @@ fail:
         drm_atomic_state_free(state);
         drm_property_unreference_blob(blob);
 
-        return;
+        return ret;
 backoff:
         drm_atomic_state_clear(state);
         drm_atomic_legacy_backoff(state);
diff --git a/drivers/gpu/drm/drm_auth.c b/drivers/gpu/drm/drm_auth.c
index 50d0baa06db0..4153e8a193af 100644
--- a/drivers/gpu/drm/drm_auth.c
+++ b/drivers/gpu/drm/drm_auth.c
@@ -30,25 +30,36 @@
 
 #include <drm/drmP.h>
 #include "drm_internal.h"
+#include "drm_legacy.h"
 
 /**
- * drm_getmagic - Get unique magic of a client
- * @dev: DRM device to operate on
- * @data: ioctl data containing the drm_auth object
- * @file_priv: DRM file that performs the operation
+ * DOC: master and authentication
  *
- * This looks up the unique magic of the passed client and returns it. If the
- * client did not have a magic assigned, yet, a new one is registered. The magic
- * is stored in the passed drm_auth object.
+ * struct &drm_master is used to track groups of clients with open
+ * primary/legacy device nodes. For every struct &drm_file which has had at
+ * least once successfully became the device master (either through the
+ * SET_MASTER IOCTL, or implicitly through opening the primary device node when
+ * no one else is the current master that time) there exists one &drm_master.
+ * This is noted in the is_master member of &drm_file. All other clients have
+ * just a pointer to the &drm_master they are associated with.
  *
- * Returns: 0 on success, negative error code on failure.
+ * In addition only one &drm_master can be the current master for a &drm_device.
+ * It can be switched through the DROP_MASTER and SET_MASTER IOCTL, or
+ * implicitly through closing/openeing the primary device node. See also
+ * drm_is_current_master().
+ *
+ * Clients can authenticate against the current master (if it matches their own)
+ * using the GETMAGIC and AUTHMAGIC IOCTLs. Together with exchanging masters,
+ * this allows controlled access to the device for an entire group of mutually
+ * trusted clients.
  */
+
 int drm_getmagic(struct drm_device *dev, void *data, struct drm_file *file_priv)
 {
         struct drm_auth *auth = data;
         int ret = 0;
 
-        mutex_lock(&dev->struct_mutex);
+        mutex_lock(&dev->master_mutex);
         if (!file_priv->magic) {
                 ret = idr_alloc(&file_priv->master->magic_map, file_priv,
                                 1, 0, GFP_KERNEL);
@@ -56,23 +67,13 @@ int drm_getmagic(struct drm_device *dev, void *data, struct drm_file *file_priv)
                         file_priv->magic = ret;
         }
         auth->magic = file_priv->magic;
-        mutex_unlock(&dev->struct_mutex);
+        mutex_unlock(&dev->master_mutex);
 
         DRM_DEBUG("%u\n", auth->magic);
 
         return ret < 0 ? ret : 0;
 }
 
-/**
- * drm_authmagic - Authenticate client with a magic
- * @dev: DRM device to operate on
- * @data: ioctl data containing the drm_auth object
- * @file_priv: DRM file that performs the operation
- *
- * This looks up a DRM client by the passed magic and authenticates it.
- *
- * Returns: 0 on success, negative error code on failure.
- */
 int drm_authmagic(struct drm_device *dev, void *data,
                   struct drm_file *file_priv)
 {
@@ -81,13 +82,253 @@ int drm_authmagic(struct drm_device *dev, void *data,
 
         DRM_DEBUG("%u\n", auth->magic);
 
-        mutex_lock(&dev->struct_mutex);
+        mutex_lock(&dev->master_mutex);
         file = idr_find(&file_priv->master->magic_map, auth->magic);
         if (file) {
                 file->authenticated = 1;
                 idr_replace(&file_priv->master->magic_map, NULL, auth->magic);
         }
-        mutex_unlock(&dev->struct_mutex);
+        mutex_unlock(&dev->master_mutex);
 
         return file ? 0 : -EINVAL;
 }
+
+static struct drm_master *drm_master_create(struct drm_device *dev)
+{
+        struct drm_master *master;
+
+        master = kzalloc(sizeof(*master), GFP_KERNEL);
+        if (!master)
+                return NULL;
+
+        kref_init(&master->refcount);
+        spin_lock_init(&master->lock.spinlock);
+        init_waitqueue_head(&master->lock.lock_queue);
+        idr_init(&master->magic_map);
+        master->dev = dev;
+
+        return master;
+}
+
+static int drm_set_master(struct drm_device *dev, struct drm_file *fpriv,
+                          bool new_master)
+{
+        int ret = 0;
+
+        dev->master = drm_master_get(fpriv->master);
+        if (dev->driver->master_set) {
+                ret = dev->driver->master_set(dev, fpriv, new_master);
+                if (unlikely(ret != 0)) {
+                        drm_master_put(&dev->master);
+                }
+        }
+
+        return ret;
+}
+
+static int drm_new_set_master(struct drm_device *dev, struct drm_file *fpriv)
+{
+        struct drm_master *old_master;
+        int ret;
+
+        lockdep_assert_held_once(&dev->master_mutex);
+
+        old_master = fpriv->master;
+        fpriv->master = drm_master_create(dev);
+        if (!fpriv->master) {
+                fpriv->master = old_master;
+                return -ENOMEM;
+        }
+
+        if (dev->driver->master_create) {
+                ret = dev->driver->master_create(dev, fpriv->master);
+                if (ret)
+                        goto out_err;
+        }
+        fpriv->is_master = 1;
+        fpriv->authenticated = 1;
+
+        ret = drm_set_master(dev, fpriv, true);
+        if (ret)
+                goto out_err;
+
+        if (old_master)
+                drm_master_put(&old_master);
+
+        return 0;
+
+out_err:
+        /* drop references and restore old master on failure */
+        drm_master_put(&fpriv->master);
+        fpriv->master = old_master;
+
+        return ret;
+}
+
+int drm_setmaster_ioctl(struct drm_device *dev, void *data,
+                        struct drm_file *file_priv)
+{
+        int ret = 0;
+
+        mutex_lock(&dev->master_mutex);
+        if (drm_is_current_master(file_priv))
+                goto out_unlock;
+
+        if (dev->master) {
+                ret = -EINVAL;
+                goto out_unlock;
+        }
+
+        if (!file_priv->master) {
+                ret = -EINVAL;
+                goto out_unlock;
+        }
+
+        if (!file_priv->is_master) {
+                ret = drm_new_set_master(dev, file_priv);
+                goto out_unlock;
+        }
+
+        ret = drm_set_master(dev, file_priv, false);
+out_unlock:
+        mutex_unlock(&dev->master_mutex);
+        return ret;
+}
+
+static void drm_drop_master(struct drm_device *dev,
+                            struct drm_file *fpriv)
+{
+        if (dev->driver->master_drop)
+                dev->driver->master_drop(dev, fpriv);
+        drm_master_put(&dev->master);
+}
+
+int drm_dropmaster_ioctl(struct drm_device *dev, void *data,
+                         struct drm_file *file_priv)
+{
+        int ret = -EINVAL;
+
+        mutex_lock(&dev->master_mutex);
+        if (!drm_is_current_master(file_priv))
+                goto out_unlock;
+
+        if (!dev->master)
+                goto out_unlock;
+
+        ret = 0;
+        drm_drop_master(dev, file_priv);
+out_unlock:
+        mutex_unlock(&dev->master_mutex);
+        return ret;
+}
+
+int drm_master_open(struct drm_file *file_priv)
+{
+        struct drm_device *dev = file_priv->minor->dev;
+        int ret = 0;
+
+        /* if there is no current master make this fd it, but do not create
+         * any master object for render clients */
+        mutex_lock(&dev->master_mutex);
+        if (!dev->master)
+                ret = drm_new_set_master(dev, file_priv);
+        else
+                file_priv->master = drm_master_get(dev->master);
+        mutex_unlock(&dev->master_mutex);
+
+        return ret;
+}
+
+void drm_master_release(struct drm_file *file_priv)
+{
+        struct drm_device *dev = file_priv->minor->dev;
+        struct drm_master *master = file_priv->master;
+
+        mutex_lock(&dev->master_mutex);
+        if (file_priv->magic)
+                idr_remove(&file_priv->master->magic_map, file_priv->magic);
+
+        if (!drm_is_current_master(file_priv))
+                goto out;
+
+        if (!drm_core_check_feature(dev, DRIVER_MODESET)) {
+                /*
+                 * Since the master is disappearing, so is the
+                 * possibility to lock.
+                 */
+                mutex_lock(&dev->struct_mutex);
+                if (master->lock.hw_lock) {
+                        if (dev->sigdata.lock == master->lock.hw_lock)
+                                dev->sigdata.lock = NULL;
+                        master->lock.hw_lock = NULL;
+                        master->lock.file_priv = NULL;
+                        wake_up_interruptible_all(&master->lock.lock_queue);
+                }
+                mutex_unlock(&dev->struct_mutex);
+        }
+
+        if (dev->master == file_priv->master)
+                drm_drop_master(dev, file_priv);
+out:
+        /* drop the master reference held by the file priv */
+        if (file_priv->master)
+                drm_master_put(&file_priv->master);
+        mutex_unlock(&dev->master_mutex);
+}
+
+/**
+ * drm_is_current_master - checks whether @priv is the current master
+ * @fpriv: DRM file private
+ *
+ * Checks whether @fpriv is current master on its device. This decides whether a
+ * client is allowed to run DRM_MASTER IOCTLs.
+ *
+ * Most of the modern IOCTL which require DRM_MASTER are for kernel modesetting
+ * - the current master is assumed to own the non-shareable display hardware.
+ */
+bool drm_is_current_master(struct drm_file *fpriv)
+{
+        return fpriv->is_master && fpriv->master == fpriv->minor->dev->master;
+}
+EXPORT_SYMBOL(drm_is_current_master);
+
+/**
+ * drm_master_get - reference a master pointer
+ * @master: struct &drm_master
+ *
+ * Increments the reference count of @master and returns a pointer to @master.
+ */
+struct drm_master *drm_master_get(struct drm_master *master)
+{
+        kref_get(&master->refcount);
+        return master;
+}
+EXPORT_SYMBOL(drm_master_get);
+
+static void drm_master_destroy(struct kref *kref)
+{
+        struct drm_master *master = container_of(kref, struct drm_master, refcount);
+        struct drm_device *dev = master->dev;
+
+        if (dev->driver->master_destroy)
+                dev->driver->master_destroy(dev, master);
+
+        drm_legacy_master_rmmaps(dev, master);
+
+        idr_destroy(&master->magic_map);
+        kfree(master->unique);
+        kfree(master);
+}
+
+/**
+ * drm_master_put - unreference and clear a master pointer
+ * @master: pointer to a pointer of struct &drm_master
+ *
+ * This decrements the &drm_master behind @master and sets it to NULL.
+ */
+void drm_master_put(struct drm_master **master)
+{
+        kref_put(&(*master)->refcount, drm_master_destroy);
+        *master = NULL;
+}
+EXPORT_SYMBOL(drm_master_put);
diff --git a/drivers/gpu/drm/drm_bridge.c b/drivers/gpu/drm/drm_bridge.c
index b3654404abd0..255543086590 100644
--- a/drivers/gpu/drm/drm_bridge.c
+++ b/drivers/gpu/drm/drm_bridge.c
@@ -36,7 +36,7 @@
  * encoder chain.
  *
  * A bridge is always attached to a single &drm_encoder at a time, but can be
- * either connected to it directly, or through an intermediate bridge:
+ * either connected to it directly, or through an intermediate bridge::
  *
  *     encoder ---> bridge B ---> bridge A
  *
diff --git a/drivers/gpu/drm/drm_bufs.c b/drivers/gpu/drm/drm_bufs.c
index 9b34158c0f77..c3a12cd8bd0d 100644
--- a/drivers/gpu/drm/drm_bufs.c
+++ b/drivers/gpu/drm/drm_bufs.c
@@ -51,7 +51,7 @@ static struct drm_map_list *drm_find_matching_map(struct drm_device *dev,
                  */
                 if (!entry->map ||
                     map->type != entry->map->type ||
-                    entry->master != dev->primary->master)
+                    entry->master != dev->master)
                         continue;
                 switch (map->type) {
                 case _DRM_SHM:
@@ -245,12 +245,12 @@ static int drm_addmap_core(struct drm_device * dev, resource_size_t offset,
                 map->offset = (unsigned long)map->handle;
                 if (map->flags & _DRM_CONTAINS_LOCK) {
                         /* Prevent a 2nd X Server from creating a 2nd lock */
-                        if (dev->primary->master->lock.hw_lock != NULL) {
+                        if (dev->master->lock.hw_lock != NULL) {
                                 vfree(map->handle);
                                 kfree(map);
                                 return -EBUSY;
                         }
-                        dev->sigdata.lock = dev->primary->master->lock.hw_lock = map->handle;   /* Pointer to lock */
+                        dev->sigdata.lock = dev->master->lock.hw_lock = map->handle;    /* Pointer to lock */
                 }
                 break;
         case _DRM_AGP: {
@@ -356,7 +356,7 @@ static int drm_addmap_core(struct drm_device * dev, resource_size_t offset,
         mutex_unlock(&dev->struct_mutex);
 
         if (!(map->flags & _DRM_DRIVER))
-                list->master = dev->primary->master;
+                list->master = dev->master;
         *maplist = list;
         return 0;
 }
diff --git a/drivers/gpu/drm/drm_cache.c b/drivers/gpu/drm/drm_cache.c
index 059f7c39c582..a7916e5f8864 100644
--- a/drivers/gpu/drm/drm_cache.c
+++ b/drivers/gpu/drm/drm_cache.c
@@ -136,6 +136,7 @@ drm_clflush_virt_range(void *addr, unsigned long length)
                 mb();
                 for (; addr < end; addr += size)
                         clflushopt(addr);
+                clflushopt(end - 1); /* force serialisation */
                 mb();
                 return;
         }
diff --git a/drivers/gpu/drm/drm_crtc.c b/drivers/gpu/drm/drm_crtc.c
index 0e3cc66aa8b7..f1d9f0569d7f 100644
--- a/drivers/gpu/drm/drm_crtc.c
+++ b/drivers/gpu/drm/drm_crtc.c
@@ -39,6 +39,7 @@
 #include <drm/drm_fourcc.h>
 #include <drm/drm_modeset_lock.h>
 #include <drm/drm_atomic.h>
+#include <drm/drm_auth.h>
 
 #include "drm_crtc_internal.h"
 #include "drm_internal.h"
@@ -239,37 +240,6 @@ const char *drm_get_subpixel_order_name(enum subpixel_order order)
 }
 EXPORT_SYMBOL(drm_get_subpixel_order_name);
 
-static char printable_char(int c)
-{
-        return isascii(c) && isprint(c) ? c : '?';
-}
-
-/**
- * drm_get_format_name - return a string for drm fourcc format
- * @format: format to compute name of
- *
- * Note that the buffer used by this function is globally shared and owned by
- * the function itself.
- *
- * FIXME: This isn't really multithreading safe.
- */
-const char *drm_get_format_name(uint32_t format)
-{
-        static char buf[32];
-
-        snprintf(buf, sizeof(buf),
-                 "%c%c%c%c %s-endian (0x%08x)",
-                 printable_char(format & 0xff),
-                 printable_char((format >> 8) & 0xff),
-                 printable_char((format >> 16) & 0xff),
-                 printable_char((format >> 24) & 0x7f),
-                 format & DRM_FORMAT_BIG_ENDIAN ? "big" : "little",
-                 format);
-
-        return buf;
-}
-EXPORT_SYMBOL(drm_get_format_name);
-
 /*
  * Internal function to assign a slot in the object idr and optionally
  * register the object into the idr.
@@ -426,6 +396,51 @@ void drm_mode_object_reference(struct drm_mode_object *obj)
 }
 EXPORT_SYMBOL(drm_mode_object_reference);
 
+/**
+ * drm_crtc_force_disable - Forcibly turn off a CRTC
+ * @crtc: CRTC to turn off
+ *
+ * Returns:
+ * Zero on success, error code on failure.
+ */
+int drm_crtc_force_disable(struct drm_crtc *crtc)
+{
+        struct drm_mode_set set = {
+                .crtc = crtc,
+        };
+
+        return drm_mode_set_config_internal(&set);
+}
+EXPORT_SYMBOL(drm_crtc_force_disable);
+
+/**
+ * drm_crtc_force_disable_all - Forcibly turn off all enabled CRTCs
+ * @dev: DRM device whose CRTCs to turn off
+ *
+ * Drivers may want to call this on unload to ensure that all displays are
+ * unlit and the GPU is in a consistent, low power state. Takes modeset locks.
+ *
+ * Returns:
+ * Zero on success, error code on failure.
+ */
+int drm_crtc_force_disable_all(struct drm_device *dev)
+{
+        struct drm_crtc *crtc;
+        int ret = 0;
+
+        drm_modeset_lock_all(dev);
+        drm_for_each_crtc(crtc, dev)
+                if (crtc->enabled) {
+                        ret = drm_crtc_force_disable(crtc);
+                        if (ret)
+                                goto out;
+                }
+out:
+        drm_modeset_unlock_all(dev);
+        return ret;
+}
+EXPORT_SYMBOL(drm_crtc_force_disable_all);
+
 static void drm_framebuffer_free(struct kref *kref)
 {
         struct drm_framebuffer *fb =
@@ -535,7 +550,7 @@ EXPORT_SYMBOL(drm_framebuffer_unregister_private);
  *
  * Cleanup framebuffer. This function is intended to be used from the drivers
  * ->destroy callback. It can also be used to clean up driver private
- *  framebuffers embedded into a larger structure.
+ * framebuffers embedded into a larger structure.
  *
  * Note that this function does not remove the fb from active usuage - if it is
  * still used anywhere, hilarity can ensue since userspace could call getfb on
@@ -574,8 +589,6 @@ void drm_framebuffer_remove(struct drm_framebuffer *fb)
         struct drm_device *dev;
         struct drm_crtc *crtc;
         struct drm_plane *plane;
-        struct drm_mode_set set;
-        int ret;
 
         if (!fb)
                 return;
@@ -605,11 +618,7 @@ void drm_framebuffer_remove(struct drm_framebuffer *fb)
                 drm_for_each_crtc(crtc, dev) {
                         if (crtc->primary->fb == fb) {
                                 /* should turn off the crtc */
-                                memset(&set, 0, sizeof(struct drm_mode_set));
-                                set.crtc = crtc;
-                                set.fb = NULL;
-                                ret = drm_mode_set_config_internal(&set);
-                                if (ret)
+                                if (drm_crtc_force_disable(crtc))
                                         DRM_ERROR("failed to reset crtc %p when fb was deleted\n", crtc);
                         }
                 }
@@ -639,6 +648,31 @@ static unsigned int drm_num_crtcs(struct drm_device *dev)
         return num;
 }
 
+static int drm_crtc_register_all(struct drm_device *dev)
+{
+        struct drm_crtc *crtc;
+        int ret = 0;
+
+        drm_for_each_crtc(crtc, dev) {
+                if (crtc->funcs->late_register)
+                        ret = crtc->funcs->late_register(crtc);
+                if (ret)
+                        return ret;
+        }
+
+        return 0;
+}
+
+static void drm_crtc_unregister_all(struct drm_device *dev)
+{
+        struct drm_crtc *crtc;
+
+        drm_for_each_crtc(crtc, dev) {
+                if (crtc->funcs->early_unregister)
+                        crtc->funcs->early_unregister(crtc);
+        }
+}
+
 /**
  * drm_crtc_init_with_planes - Initialise a new CRTC object with
  *    specified primary and cursor planes.
@@ -669,6 +703,9 @@ int drm_crtc_init_with_planes(struct drm_device *dev, struct drm_crtc *crtc,
         crtc->dev = dev;
         crtc->funcs = funcs;
 
+        INIT_LIST_HEAD(&crtc->commit_list);
+        spin_lock_init(&crtc->commit_lock);
+
         drm_modeset_lock_init(&crtc->mutex);
         ret = drm_mode_object_get(dev, &crtc->base, DRM_MODE_OBJECT_CRTC);
         if (ret)
@@ -692,7 +729,7 @@ int drm_crtc_init_with_planes(struct drm_device *dev, struct drm_crtc *crtc,
         crtc->base.properties = &crtc->properties;
 
         list_add_tail(&crtc->head, &config->crtc_list);
-        config->num_crtc++;
+        crtc->index = config->num_crtc++;
 
         crtc->primary = primary;
         crtc->cursor = cursor;
@@ -722,6 +759,11 @@ void drm_crtc_cleanup(struct drm_crtc *crtc)
 {
         struct drm_device *dev = crtc->dev;
 
+        /* Note that the crtc_list is considered to be static; should we
+         * remove the drm_crtc at runtime we would have to decrement all
+         * the indices on the drm_crtc after us in the crtc_list.
+         */
+
         kfree(crtc->gamma_store);
         crtc->gamma_store = NULL;
 
@@ -741,29 +783,6 @@ void drm_crtc_cleanup(struct drm_crtc *crtc)
 }
 EXPORT_SYMBOL(drm_crtc_cleanup);
 
-/**
- * drm_crtc_index - find the index of a registered CRTC
- * @crtc: CRTC to find index for
- *
- * Given a registered CRTC, return the index of that CRTC within a DRM
- * device's list of CRTCs.
- */
-unsigned int drm_crtc_index(struct drm_crtc *crtc)
-{
-        unsigned int index = 0;
-        struct drm_crtc *tmp;
-
-        drm_for_each_crtc(tmp, crtc->dev) {
-                if (tmp == crtc)
-                        return index;
-
-                index++;
-        }
-
-        BUG();
-}
-EXPORT_SYMBOL(drm_crtc_index);
-
 /*
  * drm_mode_remove - remove and free a mode
  * @connector: connector list to modify
@@ -909,11 +928,11 @@ int drm_connector_init(struct drm_device *dev,
         connector->dev = dev;
         connector->funcs = funcs;
 
-        connector->connector_id = ida_simple_get(&config->connector_ida, 0, 0, GFP_KERNEL);
-        if (connector->connector_id < 0) {
-                ret = connector->connector_id;
+        ret = ida_simple_get(&config->connector_ida, 0, 0, GFP_KERNEL);
+        if (ret < 0)
                 goto out_put;
-        }
+        connector->index = ret;
+        ret = 0;
 
         connector->connector_type = connector_type;
         connector->connector_type_id =
@@ -961,7 +980,7 @@ out_put_type_id:
                 ida_remove(connector_ida, connector->connector_type_id);
 out_put_id:
         if (ret)
-                ida_remove(&config->connector_ida, connector->connector_id);
+                ida_remove(&config->connector_ida, connector->index);
 out_put:
         if (ret)
                 drm_mode_object_unregister(dev, &connector->base);
@@ -984,6 +1003,12 @@ void drm_connector_cleanup(struct drm_connector *connector)
         struct drm_device *dev = connector->dev;
         struct drm_display_mode *mode, *t;
 
+        /* The connector should have been removed from userspace long before
+         * it is finally destroyed.
+         */
+        if (WARN_ON(connector->registered))
+                drm_connector_unregister(connector);
+
         if (connector->tile_group) {
                 drm_mode_put_tile_group(dev, connector->tile_group);
                 connector->tile_group = NULL;
@@ -999,7 +1024,7 @@ void drm_connector_cleanup(struct drm_connector *connector)
                    connector->connector_type_id);
 
         ida_remove(&dev->mode_config.connector_ida,
-                   connector->connector_id);
+                   connector->index);
 
         kfree(connector->display_info.bus_formats);
         drm_mode_object_unregister(dev, &connector->base);
@@ -1030,19 +1055,34 @@ int drm_connector_register(struct drm_connector *connector)
 {
         int ret;
 
+        if (connector->registered)
+                return 0;
+
         ret = drm_sysfs_connector_add(connector);
         if (ret)
                 return ret;
 
         ret = drm_debugfs_connector_add(connector);
         if (ret) {
-                drm_sysfs_connector_remove(connector);
-                return ret;
+                goto err_sysfs;
+        }
+
+        if (connector->funcs->late_register) {
+                ret = connector->funcs->late_register(connector);
+                if (ret)
+                        goto err_debugfs;
         }
 
         drm_mode_object_register(connector->dev, &connector->base);
 
+        connector->registered = true;
         return 0;
+
+err_debugfs:
+        drm_debugfs_connector_remove(connector);
+err_sysfs:
+        drm_sysfs_connector_remove(connector);
+        return ret;
 }
 EXPORT_SYMBOL(drm_connector_register);
 
@@ -1054,28 +1094,29 @@ EXPORT_SYMBOL(drm_connector_register);
  */
 void drm_connector_unregister(struct drm_connector *connector)
 {
+        if (!connector->registered)
+                return;
+
+        if (connector->funcs->early_unregister)
+                connector->funcs->early_unregister(connector);
+
         drm_sysfs_connector_remove(connector);
         drm_debugfs_connector_remove(connector);
+
+        connector->registered = false;
 }
 EXPORT_SYMBOL(drm_connector_unregister);
 
-/**
- * drm_connector_register_all - register all connectors
- * @dev: drm device
- *
- * This function registers all connectors in sysfs and other places so that
- * userspace can start to access them. Drivers can call it after calling
- * drm_dev_register() to complete the device registration, if they don't call
- * drm_connector_register() on each connector individually.
- *
- * When a device is unplugged and should be removed from userspace access,
- * call drm_connector_unregister_all(), which is the inverse of this
- * function.
- *
- * Returns:
- * Zero on success, error code on failure.
- */
-int drm_connector_register_all(struct drm_device *dev)
+static void drm_connector_unregister_all(struct drm_device *dev)
+{
+        struct drm_connector *connector;
+
+        /* FIXME: taking the mode config mutex ends up in a clash with sysfs */
+        list_for_each_entry(connector, &dev->mode_config.connector_list, head)
+                drm_connector_unregister(connector);
+}
+
+static int drm_connector_register_all(struct drm_device *dev)
 {
         struct drm_connector *connector;
         int ret;
@@ -1097,27 +1138,31 @@ err:
         drm_connector_unregister_all(dev);
         return ret;
 }
-EXPORT_SYMBOL(drm_connector_register_all);
 
-/**
- * drm_connector_unregister_all - unregister connector userspace interfaces
- * @dev: drm device
- *
- * This functions unregisters all connectors from sysfs and other places so
- * that userspace can no longer access them. Drivers should call this as the
- * first step tearing down the device instace, or when the underlying
- * physical device disappeared (e.g. USB unplug), right before calling
- * drm_dev_unregister().
- */
-void drm_connector_unregister_all(struct drm_device *dev)
+static int drm_encoder_register_all(struct drm_device *dev)
 {
-        struct drm_connector *connector;
+        struct drm_encoder *encoder;
+        int ret = 0;
 
-        /* FIXME: taking the mode config mutex ends up in a clash with sysfs */
-        list_for_each_entry(connector, &dev->mode_config.connector_list, head)
-                drm_connector_unregister(connector);
+        drm_for_each_encoder(encoder, dev) {
+                if (encoder->funcs->late_register)
+                        ret = encoder->funcs->late_register(encoder);
+                if (ret)
+                        return ret;
+        }
+
+        return 0;
+}
+
+static void drm_encoder_unregister_all(struct drm_device *dev)
+{
+        struct drm_encoder *encoder;
+
+        drm_for_each_encoder(encoder, dev) {
+                if (encoder->funcs->early_unregister)
+                        encoder->funcs->early_unregister(encoder);
+        }
 }
-EXPORT_SYMBOL(drm_connector_unregister_all);
 
 /**
  * drm_encoder_init - Init a preallocated encoder
@@ -1166,7 +1211,7 @@ int drm_encoder_init(struct drm_device *dev,
         }
 
         list_add_tail(&encoder->head, &dev->mode_config.encoder_list);
-        dev->mode_config.num_encoder++;
+        encoder->index = dev->mode_config.num_encoder++;
 
 out_put:
         if (ret)
@@ -1180,29 +1225,6 @@ out_unlock:
 EXPORT_SYMBOL(drm_encoder_init);
 
 /**
- * drm_encoder_index - find the index of a registered encoder
- * @encoder: encoder to find index for
- *
- * Given a registered encoder, return the index of that encoder within a DRM
- * device's list of encoders.
- */
-unsigned int drm_encoder_index(struct drm_encoder *encoder)
-{
-        unsigned int index = 0;
-        struct drm_encoder *tmp;
-
-        drm_for_each_encoder(tmp, encoder->dev) {
-                if (tmp == encoder)
-                        return index;
-
-                index++;
-        }
-
-        BUG();
-}
-EXPORT_SYMBOL(drm_encoder_index);
-
-/**
  * drm_encoder_cleanup - cleans up an initialised encoder
  * @encoder: encoder to cleanup
  *
@@ -1212,6 +1234,11 @@ void drm_encoder_cleanup(struct drm_encoder *encoder)
 {
         struct drm_device *dev = encoder->dev;
 
+        /* Note that the encoder_list is considered to be static; should we
+         * remove the drm_encoder at runtime we would have to decrement all
+         * the indices on the drm_encoder after us in the encoder_list.
+         */
+
         drm_modeset_lock_all(dev);
         drm_mode_object_unregister(dev, &encoder->base);
         kfree(encoder->name);
@@ -1300,7 +1327,7 @@ int drm_universal_plane_init(struct drm_device *dev, struct drm_plane *plane,
         plane->type = type;
 
         list_add_tail(&plane->head, &config->plane_list);
-        config->num_total_plane++;
+        plane->index = config->num_total_plane++;
         if (plane->type == DRM_PLANE_TYPE_OVERLAY)
                 config->num_overlay_plane++;
 
@@ -1325,6 +1352,31 @@ int drm_universal_plane_init(struct drm_device *dev, struct drm_plane *plane,
 }
 EXPORT_SYMBOL(drm_universal_plane_init);
 
+static int drm_plane_register_all(struct drm_device *dev)
+{
+        struct drm_plane *plane;
+        int ret = 0;
+
+        drm_for_each_plane(plane, dev) {
+                if (plane->funcs->late_register)
+                        ret = plane->funcs->late_register(plane);
+                if (ret)
+                        return ret;
+        }
+
+        return 0;
+}
+
+static void drm_plane_unregister_all(struct drm_device *dev)
+{
+        struct drm_plane *plane;
+
+        drm_for_each_plane(plane, dev) {
+                if (plane->funcs->early_unregister)
+                        plane->funcs->early_unregister(plane);
+        }
+}
+
 /**
  * drm_plane_init - Initialize a legacy plane
  * @dev: DRM device
@@ -1374,6 +1426,11 @@ void drm_plane_cleanup(struct drm_plane *plane)
 
         BUG_ON(list_empty(&plane->head));
 
+        /* Note that the plane_list is considered to be static; should we
+         * remove the drm_plane at runtime we would have to decrement all
+         * the indices on the drm_plane after us in the plane_list.
+         */
+
         list_del(&plane->head);
         dev->mode_config.num_total_plane--;
         if (plane->type == DRM_PLANE_TYPE_OVERLAY)
@@ -1391,29 +1448,6 @@ void drm_plane_cleanup(struct drm_plane *plane)
 EXPORT_SYMBOL(drm_plane_cleanup);
 
 /**
- * drm_plane_index - find the index of a registered plane
- * @plane: plane to find index for
- *
- * Given a registered plane, return the index of that CRTC within a DRM
- * device's list of planes.
- */
-unsigned int drm_plane_index(struct drm_plane *plane)
-{
-        unsigned int index = 0;
-        struct drm_plane *tmp;
-
-        drm_for_each_plane(tmp, plane->dev) {
-                if (tmp == plane)
-                        return index;
-
-                index++;
-        }
-
-        BUG();
-}
-EXPORT_SYMBOL(drm_plane_index);
-
-/**
  * drm_plane_from_index - find the registered plane at an index
  * @dev: DRM device
  * @idx: index of registered plane to find for
@@ -1425,13 +1459,11 @@ struct drm_plane *
 drm_plane_from_index(struct drm_device *dev, int idx)
 {
         struct drm_plane *plane;
-        unsigned int i = 0;
 
-        drm_for_each_plane(plane, dev) {
-                if (i == idx)
+        drm_for_each_plane(plane, dev)
+                if (idx == plane->index)
                         return plane;
-                i++;
-        }
+
         return NULL;
 }
 EXPORT_SYMBOL(drm_plane_from_index);
@@ -1467,6 +1499,46 @@ void drm_plane_force_disable(struct drm_plane *plane)
 }
 EXPORT_SYMBOL(drm_plane_force_disable);
 
+int drm_modeset_register_all(struct drm_device *dev)
+{
+        int ret;
+
+        ret = drm_plane_register_all(dev);
+        if (ret)
+                goto err_plane;
+
+        ret = drm_crtc_register_all(dev);
+        if  (ret)
+                goto err_crtc;
+
+        ret = drm_encoder_register_all(dev);
+        if (ret)
+                goto err_encoder;
+
+        ret = drm_connector_register_all(dev);
+        if (ret)
+                goto err_connector;
+
+        return 0;
+
+err_connector:
+        drm_encoder_unregister_all(dev);
+err_encoder:
+        drm_crtc_unregister_all(dev);
+err_crtc:
+        drm_plane_unregister_all(dev);
+err_plane:
+        return ret;
+}
+
+void drm_modeset_unregister_all(struct drm_device *dev)
+{
+        drm_connector_unregister_all(dev);
+        drm_encoder_unregister_all(dev);
+        drm_crtc_unregister_all(dev);
+        drm_plane_unregister_all(dev);
+}
+
 static int drm_mode_create_standard_properties(struct drm_device *dev)
 {
         struct drm_property *prop;
@@ -2975,6 +3047,8 @@ static int drm_mode_cursor_universal(struct drm_crtc *crtc,
                                 DRM_DEBUG_KMS("failed to wrap cursor buffer in drm framebuffer\n");
                                 return PTR_ERR(fb);
                         }
+                        fb->hot_x = req->hot_x;
+                        fb->hot_y = req->hot_y;
                 } else {
                         fb = NULL;
                 }
@@ -3581,7 +3655,7 @@ int drm_mode_getfb(struct drm_device *dev,
         r->bpp = fb->bits_per_pixel;
         r->pitch = fb->pitches[0];
         if (fb->funcs->create_handle) {
-                if (file_priv->is_master || capable(CAP_SYS_ADMIN) ||
+                if (drm_is_current_master(file_priv) || capable(CAP_SYS_ADMIN) ||
                     drm_is_control_client(file_priv)) {
                         ret = fb->funcs->create_handle(fb, file_priv,
                                                        &r->handle);
@@ -3738,6 +3812,13 @@ void drm_fb_release(struct drm_file *priv)
         }
 }
 
+static bool drm_property_type_valid(struct drm_property *property)
+{
+        if (property->flags & DRM_MODE_PROP_EXTENDED_TYPE)
+                return !(property->flags & DRM_MODE_PROP_LEGACY_TYPE);
+        return !!(property->flags & DRM_MODE_PROP_LEGACY_TYPE);
+}
+
 /**
  * drm_property_create - create a new property type
  * @dev: drm device
@@ -5138,6 +5219,9 @@ EXPORT_SYMBOL(drm_mode_connector_attach_encoder);
 int drm_mode_crtc_set_gamma_size(struct drm_crtc *crtc,
                                  int gamma_size)
 {
+        uint16_t *r_base, *g_base, *b_base;
+        int i;
+
         crtc->gamma_size = gamma_size;
 
         crtc->gamma_store = kcalloc(gamma_size, sizeof(uint16_t) * 3,
@@ -5147,6 +5231,16 @@ int drm_mode_crtc_set_gamma_size(struct drm_crtc *crtc,
                 return -ENOMEM;
         }
 
+        r_base = crtc->gamma_store;
+        g_base = r_base + gamma_size;
+        b_base = g_base + gamma_size;
+        for (i = 0; i < gamma_size; i++) {
+                r_base[i] = i << 8;
+                g_base[i] = i << 8;
+                b_base[i] = i << 8;
+        }
+
+
         return 0;
 }
 EXPORT_SYMBOL(drm_mode_crtc_set_gamma_size);
@@ -5214,7 +5308,7 @@ int drm_mode_gamma_set_ioctl(struct drm_device *dev,
                 goto out;
         }
 
-        crtc->funcs->gamma_set(crtc, r_base, g_base, b_base, 0, crtc->gamma_size);
+        ret = crtc->funcs->gamma_set(crtc, r_base, g_base, b_base, crtc->gamma_size);
 
 out:
         drm_modeset_unlock_all(dev);
@@ -5544,264 +5638,6 @@ int drm_mode_destroy_dumb_ioctl(struct drm_device *dev,
 }
 
 /**
- * drm_fb_get_bpp_depth - get the bpp/depth values for format
- * @format: pixel format (DRM_FORMAT_*)
- * @depth: storage for the depth value
- * @bpp: storage for the bpp value
- *
- * This only supports RGB formats here for compat with code that doesn't use
- * pixel formats directly yet.
- */
-void drm_fb_get_bpp_depth(uint32_t format, unsigned int *depth,
-                          int *bpp)
-{
-        switch (format) {
-        case DRM_FORMAT_C8:
-        case DRM_FORMAT_RGB332:
-        case DRM_FORMAT_BGR233:
-                *depth = 8;
-                *bpp = 8;
-                break;
-        case DRM_FORMAT_XRGB1555:
-        case DRM_FORMAT_XBGR1555:
-        case DRM_FORMAT_RGBX5551:
-        case DRM_FORMAT_BGRX5551:
-        case DRM_FORMAT_ARGB1555:
-        case DRM_FORMAT_ABGR1555:
-        case DRM_FORMAT_RGBA5551:
-        case DRM_FORMAT_BGRA5551:
-                *depth = 15;
-                *bpp = 16;
-                break;
-        case DRM_FORMAT_RGB565:
-        case DRM_FORMAT_BGR565:
-                *depth = 16;
-                *bpp = 16;
-                break;
-        case DRM_FORMAT_RGB888:
-        case DRM_FORMAT_BGR888:
-                *depth = 24;
-                *bpp = 24;
-                break;
-        case DRM_FORMAT_XRGB8888:
-        case DRM_FORMAT_XBGR8888:
-        case DRM_FORMAT_RGBX8888:
-        case DRM_FORMAT_BGRX8888:
-                *depth = 24;
-                *bpp = 32;
-                break;
-        case DRM_FORMAT_XRGB2101010:
-        case DRM_FORMAT_XBGR2101010:
-        case DRM_FORMAT_RGBX1010102:
-        case DRM_FORMAT_BGRX1010102:
-        case DRM_FORMAT_ARGB2101010:
-        case DRM_FORMAT_ABGR2101010:
-        case DRM_FORMAT_RGBA1010102:
-        case DRM_FORMAT_BGRA1010102:
-                *depth = 30;
-                *bpp = 32;
-                break;
-        case DRM_FORMAT_ARGB8888:
-        case DRM_FORMAT_ABGR8888:
-        case DRM_FORMAT_RGBA8888:
-        case DRM_FORMAT_BGRA8888:
-                *depth = 32;
-                *bpp = 32;
-                break;
-        default:
-                DRM_DEBUG_KMS("unsupported pixel format %s\n",
-                              drm_get_format_name(format));
-                *depth = 0;
-                *bpp = 0;
-                break;
-        }
-}
-EXPORT_SYMBOL(drm_fb_get_bpp_depth);
-
-/**
- * drm_format_num_planes - get the number of planes for format
- * @format: pixel format (DRM_FORMAT_*)
- *
- * Returns:
- * The number of planes used by the specified pixel format.
- */
-int drm_format_num_planes(uint32_t format)
-{
-        switch (format) {
-        case DRM_FORMAT_YUV410:
-        case DRM_FORMAT_YVU410:
-        case DRM_FORMAT_YUV411:
-        case DRM_FORMAT_YVU411:
-        case DRM_FORMAT_YUV420:
-        case DRM_FORMAT_YVU420:
-        case DRM_FORMAT_YUV422:
-        case DRM_FORMAT_YVU422:
-        case DRM_FORMAT_YUV444:
-        case DRM_FORMAT_YVU444:
-                return 3;
-        case DRM_FORMAT_NV12:
-        case DRM_FORMAT_NV21:
-        case DRM_FORMAT_NV16:
-        case DRM_FORMAT_NV61:
-        case DRM_FORMAT_NV24:
-        case DRM_FORMAT_NV42:
-                return 2;
-        default:
-                return 1;
-        }
-}
-EXPORT_SYMBOL(drm_format_num_planes);
-
-/**
- * drm_format_plane_cpp - determine the bytes per pixel value
- * @format: pixel format (DRM_FORMAT_*)
- * @plane: plane index
- *
- * Returns:
- * The bytes per pixel value for the specified plane.
- */
-int drm_format_plane_cpp(uint32_t format, int plane)
-{
-        unsigned int depth;
-        int bpp;
-
-        if (plane >= drm_format_num_planes(format))
-                return 0;
-
-        switch (format) {
-        case DRM_FORMAT_YUYV:
-        case DRM_FORMAT_YVYU:
-        case DRM_FORMAT_UYVY:
-        case DRM_FORMAT_VYUY:
-                return 2;
-        case DRM_FORMAT_NV12:
-        case DRM_FORMAT_NV21:
-        case DRM_FORMAT_NV16:
-        case DRM_FORMAT_NV61:
-        case DRM_FORMAT_NV24:
-        case DRM_FORMAT_NV42:
-                return plane ? 2 : 1;
-        case DRM_FORMAT_YUV410:
-        case DRM_FORMAT_YVU410:
-        case DRM_FORMAT_YUV411:
-        case DRM_FORMAT_YVU411:
-        case DRM_FORMAT_YUV420:
-        case DRM_FORMAT_YVU420:
-        case DRM_FORMAT_YUV422:
-        case DRM_FORMAT_YVU422:
-        case DRM_FORMAT_YUV444:
-        case DRM_FORMAT_YVU444:
-                return 1;
-        default:
-                drm_fb_get_bpp_depth(format, &depth, &bpp);
-                return bpp >> 3;
-        }
-}
-EXPORT_SYMBOL(drm_format_plane_cpp);
-
-/**
- * drm_format_horz_chroma_subsampling - get the horizontal chroma subsampling factor
- * @format: pixel format (DRM_FORMAT_*)
- *
- * Returns:
- * The horizontal chroma subsampling factor for the
- * specified pixel format.
- */
-int drm_format_horz_chroma_subsampling(uint32_t format)
-{
-        switch (format) {
-        case DRM_FORMAT_YUV411:
-        case DRM_FORMAT_YVU411:
-        case DRM_FORMAT_YUV410:
-        case DRM_FORMAT_YVU410:
-                return 4;
-        case DRM_FORMAT_YUYV:
-        case DRM_FORMAT_YVYU:
-        case DRM_FORMAT_UYVY:
-        case DRM_FORMAT_VYUY:
-        case DRM_FORMAT_NV12:
-        case DRM_FORMAT_NV21:
-        case DRM_FORMAT_NV16:
-        case DRM_FORMAT_NV61:
-        case DRM_FORMAT_YUV422:
-        case DRM_FORMAT_YVU422:
-        case DRM_FORMAT_YUV420:
-        case DRM_FORMAT_YVU420:
-                return 2;
-        default:
-                return 1;
-        }
-}
-EXPORT_SYMBOL(drm_format_horz_chroma_subsampling);
-
-/**
- * drm_format_vert_chroma_subsampling - get the vertical chroma subsampling factor
- * @format: pixel format (DRM_FORMAT_*)
- *
- * Returns:
- * The vertical chroma subsampling factor for the
- * specified pixel format.
- */
-int drm_format_vert_chroma_subsampling(uint32_t format)
-{
-        switch (format) {
-        case DRM_FORMAT_YUV410:
-        case DRM_FORMAT_YVU410:
-                return 4;
-        case DRM_FORMAT_YUV420:
-        case DRM_FORMAT_YVU420:
-        case DRM_FORMAT_NV12:
-        case DRM_FORMAT_NV21:
-                return 2;
-        default:
-                return 1;
-        }
-}
-EXPORT_SYMBOL(drm_format_vert_chroma_subsampling);
-
-/**
- * drm_format_plane_width - width of the plane given the first plane
- * @width: width of the first plane
- * @format: pixel format
- * @plane: plane index
- *
- * Returns:
- * The width of @plane, given that the width of the first plane is @width.
- */
-int drm_format_plane_width(int width, uint32_t format, int plane)
-{
-        if (plane >= drm_format_num_planes(format))
-                return 0;
-
-        if (plane == 0)
-                return width;
-
-        return width / drm_format_horz_chroma_subsampling(format);
-}
-EXPORT_SYMBOL(drm_format_plane_width);
-
-/**
- * drm_format_plane_height - height of the plane given the first plane
- * @height: height of the first plane
- * @format: pixel format
- * @plane: plane index
- *
- * Returns:
- * The height of @plane, given that the height of the first plane is @height.
- */
-int drm_format_plane_height(int height, uint32_t format, int plane)
-{
-        if (plane >= drm_format_num_planes(format))
-                return 0;
-
-        if (plane == 0)
-                return height;
-
-        return height / drm_format_vert_chroma_subsampling(format);
-}
-EXPORT_SYMBOL(drm_format_plane_height);
-
-/**
  * drm_rotation_simplify() - Try to simplify the rotation
  * @rotation: Rotation to be simplified
  * @supported_rotations: Supported rotations
@@ -6064,3 +5900,48 @@ struct drm_tile_group *drm_mode_create_tile_group(struct drm_device *dev,
         return tg;
 }
 EXPORT_SYMBOL(drm_mode_create_tile_group);
+
+/**
+ * drm_crtc_enable_color_mgmt - enable color management properties
+ * @crtc: DRM CRTC
+ * @degamma_lut_size: the size of the degamma lut (before CSC)
+ * @has_ctm: whether to attach ctm_property for CSC matrix
+ * @gamma_lut_size: the size of the gamma lut (after CSC)
+ *
+ * This function lets the driver enable the color correction
+ * properties on a CRTC. This includes 3 degamma, csc and gamma
+ * properties that userspace can set and 2 size properties to inform
+ * the userspace of the lut sizes. Each of the properties are
+ * optional. The gamma and degamma properties are only attached if
+ * their size is not 0 and ctm_property is only attached if has_ctm is
+ * true.
+ */
+void drm_crtc_enable_color_mgmt(struct drm_crtc *crtc,
+                                uint degamma_lut_size,
+                                bool has_ctm,
+                                uint gamma_lut_size)
+{
+        struct drm_device *dev = crtc->dev;
+        struct drm_mode_config *config = &dev->mode_config;
+
+        if (degamma_lut_size) {
+                drm_object_attach_property(&crtc->base,
+                                           config->degamma_lut_property, 0);
+                drm_object_attach_property(&crtc->base,
+                                           config->degamma_lut_size_property,
+                                           degamma_lut_size);
+        }
+
+        if (has_ctm)
+                drm_object_attach_property(&crtc->base,
+                                           config->ctm_property, 0);
+
+        if (gamma_lut_size) {
+                drm_object_attach_property(&crtc->base,
+                                           config->gamma_lut_property, 0);
+                drm_object_attach_property(&crtc->base,
+                                           config->gamma_lut_size_property,
+                                           gamma_lut_size);
+        }
+}
+EXPORT_SYMBOL(drm_crtc_enable_color_mgmt);
diff --git a/drivers/gpu/drm/drm_crtc_helper.c b/drivers/gpu/drm/drm_crtc_helper.c
index 26feb2f8453f..604d3ef72ffa 100644
--- a/drivers/gpu/drm/drm_crtc_helper.c
+++ b/drivers/gpu/drm/drm_crtc_helper.c
@@ -232,6 +232,9 @@ static void __drm_helper_disable_unused_functions(struct drm_device *dev)
  */
 void drm_helper_disable_unused_functions(struct drm_device *dev)
 {
+        if (drm_core_check_feature(dev, DRIVER_ATOMIC))
+                DRM_ERROR("Called for atomic driver, this is not what you want.\n");
+
         drm_modeset_lock_all(dev);
         __drm_helper_disable_unused_functions(dev);
         drm_modeset_unlock_all(dev);
@@ -1123,36 +1126,3 @@ int drm_helper_crtc_mode_set_base(struct drm_crtc *crtc, int x, int y,
         return drm_plane_helper_commit(plane, plane_state, old_fb);
 }
 EXPORT_SYMBOL(drm_helper_crtc_mode_set_base);
-
-/**
- * drm_helper_crtc_enable_color_mgmt - enable color management properties
- * @crtc: DRM CRTC
- * @degamma_lut_size: the size of the degamma lut (before CSC)
- * @gamma_lut_size: the size of the gamma lut (after CSC)
- *
- * This function lets the driver enable the color correction properties on a
- * CRTC. This includes 3 degamma, csc and gamma properties that userspace can
- * set and 2 size properties to inform the userspace of the lut sizes.
- */
-void drm_helper_crtc_enable_color_mgmt(struct drm_crtc *crtc,
-                                       int degamma_lut_size,
-                                       int gamma_lut_size)
-{
-        struct drm_device *dev = crtc->dev;
-        struct drm_mode_config *config = &dev->mode_config;
-
-        drm_object_attach_property(&crtc->base,
-                                   config->degamma_lut_property, 0);
-        drm_object_attach_property(&crtc->base,
-                                   config->ctm_property, 0);
-        drm_object_attach_property(&crtc->base,
-                                   config->gamma_lut_property, 0);
-
-        drm_object_attach_property(&crtc->base,
-                                   config->degamma_lut_size_property,
-                                   degamma_lut_size);
-        drm_object_attach_property(&crtc->base,
-                                   config->gamma_lut_size_property,
-                                   gamma_lut_size);
-}
-EXPORT_SYMBOL(drm_helper_crtc_enable_color_mgmt);
diff --git a/drivers/gpu/drm/drm_crtc_internal.h b/drivers/gpu/drm/drm_crtc_internal.h
index a78c138282ea..47a500b90fd7 100644
--- a/drivers/gpu/drm/drm_crtc_internal.h
+++ b/drivers/gpu/drm/drm_crtc_internal.h
@@ -31,14 +31,100 @@
  * and are not exported to drivers.
  */
 
+
+/* drm_crtc.c */
+void drm_connector_ida_init(void);
+void drm_connector_ida_destroy(void);
 int drm_mode_object_get(struct drm_device *dev,
                         struct drm_mode_object *obj, uint32_t obj_type);
 void drm_mode_object_unregister(struct drm_device *dev,
                                 struct drm_mode_object *object);
+bool drm_property_change_valid_get(struct drm_property *property,
+                                   uint64_t value,
+                                   struct drm_mode_object **ref);
+void drm_property_change_valid_put(struct drm_property *property,
+                                   struct drm_mode_object *ref);
+
+int drm_plane_check_pixel_format(const struct drm_plane *plane,
+                                 u32 format);
+int drm_crtc_check_viewport(const struct drm_crtc *crtc,
+                            int x, int y,
+                            const struct drm_display_mode *mode,
+                            const struct drm_framebuffer *fb);
+
+void drm_fb_release(struct drm_file *file_priv);
+void drm_property_destroy_user_blobs(struct drm_device *dev,
+                                     struct drm_file *file_priv);
+
+/* dumb buffer support IOCTLs */
+int drm_mode_create_dumb_ioctl(struct drm_device *dev,
+                               void *data, struct drm_file *file_priv);
+int drm_mode_mmap_dumb_ioctl(struct drm_device *dev,
+                             void *data, struct drm_file *file_priv);
+int drm_mode_destroy_dumb_ioctl(struct drm_device *dev,
+                                void *data, struct drm_file *file_priv);
+
+/* framebuffer IOCTLs */
+extern int drm_mode_addfb(struct drm_device *dev,
+                          void *data, struct drm_file *file_priv);
+extern int drm_mode_addfb2(struct drm_device *dev,
+                           void *data, struct drm_file *file_priv);
+int drm_mode_rmfb(struct drm_device *dev,
+                         void *data, struct drm_file *file_priv);
+int drm_mode_getfb(struct drm_device *dev,
+                   void *data, struct drm_file *file_priv);
+int drm_mode_dirtyfb_ioctl(struct drm_device *dev,
+                           void *data, struct drm_file *file_priv);
+
+/* IOCTLs */
+int drm_mode_obj_get_properties_ioctl(struct drm_device *dev, void *data,
+                                      struct drm_file *file_priv);
+int drm_mode_obj_set_property_ioctl(struct drm_device *dev, void *data,
+                                    struct drm_file *file_priv);
+
+int drm_mode_getresources(struct drm_device *dev,
+                          void *data, struct drm_file *file_priv);
+int drm_mode_getplane_res(struct drm_device *dev, void *data,
+                          struct drm_file *file_priv);
+int drm_mode_getcrtc(struct drm_device *dev,
+                     void *data, struct drm_file *file_priv);
+int drm_mode_getconnector(struct drm_device *dev,
+                          void *data, struct drm_file *file_priv);
+int drm_mode_setcrtc(struct drm_device *dev,
+                     void *data, struct drm_file *file_priv);
+int drm_mode_getplane(struct drm_device *dev,
+                      void *data, struct drm_file *file_priv);
+int drm_mode_setplane(struct drm_device *dev,
+                      void *data, struct drm_file *file_priv);
+int drm_mode_cursor_ioctl(struct drm_device *dev,
+                          void *data, struct drm_file *file_priv);
+int drm_mode_cursor2_ioctl(struct drm_device *dev,
+                           void *data, struct drm_file *file_priv);
+int drm_mode_getproperty_ioctl(struct drm_device *dev,
+                               void *data, struct drm_file *file_priv);
+int drm_mode_getblob_ioctl(struct drm_device *dev,
+                           void *data, struct drm_file *file_priv);
+int drm_mode_createblob_ioctl(struct drm_device *dev,
+                              void *data, struct drm_file *file_priv);
+int drm_mode_destroyblob_ioctl(struct drm_device *dev,
+                               void *data, struct drm_file *file_priv);
+int drm_mode_connector_property_set_ioctl(struct drm_device *dev,
+                                          void *data, struct drm_file *file_priv);
+int drm_mode_getencoder(struct drm_device *dev,
+                        void *data, struct drm_file *file_priv);
+int drm_mode_gamma_get_ioctl(struct drm_device *dev,
+                             void *data, struct drm_file *file_priv);
+int drm_mode_gamma_set_ioctl(struct drm_device *dev,
+                             void *data, struct drm_file *file_priv);
+
+int drm_mode_page_flip_ioctl(struct drm_device *dev,
+                             void *data, struct drm_file *file_priv);
 
 /* drm_atomic.c */
 int drm_atomic_get_property(struct drm_mode_object *obj,
-                           struct drm_property *property, uint64_t *val);
+                            struct drm_property *property, uint64_t *val);
 int drm_mode_atomic_ioctl(struct drm_device *dev,
                           void *data, struct drm_file *file_priv);
 
+int drm_modeset_register_all(struct drm_device *dev);
+void drm_modeset_unregister_all(struct drm_device *dev);
diff --git a/drivers/gpu/drm/drm_debugfs.c b/drivers/gpu/drm/drm_debugfs.c
index 3bcf8e6a85b3..fa10cef2ba37 100644
--- a/drivers/gpu/drm/drm_debugfs.c
+++ b/drivers/gpu/drm/drm_debugfs.c
@@ -46,11 +46,8 @@
 
 static const struct drm_info_list drm_debugfs_list[] = {
         {"name", drm_name_info, 0},
-        {"vm", drm_vm_info, 0},
         {"clients", drm_clients_info, 0},
-        {"bufs", drm_bufs_info, 0},
         {"gem_names", drm_gem_name_info, DRIVER_GEM},
-        {"vma", drm_vma_info, 0},
 };
 #define DRM_DEBUGFS_ENTRIES ARRAY_SIZE(drm_debugfs_list)
 
diff --git a/drivers/gpu/drm/drm_dp_aux_dev.c b/drivers/gpu/drm/drm_dp_aux_dev.c
index 3334baacf43d..734f86a345f6 100644
--- a/drivers/gpu/drm/drm_dp_aux_dev.c
+++ b/drivers/gpu/drm/drm_dp_aux_dev.c
@@ -355,8 +355,7 @@ int drm_dp_aux_dev_init(void)
 
         drm_dp_aux_dev_class = class_create(THIS_MODULE, "drm_dp_aux_dev");
         if (IS_ERR(drm_dp_aux_dev_class)) {
-                res = PTR_ERR(drm_dp_aux_dev_class);
-                goto out;
+                return PTR_ERR(drm_dp_aux_dev_class);
         }
         drm_dp_aux_dev_class->dev_groups = drm_dp_aux_groups;
 
diff --git a/drivers/gpu/drm/drm_dp_helper.c b/drivers/gpu/drm/drm_dp_helper.c
index eeaf5a7c3aa7..8f11b8741e42 100644
--- a/drivers/gpu/drm/drm_dp_helper.c
+++ b/drivers/gpu/drm/drm_dp_helper.c
@@ -203,7 +203,7 @@ static int drm_dp_dpcd_access(struct drm_dp_aux *aux, u8 request,
 
                 ret = aux->transfer(aux, &msg);
 
-                if (ret > 0) {
+                if (ret >= 0) {
                         native_reply = msg.reply & DP_AUX_NATIVE_REPLY_MASK;
                         if (native_reply == DP_AUX_NATIVE_REPLY_ACK) {
                                 if (ret == size)
@@ -708,8 +708,6 @@ static int drm_dp_i2c_xfer(struct i2c_adapter *adapter, struct i2c_msg *msgs,
 
         memset(&msg, 0, sizeof(msg));
 
-        mutex_lock(&aux->hw_mutex);
-
         for (i = 0; i < num; i++) {
                 msg.address = msgs[i].addr;
                 drm_dp_i2c_msg_set_request(&msg, &msgs[i]);
@@ -764,8 +762,6 @@ static int drm_dp_i2c_xfer(struct i2c_adapter *adapter, struct i2c_msg *msgs,
         msg.size = 0;
         (void)drm_dp_i2c_do_msg(aux, &msg);
 
-        mutex_unlock(&aux->hw_mutex);
-
         return err;
 }
 
@@ -774,22 +770,64 @@ static const struct i2c_algorithm drm_dp_i2c_algo = {
         .master_xfer = drm_dp_i2c_xfer,
 };
 
+static struct drm_dp_aux *i2c_to_aux(struct i2c_adapter *i2c)
+{
+        return container_of(i2c, struct drm_dp_aux, ddc);
+}
+
+static void lock_bus(struct i2c_adapter *i2c, unsigned int flags)
+{
+        mutex_lock(&i2c_to_aux(i2c)->hw_mutex);
+}
+
+static int trylock_bus(struct i2c_adapter *i2c, unsigned int flags)
+{
+        return mutex_trylock(&i2c_to_aux(i2c)->hw_mutex);
+}
+
+static void unlock_bus(struct i2c_adapter *i2c, unsigned int flags)
+{
+        mutex_unlock(&i2c_to_aux(i2c)->hw_mutex);
+}
+
 /**
- * drm_dp_aux_register() - initialise and register aux channel
+ * drm_dp_aux_init() - minimally initialise an aux channel
  * @aux: DisplayPort AUX channel
  *
- * Returns 0 on success or a negative error code on failure.
+ * If you need to use the drm_dp_aux's i2c adapter prior to registering it
+ * with the outside world, call drm_dp_aux_init() first. You must still
+ * call drm_dp_aux_register() once the connector has been registered to
+ * allow userspace access to the auxiliary DP channel.
  */
-int drm_dp_aux_register(struct drm_dp_aux *aux)
+void drm_dp_aux_init(struct drm_dp_aux *aux)
 {
-        int ret;
-
         mutex_init(&aux->hw_mutex);
 
         aux->ddc.algo = &drm_dp_i2c_algo;
         aux->ddc.algo_data = aux;
         aux->ddc.retries = 3;
 
+        aux->ddc.lock_bus = lock_bus;
+        aux->ddc.trylock_bus = trylock_bus;
+        aux->ddc.unlock_bus = unlock_bus;
+}
+EXPORT_SYMBOL(drm_dp_aux_init);
+
+/**
+ * drm_dp_aux_register() - initialise and register aux channel
+ * @aux: DisplayPort AUX channel
+ *
+ * Automatically calls drm_dp_aux_init() if this hasn't been done yet.
+ *
+ * Returns 0 on success or a negative error code on failure.
+ */
+int drm_dp_aux_register(struct drm_dp_aux *aux)
+{
+        int ret;
+
+        if (!aux->ddc.algo)
+                drm_dp_aux_init(aux);
+
         aux->ddc.class = I2C_CLASS_DDC;
         aux->ddc.owner = THIS_MODULE;
         aux->ddc.dev.parent = aux->dev;
diff --git a/drivers/gpu/drm/drm_dp_mst_topology.c b/drivers/gpu/drm/drm_dp_mst_topology.c
index 6537908050d7..04e457117980 100644
--- a/drivers/gpu/drm/drm_dp_mst_topology.c
+++ b/drivers/gpu/drm/drm_dp_mst_topology.c
@@ -1493,11 +1493,8 @@ static void process_single_down_tx_qlock(struct drm_dp_mst_topology_mgr *mgr)
         WARN_ON(!mutex_is_locked(&mgr->qlock));
 
         /* construct a chunk from the first msg in the tx_msg queue */
-        if (list_empty(&mgr->tx_msg_downq)) {
-                mgr->tx_down_in_progress = false;
+        if (list_empty(&mgr->tx_msg_downq))
                 return;
-        }
-        mgr->tx_down_in_progress = true;
 
         txmsg = list_first_entry(&mgr->tx_msg_downq, struct drm_dp_sideband_msg_tx, next);
         ret = process_single_tx_qlock(mgr, txmsg, false);
@@ -1512,10 +1509,6 @@ static void process_single_down_tx_qlock(struct drm_dp_mst_topology_mgr *mgr)
                 txmsg->state = DRM_DP_SIDEBAND_TX_TIMEOUT;
                 wake_up(&mgr->tx_waitq);
         }
-        if (list_empty(&mgr->tx_msg_downq)) {
-                mgr->tx_down_in_progress = false;
-                return;
-        }
 }
 
 /* called holding qlock */
@@ -1538,7 +1531,7 @@ static void drm_dp_queue_down_tx(struct drm_dp_mst_topology_mgr *mgr,
 {
         mutex_lock(&mgr->qlock);
         list_add_tail(&txmsg->next, &mgr->tx_msg_downq);
-        if (!mgr->tx_down_in_progress)
+        if (list_is_singular(&mgr->tx_msg_downq))
                 process_single_down_tx_qlock(mgr);
         mutex_unlock(&mgr->qlock);
 }
@@ -2372,6 +2365,7 @@ EXPORT_SYMBOL(drm_dp_mst_hpd_irq);
 
 /**
  * drm_dp_mst_detect_port() - get connection status for an MST port
+ * @connector: DRM connector for this port
  * @mgr: manager for this port
  * @port: unverified pointer to a port
  *
@@ -2887,7 +2881,7 @@ static void drm_dp_tx_work(struct work_struct *work)
         struct drm_dp_mst_topology_mgr *mgr = container_of(work, struct drm_dp_mst_topology_mgr, tx_work);
 
         mutex_lock(&mgr->qlock);
-        if (mgr->tx_down_in_progress)
+        if (!list_empty(&mgr->tx_msg_downq))
                 process_single_down_tx_qlock(mgr);
         mutex_unlock(&mgr->qlock);
 }
diff --git a/drivers/gpu/drm/drm_drv.c b/drivers/gpu/drm/drm_drv.c
index bff89226a344..be27ed36f56e 100644
--- a/drivers/gpu/drm/drm_drv.c
+++ b/drivers/gpu/drm/drm_drv.c
@@ -34,8 +34,10 @@
 #include <linux/slab.h>
 #include <drm/drmP.h>
 #include <drm/drm_core.h>
+#include "drm_crtc_internal.h"
 #include "drm_legacy.h"
 #include "drm_internal.h"
+#include "drm_crtc_internal.h"
 
 /*
  * drm_debug: Enable debug output.
@@ -93,114 +95,6 @@ void drm_ut_debug_printk(const char *function_name, const char *format, ...)
 }
 EXPORT_SYMBOL(drm_ut_debug_printk);
 
-struct drm_master *drm_master_create(struct drm_minor *minor)
-{
-        struct drm_master *master;
-
-        master = kzalloc(sizeof(*master), GFP_KERNEL);
-        if (!master)
-                return NULL;
-
-        kref_init(&master->refcount);
-        spin_lock_init(&master->lock.spinlock);
-        init_waitqueue_head(&master->lock.lock_queue);
-        idr_init(&master->magic_map);
-        master->minor = minor;
-
-        return master;
-}
-
-struct drm_master *drm_master_get(struct drm_master *master)
-{
-        kref_get(&master->refcount);
-        return master;
-}
-EXPORT_SYMBOL(drm_master_get);
-
-static void drm_master_destroy(struct kref *kref)
-{
-        struct drm_master *master = container_of(kref, struct drm_master, refcount);
-        struct drm_device *dev = master->minor->dev;
-
-        if (dev->driver->master_destroy)
-                dev->driver->master_destroy(dev, master);
-
-        drm_legacy_master_rmmaps(dev, master);
-
-        idr_destroy(&master->magic_map);
-        kfree(master->unique);
-        kfree(master);
-}
-
-void drm_master_put(struct drm_master **master)
-{
-        kref_put(&(*master)->refcount, drm_master_destroy);
-        *master = NULL;
-}
-EXPORT_SYMBOL(drm_master_put);
-
-int drm_setmaster_ioctl(struct drm_device *dev, void *data,
-                        struct drm_file *file_priv)
-{
-        int ret = 0;
-
-        mutex_lock(&dev->master_mutex);
-        if (file_priv->is_master)
-                goto out_unlock;
-
-        if (file_priv->minor->master) {
-                ret = -EINVAL;
-                goto out_unlock;
-        }
-
-        if (!file_priv->master) {
-                ret = -EINVAL;
-                goto out_unlock;
-        }
-
-        if (!file_priv->allowed_master) {
-                ret = drm_new_set_master(dev, file_priv);
-                goto out_unlock;
-        }
-
-        file_priv->minor->master = drm_master_get(file_priv->master);
-        file_priv->is_master = 1;
-        if (dev->driver->master_set) {
-                ret = dev->driver->master_set(dev, file_priv, false);
-                if (unlikely(ret != 0)) {
-                        file_priv->is_master = 0;
-                        drm_master_put(&file_priv->minor->master);
-                }
-        }
-
-out_unlock:
-        mutex_unlock(&dev->master_mutex);
-        return ret;
-}
-
-int drm_dropmaster_ioctl(struct drm_device *dev, void *data,
-                         struct drm_file *file_priv)
-{
-        int ret = -EINVAL;
-
-        mutex_lock(&dev->master_mutex);
-        if (!file_priv->is_master)
-                goto out_unlock;
-
-        if (!file_priv->minor->master)
-                goto out_unlock;
-
-        ret = 0;
-        if (dev->driver->master_drop)
-                dev->driver->master_drop(dev, file_priv, false);
-        drm_master_put(&file_priv->minor->master);
-        file_priv->is_master = 0;
-
-out_unlock:
-        mutex_unlock(&dev->master_mutex);
-        return ret;
-}
-
 /*
  * DRM Minors
  * A DRM device can provide several char-dev interfaces on the DRM-Major. Each
@@ -405,10 +299,9 @@ void drm_minor_release(struct drm_minor *minor)
  * callbacks implemented by the driver. The driver then needs to initialize all
  * the various subsystems for the drm device like memory management, vblank
  * handling, modesetting support and intial output configuration plus obviously
- * initialize all the corresponding hardware bits. An important part of this is
- * also calling drm_dev_set_unique() to set the userspace-visible unique name of
- * this device instance. Finally when everything is up and running and ready for
- * userspace the device instance can be published using drm_dev_register().
+ * initialize all the corresponding hardware bits. Finally when everything is up
+ * and running and ready for userspace the device instance can be published
+ * using drm_dev_register().
  *
  * There is also deprecated support for initalizing device instances using
  * bus-specific helpers and the ->load() callback. But due to
@@ -430,6 +323,14 @@ void drm_minor_release(struct drm_minor *minor)
  * dev_priv field of &drm_device.
  */
 
+static int drm_dev_set_unique(struct drm_device *dev, const char *name)
+{
+        kfree(dev->unique);
+        dev->unique = kstrdup(name, GFP_KERNEL);
+
+        return dev->unique ? 0 : -ENOMEM;
+}
+
 /**
  * drm_put_dev - Unregister and release a DRM device
  * @dev: DRM device
@@ -461,9 +362,7 @@ EXPORT_SYMBOL(drm_put_dev);
 void drm_unplug_dev(struct drm_device *dev)
 {
         /* for a USB device */
-        drm_minor_unregister(dev, DRM_MINOR_LEGACY);
-        drm_minor_unregister(dev, DRM_MINOR_RENDER);
-        drm_minor_unregister(dev, DRM_MINOR_CONTROL);
+        drm_dev_unregister(dev);
 
         mutex_lock(&drm_global_mutex);
 
@@ -549,11 +448,12 @@ static void drm_fs_inode_free(struct inode *inode)
 }
 
 /**
- * drm_dev_alloc - Allocate new DRM device
- * @driver: DRM driver to allocate device for
+ * drm_dev_init - Initialise new DRM device
+ * @dev: DRM device
+ * @driver: DRM driver
  * @parent: Parent device object
  *
- * Allocate and initialize a new DRM device. No device registration is done.
+ * Initialize a new DRM device. No device registration is done.
  * Call drm_dev_register() to advertice the device to user space and register it
  * with other core subsystems. This should be done last in the device
  * initialization sequence to make sure userspace can't access an inconsistent
@@ -564,19 +464,18 @@ static void drm_fs_inode_free(struct inode *inode)
  *
  * Note that for purely virtual devices @parent can be NULL.
  *
+ * Drivers that do not want to allocate their own device struct
+ * embedding struct &drm_device can call drm_dev_alloc() instead.
+ *
  * RETURNS:
- * Pointer to new DRM device, or NULL if out of memory.
+ * 0 on success, or error code on failure.
  */
-struct drm_device *drm_dev_alloc(struct drm_driver *driver,
-                                 struct device *parent)
+int drm_dev_init(struct drm_device *dev,
+                 struct drm_driver *driver,
+                 struct device *parent)
 {
-        struct drm_device *dev;
         int ret;
 
-        dev = kzalloc(sizeof(*dev), GFP_KERNEL);
-        if (!dev)
-                return NULL;
-
         kref_init(&dev->ref);
         dev->dev = parent;
         dev->driver = driver;
@@ -605,8 +504,6 @@ struct drm_device *drm_dev_alloc(struct drm_driver *driver,
                 ret = drm_minor_alloc(dev, DRM_MINOR_CONTROL);
                 if (ret)
                         goto err_minors;
-
-                WARN_ON(driver->suspend || driver->resume);
         }
 
         if (drm_core_check_feature(dev, DRIVER_RENDER)) {
@@ -619,7 +516,8 @@ struct drm_device *drm_dev_alloc(struct drm_driver *driver,
         if (ret)
                 goto err_minors;
 
-        if (drm_ht_create(&dev->map_hash, 12))
+        ret = drm_ht_create(&dev->map_hash, 12);
+        if (ret)
                 goto err_minors;
 
         drm_legacy_ctxbitmap_init(dev);
@@ -632,13 +530,13 @@ struct drm_device *drm_dev_alloc(struct drm_driver *driver,
                 }
         }
 
-        if (parent) {
-                ret = drm_dev_set_unique(dev, dev_name(parent));
-                if (ret)
-                        goto err_setunique;
-        }
+        /* Use the parent device name as DRM device unique identifier, but fall
+         * back to the driver name for virtual devices like vgem. */
+        ret = drm_dev_set_unique(dev, parent ? dev_name(parent) : driver->name);
+        if (ret)
+                goto err_setunique;
 
-        return dev;
+        return 0;
 
 err_setunique:
         if (drm_core_check_feature(dev, DRIVER_GEM))
@@ -653,8 +551,49 @@ err_minors:
         drm_fs_inode_free(dev->anon_inode);
 err_free:
         mutex_destroy(&dev->master_mutex);
-        kfree(dev);
-        return NULL;
+        return ret;
+}
+EXPORT_SYMBOL(drm_dev_init);
+
+/**
+ * drm_dev_alloc - Allocate new DRM device
+ * @driver: DRM driver to allocate device for
+ * @parent: Parent device object
+ *
+ * Allocate and initialize a new DRM device. No device registration is done.
+ * Call drm_dev_register() to advertice the device to user space and register it
+ * with other core subsystems. This should be done last in the device
+ * initialization sequence to make sure userspace can't access an inconsistent
+ * state.
+ *
+ * The initial ref-count of the object is 1. Use drm_dev_ref() and
+ * drm_dev_unref() to take and drop further ref-counts.
+ *
+ * Note that for purely virtual devices @parent can be NULL.
+ *
+ * Drivers that wish to subclass or embed struct &drm_device into their
+ * own struct should look at using drm_dev_init() instead.
+ *
+ * RETURNS:
+ * Pointer to new DRM device, or NULL if out of memory.
+ */
+struct drm_device *drm_dev_alloc(struct drm_driver *driver,
+                                 struct device *parent)
+{
+        struct drm_device *dev;
+        int ret;
+
+        dev = kzalloc(sizeof(*dev), GFP_KERNEL);
+        if (!dev)
+                return NULL;
+
+        ret = drm_dev_init(dev, driver, parent);
+        if (ret) {
+                kfree(dev);
+                return NULL;
+        }
+
+        return dev;
 }
 EXPORT_SYMBOL(drm_dev_alloc);
 
@@ -718,11 +657,7 @@ EXPORT_SYMBOL(drm_dev_unref);
  *
  * Register the DRM device @dev with the system, advertise device to user-space
  * and start normal device operation. @dev must be allocated via drm_dev_alloc()
- * previously. Right after drm_dev_register() the driver should call
- * drm_connector_register_all() to register all connectors in sysfs. This is
- * a separate call for backward compatibility with drivers still using
- * the deprecated ->load() callback, where connectors are registered from within
- * the ->load() callback.
+ * previously.
  *
  * Never call this twice on any device!
  *
@@ -759,6 +694,9 @@ int drm_dev_register(struct drm_device *dev, unsigned long flags)
                         goto err_minors;
         }
 
+        if (drm_core_check_feature(dev, DRIVER_MODESET))
+                drm_modeset_register_all(dev);
+
         ret = 0;
         goto out_unlock;
 
@@ -789,6 +727,9 @@ void drm_dev_unregister(struct drm_device *dev)
 
         drm_lastclose(dev);
 
+        if (drm_core_check_feature(dev, DRIVER_MODESET))
+                drm_modeset_unregister_all(dev);
+
         if (dev->driver->unload)
                 dev->driver->unload(dev);
 
@@ -806,26 +747,6 @@ void drm_dev_unregister(struct drm_device *dev)
 }
 EXPORT_SYMBOL(drm_dev_unregister);
 
-/**
- * drm_dev_set_unique - Set the unique name of a DRM device
- * @dev: device of which to set the unique name
- * @name: unique name
- *
- * Sets the unique name of a DRM device using the specified string. Drivers
- * can use this at driver probe time if the unique name of the devices they
- * drive is static.
- *
- * Return: 0 on success or a negative error code on failure.
- */
-int drm_dev_set_unique(struct drm_device *dev, const char *name)
-{
-        kfree(dev->unique);
-        dev->unique = kstrdup(name, GFP_KERNEL);
-
-        return dev->unique ? 0 : -ENOMEM;
-}
-EXPORT_SYMBOL(drm_dev_set_unique);
-
 /*
  * DRM Core
  * The DRM core module initializes all global DRM objects and makes them
diff --git a/drivers/gpu/drm/drm_edid_load.c b/drivers/gpu/drm/drm_edid_load.c
index 9a401aed98e0..622f788bff46 100644
--- a/drivers/gpu/drm/drm_edid_load.c
+++ b/drivers/gpu/drm/drm_edid_load.c
@@ -271,7 +271,7 @@ int drm_load_edid_firmware(struct drm_connector *connector)
          * by commas, search through the list looking for one that
          * matches the connector.
          *
-         * If there's one or more that don't't specify a connector, keep
+         * If there's one or more that doesn't specify a connector, keep
          * the last one found one as a fallback.
          */
         fwstr = kstrdup(edid_firmware, GFP_KERNEL);
diff --git a/drivers/gpu/drm/drm_fb_cma_helper.c b/drivers/gpu/drm/drm_fb_cma_helper.c
index 5075fae3c4e2..1fd6eac1400c 100644
--- a/drivers/gpu/drm/drm_fb_cma_helper.c
+++ b/drivers/gpu/drm/drm_fb_cma_helper.c
@@ -23,6 +23,7 @@
 #include <drm/drm_crtc_helper.h>
 #include <drm/drm_gem_cma_helper.h>
 #include <drm/drm_fb_cma_helper.h>
+#include <linux/dma-mapping.h>
 #include <linux/module.h>
 
 #define DEFAULT_FBDEFIO_DELAY_MS 50
@@ -52,7 +53,7 @@ struct drm_fbdev_cma {
  * will be set up automatically. dirty() is called by
  * drm_fb_helper_deferred_io() in process context (struct delayed_work).
  *
- * Example fbdev deferred io code:
+ * Example fbdev deferred io code::
  *
  *     static int driver_fbdev_fb_dirty(struct drm_framebuffer *fb,
  *                                      struct drm_file *file_priv,
@@ -162,6 +163,10 @@ static struct drm_fb_cma *drm_fb_cma_alloc(struct drm_device *dev,
  * drm_fb_cma_create_with_funcs() - helper function for the
  *                                  &drm_mode_config_funcs ->fb_create
  *                                  callback function
+ * @dev: DRM device
+ * @file_priv: drm file for the ioctl call
+ * @mode_cmd: metadata from the userspace fb creation request
+ * @funcs: vtable to be used for the new framebuffer object
  *
  * This can be used to set &drm_framebuffer_funcs for drivers that need the
  * dirty() callback. Use drm_fb_cma_create() if you don't need to change
@@ -223,6 +228,9 @@ EXPORT_SYMBOL_GPL(drm_fb_cma_create_with_funcs);
 
 /**
  * drm_fb_cma_create() - &drm_mode_config_funcs ->fb_create callback function
+ * @dev: DRM device
+ * @file_priv: drm file for the ioctl call
+ * @mode_cmd: metadata from the userspace fb creation request
  *
  * If your hardware has special alignment or pitch requirements these should be
  * checked before calling this function. Use drm_fb_cma_create_with_funcs() if
@@ -246,7 +254,7 @@ EXPORT_SYMBOL_GPL(drm_fb_cma_create);
  * This function will usually be called from the CRTC callback functions.
  */
 struct drm_gem_cma_object *drm_fb_cma_get_gem_obj(struct drm_framebuffer *fb,
-        unsigned int plane)
+                                                  unsigned int plane)
 {
         struct drm_fb_cma *fb_cma = to_fb_cma(fb);
 
@@ -258,10 +266,6 @@ struct drm_gem_cma_object *drm_fb_cma_get_gem_obj(struct drm_framebuffer *fb,
 EXPORT_SYMBOL_GPL(drm_fb_cma_get_gem_obj);
 
 #ifdef CONFIG_DEBUG_FS
-/*
- * drm_fb_cma_describe() - Helper to dump information about a single
- * CMA framebuffer object
- */
 static void drm_fb_cma_describe(struct drm_framebuffer *fb, struct seq_file *m)
 {
         struct drm_fb_cma *fb_cma = to_fb_cma(fb);
@@ -279,7 +283,9 @@ static void drm_fb_cma_describe(struct drm_framebuffer *fb, struct seq_file *m)
 
 /**
  * drm_fb_cma_debugfs_show() - Helper to list CMA framebuffer objects
- * in debugfs.
+ *                             in debugfs.
+ * @m: output file
+ * @arg: private data for the callback
  */
 int drm_fb_cma_debugfs_show(struct seq_file *m, void *arg)
 {
@@ -297,6 +303,12 @@ int drm_fb_cma_debugfs_show(struct seq_file *m, void *arg)
 EXPORT_SYMBOL_GPL(drm_fb_cma_debugfs_show);
 #endif
 
+static int drm_fb_cma_mmap(struct fb_info *info, struct vm_area_struct *vma)
+{
+        return dma_mmap_writecombine(info->device, vma, info->screen_base,
+                                     info->fix.smem_start, info->fix.smem_len);
+}
+
 static struct fb_ops drm_fbdev_cma_ops = {
         .owner          = THIS_MODULE,
         .fb_fillrect    = drm_fb_helper_sys_fillrect,
@@ -307,6 +319,7 @@ static struct fb_ops drm_fbdev_cma_ops = {
         .fb_blank       = drm_fb_helper_blank,
         .fb_pan_display = drm_fb_helper_pan_display,
         .fb_setcmap     = drm_fb_helper_setcmap,
+        .fb_mmap        = drm_fb_cma_mmap,
 };
 
 static int drm_fbdev_cma_deferred_io_mmap(struct fb_info *info,
@@ -333,6 +346,7 @@ static int drm_fbdev_cma_defio_init(struct fb_info *fbi,
         fbops = kzalloc(sizeof(*fbops), GFP_KERNEL);
         if (!fbdefio || !fbops) {
                 kfree(fbdefio);
+                kfree(fbops);
                 return -ENOMEM;
         }
 
@@ -582,3 +596,18 @@ void drm_fbdev_cma_hotplug_event(struct drm_fbdev_cma *fbdev_cma)
                 drm_fb_helper_hotplug_event(&fbdev_cma->fb_helper);
 }
 EXPORT_SYMBOL_GPL(drm_fbdev_cma_hotplug_event);
+
+/**
+ * drm_fbdev_cma_set_suspend - wrapper around drm_fb_helper_set_suspend
+ * @fbdev_cma: The drm_fbdev_cma struct, may be NULL
+ * @state: desired state, zero to resume, non-zero to suspend
+ *
+ * Calls drm_fb_helper_set_suspend, which is a wrapper around
+ * fb_set_suspend implemented by fbdev core.
+ */
+void drm_fbdev_cma_set_suspend(struct drm_fbdev_cma *fbdev_cma, int state)
+{
+        if (fbdev_cma)
+                drm_fb_helper_set_suspend(&fbdev_cma->fb_helper, state);
+}
+EXPORT_SYMBOL(drm_fbdev_cma_set_suspend);
diff --git a/drivers/gpu/drm/drm_fb_helper.c b/drivers/gpu/drm/drm_fb_helper.c
index 7c2eb75db60f..ce54e985d91b 100644
--- a/drivers/gpu/drm/drm_fb_helper.c
+++ b/drivers/gpu/drm/drm_fb_helper.c
@@ -227,7 +227,7 @@ static void drm_fb_helper_restore_lut_atomic(struct drm_crtc *crtc)
         g_base = r_base + crtc->gamma_size;
         b_base = g_base + crtc->gamma_size;
 
-        crtc->funcs->gamma_set(crtc, r_base, g_base, b_base, 0, crtc->gamma_size);
+        crtc->funcs->gamma_set(crtc, r_base, g_base, b_base, crtc->gamma_size);
 }
 
 /**
@@ -385,7 +385,7 @@ static int restore_fbdev_mode(struct drm_fb_helper *fb_helper)
 
         drm_warn_on_modeset_not_all_locked(dev);
 
-        if (fb_helper->atomic)
+        if (dev->mode_config.funcs->atomic_commit)
                 return restore_fbdev_mode_atomic(fb_helper);
 
         drm_for_each_plane(plane, dev) {
@@ -464,7 +464,7 @@ static bool drm_fb_helper_is_bound(struct drm_fb_helper *fb_helper)
 
         /* Sometimes user space wants everything disabled, so don't steal the
          * display if there's a master. */
-        if (dev->primary->master)
+        if (lockless_dereference(dev->master))
                 return false;
 
         drm_for_each_crtc(crtc, dev) {
@@ -716,8 +716,6 @@ int drm_fb_helper_init(struct drm_device *dev,
                 i++;
         }
 
-        fb_helper->atomic = !!drm_core_check_feature(dev, DRIVER_ATOMIC);
-
         return 0;
 out_free:
         drm_fb_helper_crtc_free(fb_helper);
@@ -1042,7 +1040,6 @@ static int setcolreg(struct drm_crtc *crtc, u16 red, u16 green,
 {
         struct drm_fb_helper *fb_helper = info->par;
         struct drm_framebuffer *fb = fb_helper->fb;
-        int pindex;
 
         if (info->fix.visual == FB_VISUAL_TRUECOLOR) {
                 u32 *palette;
@@ -1074,38 +1071,10 @@ static int setcolreg(struct drm_crtc *crtc, u16 red, u16 green,
                     !fb_helper->funcs->gamma_get))
                 return -EINVAL;
 
-        pindex = regno;
-
-        if (fb->bits_per_pixel == 16) {
-                pindex = regno << 3;
-
-                if (fb->depth == 16 && regno > 63)
-                        return -EINVAL;
-                if (fb->depth == 15 && regno > 31)
-                        return -EINVAL;
-
-                if (fb->depth == 16) {
-                        u16 r, g, b;
-                        int i;
-                        if (regno < 32) {
-                                for (i = 0; i < 8; i++)
-                                        fb_helper->funcs->gamma_set(crtc, red,
-                                                green, blue, pindex + i);
-                        }
+        WARN_ON(fb->bits_per_pixel != 8);
 
-                        fb_helper->funcs->gamma_get(crtc, &r,
-                                                    &g, &b,
-                                                    pindex >> 1);
+        fb_helper->funcs->gamma_set(crtc, red, green, blue, regno);
 
-                        for (i = 0; i < 4; i++)
-                                fb_helper->funcs->gamma_set(crtc, r,
-                                                            green, b,
-                                                            (pindex >> 1) + i);
-                }
-        }
-
-        if (fb->depth != 16)
-                fb_helper->funcs->gamma_set(crtc, red, green, blue, pindex);
         return 0;
 }
 
@@ -1373,7 +1342,7 @@ int drm_fb_helper_pan_display(struct fb_var_screeninfo *var,
                 return -EBUSY;
         }
 
-        if (fb_helper->atomic) {
+        if (dev->mode_config.funcs->atomic_commit) {
                 ret = pan_display_atomic(var, info);
                 goto unlock;
         }
@@ -2000,7 +1969,18 @@ static int drm_pick_crtcs(struct drm_fb_helper *fb_helper,
                 my_score++;
 
         connector_funcs = connector->helper_private;
-        encoder = connector_funcs->best_encoder(connector);
+
+        /*
+         * If the DRM device implements atomic hooks and ->best_encoder() is
+         * NULL we fallback to the default drm_atomic_helper_best_encoder()
+         * helper.
+         */
+        if (fb_helper->dev->mode_config.funcs->atomic_commit &&
+            !connector_funcs->best_encoder)
+                encoder = drm_atomic_helper_best_encoder(connector);
+        else
+                encoder = connector_funcs->best_encoder(connector);
+
         if (!encoder)
                 goto out;
 
diff --git a/drivers/gpu/drm/drm_fops.c b/drivers/gpu/drm/drm_fops.c
index 7af7f8bcb355..323c238fcac7 100644
--- a/drivers/gpu/drm/drm_fops.c
+++ b/drivers/gpu/drm/drm_fops.c
@@ -40,6 +40,7 @@
 #include <linux/module.h>
 #include "drm_legacy.h"
 #include "drm_internal.h"
+#include "drm_crtc_internal.h"
 
 /* from BKL pushdown */
 DEFINE_MUTEX(drm_global_mutex);
@@ -67,7 +68,7 @@ DEFINE_MUTEX(drm_global_mutex);
  * specific implementations. For GEM-based drivers this is drm_gem_mmap().
  *
  * No other file operations are supported by the DRM userspace API. Overall the
- * following is an example #file_operations structure:
+ * following is an example #file_operations structure::
  *
  *     static const example_drm_fops = {
  *             .owner = THIS_MODULE,
@@ -168,60 +169,6 @@ static int drm_cpu_valid(void)
 }
 
 /*
- * drm_new_set_master - Allocate a new master object and become master for the
- * associated master realm.
- *
- * @dev: The associated device.
- * @fpriv: File private identifying the client.
- *
- * This function must be called with dev::struct_mutex held.
- * Returns negative error code on failure. Zero on success.
- */
-int drm_new_set_master(struct drm_device *dev, struct drm_file *fpriv)
-{
-        struct drm_master *old_master;
-        int ret;
-
-        lockdep_assert_held_once(&dev->master_mutex);
-
-        /* create a new master */
-        fpriv->minor->master = drm_master_create(fpriv->minor);
-        if (!fpriv->minor->master)
-                return -ENOMEM;
-
-        /* take another reference for the copy in the local file priv */
-        old_master = fpriv->master;
-        fpriv->master = drm_master_get(fpriv->minor->master);
-
-        if (dev->driver->master_create) {
-                ret = dev->driver->master_create(dev, fpriv->master);
-                if (ret)
-                        goto out_err;
-        }
-        if (dev->driver->master_set) {
-                ret = dev->driver->master_set(dev, fpriv, true);
-                if (ret)
-                        goto out_err;
-        }
-
-        fpriv->is_master = 1;
-        fpriv->allowed_master = 1;
-        fpriv->authenticated = 1;
-        if (old_master)
-                drm_master_put(&old_master);
-
-        return 0;
-
-out_err:
-        /* drop both references and restore old master on failure */
-        drm_master_put(&fpriv->minor->master);
-        drm_master_put(&fpriv->master);
-        fpriv->master = old_master;
-
-        return ret;
-}
-
-/*
  * Called whenever a process opens /dev/drm.
  *
  * \param filp file pointer.
@@ -283,19 +230,11 @@ static int drm_open_helper(struct file *filp, struct drm_minor *minor)
                         goto out_prime_destroy;
         }
 
-        /* if there is no current master make this fd it, but do not create
-         * any master object for render clients */
-        mutex_lock(&dev->master_mutex);
-        if (drm_is_primary_client(priv) && !priv->minor->master) {
-                /* create a new master */
-                ret = drm_new_set_master(dev, priv);
+        if (drm_is_primary_client(priv)) {
+                ret = drm_master_open(priv);
                 if (ret)
                         goto out_close;
-        } else if (drm_is_primary_client(priv)) {
-                /* get a reference to the master */
-                priv->master = drm_master_get(priv->minor->master);
         }
-        mutex_unlock(&dev->master_mutex);
 
         mutex_lock(&dev->filelist_mutex);
         list_add(&priv->lhead, &dev->filelist);
@@ -324,7 +263,6 @@ static int drm_open_helper(struct file *filp, struct drm_minor *minor)
         return 0;
 
 out_close:
-        mutex_unlock(&dev->master_mutex);
         if (dev->driver->postclose)
                 dev->driver->postclose(dev, priv);
 out_prime_destroy:
@@ -338,18 +276,6 @@ out_prime_destroy:
         return ret;
 }
 
-static void drm_master_release(struct drm_device *dev, struct file *filp)
-{
-        struct drm_file *file_priv = filp->private_data;
-
-        if (drm_legacy_i_have_hw_lock(dev, file_priv)) {
-                DRM_DEBUG("File %p released, freeing lock for context %d\n",
-                          filp, _DRM_LOCKING_CONTEXT(file_priv->master->lock.hw_lock->lock));
-                drm_legacy_lock_free(&file_priv->master->lock,
-                                     _DRM_LOCKING_CONTEXT(file_priv->master->lock.hw_lock->lock));
-        }
-}
-
 static void drm_events_release(struct drm_file *file_priv)
 {
         struct drm_device *dev = file_priv->minor->dev;
@@ -368,7 +294,7 @@ static void drm_events_release(struct drm_file *file_priv)
         /* Remove unconsumed events */
         list_for_each_entry_safe(e, et, &file_priv->event_list, link) {
                 list_del(&e->link);
-                e->destroy(e);
+                kfree(e);
         }
 
         spin_unlock_irqrestore(&dev->event_lock, flags);
@@ -451,11 +377,6 @@ int drm_release(struct inode *inode, struct file *filp)
         list_del(&file_priv->lhead);
         mutex_unlock(&dev->filelist_mutex);
 
-        mutex_lock(&dev->struct_mutex);
-        if (file_priv->magic)
-                idr_remove(&file_priv->master->magic_map, file_priv->magic);
-        mutex_unlock(&dev->struct_mutex);
-
         if (dev->driver->preclose)
                 dev->driver->preclose(dev, file_priv);
 
@@ -468,9 +389,8 @@ int drm_release(struct inode *inode, struct file *filp)
                   (long)old_encode_dev(file_priv->minor->kdev->devt),
                   dev->open_count);
 
-        /* if the master has gone away we can't do anything with the lock */
-        if (file_priv->minor->master)
-                drm_master_release(dev, filp);
+        if (!drm_core_check_feature(dev, DRIVER_MODESET))
+                drm_legacy_lock_release(dev, filp);
 
         if (drm_core_check_feature(dev, DRIVER_HAVE_DMA))
                 drm_legacy_reclaim_buffers(dev, file_priv);
@@ -487,43 +407,12 @@ int drm_release(struct inode *inode, struct file *filp)
 
         drm_legacy_ctxbitmap_flush(dev, file_priv);
 
-        mutex_lock(&dev->master_mutex);
-
-        if (file_priv->is_master) {
-                struct drm_master *master = file_priv->master;
-
-                /*
-                 * Since the master is disappearing, so is the
-                 * possibility to lock.
-                 */
-                mutex_lock(&dev->struct_mutex);
-                if (master->lock.hw_lock) {
-                        if (dev->sigdata.lock == master->lock.hw_lock)
-                                dev->sigdata.lock = NULL;
-                        master->lock.hw_lock = NULL;
-                        master->lock.file_priv = NULL;
-                        wake_up_interruptible_all(&master->lock.lock_queue);
-                }
-                mutex_unlock(&dev->struct_mutex);
-
-                if (file_priv->minor->master == file_priv->master) {
-                        /* drop the reference held my the minor */
-                        if (dev->driver->master_drop)
-                                dev->driver->master_drop(dev, file_priv, true);
-                        drm_master_put(&file_priv->minor->master);
-                }
-        }
-
-        /* drop the master reference held by the file priv */
-        if (file_priv->master)
-                drm_master_put(&file_priv->master);
-        file_priv->is_master = 0;
-        mutex_unlock(&dev->master_mutex);
+        if (drm_is_primary_client(file_priv))
+                drm_master_release(file_priv);
 
         if (dev->driver->postclose)
                 dev->driver->postclose(dev, file_priv);
 
-
         if (drm_core_check_feature(dev, DRIVER_PRIME))
                 drm_prime_destroy_file_private(&file_priv->prime);
 
@@ -636,7 +525,7 @@ put_back_event:
                         }
 
                         ret += length;
-                        e->destroy(e);
+                        kfree(e);
                 }
         }
         mutex_unlock(&file_priv->event_read_lock);
@@ -713,9 +602,6 @@ int drm_event_reserve_init_locked(struct drm_device *dev,
         list_add(&p->pending_link, &file_priv->pending_event_list);
         p->file_priv = file_priv;
 
-        /* we *could* pass this in as arg, but everyone uses kfree: */
-        p->destroy = (void (*) (struct drm_pending_event *)) kfree;
-
         return 0;
 }
 EXPORT_SYMBOL(drm_event_reserve_init_locked);
@@ -778,7 +664,7 @@ void drm_event_cancel_free(struct drm_device *dev,
                 list_del(&p->pending_link);
         }
         spin_unlock_irqrestore(&dev->event_lock, flags);
-        p->destroy(p);
+        kfree(p);
 }
 EXPORT_SYMBOL(drm_event_cancel_free);
 
@@ -800,8 +686,19 @@ void drm_send_event_locked(struct drm_device *dev, struct drm_pending_event *e)
 {
         assert_spin_locked(&dev->event_lock);
 
+        if (e->completion) {
+                /* ->completion might disappear as soon as it signalled. */
+                complete_all(e->completion);
+                e->completion = NULL;
+        }
+
+        if (e->fence) {
+                fence_signal(e->fence);
+                fence_put(e->fence);
+        }
+
         if (!e->file_priv) {
-                e->destroy(e);
+                kfree(e);
                 return;
         }
 
diff --git a/drivers/gpu/drm/drm_fourcc.c b/drivers/gpu/drm/drm_fourcc.c
new file mode 100644
index 000000000000..0645c85d5f95
--- /dev/null
+++ b/drivers/gpu/drm/drm_fourcc.c
@@ -0,0 +1,320 @@
+/*
+ * Copyright (c) 2016 Laurent Pinchart <laurent.pinchart@ideasonboard.com>
+ *
+ * DRM core format related functions
+ *
+ * Permission to use, copy, modify, distribute, and sell this software and its
+ * documentation for any purpose is hereby granted without fee, provided that
+ * the above copyright notice appear in all copies and that both that copyright
+ * notice and this permission notice appear in supporting documentation, and
+ * that the name of the copyright holders not be used in advertising or
+ * publicity pertaining to distribution of the software without specific,
+ * written prior permission.  The copyright holders make no representations
+ * about the suitability of this software for any purpose.  It is provided "as
+ * is" without express or implied warranty.
+ *
+ * THE COPYRIGHT HOLDERS DISCLAIM ALL WARRANTIES WITH REGARD TO THIS SOFTWARE,
+ * INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS, IN NO
+ * EVENT SHALL THE COPYRIGHT HOLDERS BE LIABLE FOR ANY SPECIAL, INDIRECT OR
+ * CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE,
+ * DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER
+ * TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE
+ * OF THIS SOFTWARE.
+ */
+
+#include <linux/bug.h>
+#include <linux/ctype.h>
+#include <linux/export.h>
+#include <linux/kernel.h>
+
+#include <drm/drmP.h>
+#include <drm/drm_fourcc.h>
+
+static char printable_char(int c)
+{
+        return isascii(c) && isprint(c) ? c : '?';
+}
+
+/**
+ * drm_get_format_name - return a string for drm fourcc format
+ * @format: format to compute name of
+ *
+ * Note that the buffer used by this function is globally shared and owned by
+ * the function itself.
+ *
+ * FIXME: This isn't really multithreading safe.
+ */
+const char *drm_get_format_name(uint32_t format)
+{
+        static char buf[32];
+
+        snprintf(buf, sizeof(buf),
+                 "%c%c%c%c %s-endian (0x%08x)",
+                 printable_char(format & 0xff),
+                 printable_char((format >> 8) & 0xff),
+                 printable_char((format >> 16) & 0xff),
+                 printable_char((format >> 24) & 0x7f),
+                 format & DRM_FORMAT_BIG_ENDIAN ? "big" : "little",
+                 format);
+
+        return buf;
+}
+EXPORT_SYMBOL(drm_get_format_name);
+
+/**
+ * drm_fb_get_bpp_depth - get the bpp/depth values for format
+ * @format: pixel format (DRM_FORMAT_*)
+ * @depth: storage for the depth value
+ * @bpp: storage for the bpp value
+ *
+ * This only supports RGB formats here for compat with code that doesn't use
+ * pixel formats directly yet.
+ */
+void drm_fb_get_bpp_depth(uint32_t format, unsigned int *depth,
+                          int *bpp)
+{
+        switch (format) {
+        case DRM_FORMAT_C8:
+        case DRM_FORMAT_RGB332:
+        case DRM_FORMAT_BGR233:
+                *depth = 8;
+                *bpp = 8;
+                break;
+        case DRM_FORMAT_XRGB1555:
+        case DRM_FORMAT_XBGR1555:
+        case DRM_FORMAT_RGBX5551:
+        case DRM_FORMAT_BGRX5551:
+        case DRM_FORMAT_ARGB1555:
+        case DRM_FORMAT_ABGR1555:
+        case DRM_FORMAT_RGBA5551:
+        case DRM_FORMAT_BGRA5551:
+                *depth = 15;
+                *bpp = 16;
+                break;
+        case DRM_FORMAT_RGB565:
+        case DRM_FORMAT_BGR565:
+                *depth = 16;
+                *bpp = 16;
+                break;
+        case DRM_FORMAT_RGB888:
+        case DRM_FORMAT_BGR888:
+                *depth = 24;
+                *bpp = 24;
+                break;
+        case DRM_FORMAT_XRGB8888:
+        case DRM_FORMAT_XBGR8888:
+        case DRM_FORMAT_RGBX8888:
+        case DRM_FORMAT_BGRX8888:
+                *depth = 24;
+                *bpp = 32;
+                break;
+        case DRM_FORMAT_XRGB2101010:
+        case DRM_FORMAT_XBGR2101010:
+        case DRM_FORMAT_RGBX1010102:
+        case DRM_FORMAT_BGRX1010102:
+        case DRM_FORMAT_ARGB2101010:
+        case DRM_FORMAT_ABGR2101010:
+        case DRM_FORMAT_RGBA1010102:
+        case DRM_FORMAT_BGRA1010102:
+                *depth = 30;
+                *bpp = 32;
+                break;
+        case DRM_FORMAT_ARGB8888:
+        case DRM_FORMAT_ABGR8888:
+        case DRM_FORMAT_RGBA8888:
+        case DRM_FORMAT_BGRA8888:
+                *depth = 32;
+                *bpp = 32;
+                break;
+        default:
+                DRM_DEBUG_KMS("unsupported pixel format %s\n",
+                              drm_get_format_name(format));
+                *depth = 0;
+                *bpp = 0;
+                break;
+        }
+}
+EXPORT_SYMBOL(drm_fb_get_bpp_depth);
+
+/**
+ * drm_format_num_planes - get the number of planes for format
+ * @format: pixel format (DRM_FORMAT_*)
+ *
+ * Returns:
+ * The number of planes used by the specified pixel format.
+ */
+int drm_format_num_planes(uint32_t format)
+{
+        switch (format) {
+        case DRM_FORMAT_YUV410:
+        case DRM_FORMAT_YVU410:
+        case DRM_FORMAT_YUV411:
+        case DRM_FORMAT_YVU411:
+        case DRM_FORMAT_YUV420:
+        case DRM_FORMAT_YVU420:
+        case DRM_FORMAT_YUV422:
+        case DRM_FORMAT_YVU422:
+        case DRM_FORMAT_YUV444:
+        case DRM_FORMAT_YVU444:
+                return 3;
+        case DRM_FORMAT_NV12:
+        case DRM_FORMAT_NV21:
+        case DRM_FORMAT_NV16:
+        case DRM_FORMAT_NV61:
+        case DRM_FORMAT_NV24:
+        case DRM_FORMAT_NV42:
+                return 2;
+        default:
+                return 1;
+        }
+}
+EXPORT_SYMBOL(drm_format_num_planes);
+
+/**
+ * drm_format_plane_cpp - determine the bytes per pixel value
+ * @format: pixel format (DRM_FORMAT_*)
+ * @plane: plane index
+ *
+ * Returns:
+ * The bytes per pixel value for the specified plane.
+ */
+int drm_format_plane_cpp(uint32_t format, int plane)
+{
+        unsigned int depth;
+        int bpp;
+
+        if (plane >= drm_format_num_planes(format))
+                return 0;
+
+        switch (format) {
+        case DRM_FORMAT_YUYV:
+        case DRM_FORMAT_YVYU:
+        case DRM_FORMAT_UYVY:
+        case DRM_FORMAT_VYUY:
+                return 2;
+        case DRM_FORMAT_NV12:
+        case DRM_FORMAT_NV21:
+        case DRM_FORMAT_NV16:
+        case DRM_FORMAT_NV61:
+        case DRM_FORMAT_NV24:
+        case DRM_FORMAT_NV42:
+                return plane ? 2 : 1;
+        case DRM_FORMAT_YUV410:
+        case DRM_FORMAT_YVU410:
+        case DRM_FORMAT_YUV411:
+        case DRM_FORMAT_YVU411:
+        case DRM_FORMAT_YUV420:
+        case DRM_FORMAT_YVU420:
+        case DRM_FORMAT_YUV422:
+        case DRM_FORMAT_YVU422:
+        case DRM_FORMAT_YUV444:
+        case DRM_FORMAT_YVU444:
+                return 1;
+        default:
+                drm_fb_get_bpp_depth(format, &depth, &bpp);
+                return bpp >> 3;
+        }
+}
+EXPORT_SYMBOL(drm_format_plane_cpp);
+
+/**
+ * drm_format_horz_chroma_subsampling - get the horizontal chroma subsampling factor
+ * @format: pixel format (DRM_FORMAT_*)
+ *
+ * Returns:
+ * The horizontal chroma subsampling factor for the
+ * specified pixel format.
+ */
+int drm_format_horz_chroma_subsampling(uint32_t format)
+{
+        switch (format) {
+        case DRM_FORMAT_YUV411:
+        case DRM_FORMAT_YVU411:
+        case DRM_FORMAT_YUV410:
+        case DRM_FORMAT_YVU410:
+                return 4;
+        case DRM_FORMAT_YUYV:
+        case DRM_FORMAT_YVYU:
+        case DRM_FORMAT_UYVY:
+        case DRM_FORMAT_VYUY:
+        case DRM_FORMAT_NV12:
+        case DRM_FORMAT_NV21:
+        case DRM_FORMAT_NV16:
+        case DRM_FORMAT_NV61:
+        case DRM_FORMAT_YUV422:
+        case DRM_FORMAT_YVU422:
+        case DRM_FORMAT_YUV420:
+        case DRM_FORMAT_YVU420:
+                return 2;
+        default:
+                return 1;
+        }
+}
+EXPORT_SYMBOL(drm_format_horz_chroma_subsampling);
+
+/**
+ * drm_format_vert_chroma_subsampling - get the vertical chroma subsampling factor
+ * @format: pixel format (DRM_FORMAT_*)
+ *
+ * Returns:
+ * The vertical chroma subsampling factor for the
+ * specified pixel format.
+ */
+int drm_format_vert_chroma_subsampling(uint32_t format)
+{
+        switch (format) {
+        case DRM_FORMAT_YUV410:
+        case DRM_FORMAT_YVU410:
+                return 4;
+        case DRM_FORMAT_YUV420:
+        case DRM_FORMAT_YVU420:
+        case DRM_FORMAT_NV12:
+        case DRM_FORMAT_NV21:
+                return 2;
+        default:
+                return 1;
+        }
+}
+EXPORT_SYMBOL(drm_format_vert_chroma_subsampling);
+
+/**
+ * drm_format_plane_width - width of the plane given the first plane
+ * @width: width of the first plane
+ * @format: pixel format
+ * @plane: plane index
+ *
+ * Returns:
+ * The width of @plane, given that the width of the first plane is @width.
+ */
+int drm_format_plane_width(int width, uint32_t format, int plane)
+{
+        if (plane >= drm_format_num_planes(format))
+                return 0;
+
+        if (plane == 0)
+                return width;
+
+        return width / drm_format_horz_chroma_subsampling(format);
+}
+EXPORT_SYMBOL(drm_format_plane_width);
+
+/**
+ * drm_format_plane_height - height of the plane given the first plane
+ * @height: height of the first plane
+ * @format: pixel format
+ * @plane: plane index
+ *
+ * Returns:
+ * The height of @plane, given that the height of the first plane is @height.
+ */
+int drm_format_plane_height(int height, uint32_t format, int plane)
+{
+        if (plane >= drm_format_num_planes(format))
+                return 0;
+
+        if (plane == 0)
+                return height;
+
+        return height / drm_format_vert_chroma_subsampling(format);
+}
+EXPORT_SYMBOL(drm_format_plane_height);
diff --git a/drivers/gpu/drm/drm_gem.c b/drivers/gpu/drm/drm_gem.c
index ad89db36ca25..9134ae134667 100644
--- a/drivers/gpu/drm/drm_gem.c
+++ b/drivers/gpu/drm/drm_gem.c
@@ -787,7 +787,7 @@ EXPORT_SYMBOL(drm_gem_object_release);
  * @kref: kref of the object to free
  *
  * Called after the last reference to the object has been lost.
- * Must be called holding struct_ mutex
+ * Must be called holding &drm_device->struct_mutex.
  *
  * Frees the object
  */
diff --git a/drivers/gpu/drm/drm_info.c b/drivers/gpu/drm/drm_info.c
index 5d469b2f26f4..9ae353f4dd06 100644
--- a/drivers/gpu/drm/drm_info.c
+++ b/drivers/gpu/drm/drm_info.c
@@ -50,106 +50,24 @@ int drm_name_info(struct seq_file *m, void *data)
         struct drm_info_node *node = (struct drm_info_node *) m->private;
         struct drm_minor *minor = node->minor;
         struct drm_device *dev = minor->dev;
-        struct drm_master *master = minor->master;
-        if (!master)
-                return 0;
-
-        if (master->unique) {
-                seq_printf(m, "%s %s %s\n",
-                           dev->driver->name,
-                           dev_name(dev->dev), master->unique);
-        } else {
-                seq_printf(m, "%s %s\n",
-                           dev->driver->name, dev_name(dev->dev));
-        }
-        return 0;
-}
-
-/**
- * Called when "/proc/dri/.../vm" is read.
- *
- * Prints information about all mappings in drm_device::maplist.
- */
-int drm_vm_info(struct seq_file *m, void *data)
-{
-        struct drm_info_node *node = (struct drm_info_node *) m->private;
-        struct drm_device *dev = node->minor->dev;
-        struct drm_local_map *map;
-        struct drm_map_list *r_list;
-
-        /* Hardcoded from _DRM_FRAME_BUFFER,
-           _DRM_REGISTERS, _DRM_SHM, _DRM_AGP, and
-           _DRM_SCATTER_GATHER and _DRM_CONSISTENT */
-        const char *types[] = { "FB", "REG", "SHM", "AGP", "SG", "PCI" };
-        const char *type;
-        int i;
-
-        mutex_lock(&dev->struct_mutex);
-        seq_printf(m, "slot      offset       size type flags    address mtrr\n\n");
-        i = 0;
-        list_for_each_entry(r_list, &dev->maplist, head) {
-                map = r_list->map;
-                if (!map)
-                        continue;
-                if (map->type < 0 || map->type > 5)
-                        type = "??";
-                else
-                        type = types[map->type];
-
-                seq_printf(m, "%4d 0x%016llx 0x%08lx %4.4s  0x%02x 0x%08lx ",
-                           i,
-                           (unsigned long long)map->offset,
-                           map->size, type, map->flags,
-                           (unsigned long) r_list->user_token);
-                if (map->mtrr < 0)
-                        seq_printf(m, "none\n");
-                else
-                        seq_printf(m, "%4d\n", map->mtrr);
-                i++;
-        }
-        mutex_unlock(&dev->struct_mutex);
-        return 0;
-}
+        struct drm_master *master;
 
-/**
- * Called when "/proc/dri/.../bufs" is read.
- */
-int drm_bufs_info(struct seq_file *m, void *data)
-{
-        struct drm_info_node *node = (struct drm_info_node *) m->private;
-        struct drm_device *dev = node->minor->dev;
-        struct drm_device_dma *dma;
-        int i, seg_pages;
-
-        mutex_lock(&dev->struct_mutex);
-        dma = dev->dma;
-        if (!dma) {
-                mutex_unlock(&dev->struct_mutex);
-                return 0;
-        }
-
-        seq_printf(m, " o     size count  free   segs pages    kB\n\n");
-        for (i = 0; i <= DRM_MAX_ORDER; i++) {
-                if (dma->bufs[i].buf_count) {
-                        seg_pages = dma->bufs[i].seg_count * (1 << dma->bufs[i].page_order);
-                        seq_printf(m, "%2d %8d %5d %5d %5d %5d %5ld\n",
-                                   i,
-                                   dma->bufs[i].buf_size,
-                                   dma->bufs[i].buf_count,
-                                   0,
-                                   dma->bufs[i].seg_count,
-                                   seg_pages,
-                                   seg_pages * PAGE_SIZE / 1024);
-                }
-        }
-        seq_printf(m, "\n");
-        for (i = 0; i < dma->buf_count; i++) {
-                if (i && !(i % 32))
-                        seq_printf(m, "\n");
-                seq_printf(m, " %d", dma->buflist[i]->list);
-        }
+        mutex_lock(&dev->master_mutex);
+        master = dev->master;
+        if (!master)
+                goto out_unlock;
+
+        seq_printf(m, "%s", dev->driver->name);
+        if (dev->dev)
+                seq_printf(m, " dev=%s", dev_name(dev->dev));
+        if (master && master->unique)
+                seq_printf(m, " master=%s", master->unique);
+        if (dev->unique)
+                seq_printf(m, " unique=%s", dev->unique);
         seq_printf(m, "\n");
-        mutex_unlock(&dev->struct_mutex);
+out_unlock:
+        mutex_unlock(&dev->master_mutex);
+
         return 0;
 }
 
@@ -184,7 +102,7 @@ int drm_clients_info(struct seq_file *m, void *data)
                            task ? task->comm : "<unknown>",
                            pid_vnr(priv->pid),
                            priv->minor->index,
-                           priv->is_master ? 'y' : 'n',
+                           drm_is_current_master(priv) ? 'y' : 'n',
                            priv->authenticated ? 'y' : 'n',
                            from_kuid_munged(seq_user_ns(m), priv->uid),
                            priv->magic);
@@ -194,7 +112,6 @@ int drm_clients_info(struct seq_file *m, void *data)
         return 0;
 }
 
-
 static int drm_gem_one_name_info(int id, void *ptr, void *data)
 {
         struct drm_gem_object *obj = ptr;
diff --git a/drivers/gpu/drm/drm_internal.h b/drivers/gpu/drm/drm_internal.h
index 902cf6a15212..b86dc9b921a5 100644
--- a/drivers/gpu/drm/drm_internal.h
+++ b/drivers/gpu/drm/drm_internal.h
@@ -29,15 +29,9 @@ extern struct mutex drm_global_mutex;
 void drm_lastclose(struct drm_device *dev);
 
 /* drm_pci.c */
-int drm_pci_set_unique(struct drm_device *dev,
-                       struct drm_master *master,
-                       struct drm_unique *u);
 int drm_irq_by_busid(struct drm_device *dev, void *data,
                      struct drm_file *file_priv);
 
-/* drm_vm.c */
-int drm_vma_info(struct seq_file *m, void *data);
-
 /* drm_prime.c */
 int drm_prime_handle_to_fd_ioctl(struct drm_device *dev, void *data,
                                  struct drm_file *file_priv);
@@ -51,8 +45,6 @@ void drm_prime_remove_buf_handle_locked(struct drm_prime_file_private *prime_fpr
 
 /* drm_info.c */
 int drm_name_info(struct seq_file *m, void *data);
-int drm_vm_info(struct seq_file *m, void *data);
-int drm_bufs_info(struct seq_file *m, void *data);
 int drm_clients_info(struct seq_file *m, void* data);
 int drm_gem_name_info(struct seq_file *m, void *data);
 
@@ -67,6 +59,12 @@ int drm_getmagic(struct drm_device *dev, void *data,
                  struct drm_file *file_priv);
 int drm_authmagic(struct drm_device *dev, void *data,
                   struct drm_file *file_priv);
+int drm_setmaster_ioctl(struct drm_device *dev, void *data,
+                        struct drm_file *file_priv);
+int drm_dropmaster_ioctl(struct drm_device *dev, void *data,
+                         struct drm_file *file_priv);
+int drm_master_open(struct drm_file *file_priv);
+void drm_master_release(struct drm_file *file_priv);
 
 /* drm_sysfs.c */
 extern struct class *drm_class;
@@ -92,13 +90,6 @@ int drm_gem_open_ioctl(struct drm_device *dev, void *data,
 void drm_gem_open(struct drm_device *dev, struct drm_file *file_private);
 void drm_gem_release(struct drm_device *dev, struct drm_file *file_private);
 
-/* drm_drv.c */
-int drm_setmaster_ioctl(struct drm_device *dev, void *data,
-                        struct drm_file *file_priv);
-int drm_dropmaster_ioctl(struct drm_device *dev, void *data,
-                         struct drm_file *file_priv);
-struct drm_master *drm_master_create(struct drm_minor *minor);
-
 /* drm_debugfs.c */
 #if defined(CONFIG_DEBUG_FS)
 int drm_debugfs_init(struct drm_minor *minor, int minor_id,
diff --git a/drivers/gpu/drm/drm_ioctl.c b/drivers/gpu/drm/drm_ioctl.c
index b7a39771c152..33af4a5ddca1 100644
--- a/drivers/gpu/drm/drm_ioctl.c
+++ b/drivers/gpu/drm/drm_ioctl.c
@@ -30,6 +30,7 @@
 
 #include <drm/drmP.h>
 #include <drm/drm_core.h>
+#include <drm/drm_auth.h>
 #include "drm_legacy.h"
 #include "drm_internal.h"
 #include "drm_crtc_internal.h"
@@ -37,6 +38,64 @@
 #include <linux/pci.h>
 #include <linux/export.h>
 
+/**
+ * DOC: getunique and setversion story
+ *
+ * BEWARE THE DRAGONS! MIND THE TRAPDOORS!
+ *
+ * In an attempt to warn anyone else who's trying to figure out what's going
+ * on here, I'll try to summarize the story. First things first, let's clear up
+ * the names, because the kernel internals, libdrm and the ioctls are all named
+ * differently:
+ *
+ *  - GET_UNIQUE ioctl, implemented by drm_getunique is wrapped up in libdrm
+ *    through the drmGetBusid function.
+ *  - The libdrm drmSetBusid function is backed by the SET_UNIQUE ioctl. All
+ *    that code is nerved in the kernel with drm_invalid_op().
+ *  - The internal set_busid kernel functions and driver callbacks are
+ *    exclusively use by the SET_VERSION ioctl, because only drm 1.0 (which is
+ *    nerved) allowed userspace to set the busid through the above ioctl.
+ *  - Other ioctls and functions involved are named consistently.
+ *
+ * For anyone wondering what's the difference between drm 1.1 and 1.4: Correctly
+ * handling pci domains in the busid on ppc. Doing this correctly was only
+ * implemented in libdrm in 2010, hence can't be nerved yet. No one knows what's
+ * special with drm 1.2 and 1.3.
+ *
+ * Now the actual horror story of how device lookup in drm works. At large,
+ * there's 2 different ways, either by busid, or by device driver name.
+ *
+ * Opening by busid is fairly simple:
+ *
+ * 1. First call SET_VERSION to make sure pci domains are handled properly. As a
+ *    side-effect this fills out the unique name in the master structure.
+ * 2. Call GET_UNIQUE to read out the unique name from the master structure,
+ *    which matches the busid thanks to step 1. If it doesn't, proceed to try
+ *    the next device node.
+ *
+ * Opening by name is slightly different:
+ *
+ * 1. Directly call VERSION to get the version and to match against the driver
+ *    name returned by that ioctl. Note that SET_VERSION is not called, which
+ *    means the the unique name for the master node just opening is _not_ filled
+ *    out. This despite that with current drm device nodes are always bound to
+ *    one device, and can't be runtime assigned like with drm 1.0.
+ * 2. Match driver name. If it mismatches, proceed to the next device node.
+ * 3. Call GET_UNIQUE, and check whether the unique name has length zero (by
+ *    checking that the first byte in the string is 0). If that's not the case
+ *    libdrm skips and proceeds to the next device node. Probably this is just
+ *    copypasta from drm 1.0 times where a set unique name meant that the driver
+ *    was in use already, but that's just conjecture.
+ *
+ * Long story short: To keep the open by name logic working, GET_UNIQUE must
+ * _not_ return a unique string when SET_VERSION hasn't been called yet,
+ * otherwise libdrm breaks. Even when that unique string can't ever change, and
+ * is totally irrelevant for actually opening the device because runtime
+ * assignable device instances were only support in drm 1.0, which is long dead.
+ * But the libdrm code in drmOpenByName somehow survived, hence this can't be
+ * broken.
+ */
+
 static int drm_version(struct drm_device *dev, void *data,
                        struct drm_file *file_priv);
 
@@ -75,51 +134,6 @@ drm_unset_busid(struct drm_device *dev,
         master->unique_len = 0;
 }
 
-/*
- * Set the bus id.
- *
- * \param inode device inode.
- * \param file_priv DRM file private.
- * \param cmd command.
- * \param arg user argument, pointing to a drm_unique structure.
- * \return zero on success or a negative number on failure.
- *
- * Copies the bus id from userspace into drm_device::unique, and verifies that
- * it matches the device this DRM is attached to (EINVAL otherwise).  Deprecated
- * in interface version 1.1 and will return EBUSY when setversion has requested
- * version 1.1 or greater. Also note that KMS is all version 1.1 and later and
- * UMS was only ever supported on pci devices.
- */
-static int drm_setunique(struct drm_device *dev, void *data,
-                  struct drm_file *file_priv)
-{
-        struct drm_unique *u = data;
-        struct drm_master *master = file_priv->master;
-        int ret;
-
-        if (master->unique_len || master->unique)
-                return -EBUSY;
-
-        if (!u->unique_len || u->unique_len > 1024)
-                return -EINVAL;
-
-        if (drm_core_check_feature(dev, DRIVER_MODESET))
-                return 0;
-
-        if (WARN_ON(!dev->pdev))
-                return -EINVAL;
-
-        ret = drm_pci_set_unique(dev, master, u);
-        if (ret)
-                goto err;
-
-        return 0;
-
-err:
-        drm_unset_busid(dev, master);
-        return ret;
-}
-
 static int drm_set_busid(struct drm_device *dev, struct drm_file *file_priv)
 {
         struct drm_master *master = file_priv->master;
@@ -135,12 +149,7 @@ static int drm_set_busid(struct drm_device *dev, struct drm_file *file_priv)
                         return ret;
                 }
         } else {
-                if (WARN(dev->unique == NULL,
-                         "No drm_driver.set_busid() implementation provided by "
-                         "%ps. Use drm_dev_set_unique() to set the unique "
-                         "name explicitly.", dev->driver))
-                        return -EINVAL;
-
+                WARN_ON(!dev->unique);
                 master->unique = kstrdup(dev->unique, GFP_KERNEL);
                 if (master->unique)
                         master->unique_len = strlen(dev->unique);
@@ -473,7 +482,8 @@ int drm_ioctl_permit(u32 flags, struct drm_file *file_priv)
                 return -EACCES;
 
         /* MASTER is only for master or control clients */
-        if (unlikely((flags & DRM_MASTER) && !file_priv->is_master &&
+        if (unlikely((flags & DRM_MASTER) && 
+                     !drm_is_current_master(file_priv) &&
                      !drm_is_control_client(file_priv)))
                 return -EACCES;
 
@@ -504,7 +514,7 @@ static const struct drm_ioctl_desc drm_ioctls[] = {
         DRM_IOCTL_DEF(DRM_IOCTL_VERSION, drm_version,
                       DRM_UNLOCKED|DRM_RENDER_ALLOW|DRM_CONTROL_ALLOW),
         DRM_IOCTL_DEF(DRM_IOCTL_GET_UNIQUE, drm_getunique, 0),
-        DRM_IOCTL_DEF(DRM_IOCTL_GET_MAGIC, drm_getmagic, 0),
+        DRM_IOCTL_DEF(DRM_IOCTL_GET_MAGIC, drm_getmagic, DRM_UNLOCKED),
         DRM_IOCTL_DEF(DRM_IOCTL_IRQ_BUSID, drm_irq_by_busid, DRM_MASTER|DRM_ROOT_ONLY),
         DRM_IOCTL_DEF(DRM_IOCTL_GET_MAP, drm_legacy_getmap_ioctl, DRM_UNLOCKED),
         DRM_IOCTL_DEF(DRM_IOCTL_GET_CLIENT, drm_getclient, DRM_UNLOCKED),
@@ -513,10 +523,10 @@ static const struct drm_ioctl_desc drm_ioctls[] = {
         DRM_IOCTL_DEF(DRM_IOCTL_SET_CLIENT_CAP, drm_setclientcap, 0),
         DRM_IOCTL_DEF(DRM_IOCTL_SET_VERSION, drm_setversion, DRM_MASTER),
 
-        DRM_IOCTL_DEF(DRM_IOCTL_SET_UNIQUE, drm_setunique, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
+        DRM_IOCTL_DEF(DRM_IOCTL_SET_UNIQUE, drm_invalid_op, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
         DRM_IOCTL_DEF(DRM_IOCTL_BLOCK, drm_noop, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
         DRM_IOCTL_DEF(DRM_IOCTL_UNBLOCK, drm_noop, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
-        DRM_IOCTL_DEF(DRM_IOCTL_AUTH_MAGIC, drm_authmagic, DRM_AUTH|DRM_MASTER),
+        DRM_IOCTL_DEF(DRM_IOCTL_AUTH_MAGIC, drm_authmagic, DRM_AUTH|DRM_UNLOCKED|DRM_MASTER),
 
         DRM_IOCTL_DEF(DRM_IOCTL_ADD_MAP, drm_legacy_addmap_ioctl, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
         DRM_IOCTL_DEF(DRM_IOCTL_RM_MAP, drm_legacy_rmmap_ioctl, DRM_AUTH),
@@ -524,8 +534,8 @@ static const struct drm_ioctl_desc drm_ioctls[] = {
         DRM_IOCTL_DEF(DRM_IOCTL_SET_SAREA_CTX, drm_legacy_setsareactx, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
         DRM_IOCTL_DEF(DRM_IOCTL_GET_SAREA_CTX, drm_legacy_getsareactx, DRM_AUTH),
 
-        DRM_IOCTL_DEF(DRM_IOCTL_SET_MASTER, drm_setmaster_ioctl, DRM_ROOT_ONLY),
-        DRM_IOCTL_DEF(DRM_IOCTL_DROP_MASTER, drm_dropmaster_ioctl, DRM_ROOT_ONLY),
+        DRM_IOCTL_DEF(DRM_IOCTL_SET_MASTER, drm_setmaster_ioctl, DRM_UNLOCKED|DRM_ROOT_ONLY),
+        DRM_IOCTL_DEF(DRM_IOCTL_DROP_MASTER, drm_dropmaster_ioctl, DRM_UNLOCKED|DRM_ROOT_ONLY),
 
         DRM_IOCTL_DEF(DRM_IOCTL_ADD_CTX, drm_legacy_addctx, DRM_AUTH|DRM_ROOT_ONLY),
         DRM_IOCTL_DEF(DRM_IOCTL_RM_CTX, drm_legacy_rmctx, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
@@ -638,7 +648,7 @@ long drm_ioctl(struct file *filp,
         int retcode = -EINVAL;
         char stack_kdata[128];
         char *kdata = NULL;
-        unsigned int usize, asize, drv_size;
+        unsigned int in_size, out_size, drv_size, ksize;
         bool is_driver_ioctl;
 
         dev = file_priv->minor->dev;
@@ -661,9 +671,12 @@ long drm_ioctl(struct file *filp,
         }
 
         drv_size = _IOC_SIZE(ioctl->cmd);
-        usize = _IOC_SIZE(cmd);
-        asize = max(usize, drv_size);
-        cmd = ioctl->cmd;
+        out_size = in_size = _IOC_SIZE(cmd);
+        if ((cmd & ioctl->cmd & IOC_IN) == 0)
+                in_size = 0;
+        if ((cmd & ioctl->cmd & IOC_OUT) == 0)
+                out_size = 0;
+        ksize = max(max(in_size, out_size), drv_size);
 
         DRM_DEBUG("pid=%d, dev=0x%lx, auth=%d, %s\n",
                   task_pid_nr(current),
@@ -683,30 +696,24 @@ long drm_ioctl(struct file *filp,
         if (unlikely(retcode))
                 goto err_i1;
 
-        if (cmd & (IOC_IN | IOC_OUT)) {
-                if (asize <= sizeof(stack_kdata)) {
-                        kdata = stack_kdata;
-                } else {
-                        kdata = kmalloc(asize, GFP_KERNEL);
-                        if (!kdata) {
-                                retcode = -ENOMEM;
-                                goto err_i1;
-                        }
+        if (ksize <= sizeof(stack_kdata)) {
+                kdata = stack_kdata;
+        } else {
+                kdata = kmalloc(ksize, GFP_KERNEL);
+                if (!kdata) {
+                        retcode = -ENOMEM;
+                        goto err_i1;
                 }
-                if (asize > usize)
-                        memset(kdata + usize, 0, asize - usize);
         }
 
-        if (cmd & IOC_IN) {
-                if (copy_from_user(kdata, (void __user *)arg,
-                                   usize) != 0) {
-                        retcode = -EFAULT;
-                        goto err_i1;
-                }
-        } else if (cmd & IOC_OUT) {
-                memset(kdata, 0, usize);
+        if (copy_from_user(kdata, (void __user *)arg, in_size) != 0) {
+                retcode = -EFAULT;
+                goto err_i1;
         }
 
+        if (ksize > in_size)
+                memset(kdata + in_size, 0, ksize - in_size);
+
         /* Enforce sane locking for kms driver ioctls. Core ioctls are
          * too messy still. */
         if ((drm_core_check_feature(dev, DRIVER_MODESET) && is_driver_ioctl) ||
@@ -718,11 +725,8 @@ long drm_ioctl(struct file *filp,
                 mutex_unlock(&drm_global_mutex);
         }
 
-        if (cmd & IOC_OUT) {
-                if (copy_to_user((void __user *)arg, kdata,
-                                 usize) != 0)
-                        retcode = -EFAULT;
-        }
+        if (copy_to_user((void __user *)arg, kdata, out_size) != 0)
+                retcode = -EFAULT;
 
       err_i1:
         if (!ioctl)
@@ -749,7 +753,7 @@ EXPORT_SYMBOL(drm_ioctl);
  * shouldn't be used by any drivers.
  *
  * Returns:
- * True if the @nr corresponds to a DRM core ioctl numer, false otherwise.
+ * True if the @nr corresponds to a DRM core ioctl number, false otherwise.
  */
 bool drm_ioctl_flags(unsigned int nr, unsigned int *flags)
 {
diff --git a/drivers/gpu/drm/drm_irq.c b/drivers/gpu/drm/drm_irq.c
index 0fac801c18fe..77f357b2c386 100644
--- a/drivers/gpu/drm/drm_irq.c
+++ b/drivers/gpu/drm/drm_irq.c
@@ -42,10 +42,6 @@
 #include <linux/vgaarb.h>
 #include <linux/export.h>
 
-/* Access macro for slots in vblank timestamp ringbuffer. */
-#define vblanktimestamp(dev, pipe, count) \
-        ((dev)->vblank[pipe].time[(count) % DRM_VBLANKTIME_RBSIZE])
-
 /* Retry timestamp calculation up to 3 times to satisfy
  * drm_timestamp_precision before giving up.
  */
@@ -82,36 +78,18 @@ static void store_vblank(struct drm_device *dev, unsigned int pipe,
                          struct timeval *t_vblank, u32 last)
 {
         struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
-        u32 tslot;
 
         assert_spin_locked(&dev->vblank_time_lock);
 
         vblank->last = last;
 
-        /* All writers hold the spinlock, but readers are serialized by
-         * the latching of vblank->count below.
-         */
-        tslot = vblank->count + vblank_count_inc;
-        vblanktimestamp(dev, pipe, tslot) = *t_vblank;
-
-        /*
-         * vblank timestamp updates are protected on the write side with
-         * vblank_time_lock, but on the read side done locklessly using a
-         * sequence-lock on the vblank counter. Ensure correct ordering using
-         * memory barrriers. We need the barrier both before and also after the
-         * counter update to synchronize with the next timestamp write.
-         * The read-side barriers for this are in drm_vblank_count_and_time.
-         */
-        smp_wmb();
+        write_seqlock(&vblank->seqlock);
+        vblank->time = *t_vblank;
         vblank->count += vblank_count_inc;
-        smp_wmb();
+        write_sequnlock(&vblank->seqlock);
 }
 
-/**
- * drm_reset_vblank_timestamp - reset the last timestamp to the last vblank
- * @dev: DRM device
- * @pipe: index of CRTC for which to reset the timestamp
- *
+/*
  * Reset the stored timestamp for the current vblank count to correspond
  * to the last vblank occurred.
  *
@@ -155,11 +133,7 @@ static void drm_reset_vblank_timestamp(struct drm_device *dev, unsigned int pipe
         spin_unlock(&dev->vblank_time_lock);
 }
 
-/**
- * drm_update_vblank_count - update the master vblank counter
- * @dev: DRM device
- * @pipe: counter to update
- *
+/*
  * Call back into the driver to update the appropriate vblank counter
  * (specified by @pipe).  Deal with wraparound, if it occurred, and
  * update the last read value so we can deal with wraparound on the next
@@ -205,7 +179,7 @@ static void drm_update_vblank_count(struct drm_device *dev, unsigned int pipe,
                 const struct timeval *t_old;
                 u64 diff_ns;
 
-                t_old = &vblanktimestamp(dev, pipe, vblank->count);
+                t_old = &vblank->time;
                 diff_ns = timeval_to_ns(&t_vblank) - timeval_to_ns(t_old);
 
                 /*
@@ -239,49 +213,6 @@ static void drm_update_vblank_count(struct drm_device *dev, unsigned int pipe,
                 diff = 1;
         }
 
-        /*
-         * FIMXE: Need to replace this hack with proper seqlocks.
-         *
-         * Restrict the bump of the software vblank counter to a safe maximum
-         * value of +1 whenever there is the possibility that concurrent readers
-         * of vblank timestamps could be active at the moment, as the current
-         * implementation of the timestamp caching and updating is not safe
-         * against concurrent readers for calls to store_vblank() with a bump
-         * of anything but +1. A bump != 1 would very likely return corrupted
-         * timestamps to userspace, because the same slot in the cache could
-         * be concurrently written by store_vblank() and read by one of those
-         * readers without the read-retry logic detecting the collision.
-         *
-         * Concurrent readers can exist when we are called from the
-         * drm_vblank_off() or drm_vblank_on() functions and other non-vblank-
-         * irq callers. However, all those calls to us are happening with the
-         * vbl_lock locked to prevent drm_vblank_get(), so the vblank refcount
-         * can't increase while we are executing. Therefore a zero refcount at
-         * this point is safe for arbitrary counter bumps if we are called
-         * outside vblank irq, a non-zero count is not 100% safe. Unfortunately
-         * we must also accept a refcount of 1, as whenever we are called from
-         * drm_vblank_get() -> drm_vblank_enable() the refcount will be 1 and
-         * we must let that one pass through in order to not lose vblank counts
-         * during vblank irq off - which would completely defeat the whole
-         * point of this routine.
-         *
-         * Whenever we are called from vblank irq, we have to assume concurrent
-         * readers exist or can show up any time during our execution, even if
-         * the refcount is currently zero, as vblank irqs are usually only
-         * enabled due to the presence of readers, and because when we are called
-         * from vblank irq we can't hold the vbl_lock to protect us from sudden
-         * bumps in vblank refcount. Therefore also restrict bumps to +1 when
-         * called from vblank irq.
-         */
-        if ((diff > 1) && (atomic_read(&vblank->refcount) > 1 ||
-            (flags & DRM_CALLED_FROM_VBLIRQ))) {
-                DRM_DEBUG_VBL("clamping vblank bump to 1 on crtc %u: diffr=%u "
-                              "refcount %u, vblirq %u\n", pipe, diff,
-                              atomic_read(&vblank->refcount),
-                              (flags & DRM_CALLED_FROM_VBLIRQ) != 0);
-                diff = 1;
-        }
-
         DRM_DEBUG_VBL("updating vblank count on crtc %u:"
                       " current=%u, diff=%u, hw=%u hw_last=%u\n",
                       pipe, vblank->count, diff, cur_vblank, vblank->last);
@@ -303,6 +234,37 @@ static void drm_update_vblank_count(struct drm_device *dev, unsigned int pipe,
         store_vblank(dev, pipe, diff, &t_vblank, cur_vblank);
 }
 
+/**
+ * drm_accurate_vblank_count - retrieve the master vblank counter
+ * @crtc: which counter to retrieve
+ *
+ * This function is similar to @drm_crtc_vblank_count but this
+ * function interpolates to handle a race with vblank irq's.
+ *
+ * This is mostly useful for hardware that can obtain the scanout
+ * position, but doesn't have a frame counter.
+ */
+u32 drm_accurate_vblank_count(struct drm_crtc *crtc)
+{
+        struct drm_device *dev = crtc->dev;
+        unsigned int pipe = drm_crtc_index(crtc);
+        u32 vblank;
+        unsigned long flags;
+
+        WARN(!dev->driver->get_vblank_timestamp,
+             "This function requires support for accurate vblank timestamps.");
+
+        spin_lock_irqsave(&dev->vblank_time_lock, flags);
+
+        drm_update_vblank_count(dev, pipe, 0);
+        vblank = drm_vblank_count(dev, pipe);
+
+        spin_unlock_irqrestore(&dev->vblank_time_lock, flags);
+
+        return vblank;
+}
+EXPORT_SYMBOL(drm_accurate_vblank_count);
+
 /*
  * Disable vblank irq's on crtc, make sure that last vblank count
  * of hardware and corresponding consistent software vblank counter
@@ -417,6 +379,7 @@ int drm_vblank_init(struct drm_device *dev, unsigned int num_crtcs)
                 init_waitqueue_head(&vblank->queue);
                 setup_timer(&vblank->disable_timer, vblank_disable_fn,
                             (unsigned long)vblank);
+                seqlock_init(&vblank->seqlock);
         }
 
         DRM_INFO("Supports vblank timestamp caching Rev 2 (21.10.2013).\n");
@@ -569,7 +532,7 @@ int drm_irq_uninstall(struct drm_device *dev)
 
         /*
          * Wake up any waiters so they don't hang. This is just to paper over
-         * isssues for UMS drivers which aren't in full control of their
+         * issues for UMS drivers which aren't in full control of their
          * vblank/irq handling. KMS drivers must ensure that vblanks are all
          * disabled when uninstalling the irq handler.
          */
@@ -631,7 +594,7 @@ int drm_control(struct drm_device *dev, void *data,
                 return 0;
         if (drm_core_check_feature(dev, DRIVER_MODESET))
                 return 0;
-        /* UMS was only ever support on pci devices. */
+        /* UMS was only ever supported on pci devices. */
         if (WARN_ON(!dev->pdev))
                 return -EINVAL;
 
@@ -982,31 +945,24 @@ EXPORT_SYMBOL(drm_crtc_vblank_count);
  *
  * This is the legacy version of drm_crtc_vblank_count_and_time().
  */
-u32 drm_vblank_count_and_time(struct drm_device *dev, unsigned int pipe,
-                              struct timeval *vblanktime)
+static u32 drm_vblank_count_and_time(struct drm_device *dev, unsigned int pipe,
+                                     struct timeval *vblanktime)
 {
         struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
-        int count = DRM_TIMESTAMP_MAXRETRIES;
-        u32 cur_vblank;
+        u32 vblank_count;
+        unsigned int seq;
 
         if (WARN_ON(pipe >= dev->num_crtcs))
                 return 0;
 
-        /*
-         * Vblank timestamps are read lockless. To ensure consistency the vblank
-         * counter is rechecked and ordering is ensured using memory barriers.
-         * This works like a seqlock. The write-side barriers are in store_vblank.
-         */
         do {
-                cur_vblank = vblank->count;
-                smp_rmb();
-                *vblanktime = vblanktimestamp(dev, pipe, cur_vblank);
-                smp_rmb();
-        } while (cur_vblank != vblank->count && --count > 0);
+                seq = read_seqbegin(&vblank->seqlock);
+                vblank_count = vblank->count;
+                *vblanktime = vblank->time;
+        } while (read_seqretry(&vblank->seqlock, seq));
 
-        return cur_vblank;
+        return vblank_count;
 }
-EXPORT_SYMBOL(drm_vblank_count_and_time);
 
 /**
  * drm_crtc_vblank_count_and_time - retrieve "cooked" vblank counter value
@@ -1018,8 +974,6 @@ EXPORT_SYMBOL(drm_vblank_count_and_time);
  * vblank events since the system was booted, including lost events due to
  * modesetting activity. Returns corresponding system timestamp of the time
  * of the vblank interval that corresponds to the current vblank counter value.
- *
- * This is the native KMS version of drm_vblank_count_and_time().
  */
 u32 drm_crtc_vblank_count_and_time(struct drm_crtc *crtc,
                                    struct timeval *vblanktime)
@@ -1037,39 +991,11 @@ static void send_vblank_event(struct drm_device *dev,
         e->event.tv_sec = now->tv_sec;
         e->event.tv_usec = now->tv_usec;
 
-        drm_send_event_locked(dev, &e->base);
-
         trace_drm_vblank_event_delivered(e->base.pid, e->pipe,
                                          e->event.sequence);
-}
-
-/**
- * drm_arm_vblank_event - arm vblank event after pageflip
- * @dev: DRM device
- * @pipe: CRTC index
- * @e: the event to prepare to send
- *
- * A lot of drivers need to generate vblank events for the very next vblank
- * interrupt. For example when the page flip interrupt happens when the page
- * flip gets armed, but not when it actually executes within the next vblank
- * period. This helper function implements exactly the required vblank arming
- * behaviour.
- *
- * Caller must hold event lock. Caller must also hold a vblank reference for
- * the event @e, which will be dropped when the next vblank arrives.
- *
- * This is the legacy version of drm_crtc_arm_vblank_event().
- */
-void drm_arm_vblank_event(struct drm_device *dev, unsigned int pipe,
-                          struct drm_pending_vblank_event *e)
-{
-        assert_spin_locked(&dev->event_lock);
 
-        e->pipe = pipe;
-        e->event.sequence = drm_vblank_count(dev, pipe);
-        list_add_tail(&e->base.link, &dev->vblank_event_list);
+        drm_send_event_locked(dev, &e->base);
 }
-EXPORT_SYMBOL(drm_arm_vblank_event);
 
 /**
  * drm_crtc_arm_vblank_event - arm vblank event after pageflip
@@ -1084,32 +1010,35 @@ EXPORT_SYMBOL(drm_arm_vblank_event);
  *
  * Caller must hold event lock. Caller must also hold a vblank reference for
  * the event @e, which will be dropped when the next vblank arrives.
- *
- * This is the native KMS version of drm_arm_vblank_event().
  */
 void drm_crtc_arm_vblank_event(struct drm_crtc *crtc,
                                struct drm_pending_vblank_event *e)
 {
-        drm_arm_vblank_event(crtc->dev, drm_crtc_index(crtc), e);
+        struct drm_device *dev = crtc->dev;
+        unsigned int pipe = drm_crtc_index(crtc);
+
+        assert_spin_locked(&dev->event_lock);
+
+        e->pipe = pipe;
+        e->event.sequence = drm_vblank_count(dev, pipe);
+        list_add_tail(&e->base.link, &dev->vblank_event_list);
 }
 EXPORT_SYMBOL(drm_crtc_arm_vblank_event);
 
 /**
- * drm_send_vblank_event - helper to send vblank event after pageflip
- * @dev: DRM device
- * @pipe: CRTC index
+ * drm_crtc_send_vblank_event - helper to send vblank event after pageflip
+ * @crtc: the source CRTC of the vblank event
  * @e: the event to send
  *
  * Updates sequence # and timestamp on event, and sends it to userspace.
  * Caller must hold event lock.
- *
- * This is the legacy version of drm_crtc_send_vblank_event().
  */
-void drm_send_vblank_event(struct drm_device *dev, unsigned int pipe,
-                           struct drm_pending_vblank_event *e)
+void drm_crtc_send_vblank_event(struct drm_crtc *crtc,
+                                struct drm_pending_vblank_event *e)
 {
+        struct drm_device *dev = crtc->dev;
+        unsigned int seq, pipe = drm_crtc_index(crtc);
         struct timeval now;
-        unsigned int seq;
 
         if (dev->num_crtcs > 0) {
                 seq = drm_vblank_count_and_time(dev, pipe, &now);
@@ -1121,23 +1050,6 @@ void drm_send_vblank_event(struct drm_device *dev, unsigned int pipe,
         e->pipe = pipe;
         send_vblank_event(dev, e, seq, &now);
 }
-EXPORT_SYMBOL(drm_send_vblank_event);
-
-/**
- * drm_crtc_send_vblank_event - helper to send vblank event after pageflip
- * @crtc: the source CRTC of the vblank event
- * @e: the event to send
- *
- * Updates sequence # and timestamp on event, and sends it to userspace.
- * Caller must hold event lock.
- *
- * This is the native KMS version of drm_send_vblank_event().
- */
-void drm_crtc_send_vblank_event(struct drm_crtc *crtc,
-                                struct drm_pending_vblank_event *e)
-{
-        drm_send_vblank_event(crtc->dev, drm_crtc_index(crtc), e);
-}
 EXPORT_SYMBOL(drm_crtc_send_vblank_event);
 
 /**
@@ -1193,7 +1105,7 @@ static int drm_vblank_enable(struct drm_device *dev, unsigned int pipe)
  * Returns:
  * Zero on success or a negative error code on failure.
  */
-int drm_vblank_get(struct drm_device *dev, unsigned int pipe)
+static int drm_vblank_get(struct drm_device *dev, unsigned int pipe)
 {
         struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
         unsigned long irqflags;
@@ -1219,7 +1131,6 @@ int drm_vblank_get(struct drm_device *dev, unsigned int pipe)
 
         return ret;
 }
-EXPORT_SYMBOL(drm_vblank_get);
 
 /**
  * drm_crtc_vblank_get - get a reference count on vblank events
@@ -1228,8 +1139,6 @@ EXPORT_SYMBOL(drm_vblank_get);
  * Acquire a reference count on vblank events to avoid having them disabled
  * while in use.
  *
- * This is the native kms version of drm_vblank_get().
- *
  * Returns:
  * Zero on success or a negative error code on failure.
  */
@@ -1249,7 +1158,7 @@ EXPORT_SYMBOL(drm_crtc_vblank_get);
  *
  * This is the legacy version of drm_crtc_vblank_put().
  */
-void drm_vblank_put(struct drm_device *dev, unsigned int pipe)
+static void drm_vblank_put(struct drm_device *dev, unsigned int pipe)
 {
         struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
 
@@ -1270,7 +1179,6 @@ void drm_vblank_put(struct drm_device *dev, unsigned int pipe)
                                   jiffies + ((drm_vblank_offdelay * HZ)/1000));
         }
 }
-EXPORT_SYMBOL(drm_vblank_put);
 
 /**
  * drm_crtc_vblank_put - give up ownership of vblank events
@@ -1278,8 +1186,6 @@ EXPORT_SYMBOL(drm_vblank_put);
  *
  * Release ownership of a given vblank counter, turning off interrupts
  * if possible. Disable interrupts after drm_vblank_offdelay milliseconds.
- *
- * This is the native kms version of drm_vblank_put().
  */
 void drm_crtc_vblank_put(struct drm_crtc *crtc)
 {
@@ -1679,12 +1585,6 @@ static int drm_queue_vblank_event(struct drm_device *dev, unsigned int pipe,
 
         seq = drm_vblank_count_and_time(dev, pipe, &now);
 
-        if ((vblwait->request.type & _DRM_VBLANK_NEXTONMISS) &&
-            (seq - vblwait->request.sequence) <= (1 << 23)) {
-                vblwait->request.sequence = seq + 1;
-                vblwait->reply.sequence = vblwait->request.sequence;
-        }
-
         DRM_DEBUG("event on vblank count %d, current %d, crtc %u\n",
                   vblwait->request.sequence, seq, pipe);
 
@@ -1781,6 +1681,11 @@ int drm_wait_vblank(struct drm_device *dev, void *data,
                 goto done;
         }
 
+        if ((flags & _DRM_VBLANK_NEXTONMISS) &&
+            (seq - vblwait->request.sequence) <= (1 << 23)) {
+                vblwait->request.sequence = seq + 1;
+        }
+
         if (flags & _DRM_VBLANK_EVENT) {
                 /* must hold on to the vblank ref until the event fires
                  * drm_vblank_put will be called asynchronously
@@ -1788,14 +1693,8 @@ int drm_wait_vblank(struct drm_device *dev, void *data,
                 return drm_queue_vblank_event(dev, pipe, vblwait, file_priv);
         }
 
-        if ((flags & _DRM_VBLANK_NEXTONMISS) &&
-            (seq - vblwait->request.sequence) <= (1<<23)) {
-                vblwait->request.sequence = seq + 1;
-        }
-
         DRM_DEBUG("waiting on vblank count %d, crtc %u\n",
                   vblwait->request.sequence, pipe);
-        vblank->last_wait = vblwait->request.sequence;
         DRM_WAIT_ON(ret, vblank->queue, 3 * HZ,
                     (((drm_vblank_count(dev, pipe) -
                        vblwait->request.sequence) <= (1 << 23)) ||
diff --git a/drivers/gpu/drm/drm_legacy.h b/drivers/gpu/drm/drm_legacy.h
index d3b6ee357a2b..c6f422e879dd 100644
--- a/drivers/gpu/drm/drm_legacy.h
+++ b/drivers/gpu/drm/drm_legacy.h
@@ -88,14 +88,10 @@ struct drm_agp_mem {
         struct list_head head;
 };
 
-/*
- * Generic Userspace Locking-API
- */
-
-int drm_legacy_i_have_hw_lock(struct drm_device *d, struct drm_file *f);
+/* drm_lock.c */
 int drm_legacy_lock(struct drm_device *d, void *v, struct drm_file *f);
 int drm_legacy_unlock(struct drm_device *d, void *v, struct drm_file *f);
-int drm_legacy_lock_free(struct drm_lock_data *lock, unsigned int ctx);
+void drm_legacy_lock_release(struct drm_device *dev, struct file *filp);
 
 /* DMA support */
 int drm_legacy_dma_setup(struct drm_device *dev);
diff --git a/drivers/gpu/drm/drm_lock.c b/drivers/gpu/drm/drm_lock.c
index daa2ff12101b..48ac0ebbd663 100644
--- a/drivers/gpu/drm/drm_lock.c
+++ b/drivers/gpu/drm/drm_lock.c
@@ -41,6 +41,110 @@
 static int drm_lock_take(struct drm_lock_data *lock_data, unsigned int context);
 
 /**
+ * Take the heavyweight lock.
+ *
+ * \param lock lock pointer.
+ * \param context locking context.
+ * \return one if the lock is held, or zero otherwise.
+ *
+ * Attempt to mark the lock as held by the given context, via the \p cmpxchg instruction.
+ */
+static
+int drm_lock_take(struct drm_lock_data *lock_data,
+                  unsigned int context)
+{
+        unsigned int old, new, prev;
+        volatile unsigned int *lock = &lock_data->hw_lock->lock;
+
+        spin_lock_bh(&lock_data->spinlock);
+        do {
+                old = *lock;
+                if (old & _DRM_LOCK_HELD)
+                        new = old | _DRM_LOCK_CONT;
+                else {
+                        new = context | _DRM_LOCK_HELD |
+                                ((lock_data->user_waiters + lock_data->kernel_waiters > 1) ?
+                                 _DRM_LOCK_CONT : 0);
+                }
+                prev = cmpxchg(lock, old, new);
+        } while (prev != old);
+        spin_unlock_bh(&lock_data->spinlock);
+
+        if (_DRM_LOCKING_CONTEXT(old) == context) {
+                if (old & _DRM_LOCK_HELD) {
+                        if (context != DRM_KERNEL_CONTEXT) {
+                                DRM_ERROR("%d holds heavyweight lock\n",
+                                          context);
+                        }
+                        return 0;
+                }
+        }
+
+        if ((_DRM_LOCKING_CONTEXT(new)) == context && (new & _DRM_LOCK_HELD)) {
+                /* Have lock */
+                return 1;
+        }
+        return 0;
+}
+
+/**
+ * This takes a lock forcibly and hands it to context.  Should ONLY be used
+ * inside *_unlock to give lock to kernel before calling *_dma_schedule.
+ *
+ * \param dev DRM device.
+ * \param lock lock pointer.
+ * \param context locking context.
+ * \return always one.
+ *
+ * Resets the lock file pointer.
+ * Marks the lock as held by the given context, via the \p cmpxchg instruction.
+ */
+static int drm_lock_transfer(struct drm_lock_data *lock_data,
+                             unsigned int context)
+{
+        unsigned int old, new, prev;
+        volatile unsigned int *lock = &lock_data->hw_lock->lock;
+
+        lock_data->file_priv = NULL;
+        do {
+                old = *lock;
+                new = context | _DRM_LOCK_HELD;
+                prev = cmpxchg(lock, old, new);
+        } while (prev != old);
+        return 1;
+}
+
+static int drm_legacy_lock_free(struct drm_lock_data *lock_data,
+                                unsigned int context)
+{
+        unsigned int old, new, prev;
+        volatile unsigned int *lock = &lock_data->hw_lock->lock;
+
+        spin_lock_bh(&lock_data->spinlock);
+        if (lock_data->kernel_waiters != 0) {
+                drm_lock_transfer(lock_data, 0);
+                lock_data->idle_has_lock = 1;
+                spin_unlock_bh(&lock_data->spinlock);
+                return 1;
+        }
+        spin_unlock_bh(&lock_data->spinlock);
+
+        do {
+                old = *lock;
+                new = _DRM_LOCKING_CONTEXT(old);
+                prev = cmpxchg(lock, old, new);
+        } while (prev != old);
+
+        if (_DRM_LOCK_IS_HELD(old) && _DRM_LOCKING_CONTEXT(old) != context) {
+                DRM_ERROR("%d freed heavyweight lock held by %d\n",
+                          context, _DRM_LOCKING_CONTEXT(old));
+                return 1;
+        }
+        wake_up_interruptible(&lock_data->lock_queue);
+        return 0;
+}
+
+/**
  * Lock ioctl.
  *
  * \param inode device inode.
@@ -115,7 +219,7 @@ int drm_legacy_lock(struct drm_device *dev, void *data,
         /* don't set the block all signals on the master process for now 
          * really probably not the correct answer but lets us debug xkb
          * xserver for now */
-        if (!file_priv->is_master) {
+        if (!drm_is_current_master(file_priv)) {
                 dev->sigdata.context = lock->context;
                 dev->sigdata.lock = master->lock.hw_lock;
         }
@@ -165,120 +269,6 @@ int drm_legacy_unlock(struct drm_device *dev, void *data, struct drm_file *file_
 }
 
 /**
- * Take the heavyweight lock.
- *
- * \param lock lock pointer.
- * \param context locking context.
- * \return one if the lock is held, or zero otherwise.
- *
- * Attempt to mark the lock as held by the given context, via the \p cmpxchg instruction.
- */
-static
-int drm_lock_take(struct drm_lock_data *lock_data,
-                  unsigned int context)
-{
-        unsigned int old, new, prev;
-        volatile unsigned int *lock = &lock_data->hw_lock->lock;
-
-        spin_lock_bh(&lock_data->spinlock);
-        do {
-                old = *lock;
-                if (old & _DRM_LOCK_HELD)
-                        new = old | _DRM_LOCK_CONT;
-                else {
-                        new = context | _DRM_LOCK_HELD |
-                                ((lock_data->user_waiters + lock_data->kernel_waiters > 1) ?
-                                 _DRM_LOCK_CONT : 0);
-                }
-                prev = cmpxchg(lock, old, new);
-        } while (prev != old);
-        spin_unlock_bh(&lock_data->spinlock);
-
-        if (_DRM_LOCKING_CONTEXT(old) == context) {
-                if (old & _DRM_LOCK_HELD) {
-                        if (context != DRM_KERNEL_CONTEXT) {
-                                DRM_ERROR("%d holds heavyweight lock\n",
-                                          context);
-                        }
-                        return 0;
-                }
-        }
-
-        if ((_DRM_LOCKING_CONTEXT(new)) == context && (new & _DRM_LOCK_HELD)) {
-                /* Have lock */
-                return 1;
-        }
-        return 0;
-}
-
-/**
- * This takes a lock forcibly and hands it to context.  Should ONLY be used
- * inside *_unlock to give lock to kernel before calling *_dma_schedule.
- *
- * \param dev DRM device.
- * \param lock lock pointer.
- * \param context locking context.
- * \return always one.
- *
- * Resets the lock file pointer.
- * Marks the lock as held by the given context, via the \p cmpxchg instruction.
- */
-static int drm_lock_transfer(struct drm_lock_data *lock_data,
-                             unsigned int context)
-{
-        unsigned int old, new, prev;
-        volatile unsigned int *lock = &lock_data->hw_lock->lock;
-
-        lock_data->file_priv = NULL;
-        do {
-                old = *lock;
-                new = context | _DRM_LOCK_HELD;
-                prev = cmpxchg(lock, old, new);
-        } while (prev != old);
-        return 1;
-}
-
-/**
- * Free lock.
- *
- * \param dev DRM device.
- * \param lock lock.
- * \param context context.
- *
- * Resets the lock file pointer.
- * Marks the lock as not held, via the \p cmpxchg instruction. Wakes any task
- * waiting on the lock queue.
- */
-int drm_legacy_lock_free(struct drm_lock_data *lock_data, unsigned int context)
-{
-        unsigned int old, new, prev;
-        volatile unsigned int *lock = &lock_data->hw_lock->lock;
-
-        spin_lock_bh(&lock_data->spinlock);
-        if (lock_data->kernel_waiters != 0) {
-                drm_lock_transfer(lock_data, 0);
-                lock_data->idle_has_lock = 1;
-                spin_unlock_bh(&lock_data->spinlock);
-                return 1;
-        }
-        spin_unlock_bh(&lock_data->spinlock);
-
-        do {
-                old = *lock;
-                new = _DRM_LOCKING_CONTEXT(old);
-                prev = cmpxchg(lock, old, new);
-        } while (prev != old);
-
-        if (_DRM_LOCK_IS_HELD(old) && _DRM_LOCKING_CONTEXT(old) != context) {
-                DRM_ERROR("%d freed heavyweight lock held by %d\n",
-                          context, _DRM_LOCKING_CONTEXT(old));
-                return 1;
-        }
-        wake_up_interruptible(&lock_data->lock_queue);
-        return 0;
-}
-
-/**
  * This function returns immediately and takes the hw lock
  * with the kernel context if it is free, otherwise it gets the highest priority when and if
  * it is eventually released.
@@ -330,11 +320,27 @@ void drm_legacy_idlelock_release(struct drm_lock_data *lock_data)
 }
 EXPORT_SYMBOL(drm_legacy_idlelock_release);
 
-int drm_legacy_i_have_hw_lock(struct drm_device *dev,
-                              struct drm_file *file_priv)
+static int drm_legacy_i_have_hw_lock(struct drm_device *dev,
+                                     struct drm_file *file_priv)
 {
         struct drm_master *master = file_priv->master;
         return (file_priv->lock_count && master->lock.hw_lock &&
                 _DRM_LOCK_IS_HELD(master->lock.hw_lock->lock) &&
                 master->lock.file_priv == file_priv);
 }
+
+void drm_legacy_lock_release(struct drm_device *dev, struct file *filp)
+{
+        struct drm_file *file_priv = filp->private_data;
+
+        /* if the master has gone away we can't do anything with the lock */
+        if (!dev->master)
+                return;
+
+        if (drm_legacy_i_have_hw_lock(dev, file_priv)) {
+                DRM_DEBUG("File %p released, freeing lock for context %d\n",
+                          filp, _DRM_LOCKING_CONTEXT(file_priv->master->lock.hw_lock->lock));
+                drm_legacy_lock_free(&file_priv->master->lock,
+                                     _DRM_LOCKING_CONTEXT(file_priv->master->lock.hw_lock->lock));
+        }
+}
diff --git a/drivers/gpu/drm/drm_memory.c b/drivers/gpu/drm/drm_memory.c
index 87a8cb73366f..fc0ebd273ef8 100644
--- a/drivers/gpu/drm/drm_memory.c
+++ b/drivers/gpu/drm/drm_memory.c
@@ -44,7 +44,7 @@
 # include <asm/agp.h>
 #else
 # ifdef __powerpc__
-#  define PAGE_AGP      __pgprot(_PAGE_KERNEL | _PAGE_NO_CACHE)
+#  define PAGE_AGP      pgprot_noncached_wc(PAGE_KERNEL)
 # else
 #  define PAGE_AGP      PAGE_KERNEL
 # endif
diff --git a/drivers/gpu/drm/drm_mipi_dsi.c b/drivers/gpu/drm/drm_mipi_dsi.c
index f5d80839a90c..af0d471ee246 100644
--- a/drivers/gpu/drm/drm_mipi_dsi.c
+++ b/drivers/gpu/drm/drm_mipi_dsi.c
@@ -60,6 +60,21 @@ static int mipi_dsi_device_match(struct device *dev, struct device_driver *drv)
         return 0;
 }
 
+static int mipi_dsi_uevent(struct device *dev, struct kobj_uevent_env *env)
+{
+        struct mipi_dsi_device *dsi = to_mipi_dsi_device(dev);
+        int err;
+
+        err = of_device_uevent_modalias(dev, env);
+        if (err != -ENODEV)
+                return err;
+
+        add_uevent_var(env, "MODALIAS=%s%s", MIPI_DSI_MODULE_PREFIX,
+                       dsi->name);
+
+        return 0;
+}
+
 static const struct dev_pm_ops mipi_dsi_device_pm_ops = {
         .runtime_suspend = pm_generic_runtime_suspend,
         .runtime_resume = pm_generic_runtime_resume,
@@ -74,6 +89,7 @@ static const struct dev_pm_ops mipi_dsi_device_pm_ops = {
 static struct bus_type mipi_dsi_bus_type = {
         .name = "mipi-dsi",
         .match = mipi_dsi_device_match,
+        .uevent = mipi_dsi_uevent,
         .pm = &mipi_dsi_device_pm_ops,
 };
 
@@ -983,6 +999,28 @@ int mipi_dsi_dcs_set_tear_on(struct mipi_dsi_device *dsi,
 EXPORT_SYMBOL(mipi_dsi_dcs_set_tear_on);
 
 /**
+ * mipi_dsi_dcs_set_tear_scanline() - set the scanline to use as trigger for
+ *    the Tearing Effect output signal of the display module
+ * @dsi: DSI peripheral device
+ * @scanline: scanline to use as trigger
+ *
+ * Return: 0 on success or a negative error code on failure
+ */
+int mipi_dsi_dcs_set_tear_scanline(struct mipi_dsi_device *dsi, u16 scanline)
+{
+        u8 payload[3] = { MIPI_DCS_SET_TEAR_SCANLINE, scanline >> 8,
+                          scanline & 0xff };
+        ssize_t err;
+
+        err = mipi_dsi_generic_write(dsi, payload, sizeof(payload));
+        if (err < 0)
+                return err;
+
+        return 0;
+}
+EXPORT_SYMBOL(mipi_dsi_dcs_set_tear_scanline);
+
+/**
  * mipi_dsi_dcs_set_pixel_format() - sets the pixel format for the RGB image
  *    data used by the interface
  * @dsi: DSI peripheral device
diff --git a/drivers/gpu/drm/drm_mm.c b/drivers/gpu/drm/drm_mm.c
index 04de6fd88f8c..cb39f45d6a16 100644
--- a/drivers/gpu/drm/drm_mm.c
+++ b/drivers/gpu/drm/drm_mm.c
@@ -179,12 +179,14 @@ static void drm_mm_insert_helper(struct drm_mm_node *hole_node,
 int drm_mm_reserve_node(struct drm_mm *mm, struct drm_mm_node *node)
 {
         struct drm_mm_node *hole;
-        u64 end = node->start + node->size;
+        u64 end;
         u64 hole_start;
         u64 hole_end;
 
         BUG_ON(node == NULL);
 
+        end = node->start + node->size;
+
         /* Find the relevant hole to add our node to */
         drm_mm_for_each_hole(hole, mm, hole_start, hole_end) {
                 if (hole_start > node->start || hole_end < end)
diff --git a/drivers/gpu/drm/drm_modes.c b/drivers/gpu/drm/drm_modes.c
index e5e6f504d8cc..fc5040ae5f25 100644
--- a/drivers/gpu/drm/drm_modes.c
+++ b/drivers/gpu/drm/drm_modes.c
@@ -544,6 +544,7 @@ EXPORT_SYMBOL(drm_gtf_mode_complex);
  *
  * This function is to create the modeline based on the GTF algorithm.
  * Generalized Timing Formula is derived from:
+ *
  *      GTF Spreadsheet by Andy Morrish (1/5/97)
  *      available at http://www.vesa.org
  *
@@ -552,7 +553,8 @@ EXPORT_SYMBOL(drm_gtf_mode_complex);
  * I also refer to the function of fb_get_mode in the file of
  * drivers/video/fbmon.c
  *
- * Standard GTF parameters:
+ * Standard GTF parameters::
+ *
  *     M = 600
  *     C = 40
  *     K = 128
diff --git a/drivers/gpu/drm/drm_modeset_lock.c b/drivers/gpu/drm/drm_modeset_lock.c
index e3a4adf03e7b..61146f5b4f56 100644
--- a/drivers/gpu/drm/drm_modeset_lock.c
+++ b/drivers/gpu/drm/drm_modeset_lock.c
@@ -30,14 +30,14 @@
  *
  * As KMS moves toward more fine grained locking, and atomic ioctl where
  * userspace can indirectly control locking order, it becomes necessary
- * to use ww_mutex and acquire-contexts to avoid deadlocks.  But because
+ * to use &ww_mutex and acquire-contexts to avoid deadlocks.  But because
  * the locking is more distributed around the driver code, we want a bit
  * of extra utility/tracking out of our acquire-ctx.  This is provided
  * by drm_modeset_lock / drm_modeset_acquire_ctx.
  *
- * For basic principles of ww_mutex, see: Documentation/locking/ww-mutex-design.txt
+ * For basic principles of &ww_mutex, see: Documentation/locking/ww-mutex-design.txt
  *
- * The basic usage pattern is to:
+ * The basic usage pattern is to::
  *
  *     drm_modeset_acquire_init(&ctx)
  *     retry:
@@ -51,6 +51,13 @@
  *     ... do stuff ...
  *     drm_modeset_drop_locks(&ctx);
  *     drm_modeset_acquire_fini(&ctx);
+ *
+ *  On top of of these per-object locks using &ww_mutex there's also an overall
+ *  dev->mode_config.lock, for protecting everything else. Mostly this means
+ *  probe state of connectors, and preventing hotplug add/removal of connectors.
+ *
+ *  Finally there's a bunch of dedicated locks to protect drm core internal
+ *  lists and lookup data structures.
  */
 
 /**
diff --git a/drivers/gpu/drm/drm_pci.c b/drivers/gpu/drm/drm_pci.c
index 29d5a548d07a..b2f8f1062d5f 100644
--- a/drivers/gpu/drm/drm_pci.c
+++ b/drivers/gpu/drm/drm_pci.c
@@ -144,50 +144,6 @@ int drm_pci_set_busid(struct drm_device *dev, struct drm_master *master)
 }
 EXPORT_SYMBOL(drm_pci_set_busid);
 
-int drm_pci_set_unique(struct drm_device *dev,
-                       struct drm_master *master,
-                       struct drm_unique *u)
-{
-        int domain, bus, slot, func, ret;
-
-        master->unique_len = u->unique_len;
-        master->unique = kmalloc(master->unique_len + 1, GFP_KERNEL);
-        if (!master->unique) {
-                ret = -ENOMEM;
-                goto err;
-        }
-
-        if (copy_from_user(master->unique, u->unique, master->unique_len)) {
-                ret = -EFAULT;
-                goto err;
-        }
-
-        master->unique[master->unique_len] = '\0';
-
-        /* Return error if the busid submitted doesn't match the device's actual
-         * busid.
-         */
-        ret = sscanf(master->unique, "PCI:%d:%d:%d", &bus, &slot, &func);
-        if (ret != 3) {
-                ret = -EINVAL;
-                goto err;
-        }
-
-        domain = bus >> 8;
-        bus &= 0xff;
-
-        if ((domain != drm_get_pci_domain(dev)) ||
-            (bus != dev->pdev->bus->number) ||
-            (slot != PCI_SLOT(dev->pdev->devfn)) ||
-            (func != PCI_FUNC(dev->pdev->devfn))) {
-                ret = -EINVAL;
-                goto err;
-        }
-        return 0;
-err:
-        return ret;
-}
-
 static int drm_pci_irq_by_busid(struct drm_device *dev, struct drm_irq_busid *p)
 {
         if ((p->busnum >> 8) != drm_get_pci_domain(dev) ||
@@ -444,13 +400,6 @@ int drm_irq_by_busid(struct drm_device *dev, void *data,
 {
         return -EINVAL;
 }
-
-int drm_pci_set_unique(struct drm_device *dev,
-                       struct drm_master *master,
-                       struct drm_unique *u)
-{
-        return -EINVAL;
-}
 #endif
 
 EXPORT_SYMBOL(drm_pci_init);
diff --git a/drivers/gpu/drm/drm_plane_helper.c b/drivers/gpu/drm/drm_plane_helper.c
index 369d2898ff9e..16c4a7bd7465 100644
--- a/drivers/gpu/drm/drm_plane_helper.c
+++ b/drivers/gpu/drm/drm_plane_helper.c
@@ -115,6 +115,7 @@ static int get_connectors_for_crtc(struct drm_crtc *crtc,
  * @src: source coordinates in 16.16 fixed point
  * @dest: integer destination coordinates
  * @clip: integer clipping coordinates
+ * @rotation: plane rotation
  * @min_scale: minimum @src:@dest scaling factor in 16.16 fixed point
  * @max_scale: maximum @src:@dest scaling factor in 16.16 fixed point
  * @can_position: is it legal to position the plane such that it
@@ -134,16 +135,17 @@ static int get_connectors_for_crtc(struct drm_crtc *crtc,
  * Zero if update appears valid, error code on failure
  */
 int drm_plane_helper_check_update(struct drm_plane *plane,
-                                    struct drm_crtc *crtc,
-                                    struct drm_framebuffer *fb,
-                                    struct drm_rect *src,
-                                    struct drm_rect *dest,
-                                    const struct drm_rect *clip,
-                                    int min_scale,
-                                    int max_scale,
-                                    bool can_position,
-                                    bool can_update_disabled,
-                                    bool *visible)
+                                  struct drm_crtc *crtc,
+                                  struct drm_framebuffer *fb,
+                                  struct drm_rect *src,
+                                  struct drm_rect *dest,
+                                  const struct drm_rect *clip,
+                                  unsigned int rotation,
+                                  int min_scale,
+                                  int max_scale,
+                                  bool can_position,
+                                  bool can_update_disabled,
+                                  bool *visible)
 {
         int hscale, vscale;
 
@@ -163,6 +165,8 @@ int drm_plane_helper_check_update(struct drm_plane *plane,
                 return -EINVAL;
         }
 
+        drm_rect_rotate(src, fb->width << 16, fb->height << 16, rotation);
+
         /* Check scaling */
         hscale = drm_rect_calc_hscale(src, dest, min_scale, max_scale);
         vscale = drm_rect_calc_vscale(src, dest, min_scale, max_scale);
@@ -174,6 +178,9 @@ int drm_plane_helper_check_update(struct drm_plane *plane,
         }
 
         *visible = drm_rect_clip_scaled(src, dest, clip, hscale, vscale);
+
+        drm_rect_rotate_inv(src, fb->width << 16, fb->height << 16, rotation);
+
         if (!*visible)
                 /*
                  * Plane isn't visible; some drivers can handle this
@@ -219,10 +226,12 @@ EXPORT_SYMBOL(drm_plane_helper_check_update);
  *
  * Note that we make some assumptions about hardware limitations that may not be
  * true for all hardware --
- *   1) Primary plane cannot be repositioned.
- *   2) Primary plane cannot be scaled.
- *   3) Primary plane must cover the entire CRTC.
- *   4) Subpixel positioning is not supported.
+ *
+ * 1. Primary plane cannot be repositioned.
+ * 2. Primary plane cannot be scaled.
+ * 3. Primary plane must cover the entire CRTC.
+ * 4. Subpixel positioning is not supported.
+ *
  * Drivers for hardware that don't have these restrictions can provide their
  * own implementation rather than using this helper.
  *
@@ -265,6 +274,7 @@ int drm_primary_helper_update(struct drm_plane *plane, struct drm_crtc *crtc,
 
         ret = drm_plane_helper_check_update(plane, crtc, fb,
                                             &src, &dest, &clip,
+                                            BIT(DRM_ROTATE_0),
                                             DRM_PLANE_HELPER_NO_SCALING,
                                             DRM_PLANE_HELPER_NO_SCALING,
                                             false, false, &visible);
diff --git a/drivers/gpu/drm/drm_platform.c b/drivers/gpu/drm/drm_platform.c
index 644169e1a029..2c819ef90090 100644
--- a/drivers/gpu/drm/drm_platform.c
+++ b/drivers/gpu/drm/drm_platform.c
@@ -68,24 +68,6 @@ err_free:
         return ret;
 }
 
-int drm_platform_set_busid(struct drm_device *dev, struct drm_master *master)
-{
-        int id;
-
-        id = dev->platformdev->id;
-        if (id < 0)
-                id = 0;
-
-        master->unique = kasprintf(GFP_KERNEL, "platform:%s:%02d",
-                                                dev->platformdev->name, id);
-        if (!master->unique)
-                return -ENOMEM;
-
-        master->unique_len = strlen(master->unique);
-        return 0;
-}
-EXPORT_SYMBOL(drm_platform_set_busid);
-
 /**
  * drm_platform_init - Register a platform device with the DRM subsystem
  * @driver: DRM device driver
diff --git a/drivers/gpu/drm/drm_prime.c b/drivers/gpu/drm/drm_prime.c
index aab0f3f1f42d..780589b420a4 100644
--- a/drivers/gpu/drm/drm_prime.c
+++ b/drivers/gpu/drm/drm_prime.c
@@ -593,7 +593,7 @@ int drm_gem_prime_fd_to_handle(struct drm_device *dev,
                 get_dma_buf(dma_buf);
         }
 
-        /* drm_gem_handle_create_tail unlocks dev->object_name_lock. */
+        /* _handle_create_tail unconditionally unlocks dev->object_name_lock. */
         ret = drm_gem_handle_create_tail(file_priv, obj, handle);
         drm_gem_object_unreference_unlocked(obj);
         if (ret)
@@ -601,11 +601,10 @@ int drm_gem_prime_fd_to_handle(struct drm_device *dev,
 
         ret = drm_prime_add_buf_handle(&file_priv->prime,
                         dma_buf, *handle);
+        mutex_unlock(&file_priv->prime.lock);
         if (ret)
                 goto fail;
 
-        mutex_unlock(&file_priv->prime.lock);
-
         dma_buf_put(dma_buf);
 
         return 0;
@@ -615,11 +614,14 @@ fail:
          * to detach.. which seems ok..
          */
         drm_gem_handle_delete(file_priv, *handle);
+        dma_buf_put(dma_buf);
+        return ret;
+
 out_unlock:
         mutex_unlock(&dev->object_name_lock);
 out_put:
-        dma_buf_put(dma_buf);
         mutex_unlock(&file_priv->prime.lock);
+        dma_buf_put(dma_buf);
         return ret;
 }
 EXPORT_SYMBOL(drm_gem_prime_fd_to_handle);
diff --git a/drivers/gpu/drm/drm_probe_helper.c b/drivers/gpu/drm/drm_probe_helper.c
index 0329080d7f7c..a0df377d7d1c 100644
--- a/drivers/gpu/drm/drm_probe_helper.c
+++ b/drivers/gpu/drm/drm_probe_helper.c
@@ -82,13 +82,30 @@ drm_mode_validate_flag(const struct drm_display_mode *mode,
 
 static int drm_helper_probe_add_cmdline_mode(struct drm_connector *connector)
 {
+        struct drm_cmdline_mode *cmdline_mode;
         struct drm_display_mode *mode;
 
-        if (!connector->cmdline_mode.specified)
+        cmdline_mode = &connector->cmdline_mode;
+        if (!cmdline_mode->specified)
                 return 0;
 
+        /* Only add a GTF mode if we find no matching probed modes */
+        list_for_each_entry(mode, &connector->probed_modes, head) {
+                if (mode->hdisplay != cmdline_mode->xres ||
+                    mode->vdisplay != cmdline_mode->yres)
+                        continue;
+
+                if (cmdline_mode->refresh_specified) {
+                        /* The probed mode's vrefresh is set until later */
+                        if (drm_mode_vrefresh(mode) != cmdline_mode->refresh)
+                                continue;
+                }
+
+                return 0;
+        }
+
         mode = drm_mode_create_from_cmdline_mode(connector->dev,
-                                                 &connector->cmdline_mode);
+                                                 cmdline_mode);
         if (mode == NULL)
                 return 0;
 
diff --git a/drivers/gpu/drm/drm_scatter.c b/drivers/gpu/drm/drm_scatter.c
index 4f0f3b36d537..bf70431073f6 100644
--- a/drivers/gpu/drm/drm_scatter.c
+++ b/drivers/gpu/drm/drm_scatter.c
@@ -41,7 +41,7 @@
 static inline void *drm_vmalloc_dma(unsigned long size)
 {
 #if defined(__powerpc__) && defined(CONFIG_NOT_COHERENT_CACHE)
-        return __vmalloc(size, GFP_KERNEL, PAGE_KERNEL | _PAGE_NO_CACHE);
+        return __vmalloc(size, GFP_KERNEL, pgprot_noncached_wc(PAGE_KERNEL));
 #else
         return vmalloc_32(size);
 #endif
diff --git a/drivers/gpu/drm/drm_simple_kms_helper.c b/drivers/gpu/drm/drm_simple_kms_helper.c
new file mode 100644
index 000000000000..0db36d27e90b
--- /dev/null
+++ b/drivers/gpu/drm/drm_simple_kms_helper.c
@@ -0,0 +1,206 @@
+/*
+ * Copyright (C) 2016 Noralf Trønnes
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ */
+
+#include <drm/drmP.h>
+#include <drm/drm_atomic.h>
+#include <drm/drm_atomic_helper.h>
+#include <drm/drm_crtc_helper.h>
+#include <drm/drm_plane_helper.h>
+#include <drm/drm_simple_kms_helper.h>
+#include <linux/slab.h>
+
+/**
+ * DOC: overview
+ *
+ * This helper library provides helpers for drivers for simple display
+ * hardware.
+ *
+ * drm_simple_display_pipe_init() initializes a simple display pipeline
+ * which has only one full-screen scanout buffer feeding one output. The
+ * pipeline is represented by struct &drm_simple_display_pipe and binds
+ * together &drm_plane, &drm_crtc and &drm_encoder structures into one fixed
+ * entity. Some flexibility for code reuse is provided through a separately
+ * allocated &drm_connector object and supporting optional &drm_bridge
+ * encoder drivers.
+ */
+
+static const struct drm_encoder_funcs drm_simple_kms_encoder_funcs = {
+        .destroy = drm_encoder_cleanup,
+};
+
+static void drm_simple_kms_crtc_enable(struct drm_crtc *crtc)
+{
+        struct drm_simple_display_pipe *pipe;
+
+        pipe = container_of(crtc, struct drm_simple_display_pipe, crtc);
+        if (!pipe->funcs || !pipe->funcs->enable)
+                return;
+
+        pipe->funcs->enable(pipe, crtc->state);
+}
+
+static void drm_simple_kms_crtc_disable(struct drm_crtc *crtc)
+{
+        struct drm_simple_display_pipe *pipe;
+
+        pipe = container_of(crtc, struct drm_simple_display_pipe, crtc);
+        if (!pipe->funcs || !pipe->funcs->disable)
+                return;
+
+        pipe->funcs->disable(pipe);
+}
+
+static const struct drm_crtc_helper_funcs drm_simple_kms_crtc_helper_funcs = {
+        .disable = drm_simple_kms_crtc_disable,
+        .enable = drm_simple_kms_crtc_enable,
+};
+
+static const struct drm_crtc_funcs drm_simple_kms_crtc_funcs = {
+        .reset = drm_atomic_helper_crtc_reset,
+        .destroy = drm_crtc_cleanup,
+        .set_config = drm_atomic_helper_set_config,
+        .page_flip = drm_atomic_helper_page_flip,
+        .atomic_duplicate_state = drm_atomic_helper_crtc_duplicate_state,
+        .atomic_destroy_state = drm_atomic_helper_crtc_destroy_state,
+};
+
+static int drm_simple_kms_plane_atomic_check(struct drm_plane *plane,
+                                        struct drm_plane_state *plane_state)
+{
+        struct drm_rect src = {
+                .x1 = plane_state->src_x,
+                .y1 = plane_state->src_y,
+                .x2 = plane_state->src_x + plane_state->src_w,
+                .y2 = plane_state->src_y + plane_state->src_h,
+        };
+        struct drm_rect dest = {
+                .x1 = plane_state->crtc_x,
+                .y1 = plane_state->crtc_y,
+                .x2 = plane_state->crtc_x + plane_state->crtc_w,
+                .y2 = plane_state->crtc_y + plane_state->crtc_h,
+        };
+        struct drm_rect clip = { 0 };
+        struct drm_simple_display_pipe *pipe;
+        struct drm_crtc_state *crtc_state;
+        bool visible;
+        int ret;
+
+        pipe = container_of(plane, struct drm_simple_display_pipe, plane);
+        crtc_state = drm_atomic_get_existing_crtc_state(plane_state->state,
+                                                        &pipe->crtc);
+        if (crtc_state->enable != !!plane_state->crtc)
+                return -EINVAL; /* plane must match crtc enable state */
+
+        if (!crtc_state->enable)
+                return 0; /* nothing to check when disabling or disabled */
+
+        clip.x2 = crtc_state->adjusted_mode.hdisplay;
+        clip.y2 = crtc_state->adjusted_mode.vdisplay;
+        ret = drm_plane_helper_check_update(plane, &pipe->crtc,
+                                            plane_state->fb,
+                                            &src, &dest, &clip,
+                                            plane_state->rotation,
+                                            DRM_PLANE_HELPER_NO_SCALING,
+                                            DRM_PLANE_HELPER_NO_SCALING,
+                                            false, true, &visible);
+        if (ret)
+                return ret;
+
+        if (!visible)
+                return -EINVAL;
+
+        if (!pipe->funcs || !pipe->funcs->check)
+                return 0;
+
+        return pipe->funcs->check(pipe, plane_state, crtc_state);
+}
+
+static void drm_simple_kms_plane_atomic_update(struct drm_plane *plane,
+                                        struct drm_plane_state *pstate)
+{
+        struct drm_simple_display_pipe *pipe;
+
+        pipe = container_of(plane, struct drm_simple_display_pipe, plane);
+        if (!pipe->funcs || !pipe->funcs->update)
+                return;
+
+        pipe->funcs->update(pipe, pstate);
+}
+
+static const struct drm_plane_helper_funcs drm_simple_kms_plane_helper_funcs = {
+        .atomic_check = drm_simple_kms_plane_atomic_check,
+        .atomic_update = drm_simple_kms_plane_atomic_update,
+};
+
+static const struct drm_plane_funcs drm_simple_kms_plane_funcs = {
+        .update_plane           = drm_atomic_helper_update_plane,
+        .disable_plane          = drm_atomic_helper_disable_plane,
+        .destroy                = drm_plane_cleanup,
+        .reset                  = drm_atomic_helper_plane_reset,
+        .atomic_duplicate_state = drm_atomic_helper_plane_duplicate_state,
+        .atomic_destroy_state   = drm_atomic_helper_plane_destroy_state,
+};
+
+/**
+ * drm_simple_display_pipe_init - Initialize a simple display pipeline
+ * @dev: DRM device
+ * @pipe: simple display pipe object to initialize
+ * @funcs: callbacks for the display pipe (optional)
+ * @formats: array of supported formats (%DRM_FORMAT_*)
+ * @format_count: number of elements in @formats
+ * @connector: connector to attach and register
+ *
+ * Sets up a display pipeline which consist of a really simple
+ * plane-crtc-encoder pipe coupled with the provided connector.
+ * Teardown of a simple display pipe is all handled automatically by the drm
+ * core through calling drm_mode_config_cleanup(). Drivers afterwards need to
+ * release the memory for the structure themselves.
+ *
+ * Returns:
+ * Zero on success, negative error code on failure.
+ */
+int drm_simple_display_pipe_init(struct drm_device *dev,
+                        struct drm_simple_display_pipe *pipe,
+                        const struct drm_simple_display_pipe_funcs *funcs,
+                        const uint32_t *formats, unsigned int format_count,
+                        struct drm_connector *connector)
+{
+        struct drm_encoder *encoder = &pipe->encoder;
+        struct drm_plane *plane = &pipe->plane;
+        struct drm_crtc *crtc = &pipe->crtc;
+        int ret;
+
+        pipe->connector = connector;
+        pipe->funcs = funcs;
+
+        drm_plane_helper_add(plane, &drm_simple_kms_plane_helper_funcs);
+        ret = drm_universal_plane_init(dev, plane, 0,
+                                       &drm_simple_kms_plane_funcs,
+                                       formats, format_count,
+                                       DRM_PLANE_TYPE_PRIMARY, NULL);
+        if (ret)
+                return ret;
+
+        drm_crtc_helper_add(crtc, &drm_simple_kms_crtc_helper_funcs);
+        ret = drm_crtc_init_with_planes(dev, crtc, plane, NULL,
+                                        &drm_simple_kms_crtc_funcs, NULL);
+        if (ret)
+                return ret;
+
+        encoder->possible_crtcs = 1 << drm_crtc_index(crtc);
+        ret = drm_encoder_init(dev, encoder, &drm_simple_kms_encoder_funcs,
+                               DRM_MODE_ENCODER_NONE, NULL);
+        if (ret)
+                return ret;
+
+        return drm_mode_connector_attach_encoder(connector, encoder);
+}
+EXPORT_SYMBOL(drm_simple_display_pipe_init);
+
+MODULE_LICENSE("GPL");
diff --git a/drivers/gpu/drm/drm_sysfs.c b/drivers/gpu/drm/drm_sysfs.c
index fa7fadce8063..32dd821b7202 100644
--- a/drivers/gpu/drm/drm_sysfs.c
+++ b/drivers/gpu/drm/drm_sysfs.c
@@ -32,75 +32,6 @@ static struct device_type drm_sysfs_device_minor = {
 
 struct class *drm_class;
 
-/**
- * __drm_class_suspend - internal DRM class suspend routine
- * @dev: Linux device to suspend
- * @state: power state to enter
- *
- * Just figures out what the actual struct drm_device associated with
- * @dev is and calls its suspend hook, if present.
- */
-static int __drm_class_suspend(struct device *dev, pm_message_t state)
-{
-        if (dev->type == &drm_sysfs_device_minor) {
-                struct drm_minor *drm_minor = to_drm_minor(dev);
-                struct drm_device *drm_dev = drm_minor->dev;
-
-                if (drm_minor->type == DRM_MINOR_LEGACY &&
-                    !drm_core_check_feature(drm_dev, DRIVER_MODESET) &&
-                    drm_dev->driver->suspend)
-                        return drm_dev->driver->suspend(drm_dev, state);
-        }
-        return 0;
-}
-
-/**
- * drm_class_suspend - internal DRM class suspend hook. Simply calls
- * __drm_class_suspend() with the correct pm state.
- * @dev: Linux device to suspend
- */
-static int drm_class_suspend(struct device *dev)
-{
-        return __drm_class_suspend(dev, PMSG_SUSPEND);
-}
-
-/**
- * drm_class_freeze - internal DRM class freeze hook. Simply calls
- * __drm_class_suspend() with the correct pm state.
- * @dev: Linux device to freeze
- */
-static int drm_class_freeze(struct device *dev)
-{
-        return __drm_class_suspend(dev, PMSG_FREEZE);
-}
-
-/**
- * drm_class_resume - DRM class resume hook
- * @dev: Linux device to resume
- *
- * Just figures out what the actual struct drm_device associated with
- * @dev is and calls its resume hook, if present.
- */
-static int drm_class_resume(struct device *dev)
-{
-        if (dev->type == &drm_sysfs_device_minor) {
-                struct drm_minor *drm_minor = to_drm_minor(dev);
-                struct drm_device *drm_dev = drm_minor->dev;
-
-                if (drm_minor->type == DRM_MINOR_LEGACY &&
-                    !drm_core_check_feature(drm_dev, DRIVER_MODESET) &&
-                    drm_dev->driver->resume)
-                        return drm_dev->driver->resume(drm_dev);
-        }
-        return 0;
-}
-
-static const struct dev_pm_ops drm_class_dev_pm_ops = {
-        .suspend        = drm_class_suspend,
-        .resume         = drm_class_resume,
-        .freeze         = drm_class_freeze,
-};
-
 static char *drm_devnode(struct device *dev, umode_t *mode)
 {
         return kasprintf(GFP_KERNEL, "dri/%s", dev_name(dev));
@@ -131,8 +62,6 @@ int drm_sysfs_init(void)
         if (IS_ERR(drm_class))
                 return PTR_ERR(drm_class);
 
-        drm_class->pm = &drm_class_dev_pm_ops;
-
         err = class_create_file(drm_class, &class_attr_version.attr);
         if (err) {
                 class_destroy(drm_class);
diff --git a/drivers/gpu/drm/drm_vm.c b/drivers/gpu/drm/drm_vm.c
index ac9f4b3ec615..caa4e4ca616d 100644
--- a/drivers/gpu/drm/drm_vm.c
+++ b/drivers/gpu/drm/drm_vm.c
@@ -80,7 +80,7 @@ static pgprot_t drm_dma_prot(uint32_t map_type, struct vm_area_struct *vma)
         pgprot_t tmp = vm_get_page_prot(vma->vm_flags);
 
 #if defined(__powerpc__) && defined(CONFIG_NOT_COHERENT_CACHE)
-        tmp |= _PAGE_NO_CACHE;
+        tmp = pgprot_noncached_wc(tmp);
 #endif
         return tmp;
 }
@@ -593,7 +593,7 @@ static int drm_mmap_locked(struct file *filp, struct vm_area_struct *vma)
                          * pages and mappings in fault()
                          */
 #if defined(__powerpc__)
-                        pgprot_val(vma->vm_page_prot) |= _PAGE_NO_CACHE;
+                        vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
 #endif
                         vma->vm_ops = &drm_vm_ops;
                         break;
@@ -670,57 +670,3 @@ void drm_legacy_vma_flush(struct drm_device *dev)
                 kfree(vma);
         }
 }
-
-int drm_vma_info(struct seq_file *m, void *data)
-{
-        struct drm_info_node *node = (struct drm_info_node *) m->private;
-        struct drm_device *dev = node->minor->dev;
-        struct drm_vma_entry *pt;
-        struct vm_area_struct *vma;
-        unsigned long vma_count = 0;
-#if defined(__i386__)
-        unsigned int pgprot;
-#endif
-
-        mutex_lock(&dev->struct_mutex);
-        list_for_each_entry(pt, &dev->vmalist, head)
-                vma_count++;
-
-        seq_printf(m, "vma use count: %lu, high_memory = %pK, 0x%pK\n",
-                   vma_count, high_memory,
-                   (void *)(unsigned long)virt_to_phys(high_memory));
-
-        list_for_each_entry(pt, &dev->vmalist, head) {
-                vma = pt->vma;
-                if (!vma)
-                        continue;
-                seq_printf(m,
-                           "\n%5d 0x%pK-0x%pK %c%c%c%c%c%c 0x%08lx000",
-                           pt->pid,
-                           (void *)vma->vm_start, (void *)vma->vm_end,
-                           vma->vm_flags & VM_READ ? 'r' : '-',
-                           vma->vm_flags & VM_WRITE ? 'w' : '-',
-                           vma->vm_flags & VM_EXEC ? 'x' : '-',
-                           vma->vm_flags & VM_MAYSHARE ? 's' : 'p',
-                           vma->vm_flags & VM_LOCKED ? 'l' : '-',
-                           vma->vm_flags & VM_IO ? 'i' : '-',
-                           vma->vm_pgoff);
-
-#if defined(__i386__)
-                pgprot = pgprot_val(vma->vm_page_prot);
-                seq_printf(m, " %c%c%c%c%c%c%c%c%c",
-                           pgprot & _PAGE_PRESENT ? 'p' : '-',
-                           pgprot & _PAGE_RW ? 'w' : 'r',
-                           pgprot & _PAGE_USER ? 'u' : 's',
-                           pgprot & _PAGE_PWT ? 't' : 'b',
-                           pgprot & _PAGE_PCD ? 'u' : 'c',
-                           pgprot & _PAGE_ACCESSED ? 'a' : '-',
-                           pgprot & _PAGE_DIRTY ? 'd' : '-',
-                           pgprot & _PAGE_PSE ? 'm' : 'k',
-                           pgprot & _PAGE_GLOBAL ? 'g' : 'l');
-#endif
-                seq_printf(m, "\n");
-        }
-        mutex_unlock(&dev->struct_mutex);
-        return 0;
-}
diff --git a/drivers/gpu/drm/drm_vma_manager.c b/drivers/gpu/drm/drm_vma_manager.c
index 2f2ecde8285b..f306c8855978 100644
--- a/drivers/gpu/drm/drm_vma_manager.c
+++ b/drivers/gpu/drm/drm_vma_manager.c
@@ -127,6 +127,9 @@ EXPORT_SYMBOL(drm_vma_offset_manager_destroy);
  * used to implement weakly referenced lookups using kref_get_unless_zero().
  *
  * Example:
+ *
+ * ::
+ *
  *     drm_vma_offset_lock_lookup(mgr);
  *     node = drm_vma_offset_lookup_locked(mgr);
  *     if (node)
diff --git a/drivers/gpu/drm/etnaviv/etnaviv_drv.c b/drivers/gpu/drm/etnaviv/etnaviv_drv.c
index 3d4f56df8359..ffd1b32caa8d 100644
--- a/drivers/gpu/drm/etnaviv/etnaviv_drv.c
+++ b/drivers/gpu/drm/etnaviv/etnaviv_drv.c
@@ -91,10 +91,8 @@ static void load_gpu(struct drm_device *dev)
                         int ret;
 
                         ret = etnaviv_gpu_init(g);
-                        if (ret) {
-                                dev_err(g->dev, "hw init failed: %d\n", ret);
+                        if (ret)
                                 priv->gpu[i] = NULL;
-                        }
                 }
         }
 }
@@ -496,7 +494,6 @@ static struct drm_driver etnaviv_drm_driver = {
                                 DRIVER_RENDER,
         .open               = etnaviv_open,
         .preclose           = etnaviv_preclose,
-        .set_busid          = drm_platform_set_busid,
         .gem_free_object_unlocked = etnaviv_gem_free_object,
         .gem_vm_ops         = &vm_ops,
         .prime_handle_to_fd = drm_gem_prime_handle_to_fd,
diff --git a/drivers/gpu/drm/etnaviv/etnaviv_gem.c b/drivers/gpu/drm/etnaviv/etnaviv_gem.c
index 8c6f750634af..5ce3603e6eac 100644
--- a/drivers/gpu/drm/etnaviv/etnaviv_gem.c
+++ b/drivers/gpu/drm/etnaviv/etnaviv_gem.c
@@ -535,8 +535,7 @@ void etnaviv_gem_describe_objects(struct etnaviv_drm_private *priv,
 
 static void etnaviv_gem_shmem_release(struct etnaviv_gem_object *etnaviv_obj)
 {
-        if (etnaviv_obj->vaddr)
-                vunmap(etnaviv_obj->vaddr);
+        vunmap(etnaviv_obj->vaddr);
         put_pages(etnaviv_obj);
 }
 
@@ -670,9 +669,7 @@ static struct drm_gem_object *__etnaviv_gem_new(struct drm_device *dev,
         return obj;
 
 fail:
-        if (obj)
-                drm_gem_object_unreference_unlocked(obj);
-
+        drm_gem_object_unreference_unlocked(obj);
         return ERR_PTR(ret);
 }
 
@@ -916,15 +913,12 @@ int etnaviv_gem_new_userptr(struct drm_device *dev, struct drm_file *file,
         get_task_struct(current);
 
         ret = etnaviv_gem_obj_add(dev, &etnaviv_obj->base);
-        if (ret) {
-                drm_gem_object_unreference_unlocked(&etnaviv_obj->base);
-                return ret;
-        }
+        if (ret)
+                goto unreference;
 
         ret = drm_gem_handle_create(file, &etnaviv_obj->base, handle);
-
+unreference:
         /* drop reference from allocate - handle holds it now */
         drm_gem_object_unreference_unlocked(&etnaviv_obj->base);
-
         return ret;
 }
diff --git a/drivers/gpu/drm/etnaviv/etnaviv_gpu.c b/drivers/gpu/drm/etnaviv/etnaviv_gpu.c
index ff6aa5dfb2d7..87ef34150d46 100644
--- a/drivers/gpu/drm/etnaviv/etnaviv_gpu.c
+++ b/drivers/gpu/drm/etnaviv/etnaviv_gpu.c
@@ -487,6 +487,47 @@ static int etnaviv_hw_reset(struct etnaviv_gpu *gpu)
         return 0;
 }
 
+static void etnaviv_gpu_enable_mlcg(struct etnaviv_gpu *gpu)
+{
+        u32 pmc, ppc;
+
+        /* enable clock gating */
+        ppc = gpu_read(gpu, VIVS_PM_POWER_CONTROLS);
+        ppc |= VIVS_PM_POWER_CONTROLS_ENABLE_MODULE_CLOCK_GATING;
+
+        /* Disable stall module clock gating for 4.3.0.1 and 4.3.0.2 revs */
+        if (gpu->identity.revision == 0x4301 ||
+            gpu->identity.revision == 0x4302)
+                ppc |= VIVS_PM_POWER_CONTROLS_DISABLE_STALL_MODULE_CLOCK_GATING;
+
+        gpu_write(gpu, VIVS_PM_POWER_CONTROLS, ppc);
+
+        pmc = gpu_read(gpu, VIVS_PM_MODULE_CONTROLS);
+
+        /* Disable PA clock gating for GC400+ except for GC420 */
+        if (gpu->identity.model >= chipModel_GC400 &&
+            gpu->identity.model != chipModel_GC420)
+                pmc |= VIVS_PM_MODULE_CONTROLS_DISABLE_MODULE_CLOCK_GATING_PA;
+
+        /*
+         * Disable PE clock gating on revs < 5.0.0.0 when HZ is
+         * present without a bug fix.
+         */
+        if (gpu->identity.revision < 0x5000 &&
+            gpu->identity.minor_features0 & chipMinorFeatures0_HZ &&
+            !(gpu->identity.minor_features1 &
+              chipMinorFeatures1_DISABLE_PE_GATING))
+                pmc |= VIVS_PM_MODULE_CONTROLS_DISABLE_MODULE_CLOCK_GATING_PE;
+
+        if (gpu->identity.revision < 0x5422)
+                pmc |= BIT(15); /* Unknown bit */
+
+        pmc |= VIVS_PM_MODULE_CONTROLS_DISABLE_MODULE_CLOCK_GATING_RA_HZ;
+        pmc |= VIVS_PM_MODULE_CONTROLS_DISABLE_MODULE_CLOCK_GATING_RA_EZ;
+
+        gpu_write(gpu, VIVS_PM_MODULE_CONTROLS, pmc);
+}
+
 static void etnaviv_gpu_hw_init(struct etnaviv_gpu *gpu)
 {
         u16 prefetch;
@@ -506,6 +547,9 @@ static void etnaviv_gpu_hw_init(struct etnaviv_gpu *gpu)
                 gpu_write(gpu, VIVS_MC_DEBUG_MEMORY, mc_memory_debug);
         }
 
+        /* enable module-level clock gating */
+        etnaviv_gpu_enable_mlcg(gpu);
+
         /*
          * Update GPU AXI cache atttribute to "cacheable, no allocate".
          * This is necessary to prevent the iMX6 SoC locking up.
@@ -553,8 +597,10 @@ int etnaviv_gpu_init(struct etnaviv_gpu *gpu)
         bool mmuv2;
 
         ret = pm_runtime_get_sync(gpu->dev);
-        if (ret < 0)
+        if (ret < 0) {
+                dev_err(gpu->dev, "Failed to enable GPU power domain\n");
                 return ret;
+        }
 
         etnaviv_hw_identify(gpu);
 
@@ -591,8 +637,10 @@ int etnaviv_gpu_init(struct etnaviv_gpu *gpu)
         }
 
         ret = etnaviv_hw_reset(gpu);
-        if (ret)
+        if (ret) {
+                dev_err(gpu->dev, "GPU reset failed\n");
                 goto fail;
+        }
 
         /* Setup IOMMU.. eventually we will (I think) do this once per context
          * and have separate page tables per context.  For now, to keep things
@@ -610,12 +658,14 @@ int etnaviv_gpu_init(struct etnaviv_gpu *gpu)
         }
 
         if (!iommu) {
+                dev_err(gpu->dev, "Failed to allocate GPU IOMMU domain\n");
                 ret = -ENOMEM;
                 goto fail;
         }
 
         gpu->mmu = etnaviv_iommu_new(gpu, iommu, version);
         if (!gpu->mmu) {
+                dev_err(gpu->dev, "Failed to instantiate GPU IOMMU\n");
                 iommu_domain_free(iommu);
                 ret = -ENOMEM;
                 goto fail;
diff --git a/drivers/gpu/drm/etnaviv/etnaviv_gpu.h b/drivers/gpu/drm/etnaviv/etnaviv_gpu.h
index f5321e2f25ff..a69cdd526bf8 100644
--- a/drivers/gpu/drm/etnaviv/etnaviv_gpu.h
+++ b/drivers/gpu/drm/etnaviv/etnaviv_gpu.h
@@ -125,7 +125,7 @@ struct etnaviv_gpu {
         u32 completed_fence;
         u32 retired_fence;
         wait_queue_head_t fence_event;
-        unsigned int fence_context;
+        u64 fence_context;
         spinlock_t fence_spinlock;
 
         /* worker for handling active-list retiring: */
diff --git a/drivers/gpu/drm/etnaviv/state_hi.xml.h b/drivers/gpu/drm/etnaviv/state_hi.xml.h
index 6a7de5f1454a..807a3d9e0dd5 100644
--- a/drivers/gpu/drm/etnaviv/state_hi.xml.h
+++ b/drivers/gpu/drm/etnaviv/state_hi.xml.h
@@ -218,6 +218,13 @@ Copyright (C) 2015
 #define VIVS_PM_MODULE_CONTROLS_DISABLE_MODULE_CLOCK_GATING_FE  0x00000001
 #define VIVS_PM_MODULE_CONTROLS_DISABLE_MODULE_CLOCK_GATING_DE  0x00000002
 #define VIVS_PM_MODULE_CONTROLS_DISABLE_MODULE_CLOCK_GATING_PE  0x00000004
+#define VIVS_PM_MODULE_CONTROLS_DISABLE_MODULE_CLOCK_GATING_SH  0x00000008
+#define VIVS_PM_MODULE_CONTROLS_DISABLE_MODULE_CLOCK_GATING_PA  0x00000010
+#define VIVS_PM_MODULE_CONTROLS_DISABLE_MODULE_CLOCK_GATING_SE  0x00000020
+#define VIVS_PM_MODULE_CONTROLS_DISABLE_MODULE_CLOCK_GATING_RA  0x00000040
+#define VIVS_PM_MODULE_CONTROLS_DISABLE_MODULE_CLOCK_GATING_TX  0x00000080
+#define VIVS_PM_MODULE_CONTROLS_DISABLE_MODULE_CLOCK_GATING_RA_EZ       0x00010000
+#define VIVS_PM_MODULE_CONTROLS_DISABLE_MODULE_CLOCK_GATING_RA_HZ       0x00020000
 
 #define VIVS_PM_MODULE_STATUS                                   0x00000108
 #define VIVS_PM_MODULE_STATUS_MODULE_CLOCK_GATED_FE             0x00000001
diff --git a/drivers/gpu/drm/exynos/Kconfig b/drivers/gpu/drm/exynos/Kconfig
index d814b3048ee5..83f61c513b7e 100644
--- a/drivers/gpu/drm/exynos/Kconfig
+++ b/drivers/gpu/drm/exynos/Kconfig
@@ -2,10 +2,6 @@ config DRM_EXYNOS
         tristate "DRM Support for Samsung SoC EXYNOS Series"
         depends on OF && DRM && (ARCH_S3C64XX || ARCH_EXYNOS || ARCH_MULTIPLATFORM)
         select DRM_KMS_HELPER
-        select DRM_KMS_FB_HELPER
-        select FB_CFB_FILLRECT
-        select FB_CFB_COPYAREA
-        select FB_CFB_IMAGEBLIT
         select VIDEOMODE_HELPERS
         help
           Choose this option if you have a Samsung SoC EXYNOS chipset.
@@ -15,7 +11,7 @@ if DRM_EXYNOS
 
 config DRM_EXYNOS_IOMMU
         bool
-        depends on EXYNOS_IOMMU && ARM_DMA_USE_IOMMU
+        depends on EXYNOS_IOMMU
         default y
 
 comment "CRTCs"
diff --git a/drivers/gpu/drm/exynos/exynos_dp.c b/drivers/gpu/drm/exynos/exynos_dp.c
index 4c1fb3f8b5a6..4f0850585b8e 100644
--- a/drivers/gpu/drm/exynos/exynos_dp.c
+++ b/drivers/gpu/drm/exynos/exynos_dp.c
@@ -67,10 +67,10 @@ static int exynos_dp_poweroff(struct analogix_dp_plat_data *plat_data)
         return exynos_dp_crtc_clock_enable(plat_data, false);
 }
 
-static int exynos_dp_get_modes(struct analogix_dp_plat_data *plat_data)
+static int exynos_dp_get_modes(struct analogix_dp_plat_data *plat_data,
+                               struct drm_connector *connector)
 {
         struct exynos_dp_device *dp = to_dp(plat_data);
-        struct drm_connector *connector = dp->connector;
         struct drm_display_mode *mode;
         int num_modes = 0;
 
diff --git a/drivers/gpu/drm/exynos/exynos_drm_dpi.c b/drivers/gpu/drm/exynos/exynos_drm_dpi.c
index 5e38e749ac17..ad6b73c7fc59 100644
--- a/drivers/gpu/drm/exynos/exynos_drm_dpi.c
+++ b/drivers/gpu/drm/exynos/exynos_drm_dpi.c
@@ -93,17 +93,8 @@ static int exynos_dpi_get_modes(struct drm_connector *connector)
         return 0;
 }
 
-static struct drm_encoder *
-exynos_dpi_best_encoder(struct drm_connector *connector)
-{
-        struct exynos_dpi *ctx = connector_to_dpi(connector);
-
-        return &ctx->encoder;
-}
-
 static const struct drm_connector_helper_funcs exynos_dpi_connector_helper_funcs = {
         .get_modes = exynos_dpi_get_modes,
-        .best_encoder = exynos_dpi_best_encoder,
 };
 
 static int exynos_dpi_create_connector(struct drm_encoder *encoder)
diff --git a/drivers/gpu/drm/exynos/exynos_drm_drv.c b/drivers/gpu/drm/exynos/exynos_drm_drv.c
index 2dd820e23b0c..877d2efa28e2 100644
--- a/drivers/gpu/drm/exynos/exynos_drm_drv.c
+++ b/drivers/gpu/drm/exynos/exynos_drm_drv.c
@@ -159,12 +159,7 @@ static int exynos_drm_load(struct drm_device *dev, unsigned long flags)
         DRM_INFO("Exynos DRM: using %s device for DMA mapping operations\n",
                  dev_name(private->dma_dev));
 
-        /*
-         * create mapping to manage iommu table and set a pointer to iommu
-         * mapping structure to iommu_mapping of private data.
-         * also this iommu_mapping can be used to check if iommu is supported
-         * or not.
-         */
+        /* create common IOMMU mapping for all devices attached to Exynos DRM */
         ret = drm_create_iommu_mapping(dev);
         if (ret < 0) {
                 DRM_ERROR("failed to create iommu mapping.\n");
@@ -267,6 +262,8 @@ int exynos_atomic_commit(struct drm_device *dev, struct drm_atomic_state *state,
 {
         struct exynos_drm_private *priv = dev->dev_private;
         struct exynos_atomic_commit *commit;
+        struct drm_crtc *crtc;
+        struct drm_crtc_state *crtc_state;
         int i, ret;
 
         commit = kzalloc(sizeof(*commit), GFP_KERNEL);
@@ -288,10 +285,8 @@ int exynos_atomic_commit(struct drm_device *dev, struct drm_atomic_state *state,
         /* Wait until all affected CRTCs have completed previous commits and
          * mark them as pending.
          */
-        for (i = 0; i < dev->mode_config.num_crtc; ++i) {
-                if (state->crtcs[i])
-                        commit->crtcs |= 1 << drm_crtc_index(state->crtcs[i]);
-        }
+        for_each_crtc_in_state(state, crtc, crtc_state, i)
+                commit->crtcs |= drm_crtc_mask(crtc);
 
         wait_event(priv->wait, !commit_is_pending(priv, commit->crtcs));
 
@@ -299,7 +294,7 @@ int exynos_atomic_commit(struct drm_device *dev, struct drm_atomic_state *state,
         priv->pending |= commit->crtcs;
         spin_unlock(&priv->lock);
 
-        drm_atomic_helper_swap_state(dev, state);
+        drm_atomic_helper_swap_state(state, true);
 
         if (nonblock)
                 schedule_work(&commit->work);
@@ -407,7 +402,6 @@ static struct drm_driver exynos_drm_driver = {
         .preclose               = exynos_drm_preclose,
         .lastclose              = exynos_drm_lastclose,
         .postclose              = exynos_drm_postclose,
-        .set_busid              = drm_platform_set_busid,
         .get_vblank_counter     = drm_vblank_no_hw_counter,
         .enable_vblank          = exynos_drm_crtc_enable_vblank,
         .disable_vblank         = exynos_drm_crtc_disable_vblank,
diff --git a/drivers/gpu/drm/exynos/exynos_drm_drv.h b/drivers/gpu/drm/exynos/exynos_drm_drv.h
index cc33ec9296e7..b39d521f093d 100644
--- a/drivers/gpu/drm/exynos/exynos_drm_drv.h
+++ b/drivers/gpu/drm/exynos/exynos_drm_drv.h
@@ -224,8 +224,6 @@ struct exynos_drm_private {
         struct drm_property *plane_zpos_property;
 
         struct device *dma_dev;
-        unsigned long da_start;
-        unsigned long da_space_size;
         void *mapping;
 
         unsigned int pipe;
diff --git a/drivers/gpu/drm/exynos/exynos_drm_dsi.c b/drivers/gpu/drm/exynos/exynos_drm_dsi.c
index 601ecf8006a7..e07cb1fe4860 100644
--- a/drivers/gpu/drm/exynos/exynos_drm_dsi.c
+++ b/drivers/gpu/drm/exynos/exynos_drm_dsi.c
@@ -1566,17 +1566,8 @@ static int exynos_dsi_get_modes(struct drm_connector *connector)
         return 0;
 }
 
-static struct drm_encoder *
-exynos_dsi_best_encoder(struct drm_connector *connector)
-{
-        struct exynos_dsi *dsi = connector_to_dsi(connector);
-
-        return &dsi->encoder;
-}
-
 static const struct drm_connector_helper_funcs exynos_dsi_connector_helper_funcs = {
         .get_modes = exynos_dsi_get_modes,
-        .best_encoder = exynos_dsi_best_encoder,
 };
 
 static int exynos_dsi_create_connector(struct drm_encoder *encoder)
diff --git a/drivers/gpu/drm/exynos/exynos_drm_fbdev.c b/drivers/gpu/drm/exynos/exynos_drm_fbdev.c
index 67dcd6831291..fb49443bfd32 100644
--- a/drivers/gpu/drm/exynos/exynos_drm_fbdev.c
+++ b/drivers/gpu/drm/exynos/exynos_drm_fbdev.c
@@ -269,8 +269,7 @@ static void exynos_drm_fbdev_destroy(struct drm_device *dev,
         struct exynos_drm_gem *exynos_gem = exynos_fbd->exynos_gem;
         struct drm_framebuffer *fb;
 
-        if (exynos_gem->kvaddr)
-                vunmap(exynos_gem->kvaddr);
+        vunmap(exynos_gem->kvaddr);
 
         /* release drm framebuffer and real buffer */
         if (fb_helper->fb && fb_helper->fb->funcs) {
diff --git a/drivers/gpu/drm/exynos/exynos_drm_iommu.c b/drivers/gpu/drm/exynos/exynos_drm_iommu.c
index 7ca09ee19656..0f373702414e 100644
--- a/drivers/gpu/drm/exynos/exynos_drm_iommu.c
+++ b/drivers/gpu/drm/exynos/exynos_drm_iommu.c
@@ -14,13 +14,27 @@
 
 #include <linux/dma-mapping.h>
 #include <linux/iommu.h>
-#include <linux/kref.h>
-
-#include <asm/dma-iommu.h>
 
 #include "exynos_drm_drv.h"
 #include "exynos_drm_iommu.h"
 
+static inline int configure_dma_max_seg_size(struct device *dev)
+{
+        if (!dev->dma_parms)
+                dev->dma_parms = kzalloc(sizeof(*dev->dma_parms), GFP_KERNEL);
+        if (!dev->dma_parms)
+                return -ENOMEM;
+
+        dma_set_max_seg_size(dev, DMA_BIT_MASK(32));
+        return 0;
+}
+
+static inline void clear_dma_max_seg_size(struct device *dev)
+{
+        kfree(dev->dma_parms);
+        dev->dma_parms = NULL;
+}
+
 /*
  * drm_create_iommu_mapping - create a mapping structure
  *
@@ -28,38 +42,22 @@
  */
 int drm_create_iommu_mapping(struct drm_device *drm_dev)
 {
-        struct dma_iommu_mapping *mapping = NULL;
         struct exynos_drm_private *priv = drm_dev->dev_private;
 
-        if (!priv->da_start)
-                priv->da_start = EXYNOS_DEV_ADDR_START;
-        if (!priv->da_space_size)
-                priv->da_space_size = EXYNOS_DEV_ADDR_SIZE;
-
-        mapping = arm_iommu_create_mapping(&platform_bus_type, priv->da_start,
-                                                priv->da_space_size);
-
-        if (IS_ERR(mapping))
-                return PTR_ERR(mapping);
-
-        priv->mapping = mapping;
-
-        return 0;
+        return __exynos_iommu_create_mapping(priv, EXYNOS_DEV_ADDR_START,
+                                             EXYNOS_DEV_ADDR_SIZE);
 }
 
 /*
  * drm_release_iommu_mapping - release iommu mapping structure
  *
  * @drm_dev: DRM device
- *
- * if mapping->kref becomes 0 then all things related to iommu mapping
- * will be released
  */
 void drm_release_iommu_mapping(struct drm_device *drm_dev)
 {
         struct exynos_drm_private *priv = drm_dev->dev_private;
 
-        arm_iommu_release_mapping(priv->mapping);
+        __exynos_iommu_release_mapping(priv);
 }
 
 /*
@@ -77,25 +75,19 @@ int drm_iommu_attach_device(struct drm_device *drm_dev,
         struct exynos_drm_private *priv = drm_dev->dev_private;
         int ret;
 
-        if (!priv->mapping)
-                return 0;
-
-        subdrv_dev->dma_parms = devm_kzalloc(subdrv_dev,
-                                        sizeof(*subdrv_dev->dma_parms),
-                                        GFP_KERNEL);
-        if (!subdrv_dev->dma_parms)
-                return -ENOMEM;
-
-        dma_set_max_seg_size(subdrv_dev, 0xffffffffu);
-
-        if (subdrv_dev->archdata.mapping)
-                arm_iommu_detach_device(subdrv_dev);
+        if (get_dma_ops(priv->dma_dev) != get_dma_ops(subdrv_dev)) {
+                DRM_ERROR("Device %s lacks support for IOMMU\n",
+                          dev_name(subdrv_dev));
+                return -EINVAL;
+        }
 
-        ret = arm_iommu_attach_device(subdrv_dev, priv->mapping);
-        if (ret < 0) {
-                DRM_DEBUG_KMS("failed iommu attach.\n");
+        ret = configure_dma_max_seg_size(subdrv_dev);
+        if (ret)
                 return ret;
-        }
+
+        ret = __exynos_iommu_attach(priv, subdrv_dev);
+        if (ret)
+                clear_dma_max_seg_size(subdrv_dev);
 
         return 0;
 }
@@ -113,10 +105,7 @@ void drm_iommu_detach_device(struct drm_device *drm_dev,
                                 struct device *subdrv_dev)
 {
         struct exynos_drm_private *priv = drm_dev->dev_private;
-        struct dma_iommu_mapping *mapping = priv->mapping;
-
-        if (!mapping || !mapping->domain)
-                return;
 
-        arm_iommu_detach_device(subdrv_dev);
+        __exynos_iommu_detach(priv, subdrv_dev);
+        clear_dma_max_seg_size(subdrv_dev);
 }
diff --git a/drivers/gpu/drm/exynos/exynos_drm_iommu.h b/drivers/gpu/drm/exynos/exynos_drm_iommu.h
index 5ffebe02ee4d..c8de4913fdbe 100644
--- a/drivers/gpu/drm/exynos/exynos_drm_iommu.h
+++ b/drivers/gpu/drm/exynos/exynos_drm_iommu.h
@@ -17,6 +17,97 @@
 
 #ifdef CONFIG_DRM_EXYNOS_IOMMU
 
+#if defined(CONFIG_ARM_DMA_USE_IOMMU)
+#include <asm/dma-iommu.h>
+
+static inline int __exynos_iommu_create_mapping(struct exynos_drm_private *priv,
+                                        unsigned long start, unsigned long size)
+{
+        priv->mapping = arm_iommu_create_mapping(&platform_bus_type, start,
+                                                 size);
+        return IS_ERR(priv->mapping);
+}
+
+static inline void
+__exynos_iommu_release_mapping(struct exynos_drm_private *priv)
+{
+        arm_iommu_release_mapping(priv->mapping);
+}
+
+static inline int __exynos_iommu_attach(struct exynos_drm_private *priv,
+                                        struct device *dev)
+{
+        if (dev->archdata.mapping)
+                arm_iommu_detach_device(dev);
+
+        return arm_iommu_attach_device(dev, priv->mapping);
+}
+
+static inline void __exynos_iommu_detach(struct exynos_drm_private *priv,
+                                         struct device *dev)
+{
+        arm_iommu_detach_device(dev);
+}
+
+#elif defined(CONFIG_IOMMU_DMA)
+#include <linux/dma-iommu.h>
+
+static inline int __exynos_iommu_create_mapping(struct exynos_drm_private *priv,
+                                        unsigned long start, unsigned long size)
+{
+        struct iommu_domain *domain;
+        int ret;
+
+        domain = iommu_domain_alloc(priv->dma_dev->bus);
+        if (!domain)
+                return -ENOMEM;
+
+        ret = iommu_get_dma_cookie(domain);
+        if (ret)
+                goto free_domain;
+
+        ret = iommu_dma_init_domain(domain, start, size);
+        if (ret)
+                goto put_cookie;
+
+        priv->mapping = domain;
+        return 0;
+
+put_cookie:
+        iommu_put_dma_cookie(domain);
+free_domain:
+        iommu_domain_free(domain);
+        return ret;
+}
+
+static inline void __exynos_iommu_release_mapping(struct exynos_drm_private *priv)
+{
+        struct iommu_domain *domain = priv->mapping;
+
+        iommu_put_dma_cookie(domain);
+        iommu_domain_free(domain);
+        priv->mapping = NULL;
+}
+
+static inline int __exynos_iommu_attach(struct exynos_drm_private *priv,
+                                        struct device *dev)
+{
+        struct iommu_domain *domain = priv->mapping;
+
+        return iommu_attach_device(domain, dev);
+}
+
+static inline void __exynos_iommu_detach(struct exynos_drm_private *priv,
+                                         struct device *dev)
+{
+        struct iommu_domain *domain = priv->mapping;
+
+        iommu_detach_device(domain, dev);
+}
+#else
+#error Unsupported architecture and IOMMU/DMA-mapping glue code
+#endif
+
 int drm_create_iommu_mapping(struct drm_device *drm_dev);
 
 void drm_release_iommu_mapping(struct drm_device *drm_dev);
diff --git a/drivers/gpu/drm/exynos/exynos_drm_vidi.c b/drivers/gpu/drm/exynos/exynos_drm_vidi.c
index 608b0afa337f..e8f6c92b2a36 100644
--- a/drivers/gpu/drm/exynos/exynos_drm_vidi.c
+++ b/drivers/gpu/drm/exynos/exynos_drm_vidi.c
@@ -378,16 +378,8 @@ static int vidi_get_modes(struct drm_connector *connector)
         return drm_add_edid_modes(connector, edid);
 }
 
-static struct drm_encoder *vidi_best_encoder(struct drm_connector *connector)
-{
-        struct vidi_context *ctx = ctx_from_connector(connector);
-
-        return &ctx->encoder;
-}
-
 static const struct drm_connector_helper_funcs vidi_connector_helper_funcs = {
         .get_modes = vidi_get_modes,
-        .best_encoder = vidi_best_encoder,
 };
 
 static int vidi_create_connector(struct drm_encoder *encoder)
diff --git a/drivers/gpu/drm/exynos/exynos_hdmi.c b/drivers/gpu/drm/exynos/exynos_hdmi.c
index 58de5a430508..2275efe41acd 100644
--- a/drivers/gpu/drm/exynos/exynos_hdmi.c
+++ b/drivers/gpu/drm/exynos/exynos_hdmi.c
@@ -937,17 +937,9 @@ static int hdmi_mode_valid(struct drm_connector *connector,
         return MODE_OK;
 }
 
-static struct drm_encoder *hdmi_best_encoder(struct drm_connector *connector)
-{
-        struct hdmi_context *hdata = connector_to_hdmi(connector);
-
-        return &hdata->encoder;
-}
-
 static const struct drm_connector_helper_funcs hdmi_connector_helper_funcs = {
         .get_modes = hdmi_get_modes,
         .mode_valid = hdmi_mode_valid,
-        .best_encoder = hdmi_best_encoder,
 };
 
 static int hdmi_create_connector(struct drm_encoder *encoder)
@@ -1828,6 +1820,7 @@ static int hdmi_probe(struct platform_device *pdev)
                 DRM_ERROR("Failed to find ddc node in device tree\n");
                 return -ENODEV;
         }
+        of_node_put(dev->of_node);
 
 out_get_ddc_adpt:
         hdata->ddc_adpt = of_find_i2c_adapter_by_node(ddc_node);
@@ -1846,6 +1839,7 @@ out_get_ddc_adpt:
                 ret = -ENODEV;
                 goto err_ddc;
         }
+        of_node_put(dev->of_node);
 
 out_get_phy_port:
         if (hdata->drv_data->is_apb_phy) {
diff --git a/drivers/gpu/drm/fsl-dcu/Kconfig b/drivers/gpu/drm/fsl-dcu/Kconfig
index b9c714de6e40..14a72c4c496d 100644
--- a/drivers/gpu/drm/fsl-dcu/Kconfig
+++ b/drivers/gpu/drm/fsl-dcu/Kconfig
@@ -5,12 +5,7 @@ config DRM_FSL_DCU
         select BACKLIGHT_LCD_SUPPORT
         select DRM_KMS_HELPER
         select DRM_KMS_CMA_HELPER
-        select DRM_KMS_FB_HELPER
         select DRM_PANEL
-        select FB_SYS_FILLRECT
-        select FB_SYS_COPYAREA
-        select FB_SYS_IMAGEBLIT
-        select FB_SYS_FOPS
         select REGMAP_MMIO
         select VIDEOMODE_HELPERS
         help
diff --git a/drivers/gpu/drm/fsl-dcu/fsl_dcu_drm_crtc.c b/drivers/gpu/drm/fsl-dcu/fsl_dcu_drm_crtc.c
index 89c0084c2814..3371635cd4d7 100644
--- a/drivers/gpu/drm/fsl-dcu/fsl_dcu_drm_crtc.c
+++ b/drivers/gpu/drm/fsl-dcu/fsl_dcu_drm_crtc.c
@@ -22,20 +22,21 @@
 #include "fsl_dcu_drm_drv.h"
 #include "fsl_dcu_drm_plane.h"
 
-static void fsl_dcu_drm_crtc_atomic_begin(struct drm_crtc *crtc,
+static void fsl_dcu_drm_crtc_atomic_flush(struct drm_crtc *crtc,
                                           struct drm_crtc_state *old_crtc_state)
 {
-}
+        struct drm_pending_vblank_event *event = crtc->state->event;
 
-static int fsl_dcu_drm_crtc_atomic_check(struct drm_crtc *crtc,
-                                         struct drm_crtc_state *state)
-{
-        return 0;
-}
+        if (event) {
+                crtc->state->event = NULL;
 
-static void fsl_dcu_drm_crtc_atomic_flush(struct drm_crtc *crtc,
-                                          struct drm_crtc_state *old_crtc_state)
-{
+                spin_lock_irq(&crtc->dev->event_lock);
+                if (drm_crtc_vblank_get(crtc) == 0)
+                        drm_crtc_arm_vblank_event(crtc, event);
+                else
+                        drm_crtc_send_vblank_event(crtc, event);
+                spin_unlock_irq(&crtc->dev->event_lock);
+        }
 }
 
 static void fsl_dcu_drm_disable_crtc(struct drm_crtc *crtc)
@@ -43,6 +44,8 @@ static void fsl_dcu_drm_disable_crtc(struct drm_crtc *crtc)
         struct drm_device *dev = crtc->dev;
         struct fsl_dcu_drm_device *fsl_dev = dev->dev_private;
 
+        drm_crtc_vblank_off(crtc);
+
         regmap_update_bits(fsl_dev->regmap, DCU_DCU_MODE,
                            DCU_MODE_DCU_MODE_MASK,
                            DCU_MODE_DCU_MODE(DCU_MODE_OFF));
@@ -60,6 +63,8 @@ static void fsl_dcu_drm_crtc_enable(struct drm_crtc *crtc)
                            DCU_MODE_DCU_MODE(DCU_MODE_NORMAL));
         regmap_write(fsl_dev->regmap, DCU_UPDATE_MODE,
                      DCU_UPDATE_MODE_READREG);
+
+        drm_crtc_vblank_on(crtc);
 }
 
 static void fsl_dcu_drm_crtc_mode_set_nofb(struct drm_crtc *crtc)
@@ -117,8 +122,6 @@ static void fsl_dcu_drm_crtc_mode_set_nofb(struct drm_crtc *crtc)
 }
 
 static const struct drm_crtc_helper_funcs fsl_dcu_drm_crtc_helper_funcs = {
-        .atomic_begin = fsl_dcu_drm_crtc_atomic_begin,
-        .atomic_check = fsl_dcu_drm_crtc_atomic_check,
         .atomic_flush = fsl_dcu_drm_crtc_atomic_flush,
         .disable = fsl_dcu_drm_disable_crtc,
         .enable = fsl_dcu_drm_crtc_enable,
@@ -138,9 +141,10 @@ int fsl_dcu_drm_crtc_create(struct fsl_dcu_drm_device *fsl_dev)
 {
         struct drm_plane *primary;
         struct drm_crtc *crtc = &fsl_dev->crtc;
-        unsigned int i, j, reg_num;
         int ret;
 
+        fsl_dcu_drm_init_planes(fsl_dev->drm);
+
         primary = fsl_dcu_drm_primary_create_plane(fsl_dev->drm);
         if (!primary)
                 return -ENOMEM;
@@ -154,19 +158,5 @@ int fsl_dcu_drm_crtc_create(struct fsl_dcu_drm_device *fsl_dev)
 
         drm_crtc_helper_add(crtc, &fsl_dcu_drm_crtc_helper_funcs);
 
-        if (!strcmp(fsl_dev->soc->name, "ls1021a"))
-                reg_num = LS1021A_LAYER_REG_NUM;
-        else
-                reg_num = VF610_LAYER_REG_NUM;
-        for (i = 0; i < fsl_dev->soc->total_layer; i++) {
-                for (j = 1; j <= reg_num; j++)
-                        regmap_write(fsl_dev->regmap, DCU_CTRLDESCLN(i, j), 0);
-        }
-        regmap_update_bits(fsl_dev->regmap, DCU_DCU_MODE,
-                           DCU_MODE_DCU_MODE_MASK,
-                           DCU_MODE_DCU_MODE(DCU_MODE_OFF));
-        regmap_write(fsl_dev->regmap, DCU_UPDATE_MODE,
-                     DCU_UPDATE_MODE_READREG);
-
         return 0;
 }
diff --git a/drivers/gpu/drm/fsl-dcu/fsl_dcu_drm_drv.c b/drivers/gpu/drm/fsl-dcu/fsl_dcu_drm_drv.c
index dc723f7ead7d..7882387f9bff 100644
--- a/drivers/gpu/drm/fsl-dcu/fsl_dcu_drm_drv.c
+++ b/drivers/gpu/drm/fsl-dcu/fsl_dcu_drm_drv.c
@@ -11,6 +11,7 @@
 
 #include <linux/clk.h>
 #include <linux/clk-provider.h>
+#include <linux/console.h>
 #include <linux/io.h>
 #include <linux/mfd/syscon.h>
 #include <linux/mm.h>
@@ -22,6 +23,7 @@
 #include <linux/regmap.h>
 
 #include <drm/drmP.h>
+#include <drm/drm_atomic_helper.h>
 #include <drm/drm_crtc_helper.h>
 #include <drm/drm_fb_cma_helper.h>
 #include <drm/drm_gem_cma_helper.h>
@@ -42,10 +44,8 @@ static const struct regmap_config fsl_dcu_regmap_config = {
         .reg_bits = 32,
         .reg_stride = 4,
         .val_bits = 32,
-        .cache_type = REGCACHE_FLAT,
 
         .volatile_reg = fsl_dcu_drm_is_volatile_reg,
-        .max_register = 0x11fc,
 };
 
 static int fsl_dcu_drm_irq_init(struct drm_device *dev)
@@ -199,7 +199,7 @@ static struct drm_driver fsl_dcu_drm_driver = {
         .get_vblank_counter     = drm_vblank_no_hw_counter,
         .enable_vblank          = fsl_dcu_drm_enable_vblank,
         .disable_vblank         = fsl_dcu_drm_disable_vblank,
-        .gem_free_object        = drm_gem_cma_free_object,
+        .gem_free_object_unlocked = drm_gem_cma_free_object,
         .gem_vm_ops             = &drm_gem_cma_vm_ops,
         .prime_handle_to_fd     = drm_gem_prime_handle_to_fd,
         .prime_fd_to_handle     = drm_gem_prime_fd_to_handle,
@@ -229,11 +229,26 @@ static int fsl_dcu_drm_pm_suspend(struct device *dev)
         if (!fsl_dev)
                 return 0;
 
+        disable_irq(fsl_dev->irq);
         drm_kms_helper_poll_disable(fsl_dev->drm);
-        regcache_cache_only(fsl_dev->regmap, true);
-        regcache_mark_dirty(fsl_dev->regmap);
-        clk_disable(fsl_dev->clk);
-        clk_unprepare(fsl_dev->clk);
+
+        console_lock();
+        drm_fbdev_cma_set_suspend(fsl_dev->fbdev, 1);
+        console_unlock();
+
+        fsl_dev->state = drm_atomic_helper_suspend(fsl_dev->drm);
+        if (IS_ERR(fsl_dev->state)) {
+                console_lock();
+                drm_fbdev_cma_set_suspend(fsl_dev->fbdev, 0);
+                console_unlock();
+
+                drm_kms_helper_poll_enable(fsl_dev->drm);
+                enable_irq(fsl_dev->irq);
+                return PTR_ERR(fsl_dev->state);
+        }
+
+        clk_disable_unprepare(fsl_dev->pix_clk);
+        clk_disable_unprepare(fsl_dev->clk);
 
         return 0;
 }
@@ -246,21 +261,27 @@ static int fsl_dcu_drm_pm_resume(struct device *dev)
         if (!fsl_dev)
                 return 0;
 
-        ret = clk_enable(fsl_dev->clk);
+        ret = clk_prepare_enable(fsl_dev->clk);
         if (ret < 0) {
                 dev_err(dev, "failed to enable dcu clk\n");
-                clk_unprepare(fsl_dev->clk);
                 return ret;
         }
-        ret = clk_prepare(fsl_dev->clk);
+
+        ret = clk_prepare_enable(fsl_dev->pix_clk);
         if (ret < 0) {
-                dev_err(dev, "failed to prepare dcu clk\n");
+                dev_err(dev, "failed to enable pix clk\n");
                 return ret;
         }
 
+        fsl_dcu_drm_init_planes(fsl_dev->drm);
+        drm_atomic_helper_resume(fsl_dev->drm, fsl_dev->state);
+
+        console_lock();
+        drm_fbdev_cma_set_suspend(fsl_dev->fbdev, 0);
+        console_unlock();
+
         drm_kms_helper_poll_enable(fsl_dev->drm);
-        regcache_cache_only(fsl_dev->regmap, false);
-        regcache_sync(fsl_dev->regmap);
+        enable_irq(fsl_dev->irq);
 
         return 0;
 }
@@ -274,12 +295,14 @@ static const struct fsl_dcu_soc_data fsl_dcu_ls1021a_data = {
         .name = "ls1021a",
         .total_layer = 16,
         .max_layer = 4,
+        .layer_regs = LS1021A_LAYER_REG_NUM,
 };
 
 static const struct fsl_dcu_soc_data fsl_dcu_vf610_data = {
         .name = "vf610",
         .total_layer = 64,
         .max_layer = 6,
+        .layer_regs = VF610_LAYER_REG_NUM,
 };
 
 static const struct of_device_id fsl_dcu_of_match[] = {
diff --git a/drivers/gpu/drm/fsl-dcu/fsl_dcu_drm_drv.h b/drivers/gpu/drm/fsl-dcu/fsl_dcu_drm_drv.h
index c275f900ff23..3b371fe7491e 100644
--- a/drivers/gpu/drm/fsl-dcu/fsl_dcu_drm_drv.h
+++ b/drivers/gpu/drm/fsl-dcu/fsl_dcu_drm_drv.h
@@ -175,6 +175,7 @@ struct fsl_dcu_soc_data {
         unsigned int total_layer;
         /*max layer number DCU supported*/
         unsigned int max_layer;
+        unsigned int layer_regs;
 };
 
 struct fsl_dcu_drm_device {
@@ -193,6 +194,7 @@ struct fsl_dcu_drm_device {
         struct drm_encoder encoder;
         struct fsl_dcu_drm_connector connector;
         const struct fsl_dcu_soc_data *soc;
+        struct drm_atomic_state *state;
 };
 
 void fsl_dcu_fbdev_init(struct drm_device *dev);
diff --git a/drivers/gpu/drm/fsl-dcu/fsl_dcu_drm_kms.c b/drivers/gpu/drm/fsl-dcu/fsl_dcu_drm_kms.c
index c564ec612b59..d9d6cc1c8e39 100644
--- a/drivers/gpu/drm/fsl-dcu/fsl_dcu_drm_kms.c
+++ b/drivers/gpu/drm/fsl-dcu/fsl_dcu_drm_kms.c
@@ -37,23 +37,22 @@ int fsl_dcu_drm_modeset_init(struct fsl_dcu_drm_device *fsl_dev)
 
         ret = fsl_dcu_drm_crtc_create(fsl_dev);
         if (ret)
-                return ret;
+                goto err;
 
         ret = fsl_dcu_drm_encoder_create(fsl_dev, &fsl_dev->crtc);
         if (ret)
-                goto fail_encoder;
+                goto err;
 
-        ret = fsl_dcu_drm_connector_create(fsl_dev, &fsl_dev->encoder);
+        ret = fsl_dcu_create_outputs(fsl_dev);
         if (ret)
-                goto fail_connector;
+                goto err;
 
         drm_mode_config_reset(fsl_dev->drm);
         drm_kms_helper_poll_init(fsl_dev->drm);
 
         return 0;
-fail_encoder:
-        fsl_dev->crtc.funcs->destroy(&fsl_dev->crtc);
-fail_connector:
-        fsl_dev->encoder.funcs->destroy(&fsl_dev->encoder);
+
+err:
+        drm_mode_config_cleanup(fsl_dev->drm);
         return ret;
 }
diff --git a/drivers/gpu/drm/fsl-dcu/fsl_dcu_drm_output.h b/drivers/gpu/drm/fsl-dcu/fsl_dcu_drm_output.h
index 7093109fbc21..5a7b88e19e44 100644
--- a/drivers/gpu/drm/fsl-dcu/fsl_dcu_drm_output.h
+++ b/drivers/gpu/drm/fsl-dcu/fsl_dcu_drm_output.h
@@ -25,9 +25,8 @@ to_fsl_dcu_connector(struct drm_connector *con)
                      : NULL;
 }
 
-int fsl_dcu_drm_connector_create(struct fsl_dcu_drm_device *fsl_dev,
-                                 struct drm_encoder *encoder);
 int fsl_dcu_drm_encoder_create(struct fsl_dcu_drm_device *fsl_dev,
                                struct drm_crtc *crtc);
+int fsl_dcu_create_outputs(struct fsl_dcu_drm_device *fsl_dev);
 
 #endif /* __FSL_DCU_DRM_CONNECTOR_H__ */
diff --git a/drivers/gpu/drm/fsl-dcu/fsl_dcu_drm_plane.c b/drivers/gpu/drm/fsl-dcu/fsl_dcu_drm_plane.c
index 274558b3b32b..e50467a0deb0 100644
--- a/drivers/gpu/drm/fsl-dcu/fsl_dcu_drm_plane.c
+++ b/drivers/gpu/drm/fsl-dcu/fsl_dcu_drm_plane.c
@@ -217,6 +217,22 @@ static const u32 fsl_dcu_drm_plane_formats[] = {
         DRM_FORMAT_YUV422,
 };
 
+void fsl_dcu_drm_init_planes(struct drm_device *dev)
+{
+        struct fsl_dcu_drm_device *fsl_dev = dev->dev_private;
+        int i, j;
+
+        for (i = 0; i < fsl_dev->soc->total_layer; i++) {
+                for (j = 1; j <= fsl_dev->soc->layer_regs; j++)
+                        regmap_write(fsl_dev->regmap, DCU_CTRLDESCLN(i, j), 0);
+        }
+        regmap_update_bits(fsl_dev->regmap, DCU_DCU_MODE,
+                           DCU_MODE_DCU_MODE_MASK,
+                           DCU_MODE_DCU_MODE(DCU_MODE_OFF));
+        regmap_write(fsl_dev->regmap, DCU_UPDATE_MODE,
+                     DCU_UPDATE_MODE_READREG);
+}
+
 struct drm_plane *fsl_dcu_drm_primary_create_plane(struct drm_device *dev)
 {
         struct drm_plane *primary;
diff --git a/drivers/gpu/drm/fsl-dcu/fsl_dcu_drm_plane.h b/drivers/gpu/drm/fsl-dcu/fsl_dcu_drm_plane.h
index d657f088d859..8ee45f813ee8 100644
--- a/drivers/gpu/drm/fsl-dcu/fsl_dcu_drm_plane.h
+++ b/drivers/gpu/drm/fsl-dcu/fsl_dcu_drm_plane.h
@@ -12,6 +12,7 @@
 #ifndef __FSL_DCU_DRM_PLANE_H__
 #define __FSL_DCU_DRM_PLANE_H__
 
+void fsl_dcu_drm_init_planes(struct drm_device *dev);
 struct drm_plane *fsl_dcu_drm_primary_create_plane(struct drm_device *dev);
 
 #endif /* __FSL_DCU_DRM_PLANE_H__ */
diff --git a/drivers/gpu/drm/fsl-dcu/fsl_dcu_drm_rgb.c b/drivers/gpu/drm/fsl-dcu/fsl_dcu_drm_rgb.c
index 98c998da91eb..26edcc899712 100644
--- a/drivers/gpu/drm/fsl-dcu/fsl_dcu_drm_rgb.c
+++ b/drivers/gpu/drm/fsl-dcu/fsl_dcu_drm_rgb.c
@@ -10,6 +10,7 @@
  */
 
 #include <linux/backlight.h>
+#include <linux/of_graph.h>
 
 #include <drm/drmP.h>
 #include <drm/drm_atomic_helper.h>
@@ -102,14 +103,6 @@ static const struct drm_connector_funcs fsl_dcu_drm_connector_funcs = {
         .reset = drm_atomic_helper_connector_reset,
 };
 
-static struct drm_encoder *
-fsl_dcu_drm_connector_best_encoder(struct drm_connector *connector)
-{
-        struct fsl_dcu_drm_connector *fsl_con = to_fsl_dcu_connector(connector);
-
-        return fsl_con->encoder;
-}
-
 static int fsl_dcu_drm_connector_get_modes(struct drm_connector *connector)
 {
         struct fsl_dcu_drm_connector *fsl_connector;
@@ -136,17 +129,16 @@ static int fsl_dcu_drm_connector_mode_valid(struct drm_connector *connector,
 }
 
 static const struct drm_connector_helper_funcs connector_helper_funcs = {
-        .best_encoder = fsl_dcu_drm_connector_best_encoder,
         .get_modes = fsl_dcu_drm_connector_get_modes,
         .mode_valid = fsl_dcu_drm_connector_mode_valid,
 };
 
-int fsl_dcu_drm_connector_create(struct fsl_dcu_drm_device *fsl_dev,
-                                 struct drm_encoder *encoder)
+static int fsl_dcu_attach_panel(struct fsl_dcu_drm_device *fsl_dev,
+                                 struct drm_panel *panel)
 {
+        struct drm_encoder *encoder = &fsl_dev->encoder;
         struct drm_connector *connector = &fsl_dev->connector.base;
         struct drm_mode_config *mode_config = &fsl_dev->drm->mode_config;
-        struct device_node *panel_node;
         int ret;
 
         fsl_dev->connector.encoder = encoder;
@@ -170,21 +162,7 @@ int fsl_dcu_drm_connector_create(struct fsl_dcu_drm_device *fsl_dev,
                                       mode_config->dpms_property,
                                       DRM_MODE_DPMS_OFF);
 
-        panel_node = of_parse_phandle(fsl_dev->np, "fsl,panel", 0);
-        if (!panel_node) {
-                dev_err(fsl_dev->dev, "fsl,panel property not found\n");
-                ret = -ENODEV;
-                goto err_sysfs;
-        }
-
-        fsl_dev->connector.panel = of_drm_find_panel(panel_node);
-        if (!fsl_dev->connector.panel) {
-                ret = -EPROBE_DEFER;
-                goto err_panel;
-        }
-        of_node_put(panel_node);
-
-        ret = drm_panel_attach(fsl_dev->connector.panel, connector);
+        ret = drm_panel_attach(panel, connector);
         if (ret) {
                 dev_err(fsl_dev->dev, "failed to attach panel\n");
                 goto err_sysfs;
@@ -192,11 +170,62 @@ int fsl_dcu_drm_connector_create(struct fsl_dcu_drm_device *fsl_dev,
 
         return 0;
 
-err_panel:
-        of_node_put(panel_node);
 err_sysfs:
         drm_connector_unregister(connector);
 err_cleanup:
         drm_connector_cleanup(connector);
         return ret;
 }
+
+static int fsl_dcu_attach_endpoint(struct fsl_dcu_drm_device *fsl_dev,
+                                   const struct of_endpoint *ep)
+{
+        struct drm_bridge *bridge;
+        struct device_node *np;
+
+        np = of_graph_get_remote_port_parent(ep->local_node);
+
+        fsl_dev->connector.panel = of_drm_find_panel(np);
+        if (fsl_dev->connector.panel) {
+                of_node_put(np);
+                return fsl_dcu_attach_panel(fsl_dev, fsl_dev->connector.panel);
+        }
+
+        bridge = of_drm_find_bridge(np);
+        of_node_put(np);
+        if (!bridge)
+                return -ENODEV;
+
+        fsl_dev->encoder.bridge = bridge;
+        bridge->encoder = &fsl_dev->encoder;
+
+        return drm_bridge_attach(fsl_dev->drm, bridge);
+}
+
+int fsl_dcu_create_outputs(struct fsl_dcu_drm_device *fsl_dev)
+{
+        struct of_endpoint ep;
+        struct device_node *ep_node, *panel_node;
+        int ret;
+
+        /* This is for backward compatibility */
+        panel_node = of_parse_phandle(fsl_dev->np, "fsl,panel", 0);
+        if (panel_node) {
+                fsl_dev->connector.panel = of_drm_find_panel(panel_node);
+                of_node_put(panel_node);
+                if (!fsl_dev->connector.panel)
+                        return -EPROBE_DEFER;
+                return fsl_dcu_attach_panel(fsl_dev, fsl_dev->connector.panel);
+        }
+
+        ep_node = of_graph_get_next_endpoint(fsl_dev->np, NULL);
+        if (!ep_node)
+                return -ENODEV;
+
+        ret = of_graph_parse_endpoint(ep_node, &ep);
+        of_node_put(ep_node);
+        if (ret)
+                return -ENODEV;
+
+        return fsl_dcu_attach_endpoint(fsl_dev, &ep);
+}
diff --git a/drivers/gpu/drm/fsl-dcu/fsl_tcon.c b/drivers/gpu/drm/fsl-dcu/fsl_tcon.c
index bbe34f1c0505..bca09ea24632 100644
--- a/drivers/gpu/drm/fsl-dcu/fsl_tcon.c
+++ b/drivers/gpu/drm/fsl-dcu/fsl_tcon.c
@@ -92,6 +92,7 @@ struct fsl_tcon *fsl_tcon_init(struct device *dev)
                 goto err_node_put;
         }
 
+        of_node_put(np);
         clk_prepare_enable(tcon->ipg_clk);
 
         dev_info(dev, "Using TCON in bypass mode\n");
diff --git a/drivers/gpu/drm/gma500/Kconfig b/drivers/gpu/drm/gma500/Kconfig
index 17f928ec84ea..8906d67494fc 100644
--- a/drivers/gpu/drm/gma500/Kconfig
+++ b/drivers/gpu/drm/gma500/Kconfig
@@ -1,11 +1,7 @@
 config DRM_GMA500
         tristate "Intel GMA5/600 KMS Framebuffer"
         depends on DRM && PCI && X86
-        select FB_CFB_COPYAREA
-        select FB_CFB_FILLRECT
-        select FB_CFB_IMAGEBLIT
         select DRM_KMS_HELPER
-        select DRM_KMS_FB_HELPER
         select DRM_TTM
         # GMA500 depends on ACPI_VIDEO when ACPI is enabled, just like i915
         select ACPI_VIDEO if ACPI
diff --git a/drivers/gpu/drm/gma500/cdv_intel_hdmi.c b/drivers/gpu/drm/gma500/cdv_intel_hdmi.c
index 28f9d90988ff..563f193fcfac 100644
--- a/drivers/gpu/drm/gma500/cdv_intel_hdmi.c
+++ b/drivers/gpu/drm/gma500/cdv_intel_hdmi.c
@@ -246,8 +246,7 @@ static void cdv_hdmi_destroy(struct drm_connector *connector)
 {
         struct gma_encoder *gma_encoder = gma_attached_encoder(connector);
 
-        if (gma_encoder->i2c_bus)
-                psb_intel_i2c_destroy(gma_encoder->i2c_bus);
+        psb_intel_i2c_destroy(gma_encoder->i2c_bus);
         drm_connector_unregister(connector);
         drm_connector_cleanup(connector);
         kfree(connector);
diff --git a/drivers/gpu/drm/gma500/cdv_intel_lvds.c b/drivers/gpu/drm/gma500/cdv_intel_lvds.c
index 813ef23a8054..38dc89083148 100644
--- a/drivers/gpu/drm/gma500/cdv_intel_lvds.c
+++ b/drivers/gpu/drm/gma500/cdv_intel_lvds.c
@@ -444,8 +444,7 @@ static void cdv_intel_lvds_destroy(struct drm_connector *connector)
 {
         struct gma_encoder *gma_encoder = gma_attached_encoder(connector);
 
-        if (gma_encoder->i2c_bus)
-                psb_intel_i2c_destroy(gma_encoder->i2c_bus);
+        psb_intel_i2c_destroy(gma_encoder->i2c_bus);
         drm_connector_unregister(connector);
         drm_connector_cleanup(connector);
         kfree(connector);
@@ -780,12 +779,10 @@ out:
 failed_find:
         mutex_unlock(&dev->mode_config.mutex);
         printk(KERN_ERR "Failed find\n");
-        if (gma_encoder->ddc_bus)
-                psb_intel_i2c_destroy(gma_encoder->ddc_bus);
+        psb_intel_i2c_destroy(gma_encoder->ddc_bus);
 failed_ddc:
         printk(KERN_ERR "Failed DDC\n");
-        if (gma_encoder->i2c_bus)
-                psb_intel_i2c_destroy(gma_encoder->i2c_bus);
+        psb_intel_i2c_destroy(gma_encoder->i2c_bus);
 failed_blc_i2c:
         printk(KERN_ERR "Failed BLC\n");
         drm_encoder_cleanup(encoder);
diff --git a/drivers/gpu/drm/gma500/framebuffer.c b/drivers/gpu/drm/gma500/framebuffer.c
index 7440bf90ac9c..0fcdce0817de 100644
--- a/drivers/gpu/drm/gma500/framebuffer.c
+++ b/drivers/gpu/drm/gma500/framebuffer.c
@@ -184,12 +184,6 @@ static int psbfb_mmap(struct fb_info *info, struct vm_area_struct *vma)
         return 0;
 }
 
-static int psbfb_ioctl(struct fb_info *info, unsigned int cmd,
-                                                unsigned long arg)
-{
-        return -ENOTTY;
-}
-
 static struct fb_ops psbfb_ops = {
         .owner = THIS_MODULE,
         .fb_check_var = drm_fb_helper_check_var,
@@ -201,7 +195,6 @@ static struct fb_ops psbfb_ops = {
         .fb_imageblit = drm_fb_helper_cfb_imageblit,
         .fb_mmap = psbfb_mmap,
         .fb_sync = psbfb_sync,
-        .fb_ioctl = psbfb_ioctl,
 };
 
 static struct fb_ops psbfb_roll_ops = {
@@ -215,7 +208,6 @@ static struct fb_ops psbfb_roll_ops = {
         .fb_imageblit = drm_fb_helper_cfb_imageblit,
         .fb_pan_display = psbfb_pan,
         .fb_mmap = psbfb_mmap,
-        .fb_ioctl = psbfb_ioctl,
 };
 
 static struct fb_ops psbfb_unaccel_ops = {
@@ -228,7 +220,6 @@ static struct fb_ops psbfb_unaccel_ops = {
         .fb_copyarea = drm_fb_helper_cfb_copyarea,
         .fb_imageblit = drm_fb_helper_cfb_imageblit,
         .fb_mmap = psbfb_mmap,
-        .fb_ioctl = psbfb_ioctl,
 };
 
 /**
diff --git a/drivers/gpu/drm/gma500/gma_display.c b/drivers/gpu/drm/gma500/gma_display.c
index c95406e6f44d..1a1cf7a3b5ef 100644
--- a/drivers/gpu/drm/gma500/gma_display.c
+++ b/drivers/gpu/drm/gma500/gma_display.c
@@ -175,20 +175,21 @@ void gma_crtc_load_lut(struct drm_crtc *crtc)
         }
 }
 
-void gma_crtc_gamma_set(struct drm_crtc *crtc, u16 *red, u16 *green, u16 *blue,
-                        u32 start, u32 size)
+int gma_crtc_gamma_set(struct drm_crtc *crtc, u16 *red, u16 *green, u16 *blue,
+                       u32 size)
 {
         struct gma_crtc *gma_crtc = to_gma_crtc(crtc);
         int i;
-        int end = (start + size > 256) ? 256 : start + size;
 
-        for (i = start; i < end; i++) {
+        for (i = 0; i < size; i++) {
                 gma_crtc->lut_r[i] = red[i] >> 8;
                 gma_crtc->lut_g[i] = green[i] >> 8;
                 gma_crtc->lut_b[i] = blue[i] >> 8;
         }
 
         gma_crtc_load_lut(crtc);
+
+        return 0;
 }
 
 /**
@@ -281,7 +282,7 @@ void gma_crtc_dpms(struct drm_crtc *crtc, int mode)
                 REG_WRITE(VGACNTRL, VGA_DISP_DISABLE);
 
                 /* Turn off vblank interrupts */
-                drm_vblank_off(dev, pipe);
+                drm_crtc_vblank_off(crtc);
 
                 /* Wait for vblank for the disable to take effect */
                 gma_wait_for_vblank(dev);
diff --git a/drivers/gpu/drm/gma500/gma_display.h b/drivers/gpu/drm/gma500/gma_display.h
index b2491c65f053..e72dd08b701b 100644
--- a/drivers/gpu/drm/gma500/gma_display.h
+++ b/drivers/gpu/drm/gma500/gma_display.h
@@ -72,8 +72,8 @@ extern int gma_crtc_cursor_set(struct drm_crtc *crtc,
                                uint32_t width, uint32_t height);
 extern int gma_crtc_cursor_move(struct drm_crtc *crtc, int x, int y);
 extern void gma_crtc_load_lut(struct drm_crtc *crtc);
-extern void gma_crtc_gamma_set(struct drm_crtc *crtc, u16 *red, u16 *green,
-                               u16 *blue, u32 start, u32 size);
+extern int gma_crtc_gamma_set(struct drm_crtc *crtc, u16 *red, u16 *green,
+                              u16 *blue, u32 size);
 extern void gma_crtc_dpms(struct drm_crtc *crtc, int mode);
 extern void gma_crtc_prepare(struct drm_crtc *crtc);
 extern void gma_crtc_commit(struct drm_crtc *crtc);
diff --git a/drivers/gpu/drm/gma500/psb_drv.c b/drivers/gpu/drm/gma500/psb_drv.c
index 82b8ce418b27..50eb944fb78a 100644
--- a/drivers/gpu/drm/gma500/psb_drv.c
+++ b/drivers/gpu/drm/gma500/psb_drv.c
@@ -210,10 +210,8 @@ static int psb_driver_unload(struct drm_device *dev)
                         iounmap(dev_priv->aux_reg);
                         dev_priv->aux_reg = NULL;
                 }
-                if (dev_priv->aux_pdev)
-                        pci_dev_put(dev_priv->aux_pdev);
-                if (dev_priv->lpc_pdev)
-                        pci_dev_put(dev_priv->lpc_pdev);
+                pci_dev_put(dev_priv->aux_pdev);
+                pci_dev_put(dev_priv->lpc_pdev);
 
                 /* Destroy VBT data */
                 psb_intel_destroy_bios(dev);
diff --git a/drivers/gpu/drm/gma500/psb_intel_display.c b/drivers/gpu/drm/gma500/psb_intel_display.c
index 398015be87e4..7b6c84925098 100644
--- a/drivers/gpu/drm/gma500/psb_intel_display.c
+++ b/drivers/gpu/drm/gma500/psb_intel_display.c
@@ -491,7 +491,6 @@ void psb_intel_crtc_init(struct drm_device *dev, int pipe,
         struct drm_psb_private *dev_priv = dev->dev_private;
         struct gma_crtc *gma_crtc;
         int i;
-        uint16_t *r_base, *g_base, *b_base;
 
         /* We allocate a extra array of drm_connector pointers
          * for fbdev after the crtc */
@@ -519,16 +518,10 @@ void psb_intel_crtc_init(struct drm_device *dev, int pipe,
         gma_crtc->pipe = pipe;
         gma_crtc->plane = pipe;
 
-        r_base = gma_crtc->base.gamma_store;
-        g_base = r_base + 256;
-        b_base = g_base + 256;
         for (i = 0; i < 256; i++) {
                 gma_crtc->lut_r[i] = i;
                 gma_crtc->lut_g[i] = i;
                 gma_crtc->lut_b[i] = i;
-                r_base[i] = i << 8;
-                g_base[i] = i << 8;
-                b_base[i] = i << 8;
 
                 gma_crtc->lut_adj[i] = 0;
         }
diff --git a/drivers/gpu/drm/gma500/psb_intel_lvds.c b/drivers/gpu/drm/gma500/psb_intel_lvds.c
index b1b93317d054..e55733ca46d2 100644
--- a/drivers/gpu/drm/gma500/psb_intel_lvds.c
+++ b/drivers/gpu/drm/gma500/psb_intel_lvds.c
@@ -561,8 +561,7 @@ void psb_intel_lvds_destroy(struct drm_connector *connector)
         struct gma_encoder *gma_encoder = gma_attached_encoder(connector);
         struct psb_intel_lvds_priv *lvds_priv = gma_encoder->dev_priv;
 
-        if (lvds_priv->ddc_bus)
-                psb_intel_i2c_destroy(lvds_priv->ddc_bus);
+        psb_intel_i2c_destroy(lvds_priv->ddc_bus);
         drm_connector_unregister(connector);
         drm_connector_cleanup(connector);
         kfree(connector);
@@ -835,11 +834,9 @@ out:
 
 failed_find:
         mutex_unlock(&dev->mode_config.mutex);
-        if (lvds_priv->ddc_bus)
-                psb_intel_i2c_destroy(lvds_priv->ddc_bus);
+        psb_intel_i2c_destroy(lvds_priv->ddc_bus);
 failed_ddc:
-        if (lvds_priv->i2c_bus)
-                psb_intel_i2c_destroy(lvds_priv->i2c_bus);
+        psb_intel_i2c_destroy(lvds_priv->i2c_bus);
 failed_blc_i2c:
         drm_encoder_cleanup(encoder);
         drm_connector_cleanup(connector);
diff --git a/drivers/gpu/drm/hisilicon/kirin/Kconfig b/drivers/gpu/drm/hisilicon/kirin/Kconfig
index ea0df6115f7e..499f64405dac 100644
--- a/drivers/gpu/drm/hisilicon/kirin/Kconfig
+++ b/drivers/gpu/drm/hisilicon/kirin/Kconfig
@@ -4,6 +4,7 @@ config DRM_HISI_KIRIN
         select DRM_KMS_HELPER
         select DRM_GEM_CMA_HELPER
         select DRM_KMS_CMA_HELPER
+        select HISI_KIRIN_DW_DSI
         help
           Choose this option if you have a hisilicon Kirin chipsets(hi6220).
           If M is selected the module will be called kirin-drm.
diff --git a/drivers/gpu/drm/hisilicon/kirin/kirin_drm_ade.c b/drivers/gpu/drm/hisilicon/kirin/kirin_drm_ade.c
index fba6372d060e..c3707d47cd89 100644
--- a/drivers/gpu/drm/hisilicon/kirin/kirin_drm_ade.c
+++ b/drivers/gpu/drm/hisilicon/kirin/kirin_drm_ade.c
@@ -487,6 +487,7 @@ static void ade_crtc_enable(struct drm_crtc *crtc)
         ade_set_medianoc_qos(acrtc);
         ade_display_enable(acrtc);
         ade_dump_regs(ctx->base);
+        drm_crtc_vblank_on(crtc);
         acrtc->enable = true;
 }
 
@@ -498,17 +499,11 @@ static void ade_crtc_disable(struct drm_crtc *crtc)
         if (!acrtc->enable)
                 return;
 
+        drm_crtc_vblank_off(crtc);
         ade_power_down(ctx);
         acrtc->enable = false;
 }
 
-static int ade_crtc_atomic_check(struct drm_crtc *crtc,
-                                 struct drm_crtc_state *state)
-{
-        /* do nothing */
-        return 0;
-}
-
 static void ade_crtc_mode_set_nofb(struct drm_crtc *crtc)
 {
         struct ade_crtc *acrtc = to_ade_crtc(crtc);
@@ -537,6 +532,7 @@ static void ade_crtc_atomic_flush(struct drm_crtc *crtc,
 {
         struct ade_crtc *acrtc = to_ade_crtc(crtc);
         struct ade_hw_ctx *ctx = acrtc->ctx;
+        struct drm_pending_vblank_event *event = crtc->state->event;
         void __iomem *base = ctx->base;
 
         /* only crtc is enabled regs take effect */
@@ -545,12 +541,22 @@ static void ade_crtc_atomic_flush(struct drm_crtc *crtc,
                 /* flush ade registers */
                 writel(ADE_ENABLE, base + ADE_EN);
         }
+
+        if (event) {
+                crtc->state->event = NULL;
+
+                spin_lock_irq(&crtc->dev->event_lock);
+                if (drm_crtc_vblank_get(crtc) == 0)
+                        drm_crtc_arm_vblank_event(crtc, event);
+                else
+                        drm_crtc_send_vblank_event(crtc, event);
+                spin_unlock_irq(&crtc->dev->event_lock);
+        }
 }
 
 static const struct drm_crtc_helper_funcs ade_crtc_helper_funcs = {
         .enable         = ade_crtc_enable,
         .disable        = ade_crtc_disable,
-        .atomic_check   = ade_crtc_atomic_check,
         .mode_set_nofb  = ade_crtc_mode_set_nofb,
         .atomic_begin   = ade_crtc_atomic_begin,
         .atomic_flush   = ade_crtc_atomic_flush,
@@ -961,21 +967,21 @@ static int ade_dts_parse(struct platform_device *pdev, struct ade_hw_ctx *ctx)
         }
 
         ctx->ade_core_clk = devm_clk_get(dev, "clk_ade_core");
-        if (!ctx->ade_core_clk) {
+        if (IS_ERR(ctx->ade_core_clk)) {
                 DRM_ERROR("failed to parse clk ADE_CORE\n");
-                return -ENODEV;
+                return PTR_ERR(ctx->ade_core_clk);
         }
 
         ctx->media_noc_clk = devm_clk_get(dev, "clk_codec_jpeg");
-        if (!ctx->media_noc_clk) {
+        if (IS_ERR(ctx->media_noc_clk)) {
                 DRM_ERROR("failed to parse clk CODEC_JPEG\n");
-            return -ENODEV;
+                return PTR_ERR(ctx->media_noc_clk);
         }
 
         ctx->ade_pix_clk = devm_clk_get(dev, "clk_ade_pix");
-        if (!ctx->ade_pix_clk) {
+        if (IS_ERR(ctx->ade_pix_clk)) {
                 DRM_ERROR("failed to parse clk ADE_PIX\n");
-            return -ENODEV;
+                return PTR_ERR(ctx->ade_pix_clk);
         }
 
         return 0;
diff --git a/drivers/gpu/drm/hisilicon/kirin/kirin_drm_drv.c b/drivers/gpu/drm/hisilicon/kirin/kirin_drm_drv.c
index 3f94785fbcca..1edd9bc80294 100644
--- a/drivers/gpu/drm/hisilicon/kirin/kirin_drm_drv.c
+++ b/drivers/gpu/drm/hisilicon/kirin/kirin_drm_drv.c
@@ -171,9 +171,8 @@ static struct drm_driver kirin_drm_driver = {
         .driver_features        = DRIVER_GEM | DRIVER_MODESET | DRIVER_PRIME |
                                   DRIVER_ATOMIC | DRIVER_HAVE_IRQ,
         .fops                   = &kirin_drm_fops,
-        .set_busid              = drm_platform_set_busid,
 
-        .gem_free_object        = drm_gem_cma_free_object,
+        .gem_free_object_unlocked = drm_gem_cma_free_object,
         .gem_vm_ops             = &drm_gem_cma_vm_ops,
         .dumb_create            = kirin_gem_cma_dumb_create,
         .dumb_map_offset        = drm_gem_cma_dumb_map_offset,
@@ -221,19 +220,12 @@ static int kirin_drm_bind(struct device *dev)
         if (ret)
                 goto err_kms_cleanup;
 
-        /* connectors should be registered after drm device register */
-        ret = drm_connector_register_all(drm_dev);
-        if (ret)
-                goto err_drm_dev_unregister;
-
         DRM_INFO("Initialized %s %d.%d.%d %s on minor %d\n",
                  driver->name, driver->major, driver->minor, driver->patchlevel,
                  driver->date, drm_dev->primary->index);
 
         return 0;
 
-err_drm_dev_unregister:
-        drm_dev_unregister(drm_dev);
 err_kms_cleanup:
         kirin_drm_kms_cleanup(drm_dev);
 err_drm_dev_unref:
@@ -246,7 +238,6 @@ static void kirin_drm_unbind(struct device *dev)
 {
         struct drm_device *drm_dev = dev_get_drvdata(dev);
 
-        drm_connector_unregister_all(drm_dev);
         drm_dev_unregister(drm_dev);
         kirin_drm_kms_cleanup(drm_dev);
         drm_dev_unref(drm_dev);
diff --git a/drivers/gpu/drm/i2c/Kconfig b/drivers/gpu/drm/i2c/Kconfig
index 22c7ed63a001..4d341db462a2 100644
--- a/drivers/gpu/drm/i2c/Kconfig
+++ b/drivers/gpu/drm/i2c/Kconfig
@@ -1,12 +1,6 @@
 menu "I2C encoder or helper chips"
      depends on DRM && DRM_KMS_HELPER && I2C
 
-config DRM_I2C_ADV7511
-        tristate "AV7511 encoder"
-        select REGMAP_I2C
-        help
-          Support for the Analog Device ADV7511(W) and ADV7513 HDMI encoders.
-
 config DRM_I2C_CH7006
         tristate "Chrontel ch7006 TV encoder"
         default m if DRM_NOUVEAU
diff --git a/drivers/gpu/drm/i2c/Makefile b/drivers/gpu/drm/i2c/Makefile
index 2c72eb584ab7..43aa33baebed 100644
--- a/drivers/gpu/drm/i2c/Makefile
+++ b/drivers/gpu/drm/i2c/Makefile
@@ -1,7 +1,5 @@
 ccflags-y := -Iinclude/drm
 
-obj-$(CONFIG_DRM_I2C_ADV7511) += adv7511.o
-
 ch7006-y := ch7006_drv.o ch7006_mode.o
 obj-$(CONFIG_DRM_I2C_CH7006) += ch7006.o
 
diff --git a/drivers/gpu/drm/i2c/ch7006_drv.c b/drivers/gpu/drm/i2c/ch7006_drv.c
index 0594c45f7164..e9e8ae2ec06b 100644
--- a/drivers/gpu/drm/i2c/ch7006_drv.c
+++ b/drivers/gpu/drm/i2c/ch7006_drv.c
@@ -361,13 +361,8 @@ static int ch7006_encoder_set_property(struct drm_encoder *encoder,
 
                 /* Disable the crtc to ensure a full modeset is
                  * performed whenever it's turned on again. */
-                if (crtc) {
-                        struct drm_mode_set modeset = {
-                                .crtc = crtc,
-                        };
-
-                        drm_mode_set_config_internal(&modeset);
-                }
+                if (crtc)
+                        drm_crtc_force_disable(crtc);
         }
 
         return 0;
diff --git a/drivers/gpu/drm/i915/Kconfig b/drivers/gpu/drm/i915/Kconfig
index 29a32b11953b..7769e469118f 100644
--- a/drivers/gpu/drm/i915/Kconfig
+++ b/drivers/gpu/drm/i915/Kconfig
@@ -57,6 +57,28 @@ config DRM_I915_USERPTR
 
           If in doubt, say "Y".
 
+config DRM_I915_GVT
+        bool "Enable Intel GVT-g graphics virtualization host support"
+        depends on DRM_I915
+        default n
+        help
+          Choose this option if you want to enable Intel GVT-g graphics
+          virtualization technology host support with integrated graphics.
+          With GVT-g, it's possible to have one integrated graphics
+          device shared by multiple VMs under different hypervisors.
+
+          Note that at least one hypervisor like Xen or KVM is required for
+          this driver to work, and it only supports newer device from
+          Broadwell+. For further information and setup guide, you can
+          visit: http://01.org/igvt-g.
+
+          Now it's just a stub to support the modifications of i915 for
+          GVT device model. It requires at least one MPT modules for Xen/KVM
+          and other components of GVT device model to work. Use it under
+          you own risk.
+
+          If in doubt, say "N".
+
 menu "drm/i915 Debugging"
 depends on DRM_I915
 depends on EXPERT
diff --git a/drivers/gpu/drm/i915/Kconfig.debug b/drivers/gpu/drm/i915/Kconfig.debug
index 8f404103341d..cee87bfd10c4 100644
--- a/drivers/gpu/drm/i915/Kconfig.debug
+++ b/drivers/gpu/drm/i915/Kconfig.debug
@@ -18,6 +18,9 @@ config DRM_I915_WERROR
 config DRM_I915_DEBUG
         bool "Enable additional driver debugging"
         depends on DRM_I915
+        select PREEMPT_COUNT
+        select X86_MSR # used by igt/pm_rpm
+        select DRM_VGEM # used by igt/prime_vgem (dmabuf interop checks)
         default n
         help
           Choose this option to turn on extra driver debugging that may affect
diff --git a/drivers/gpu/drm/i915/Makefile b/drivers/gpu/drm/i915/Makefile
index 0b88ba0f3c1f..684fc1cd08fa 100644
--- a/drivers/gpu/drm/i915/Makefile
+++ b/drivers/gpu/drm/i915/Makefile
@@ -10,9 +10,11 @@ subdir-ccflags-$(CONFIG_DRM_I915_WERROR) := -Werror
 i915-y := i915_drv.o \
           i915_irq.o \
           i915_params.o \
+          i915_pci.o \
           i915_suspend.o \
           i915_sysfs.o \
           intel_csr.o \
+          intel_device_info.o \
           intel_pm.o \
           intel_runtime_pm.o
 
@@ -37,6 +39,7 @@ i915-y += i915_cmd_parser.o \
           i915_gem_userptr.o \
           i915_gpu_error.o \
           i915_trace_points.o \
+          intel_breadcrumbs.o \
           intel_lrc.o \
           intel_mocs.o \
           intel_ringbuffer.o \
@@ -59,6 +62,7 @@ i915-y += intel_audio.o \
           intel_bios.o \
           intel_color.o \
           intel_display.o \
+          intel_dpio_phy.o \
           intel_dpll_mgr.o \
           intel_fbc.o \
           intel_fifo_underrun.o \
@@ -81,10 +85,12 @@ i915-y += dvo_ch7017.o \
           dvo_tfp410.o \
           intel_crt.o \
           intel_ddi.o \
+          intel_dp_aux_backlight.o \
           intel_dp_link_training.o \
           intel_dp_mst.o \
           intel_dp.o \
           intel_dsi.o \
+          intel_dsi_dcs_backlight.o \
           intel_dsi_panel_vbt.o \
           intel_dsi_pll.o \
           intel_dvo.o \
@@ -98,8 +104,10 @@ i915-y += dvo_ch7017.o \
 # virtual gpu code
 i915-y += i915_vgpu.o
 
-# legacy horrors
-i915-y += i915_dma.o
+ifeq ($(CONFIG_DRM_I915_GVT),y)
+i915-y += intel_gvt.o
+include $(src)/gvt/Makefile
+endif
 
 obj-$(CONFIG_DRM_I915)  += i915.o
 
diff --git a/drivers/gpu/drm/i915/gvt/Makefile b/drivers/gpu/drm/i915/gvt/Makefile
new file mode 100644
index 000000000000..d0f21a6ad60d
--- /dev/null
+++ b/drivers/gpu/drm/i915/gvt/Makefile
@@ -0,0 +1,5 @@
+GVT_DIR := gvt
+GVT_SOURCE := gvt.o
+
+ccflags-y                      += -I$(src) -I$(src)/$(GVT_DIR) -Wall
+i915-y                         += $(addprefix $(GVT_DIR)/, $(GVT_SOURCE))
diff --git a/drivers/gpu/drm/i915/gvt/debug.h b/drivers/gpu/drm/i915/gvt/debug.h
new file mode 100644
index 000000000000..7ef412be665f
--- /dev/null
+++ b/drivers/gpu/drm/i915/gvt/debug.h
@@ -0,0 +1,34 @@
+/*
+ * Copyright(c) 2011-2016 Intel Corporation. All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#ifndef __GVT_DEBUG_H__
+#define __GVT_DEBUG_H__
+
+#define gvt_dbg_core(fmt, args...) \
+        DRM_DEBUG_DRIVER("gvt: core: "fmt, ##args)
+
+/*
+ * Other GVT debug stuff will be introduced in the GVT device model patches.
+ */
+
+#endif
diff --git a/drivers/gpu/drm/i915/gvt/gvt.c b/drivers/gpu/drm/i915/gvt/gvt.c
new file mode 100644
index 000000000000..927f4579f5b6
--- /dev/null
+++ b/drivers/gpu/drm/i915/gvt/gvt.c
@@ -0,0 +1,145 @@
+/*
+ * Copyright(c) 2011-2016 Intel Corporation. All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#include <linux/types.h>
+#include <xen/xen.h>
+
+#include "i915_drv.h"
+
+struct intel_gvt_host intel_gvt_host;
+
+static const char * const supported_hypervisors[] = {
+        [INTEL_GVT_HYPERVISOR_XEN] = "XEN",
+        [INTEL_GVT_HYPERVISOR_KVM] = "KVM",
+};
+
+/**
+ * intel_gvt_init_host - Load MPT modules and detect if we're running in host
+ * @gvt: intel gvt device
+ *
+ * This function is called at the driver loading stage. If failed to find a
+ * loadable MPT module or detect currently we're running in a VM, then GVT-g
+ * will be disabled
+ *
+ * Returns:
+ * Zero on success, negative error code if failed.
+ *
+ */
+int intel_gvt_init_host(void)
+{
+        if (intel_gvt_host.initialized)
+                return 0;
+
+        /* Xen DOM U */
+        if (xen_domain() && !xen_initial_domain())
+                return -ENODEV;
+
+        /* Try to load MPT modules for hypervisors */
+        if (xen_initial_domain()) {
+                /* In Xen dom0 */
+                intel_gvt_host.mpt = try_then_request_module(
+                                symbol_get(xengt_mpt), "xengt");
+                intel_gvt_host.hypervisor_type = INTEL_GVT_HYPERVISOR_XEN;
+        } else {
+                /* not in Xen. Try KVMGT */
+                intel_gvt_host.mpt = try_then_request_module(
+                                symbol_get(kvmgt_mpt), "kvm");
+                intel_gvt_host.hypervisor_type = INTEL_GVT_HYPERVISOR_KVM;
+        }
+
+        /* Fail to load MPT modules - bail out */
+        if (!intel_gvt_host.mpt)
+                return -EINVAL;
+
+        /* Try to detect if we're running in host instead of VM. */
+        if (!intel_gvt_hypervisor_detect_host())
+                return -ENODEV;
+
+        gvt_dbg_core("Running with hypervisor %s in host mode\n",
+                        supported_hypervisors[intel_gvt_host.hypervisor_type]);
+
+        intel_gvt_host.initialized = true;
+        return 0;
+}
+
+static void init_device_info(struct intel_gvt *gvt)
+{
+        if (IS_BROADWELL(gvt->dev_priv))
+                gvt->device_info.max_support_vgpus = 8;
+        /* This function will grow large in GVT device model patches. */
+}
+
+/**
+ * intel_gvt_clean_device - clean a GVT device
+ * @gvt: intel gvt device
+ *
+ * This function is called at the driver unloading stage, to free the
+ * resources owned by a GVT device.
+ *
+ */
+void intel_gvt_clean_device(struct drm_i915_private *dev_priv)
+{
+        struct intel_gvt *gvt = &dev_priv->gvt;
+
+        if (WARN_ON(!gvt->initialized))
+                return;
+
+        /* Other de-initialization of GVT components will be introduced. */
+
+        gvt->initialized = false;
+}
+
+/**
+ * intel_gvt_init_device - initialize a GVT device
+ * @dev_priv: drm i915 private data
+ *
+ * This function is called at the initialization stage, to initialize
+ * necessary GVT components.
+ *
+ * Returns:
+ * Zero on success, negative error code if failed.
+ *
+ */
+int intel_gvt_init_device(struct drm_i915_private *dev_priv)
+{
+        struct intel_gvt *gvt = &dev_priv->gvt;
+        /*
+         * Cannot initialize GVT device without intel_gvt_host gets
+         * initialized first.
+         */
+        if (WARN_ON(!intel_gvt_host.initialized))
+                return -EINVAL;
+
+        if (WARN_ON(gvt->initialized))
+                return -EEXIST;
+
+        gvt_dbg_core("init gvt device\n");
+
+        init_device_info(gvt);
+        /*
+         * Other initialization of GVT components will be introduce here.
+         */
+        gvt_dbg_core("gvt device creation is done\n");
+        gvt->initialized = true;
+        return 0;
+}
diff --git a/drivers/gpu/drm/i915/gvt/gvt.h b/drivers/gpu/drm/i915/gvt/gvt.h
new file mode 100644
index 000000000000..fb619a6e519d
--- /dev/null
+++ b/drivers/gpu/drm/i915/gvt/gvt.h
@@ -0,0 +1,69 @@
+/*
+ * Copyright(c) 2011-2016 Intel Corporation. All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#ifndef _GVT_H_
+#define _GVT_H_
+
+#include "debug.h"
+#include "hypercall.h"
+
+#define GVT_MAX_VGPU 8
+
+enum {
+        INTEL_GVT_HYPERVISOR_XEN = 0,
+        INTEL_GVT_HYPERVISOR_KVM,
+};
+
+struct intel_gvt_host {
+        bool initialized;
+        int hypervisor_type;
+        struct intel_gvt_mpt *mpt;
+};
+
+extern struct intel_gvt_host intel_gvt_host;
+
+/* Describe per-platform limitations. */
+struct intel_gvt_device_info {
+        u32 max_support_vgpus;
+        /* This data structure will grow bigger in GVT device model patches */
+};
+
+struct intel_vgpu {
+        struct intel_gvt *gvt;
+        int id;
+        unsigned long handle; /* vGPU handle used by hypervisor MPT modules */
+};
+
+struct intel_gvt {
+        struct mutex lock;
+        bool initialized;
+
+        struct drm_i915_private *dev_priv;
+        struct idr vgpu_idr;    /* vGPU IDR pool */
+
+        struct intel_gvt_device_info device_info;
+};
+
+#include "mpt.h"
+
+#endif
diff --git a/drivers/gpu/drm/i915/gvt/hypercall.h b/drivers/gpu/drm/i915/gvt/hypercall.h
new file mode 100644
index 000000000000..254df8bf1f35
--- /dev/null
+++ b/drivers/gpu/drm/i915/gvt/hypercall.h
@@ -0,0 +1,38 @@
+/*
+ * Copyright(c) 2011-2016 Intel Corporation. All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#ifndef _GVT_HYPERCALL_H_
+#define _GVT_HYPERCALL_H_
+
+/*
+ * Specific GVT-g MPT modules function collections. Currently GVT-g supports
+ * both Xen and KVM by providing dedicated hypervisor-related MPT modules.
+ */
+struct intel_gvt_mpt {
+        int (*detect_host)(void);
+};
+
+extern struct intel_gvt_mpt xengt_mpt;
+extern struct intel_gvt_mpt kvmgt_mpt;
+
+#endif /* _GVT_HYPERCALL_H_ */
diff --git a/drivers/gpu/drm/i915/gvt/mpt.h b/drivers/gpu/drm/i915/gvt/mpt.h
new file mode 100644
index 000000000000..03601e3ffa7c
--- /dev/null
+++ b/drivers/gpu/drm/i915/gvt/mpt.h
@@ -0,0 +1,49 @@
+/*
+ * Copyright(c) 2011-2016 Intel Corporation. All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#ifndef _GVT_MPT_H_
+#define _GVT_MPT_H_
+
+/**
+ * DOC: Hypervisor Service APIs for GVT-g Core Logic
+ *
+ * This is the glue layer between specific hypervisor MPT modules and GVT-g core
+ * logic. Each kind of hypervisor MPT module provides a collection of function
+ * callbacks and will be attached to GVT host when the driver is loading.
+ * GVT-g core logic will call these APIs to request specific services from
+ * hypervisor.
+ */
+
+/**
+ * intel_gvt_hypervisor_detect_host - check if GVT-g is running within
+ * hypervisor host/privilged domain
+ *
+ * Returns:
+ * Zero on success, -ENODEV if current kernel is running inside a VM
+ */
+static inline int intel_gvt_hypervisor_detect_host(void)
+{
+        return intel_gvt_host.mpt->detect_host();
+}
+
+#endif /* _GVT_MPT_H_ */
diff --git a/drivers/gpu/drm/i915/i915_cmd_parser.c b/drivers/gpu/drm/i915/i915_cmd_parser.c
index a337f33bec5b..b0fd6a7b0603 100644
--- a/drivers/gpu/drm/i915/i915_cmd_parser.c
+++ b/drivers/gpu/drm/i915/i915_cmd_parser.c
@@ -215,7 +215,8 @@ static const struct drm_i915_cmd_descriptor hsw_render_cmds[] = {
         CMD(  MI_RS_CONTEXT,                    SMI,    F,  1,      S  ),
         CMD(  MI_LOAD_SCAN_LINES_INCL,          SMI,   !F,  0x3F,   M  ),
         CMD(  MI_LOAD_SCAN_LINES_EXCL,          SMI,   !F,  0x3F,   R  ),
-        CMD(  MI_LOAD_REGISTER_REG,             SMI,   !F,  0xFF,   R  ),
+        CMD(  MI_LOAD_REGISTER_REG,             SMI,   !F,  0xFF,   W,
+              .reg = { .offset = 1, .mask = 0x007FFFFC, .step = 1 }    ),
         CMD(  MI_RS_STORE_DATA_IMM,             SMI,   !F,  0xFF,   S  ),
         CMD(  MI_LOAD_URB_MEM,                  SMI,   !F,  0xFF,   S  ),
         CMD(  MI_STORE_URB_MEM,                 SMI,   !F,  0xFF,   S  ),
@@ -736,7 +737,7 @@ static void fini_hash_table(struct intel_engine_cs *engine)
 
 /**
  * i915_cmd_parser_init_ring() - set cmd parser related fields for a ringbuffer
- * @ring: the ringbuffer to initialize
+ * @engine: the engine to initialize
  *
  * Optionally initializes fields related to batch buffer command parsing in the
  * struct intel_engine_cs based on whether the platform requires software
@@ -750,12 +751,12 @@ int i915_cmd_parser_init_ring(struct intel_engine_cs *engine)
         int cmd_table_count;
         int ret;
 
-        if (!IS_GEN7(engine->dev))
+        if (!IS_GEN7(engine->i915))
                 return 0;
 
         switch (engine->id) {
         case RCS:
-                if (IS_HASWELL(engine->dev)) {
+                if (IS_HASWELL(engine->i915)) {
                         cmd_tables = hsw_render_ring_cmds;
                         cmd_table_count =
                                 ARRAY_SIZE(hsw_render_ring_cmds);
@@ -764,7 +765,7 @@ int i915_cmd_parser_init_ring(struct intel_engine_cs *engine)
                         cmd_table_count = ARRAY_SIZE(gen7_render_cmds);
                 }
 
-                if (IS_HASWELL(engine->dev)) {
+                if (IS_HASWELL(engine->i915)) {
                         engine->reg_tables = hsw_render_reg_tables;
                         engine->reg_table_count = ARRAY_SIZE(hsw_render_reg_tables);
                 } else {
@@ -780,7 +781,7 @@ int i915_cmd_parser_init_ring(struct intel_engine_cs *engine)
                 engine->get_cmd_length_mask = gen7_bsd_get_cmd_length_mask;
                 break;
         case BCS:
-                if (IS_HASWELL(engine->dev)) {
+                if (IS_HASWELL(engine->i915)) {
                         cmd_tables = hsw_blt_ring_cmds;
                         cmd_table_count = ARRAY_SIZE(hsw_blt_ring_cmds);
                 } else {
@@ -788,7 +789,7 @@ int i915_cmd_parser_init_ring(struct intel_engine_cs *engine)
                         cmd_table_count = ARRAY_SIZE(gen7_blt_cmds);
                 }
 
-                if (IS_HASWELL(engine->dev)) {
+                if (IS_HASWELL(engine->i915)) {
                         engine->reg_tables = hsw_blt_reg_tables;
                         engine->reg_table_count = ARRAY_SIZE(hsw_blt_reg_tables);
                 } else {
@@ -829,7 +830,7 @@ int i915_cmd_parser_init_ring(struct intel_engine_cs *engine)
 
 /**
  * i915_cmd_parser_fini_ring() - clean up cmd parser related fields
- * @ring: the ringbuffer to clean up
+ * @engine: the engine to clean up
  *
  * Releases any resources related to command parsing that may have been
  * initialized for the specified ring.
@@ -1023,7 +1024,7 @@ unpin_src:
 
 /**
  * i915_needs_cmd_parser() - should a given ring use software command parsing?
- * @ring: the ring in question
+ * @engine: the engine in question
  *
  * Only certain platforms require software batch buffer command parsing, and
  * only when enabled via module parameter.
@@ -1035,7 +1036,7 @@ bool i915_needs_cmd_parser(struct intel_engine_cs *engine)
         if (!engine->needs_cmd_parser)
                 return false;
 
-        if (!USES_PPGTT(engine->dev))
+        if (!USES_PPGTT(engine->i915))
                 return false;
 
         return (i915.enable_cmd_parser == 1);
@@ -1098,6 +1099,11 @@ static bool check_cmd(const struct intel_engine_cs *engine,
                                         return false;
                                 }
 
+                                if (desc->cmd.value == MI_LOAD_REGISTER_REG) {
+                                        DRM_DEBUG_DRIVER("CMD: Rejected LRR to OACONTROL\n");
+                                        return false;
+                                }
+
                                 if (desc->cmd.value == MI_LOAD_REGISTER_IMM(1))
                                         *oacontrol_set = (cmd[offset + 1] != 0);
                         }
@@ -1113,6 +1119,12 @@ static bool check_cmd(const struct intel_engine_cs *engine,
                                         return false;
                                 }
 
+                                if (desc->cmd.value == MI_LOAD_REGISTER_REG) {
+                                        DRM_DEBUG_DRIVER("CMD: Rejected LRR to masked register 0x%08X\n",
+                                                         reg_addr);
+                                        return false;
+                                }
+
                                 if (desc->cmd.value == MI_LOAD_REGISTER_IMM(1) &&
                                     (offset + 2 > length ||
                                      (cmd[offset + 1] & reg->mask) != reg->value)) {
@@ -1164,7 +1176,7 @@ static bool check_cmd(const struct intel_engine_cs *engine,
 
 /**
  * i915_parse_cmds() - parse a submitted batch buffer for privilege violations
- * @ring: the ring on which the batch is to execute
+ * @engine: the engine on which the batch is to execute
  * @batch_obj: the batch buffer in question
  * @shadow_batch_obj: copy of the batch buffer in question
  * @batch_start_offset: byte offset in the batch at which execution starts
@@ -1269,14 +1281,28 @@ int i915_parse_cmds(struct intel_engine_cs *engine,
 
 /**
  * i915_cmd_parser_get_version() - get the cmd parser version number
+ * @dev_priv: i915 device private
  *
  * The cmd parser maintains a simple increasing integer version number suitable
  * for passing to userspace clients to determine what operations are permitted.
  *
  * Return: the current version number of the cmd parser
  */
-int i915_cmd_parser_get_version(void)
+int i915_cmd_parser_get_version(struct drm_i915_private *dev_priv)
 {
+        struct intel_engine_cs *engine;
+        bool active = false;
+
+        /* If the command parser is not enabled, report 0 - unsupported */
+        for_each_engine(engine, dev_priv) {
+                if (i915_needs_cmd_parser(engine)) {
+                        active = true;
+                        break;
+                }
+        }
+        if (!active)
+                return 0;
+
         /*
          * Command parser version history
          *
@@ -1288,6 +1314,7 @@ int i915_cmd_parser_get_version(void)
          * 4. L3 atomic chicken bits of HSW_SCRATCH1 and HSW_ROW_CHICKEN3.
          * 5. GPGPU dispatch compute indirect registers.
          * 6. TIMESTAMP register and Haswell CS GPR registers
+         * 7. Allow MI_LOAD_REGISTER_REG between whitelisted registers.
          */
-        return 6;
+        return 7;
 }
diff --git a/drivers/gpu/drm/i915/i915_debugfs.c b/drivers/gpu/drm/i915/i915_debugfs.c
index 2a6e12956baf..844fea795bae 100644
--- a/drivers/gpu/drm/i915/i915_debugfs.c
+++ b/drivers/gpu/drm/i915/i915_debugfs.c
@@ -89,17 +89,17 @@ static int i915_capabilities(struct seq_file *m, void *data)
         return 0;
 }
 
-static const char get_active_flag(struct drm_i915_gem_object *obj)
+static char get_active_flag(struct drm_i915_gem_object *obj)
 {
         return obj->active ? '*' : ' ';
 }
 
-static const char get_pin_flag(struct drm_i915_gem_object *obj)
+static char get_pin_flag(struct drm_i915_gem_object *obj)
 {
         return obj->pin_display ? 'p' : ' ';
 }
 
-static const char get_tiling_flag(struct drm_i915_gem_object *obj)
+static char get_tiling_flag(struct drm_i915_gem_object *obj)
 {
         switch (obj->tiling_mode) {
         default:
@@ -109,12 +109,12 @@ static const char get_tiling_flag(struct drm_i915_gem_object *obj)
         }
 }
 
-static inline const char get_global_flag(struct drm_i915_gem_object *obj)
+static char get_global_flag(struct drm_i915_gem_object *obj)
 {
         return i915_gem_obj_to_ggtt(obj) ? 'g' : ' ';
 }
 
-static inline const char get_pin_mapped_flag(struct drm_i915_gem_object *obj)
+static char get_pin_mapped_flag(struct drm_i915_gem_object *obj)
 {
         return obj->mapping ? 'M' : ' ';
 }
@@ -199,13 +199,6 @@ describe_obj(struct seq_file *m, struct drm_i915_gem_object *obj)
                 seq_printf(m, " (frontbuffer: 0x%03x)", obj->frontbuffer_bits);
 }
 
-static void describe_ctx(struct seq_file *m, struct intel_context *ctx)
-{
-        seq_putc(m, ctx->legacy_hw_ctx.initialized ? 'I' : 'i');
-        seq_putc(m, ctx->remap_slice ? 'R' : 'r');
-        seq_putc(m, ' ');
-}
-
 static int i915_gem_object_list_info(struct seq_file *m, void *data)
 {
         struct drm_info_node *node = m->private;
@@ -272,7 +265,7 @@ static int i915_gem_stolen_list_info(struct seq_file *m, void *data)
 {
         struct drm_info_node *node = m->private;
         struct drm_device *dev = node->minor->dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         struct drm_i915_gem_object *obj;
         u64 total_obj_size, total_gtt_size;
         LIST_HEAD(stolen);
@@ -424,6 +417,42 @@ static void print_batch_pool_stats(struct seq_file *m,
         print_file_stats(m, "[k]batch pool", stats);
 }
 
+static int per_file_ctx_stats(int id, void *ptr, void *data)
+{
+        struct i915_gem_context *ctx = ptr;
+        int n;
+
+        for (n = 0; n < ARRAY_SIZE(ctx->engine); n++) {
+                if (ctx->engine[n].state)
+                        per_file_stats(0, ctx->engine[n].state, data);
+                if (ctx->engine[n].ringbuf)
+                        per_file_stats(0, ctx->engine[n].ringbuf->obj, data);
+        }
+
+        return 0;
+}
+
+static void print_context_stats(struct seq_file *m,
+                                struct drm_i915_private *dev_priv)
+{
+        struct file_stats stats;
+        struct drm_file *file;
+
+        memset(&stats, 0, sizeof(stats));
+
+        mutex_lock(&dev_priv->drm.struct_mutex);
+        if (dev_priv->kernel_context)
+                per_file_ctx_stats(0, dev_priv->kernel_context, &stats);
+
+        list_for_each_entry(file, &dev_priv->drm.filelist, lhead) {
+                struct drm_i915_file_private *fpriv = file->driver_priv;
+                idr_for_each(&fpriv->context_idr, per_file_ctx_stats, &stats);
+        }
+        mutex_unlock(&dev_priv->drm.struct_mutex);
+
+        print_file_stats(m, "[k]contexts", stats);
+}
+
 #define count_vmas(list, member) do { \
         list_for_each_entry(vma, list, member) { \
                 size += i915_gem_obj_total_ggtt_size(vma->obj); \
@@ -528,10 +557,10 @@ static int i915_gem_object_info(struct seq_file *m, void* data)
 
         seq_putc(m, '\n');
         print_batch_pool_stats(m, dev_priv);
-
         mutex_unlock(&dev->struct_mutex);
 
         mutex_lock(&dev->filelist_mutex);
+        print_context_stats(m, dev_priv);
         list_for_each_entry_reverse(file, &dev->filelist, lhead) {
                 struct file_stats stats;
                 struct task_struct *task;
@@ -562,7 +591,7 @@ static int i915_gem_gtt_info(struct seq_file *m, void *data)
         struct drm_info_node *node = m->private;
         struct drm_device *dev = node->minor->dev;
         uintptr_t list = (uintptr_t) node->info_ent->data;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         struct drm_i915_gem_object *obj;
         u64 total_obj_size, total_gtt_size;
         int count, ret;
@@ -596,7 +625,7 @@ static int i915_gem_pageflip_info(struct seq_file *m, void *data)
 {
         struct drm_info_node *node = m->private;
         struct drm_device *dev = node->minor->dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         struct intel_crtc *crtc;
         int ret;
 
@@ -607,18 +636,20 @@ static int i915_gem_pageflip_info(struct seq_file *m, void *data)
         for_each_intel_crtc(dev, crtc) {
                 const char pipe = pipe_name(crtc->pipe);
                 const char plane = plane_name(crtc->plane);
-                struct intel_unpin_work *work;
+                struct intel_flip_work *work;
 
                 spin_lock_irq(&dev->event_lock);
-                work = crtc->unpin_work;
+                work = crtc->flip_work;
                 if (work == NULL) {
                         seq_printf(m, "No flip due on pipe %c (plane %c)\n",
                                    pipe, plane);
                 } else {
+                        u32 pending;
                         u32 addr;
 
-                        if (atomic_read(&work->pending) < INTEL_FLIP_COMPLETE) {
-                                seq_printf(m, "Flip queued on pipe %c (plane %c)\n",
+                        pending = atomic_read(&work->pending);
+                        if (pending) {
+                                seq_printf(m, "Flip ioctl preparing on pipe %c (plane %c)\n",
                                            pipe, plane);
                         } else {
                                 seq_printf(m, "Flip pending (waiting for vsync) on pipe %c (plane %c)\n",
@@ -631,18 +662,14 @@ static int i915_gem_pageflip_info(struct seq_file *m, void *data)
                                            engine->name,
                                            i915_gem_request_get_seqno(work->flip_queued_req),
                                            dev_priv->next_seqno,
-                                           engine->get_seqno(engine),
-                                           i915_gem_request_completed(work->flip_queued_req, true));
+                                           intel_engine_get_seqno(engine),
+                                           i915_gem_request_completed(work->flip_queued_req));
                         } else
                                 seq_printf(m, "Flip not associated with any ring\n");
                         seq_printf(m, "Flip queued on frame %d, (was ready on frame %d), now %d\n",
                                    work->flip_queued_vblank,
                                    work->flip_ready_vblank,
-                                   drm_crtc_vblank_count(&crtc->base));
-                        if (work->enable_stall_check)
-                                seq_puts(m, "Stall check enabled, ");
-                        else
-                                seq_puts(m, "Stall check waiting for page flip ioctl, ");
+                                   intel_crtc_get_vblank_counter(crtc));
                         seq_printf(m, "%d prepares\n", atomic_read(&work->pending));
 
                         if (INTEL_INFO(dev)->gen >= 4)
@@ -668,7 +695,7 @@ static int i915_gem_batch_pool_info(struct seq_file *m, void *data)
 {
         struct drm_info_node *node = m->private;
         struct drm_device *dev = node->minor->dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         struct drm_i915_gem_object *obj;
         struct intel_engine_cs *engine;
         int total = 0;
@@ -713,7 +740,7 @@ static int i915_gem_request_info(struct seq_file *m, void *data)
 {
         struct drm_info_node *node = m->private;
         struct drm_device *dev = node->minor->dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         struct intel_engine_cs *engine;
         struct drm_i915_gem_request *req;
         int ret, any;
@@ -761,17 +788,29 @@ static int i915_gem_request_info(struct seq_file *m, void *data)
 static void i915_ring_seqno_info(struct seq_file *m,
                                  struct intel_engine_cs *engine)
 {
+        struct intel_breadcrumbs *b = &engine->breadcrumbs;
+        struct rb_node *rb;
+
         seq_printf(m, "Current sequence (%s): %x\n",
-                   engine->name, engine->get_seqno(engine));
-        seq_printf(m, "Current user interrupts (%s): %x\n",
-                   engine->name, READ_ONCE(engine->user_interrupts));
+                   engine->name, intel_engine_get_seqno(engine));
+        seq_printf(m, "Current user interrupts (%s): %lx\n",
+                   engine->name, READ_ONCE(engine->breadcrumbs.irq_wakeups));
+
+        spin_lock(&b->lock);
+        for (rb = rb_first(&b->waiters); rb; rb = rb_next(rb)) {
+                struct intel_wait *w = container_of(rb, typeof(*w), node);
+
+                seq_printf(m, "Waiting (%s): %s [%d] on %x\n",
+                           engine->name, w->tsk->comm, w->tsk->pid, w->seqno);
+        }
+        spin_unlock(&b->lock);
 }
 
 static int i915_gem_seqno_info(struct seq_file *m, void *data)
 {
         struct drm_info_node *node = m->private;
         struct drm_device *dev = node->minor->dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         struct intel_engine_cs *engine;
         int ret;
 
@@ -794,7 +833,7 @@ static int i915_interrupt_info(struct seq_file *m, void *data)
 {
         struct drm_info_node *node = m->private;
         struct drm_device *dev = node->minor->dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         struct intel_engine_cs *engine;
         int ret, i, pipe;
 
@@ -985,7 +1024,7 @@ static int i915_gem_fence_regs_info(struct seq_file *m, void *data)
 {
         struct drm_info_node *node = m->private;
         struct drm_device *dev = node->minor->dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         int i, ret;
 
         ret = mutex_lock_interruptible(&dev->struct_mutex);
@@ -1013,7 +1052,7 @@ static int i915_hws_info(struct seq_file *m, void *data)
 {
         struct drm_info_node *node = m->private;
         struct drm_device *dev = node->minor->dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         struct intel_engine_cs *engine;
         const u32 *hws;
         int i;
@@ -1124,7 +1163,7 @@ static int
 i915_next_seqno_get(void *data, u64 *val)
 {
         struct drm_device *dev = data;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         int ret;
 
         ret = mutex_lock_interruptible(&dev->struct_mutex);
@@ -1161,7 +1200,7 @@ static int i915_frequency_info(struct seq_file *m, void *unused)
 {
         struct drm_info_node *node = m->private;
         struct drm_device *dev = node->minor->dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         int ret = 0;
 
         intel_runtime_pm_get(dev_priv);
@@ -1281,6 +1320,7 @@ static int i915_frequency_info(struct seq_file *m, void *unused)
                 }
                 seq_printf(m, "PM IER=0x%08x IMR=0x%08x ISR=0x%08x IIR=0x%08x, MASK=0x%08x\n",
                            pm_ier, pm_imr, pm_isr, pm_iir, pm_mask);
+                seq_printf(m, "pm_intr_keep: 0x%08x\n", dev_priv->rps.pm_intr_keep);
                 seq_printf(m, "GT_PERF_STATUS: 0x%08x\n", gt_perf_status);
                 seq_printf(m, "Render p-state ratio: %d\n",
                            (gt_perf_status & (IS_GEN9(dev) ? 0x1ff00 : 0xff00)) >> 8);
@@ -1363,7 +1403,7 @@ static int i915_hangcheck_info(struct seq_file *m, void *unused)
 {
         struct drm_info_node *node = m->private;
         struct drm_device *dev = node->minor->dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         struct intel_engine_cs *engine;
         u64 acthd[I915_NUM_ENGINES];
         u32 seqno[I915_NUM_ENGINES];
@@ -1380,10 +1420,10 @@ static int i915_hangcheck_info(struct seq_file *m, void *unused)
 
         for_each_engine_id(engine, dev_priv, id) {
                 acthd[id] = intel_ring_get_active_head(engine);
-                seqno[id] = engine->get_seqno(engine);
+                seqno[id] = intel_engine_get_seqno(engine);
         }
 
-        i915_get_extra_instdone(dev, instdone);
+        i915_get_extra_instdone(dev_priv, instdone);
 
         intel_runtime_pm_put(dev_priv);
 
@@ -1400,9 +1440,11 @@ static int i915_hangcheck_info(struct seq_file *m, void *unused)
                            engine->hangcheck.seqno,
                            seqno[id],
                            engine->last_submitted_seqno);
-                seq_printf(m, "\tuser interrupts = %x [current %x]\n",
+                seq_printf(m, "\twaiters? %d\n",
+                           intel_engine_has_waiter(engine));
+                seq_printf(m, "\tuser interrupts = %lx [current %lx]\n",
                            engine->hangcheck.user_interrupts,
-                           READ_ONCE(engine->user_interrupts));
+                           READ_ONCE(engine->breadcrumbs.irq_wakeups));
                 seq_printf(m, "\tACTHD = 0x%08llx [current 0x%08llx]\n",
                            (long long)engine->hangcheck.acthd,
                            (long long)acthd[id]);
@@ -1432,7 +1474,7 @@ static int ironlake_drpc_info(struct seq_file *m)
 {
         struct drm_info_node *node = m->private;
         struct drm_device *dev = node->minor->dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         u32 rgvmodectl, rstdbyctl;
         u16 crstandvid;
         int ret;
@@ -1500,7 +1542,7 @@ static int i915_forcewake_domains(struct seq_file *m, void *data)
 {
         struct drm_info_node *node = m->private;
         struct drm_device *dev = node->minor->dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         struct intel_uncore_forcewake_domain *fw_domain;
 
         spin_lock_irq(&dev_priv->uncore.lock);
@@ -1518,7 +1560,7 @@ static int vlv_drpc_info(struct seq_file *m)
 {
         struct drm_info_node *node = m->private;
         struct drm_device *dev = node->minor->dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         u32 rpmodectl1, rcctl1, pw_status;
 
         intel_runtime_pm_get(dev_priv);
@@ -1558,7 +1600,7 @@ static int gen6_drpc_info(struct seq_file *m)
 {
         struct drm_info_node *node = m->private;
         struct drm_device *dev = node->minor->dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         u32 rpmodectl1, gt_core_status, rcctl1, rc6vids = 0;
         unsigned forcewake_count;
         int count = 0, ret;
@@ -1670,7 +1712,7 @@ static int i915_frontbuffer_tracking(struct seq_file *m, void *unused)
 {
         struct drm_info_node *node = m->private;
         struct drm_device *dev = node->minor->dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
 
         seq_printf(m, "FB tracking busy bits: 0x%08x\n",
                    dev_priv->fb_tracking.busy_bits);
@@ -1685,7 +1727,7 @@ static int i915_fbc_status(struct seq_file *m, void *unused)
 {
         struct drm_info_node *node = m->private;
         struct drm_device *dev = node->minor->dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
 
         if (!HAS_FBC(dev)) {
                 seq_puts(m, "FBC unsupported on this chipset\n");
@@ -1715,7 +1757,7 @@ static int i915_fbc_status(struct seq_file *m, void *unused)
 static int i915_fbc_fc_get(void *data, u64 *val)
 {
         struct drm_device *dev = data;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
 
         if (INTEL_INFO(dev)->gen < 7 || !HAS_FBC(dev))
                 return -ENODEV;
@@ -1728,7 +1770,7 @@ static int i915_fbc_fc_get(void *data, u64 *val)
 static int i915_fbc_fc_set(void *data, u64 val)
 {
         struct drm_device *dev = data;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         u32 reg;
 
         if (INTEL_INFO(dev)->gen < 7 || !HAS_FBC(dev))
@@ -1755,7 +1797,7 @@ static int i915_ips_status(struct seq_file *m, void *unused)
 {
         struct drm_info_node *node = m->private;
         struct drm_device *dev = node->minor->dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
 
         if (!HAS_IPS(dev)) {
                 seq_puts(m, "not supported\n");
@@ -1785,7 +1827,7 @@ static int i915_sr_status(struct seq_file *m, void *unused)
 {
         struct drm_info_node *node = m->private;
         struct drm_device *dev = node->minor->dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         bool sr_enabled = false;
 
         intel_runtime_pm_get(dev_priv);
@@ -1814,7 +1856,7 @@ static int i915_emon_status(struct seq_file *m, void *unused)
 {
         struct drm_info_node *node = m->private;
         struct drm_device *dev = node->minor->dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         unsigned long temp, chipset, gfx;
         int ret;
 
@@ -1842,7 +1884,7 @@ static int i915_ring_freq_table(struct seq_file *m, void *unused)
 {
         struct drm_info_node *node = m->private;
         struct drm_device *dev = node->minor->dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         int ret = 0;
         int gpu_freq, ia_freq;
         unsigned int max_gpu_freq, min_gpu_freq;
@@ -1897,7 +1939,7 @@ static int i915_opregion(struct seq_file *m, void *unused)
 {
         struct drm_info_node *node = m->private;
         struct drm_device *dev = node->minor->dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         struct intel_opregion *opregion = &dev_priv->opregion;
         int ret;
 
@@ -1918,7 +1960,7 @@ static int i915_vbt(struct seq_file *m, void *unused)
 {
         struct drm_info_node *node = m->private;
         struct drm_device *dev = node->minor->dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         struct intel_opregion *opregion = &dev_priv->opregion;
 
         if (opregion->vbt)
@@ -1940,19 +1982,19 @@ static int i915_gem_framebuffer_info(struct seq_file *m, void *data)
                 return ret;
 
 #ifdef CONFIG_DRM_FBDEV_EMULATION
-       if (to_i915(dev)->fbdev) {
-               fbdev_fb = to_intel_framebuffer(to_i915(dev)->fbdev->helper.fb);
-
-               seq_printf(m, "fbcon size: %d x %d, depth %d, %d bpp, modifier 0x%llx, refcount %d, obj ",
-                         fbdev_fb->base.width,
-                         fbdev_fb->base.height,
-                         fbdev_fb->base.depth,
-                         fbdev_fb->base.bits_per_pixel,
-                         fbdev_fb->base.modifier[0],
-                         drm_framebuffer_read_refcount(&fbdev_fb->base));
-               describe_obj(m, fbdev_fb->obj);
-               seq_putc(m, '\n');
-       }
+        if (to_i915(dev)->fbdev) {
+                fbdev_fb = to_intel_framebuffer(to_i915(dev)->fbdev->helper.fb);
+
+                seq_printf(m, "fbcon size: %d x %d, depth %d, %d bpp, modifier 0x%llx, refcount %d, obj ",
+                           fbdev_fb->base.width,
+                           fbdev_fb->base.height,
+                           fbdev_fb->base.depth,
+                           fbdev_fb->base.bits_per_pixel,
+                           fbdev_fb->base.modifier[0],
+                           drm_framebuffer_read_refcount(&fbdev_fb->base));
+                describe_obj(m, fbdev_fb->obj);
+                seq_putc(m, '\n');
+        }
 #endif
 
         mutex_lock(&dev->mode_config.fb_lock);
@@ -1989,10 +2031,9 @@ static int i915_context_status(struct seq_file *m, void *unused)
 {
         struct drm_info_node *node = m->private;
         struct drm_device *dev = node->minor->dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         struct intel_engine_cs *engine;
-        struct intel_context *ctx;
-        enum intel_engine_id id;
+        struct i915_gem_context *ctx;
         int ret;
 
         ret = mutex_lock_interruptible(&dev->struct_mutex);
@@ -2000,32 +2041,36 @@ static int i915_context_status(struct seq_file *m, void *unused)
                 return ret;
 
         list_for_each_entry(ctx, &dev_priv->context_list, link) {
-                if (!i915.enable_execlists &&
-                    ctx->legacy_hw_ctx.rcs_state == NULL)
-                        continue;
-
-                seq_puts(m, "HW context ");
-                describe_ctx(m, ctx);
-                if (ctx == dev_priv->kernel_context)
-                        seq_printf(m, "(kernel context) ");
+                seq_printf(m, "HW context %u ", ctx->hw_id);
+                if (IS_ERR(ctx->file_priv)) {
+                        seq_puts(m, "(deleted) ");
+                } else if (ctx->file_priv) {
+                        struct pid *pid = ctx->file_priv->file->pid;
+                        struct task_struct *task;
 
-                if (i915.enable_execlists) {
-                        seq_putc(m, '\n');
-                        for_each_engine_id(engine, dev_priv, id) {
-                                struct drm_i915_gem_object *ctx_obj =
-                                        ctx->engine[id].state;
-                                struct intel_ringbuffer *ringbuf =
-                                        ctx->engine[id].ringbuf;
-
-                                seq_printf(m, "%s: ", engine->name);
-                                if (ctx_obj)
-                                        describe_obj(m, ctx_obj);
-                                if (ringbuf)
-                                        describe_ctx_ringbuf(m, ringbuf);
-                                seq_putc(m, '\n');
+                        task = get_pid_task(pid, PIDTYPE_PID);
+                        if (task) {
+                                seq_printf(m, "(%s [%d]) ",
+                                           task->comm, task->pid);
+                                put_task_struct(task);
                         }
                 } else {
-                        describe_obj(m, ctx->legacy_hw_ctx.rcs_state);
+                        seq_puts(m, "(kernel) ");
+                }
+
+                seq_putc(m, ctx->remap_slice ? 'R' : 'r');
+                seq_putc(m, '\n');
+
+                for_each_engine(engine, dev_priv) {
+                        struct intel_context *ce = &ctx->engine[engine->id];
+
+                        seq_printf(m, "%s: ", engine->name);
+                        seq_putc(m, ce->initialised ? 'I' : 'i');
+                        if (ce->state)
+                                describe_obj(m, ce->state);
+                        if (ce->ringbuf)
+                                describe_ctx_ringbuf(m, ce->ringbuf);
+                        seq_putc(m, '\n');
                 }
 
                 seq_putc(m, '\n');
@@ -2037,24 +2082,22 @@ static int i915_context_status(struct seq_file *m, void *unused)
 }
 
 static void i915_dump_lrc_obj(struct seq_file *m,
-                              struct intel_context *ctx,
+                              struct i915_gem_context *ctx,
                               struct intel_engine_cs *engine)
 {
+        struct drm_i915_gem_object *ctx_obj = ctx->engine[engine->id].state;
         struct page *page;
         uint32_t *reg_state;
         int j;
-        struct drm_i915_gem_object *ctx_obj = ctx->engine[engine->id].state;
         unsigned long ggtt_offset = 0;
 
+        seq_printf(m, "CONTEXT: %s %u\n", engine->name, ctx->hw_id);
+
         if (ctx_obj == NULL) {
-                seq_printf(m, "Context on %s with no gem object\n",
-                           engine->name);
+                seq_puts(m, "\tNot allocated\n");
                 return;
         }
 
-        seq_printf(m, "CONTEXT: %s %u\n", engine->name,
-                   intel_execlists_ctx_id(ctx, engine));
-
         if (!i915_gem_obj_ggtt_bound(ctx_obj))
                 seq_puts(m, "\tNot bound in GGTT\n");
         else
@@ -2085,9 +2128,9 @@ static int i915_dump_lrc(struct seq_file *m, void *unused)
 {
         struct drm_info_node *node = (struct drm_info_node *) m->private;
         struct drm_device *dev = node->minor->dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         struct intel_engine_cs *engine;
-        struct intel_context *ctx;
+        struct i915_gem_context *ctx;
         int ret;
 
         if (!i915.enable_execlists) {
@@ -2100,10 +2143,8 @@ static int i915_dump_lrc(struct seq_file *m, void *unused)
                 return ret;
 
         list_for_each_entry(ctx, &dev_priv->context_list, link)
-                if (ctx != dev_priv->kernel_context) {
-                        for_each_engine(engine, dev_priv)
-                                i915_dump_lrc_obj(m, ctx, engine);
-                }
+                for_each_engine(engine, dev_priv)
+                        i915_dump_lrc_obj(m, ctx, engine);
 
         mutex_unlock(&dev->struct_mutex);
 
@@ -2114,7 +2155,7 @@ static int i915_execlists(struct seq_file *m, void *data)
 {
         struct drm_info_node *node = (struct drm_info_node *)m->private;
         struct drm_device *dev = node->minor->dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         struct intel_engine_cs *engine;
         u32 status_pointer;
         u8 read_pointer;
@@ -2174,8 +2215,8 @@ static int i915_execlists(struct seq_file *m, void *data)
 
                 seq_printf(m, "\t%d requests in queue\n", count);
                 if (head_req) {
-                        seq_printf(m, "\tHead request id: %u\n",
-                                   intel_execlists_ctx_id(head_req->ctx, engine));
+                        seq_printf(m, "\tHead request context: %u\n",
+                                   head_req->ctx->hw_id);
                         seq_printf(m, "\tHead request tail: %u\n",
                                    head_req->tail);
                 }
@@ -2217,7 +2258,7 @@ static int i915_swizzle_info(struct seq_file *m, void *data)
 {
         struct drm_info_node *node = m->private;
         struct drm_device *dev = node->minor->dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         int ret;
 
         ret = mutex_lock_interruptible(&dev->struct_mutex);
@@ -2269,7 +2310,7 @@ static int i915_swizzle_info(struct seq_file *m, void *data)
 
 static int per_file_ctx(int id, void *ptr, void *data)
 {
-        struct intel_context *ctx = ptr;
+        struct i915_gem_context *ctx = ptr;
         struct seq_file *m = data;
         struct i915_hw_ppgtt *ppgtt = ctx->ppgtt;
 
@@ -2290,7 +2331,7 @@ static int per_file_ctx(int id, void *ptr, void *data)
 
 static void gen8_ppgtt_info(struct seq_file *m, struct drm_device *dev)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         struct intel_engine_cs *engine;
         struct i915_hw_ppgtt *ppgtt = dev_priv->mm.aliasing_ppgtt;
         int i;
@@ -2311,15 +2352,15 @@ static void gen8_ppgtt_info(struct seq_file *m, struct drm_device *dev)
 
 static void gen6_ppgtt_info(struct seq_file *m, struct drm_device *dev)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         struct intel_engine_cs *engine;
 
-        if (INTEL_INFO(dev)->gen == 6)
+        if (IS_GEN6(dev_priv))
                 seq_printf(m, "GFX_MODE: 0x%08x\n", I915_READ(GFX_MODE));
 
         for_each_engine(engine, dev_priv) {
                 seq_printf(m, "%s\n", engine->name);
-                if (INTEL_INFO(dev)->gen == 7)
+                if (IS_GEN7(dev_priv))
                         seq_printf(m, "GFX_MODE: 0x%08x\n",
                                    I915_READ(RING_MODE_GEN7(engine)));
                 seq_printf(m, "PP_DIR_BASE: 0x%08x\n",
@@ -2345,7 +2386,7 @@ static int i915_ppgtt_info(struct seq_file *m, void *data)
 {
         struct drm_info_node *node = m->private;
         struct drm_device *dev = node->minor->dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         struct drm_file *file;
 
         int ret = mutex_lock_interruptible(&dev->struct_mutex);
@@ -2388,7 +2429,7 @@ static int count_irq_waiters(struct drm_i915_private *i915)
         int count = 0;
 
         for_each_engine(engine, i915)
-                count += engine->irq_refcount;
+                count += intel_engine_has_waiter(engine);
 
         return count;
 }
@@ -2397,11 +2438,12 @@ static int i915_rps_boost_info(struct seq_file *m, void *data)
 {
         struct drm_info_node *node = m->private;
         struct drm_device *dev = node->minor->dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         struct drm_file *file;
 
         seq_printf(m, "RPS enabled? %d\n", dev_priv->rps.enabled);
-        seq_printf(m, "GPU busy? %d\n", dev_priv->mm.busy);
+        seq_printf(m, "GPU busy? %s [%x]\n",
+                   yesno(dev_priv->gt.awake), dev_priv->gt.active_engines);
         seq_printf(m, "CPU waiting? %d\n", count_irq_waiters(dev_priv));
         seq_printf(m, "Frequency requested %d; min hard:%d, soft:%d; max soft:%d, hard:%d\n",
                    intel_gpu_freq(dev_priv, dev_priv->rps.cur_freq),
@@ -2442,7 +2484,7 @@ static int i915_llc(struct seq_file *m, void *data)
 {
         struct drm_info_node *node = m->private;
         struct drm_device *dev = node->minor->dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         const bool edram = INTEL_GEN(dev_priv) > 8;
 
         seq_printf(m, "LLC: %s\n", yesno(HAS_LLC(dev)));
@@ -2455,7 +2497,7 @@ static int i915_llc(struct seq_file *m, void *data)
 static int i915_guc_load_status_info(struct seq_file *m, void *data)
 {
         struct drm_info_node *node = m->private;
-        struct drm_i915_private *dev_priv = node->minor->dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(node->minor->dev);
         struct intel_guc_fw *guc_fw = &dev_priv->guc.guc_fw;
         u32 tmp, i;
 
@@ -2510,15 +2552,16 @@ static void i915_guc_client_info(struct seq_file *m,
         seq_printf(m, "\tWQ size %d, offset: 0x%x, tail %d\n",
                 client->wq_size, client->wq_offset, client->wq_tail);
 
+        seq_printf(m, "\tWork queue full: %u\n", client->no_wq_space);
         seq_printf(m, "\tFailed to queue: %u\n", client->q_fail);
         seq_printf(m, "\tFailed doorbell: %u\n", client->b_fail);
         seq_printf(m, "\tLast submission result: %d\n", client->retcode);
 
         for_each_engine(engine, dev_priv) {
                 seq_printf(m, "\tSubmissions: %llu %s\n",
-                                client->submissions[engine->guc_id],
+                                client->submissions[engine->id],
                                 engine->name);
-                tot += client->submissions[engine->guc_id];
+                tot += client->submissions[engine->id];
         }
         seq_printf(m, "\tTotal: %llu\n", tot);
 }
@@ -2527,7 +2570,7 @@ static int i915_guc_info(struct seq_file *m, void *data)
 {
         struct drm_info_node *node = m->private;
         struct drm_device *dev = node->minor->dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         struct intel_guc guc;
         struct i915_guc_client client = {};
         struct intel_engine_cs *engine;
@@ -2546,6 +2589,10 @@ static int i915_guc_info(struct seq_file *m, void *data)
 
         mutex_unlock(&dev->struct_mutex);
 
+        seq_printf(m, "Doorbell map:\n");
+        seq_printf(m, "\t%*pb\n", GUC_MAX_DOORBELLS, guc.doorbell_bitmap);
+        seq_printf(m, "Doorbell next cacheline: 0x%x\n\n", guc.db_cacheline);
+
         seq_printf(m, "GuC total action count: %llu\n", guc.action_count);
         seq_printf(m, "GuC action failure count: %u\n", guc.action_fail);
         seq_printf(m, "GuC last action command: 0x%x\n", guc.action_cmd);
@@ -2555,9 +2602,9 @@ static int i915_guc_info(struct seq_file *m, void *data)
         seq_printf(m, "\nGuC submissions:\n");
         for_each_engine(engine, dev_priv) {
                 seq_printf(m, "\t%-24s: %10llu, last seqno 0x%08x\n",
-                        engine->name, guc.submissions[engine->guc_id],
-                        guc.last_seqno[engine->guc_id]);
-                total += guc.submissions[engine->guc_id];
+                        engine->name, guc.submissions[engine->id],
+                        guc.last_seqno[engine->id]);
+                total += guc.submissions[engine->id];
         }
         seq_printf(m, "\t%s: %llu\n", "Total", total);
 
@@ -2573,7 +2620,7 @@ static int i915_guc_log_dump(struct seq_file *m, void *data)
 {
         struct drm_info_node *node = m->private;
         struct drm_device *dev = node->minor->dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         struct drm_i915_gem_object *log_obj = dev_priv->guc.log_obj;
         u32 *log;
         int i = 0, pg;
@@ -2601,7 +2648,7 @@ static int i915_edp_psr_status(struct seq_file *m, void *data)
 {
         struct drm_info_node *node = m->private;
         struct drm_device *dev = node->minor->dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         u32 psrperf = 0;
         u32 stat[3];
         enum pipe pipe;
@@ -2669,7 +2716,6 @@ static int i915_sink_crc(struct seq_file *m, void *data)
 {
         struct drm_info_node *node = m->private;
         struct drm_device *dev = node->minor->dev;
-        struct intel_encoder *encoder;
         struct intel_connector *connector;
         struct intel_dp *intel_dp = NULL;
         int ret;
@@ -2677,18 +2723,19 @@ static int i915_sink_crc(struct seq_file *m, void *data)
 
         drm_modeset_lock_all(dev);
         for_each_intel_connector(dev, connector) {
+                struct drm_crtc *crtc;
 
-                if (connector->base.dpms != DRM_MODE_DPMS_ON)
+                if (!connector->base.state->best_encoder)
                         continue;
 
-                if (!connector->base.encoder)
+                crtc = connector->base.state->crtc;
+                if (!crtc->state->active)
                         continue;
 
-                encoder = to_intel_encoder(connector->base.encoder);
-                if (encoder->type != INTEL_OUTPUT_EDP)
+                if (connector->base.connector_type != DRM_MODE_CONNECTOR_eDP)
                         continue;
 
-                intel_dp = enc_to_intel_dp(&encoder->base);
+                intel_dp = enc_to_intel_dp(connector->base.state->best_encoder);
 
                 ret = intel_dp_sink_crc(intel_dp, crc);
                 if (ret)
@@ -2709,7 +2756,7 @@ static int i915_energy_uJ(struct seq_file *m, void *data)
 {
         struct drm_info_node *node = m->private;
         struct drm_device *dev = node->minor->dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         u64 power;
         u32 units;
 
@@ -2735,12 +2782,12 @@ static int i915_runtime_pm_status(struct seq_file *m, void *unused)
 {
         struct drm_info_node *node = m->private;
         struct drm_device *dev = node->minor->dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
 
         if (!HAS_RUNTIME_PM(dev_priv))
                 seq_puts(m, "Runtime power management not supported\n");
 
-        seq_printf(m, "GPU idle: %s\n", yesno(!dev_priv->mm.busy));
+        seq_printf(m, "GPU idle: %s\n", yesno(!dev_priv->gt.awake));
         seq_printf(m, "IRQs disabled: %s\n",
                    yesno(!intel_irqs_enabled(dev_priv)));
 #ifdef CONFIG_PM
@@ -2750,8 +2797,8 @@ static int i915_runtime_pm_status(struct seq_file *m, void *unused)
         seq_printf(m, "Device Power Management (CONFIG_PM) disabled\n");
 #endif
         seq_printf(m, "PCI device power state: %s [%d]\n",
-                   pci_power_name(dev_priv->dev->pdev->current_state),
-                   dev_priv->dev->pdev->current_state);
+                   pci_power_name(dev_priv->drm.pdev->current_state),
+                   dev_priv->drm.pdev->current_state);
 
         return 0;
 }
@@ -2760,7 +2807,7 @@ static int i915_power_domain_info(struct seq_file *m, void *unused)
 {
         struct drm_info_node *node = m->private;
         struct drm_device *dev = node->minor->dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         struct i915_power_domains *power_domains = &dev_priv->power_domains;
         int i;
 
@@ -2795,7 +2842,7 @@ static int i915_dmc_info(struct seq_file *m, void *unused)
 {
         struct drm_info_node *node = m->private;
         struct drm_device *dev = node->minor->dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         struct intel_csr *csr;
 
         if (!HAS_CSR(dev)) {
@@ -2918,7 +2965,7 @@ static void intel_dp_info(struct seq_file *m,
 
         seq_printf(m, "\tDPCD rev: %x\n", intel_dp->dpcd[DP_DPCD_REV]);
         seq_printf(m, "\taudio support: %s\n", yesno(intel_dp->has_audio));
-        if (intel_encoder->type == INTEL_OUTPUT_EDP)
+        if (intel_connector->base.connector_type == DRM_MODE_CONNECTOR_eDP)
                 intel_panel_info(m, &intel_connector->panel);
 }
 
@@ -2957,14 +3004,26 @@ static void intel_connector_info(struct seq_file *m,
                 seq_printf(m, "\tCEA rev: %d\n",
                            connector->display_info.cea_rev);
         }
-        if (intel_encoder) {
-                if (intel_encoder->type == INTEL_OUTPUT_DISPLAYPORT ||
-                    intel_encoder->type == INTEL_OUTPUT_EDP)
-                        intel_dp_info(m, intel_connector);
-                else if (intel_encoder->type == INTEL_OUTPUT_HDMI)
-                        intel_hdmi_info(m, intel_connector);
-                else if (intel_encoder->type == INTEL_OUTPUT_LVDS)
+
+        if (!intel_encoder || intel_encoder->type == INTEL_OUTPUT_DP_MST)
+                return;
+
+        switch (connector->connector_type) {
+        case DRM_MODE_CONNECTOR_DisplayPort:
+        case DRM_MODE_CONNECTOR_eDP:
+                intel_dp_info(m, intel_connector);
+                break;
+        case DRM_MODE_CONNECTOR_LVDS:
+                if (intel_encoder->type == INTEL_OUTPUT_LVDS)
                         intel_lvds_info(m, intel_connector);
+                break;
+        case DRM_MODE_CONNECTOR_HDMIA:
+                if (intel_encoder->type == INTEL_OUTPUT_HDMI ||
+                    intel_encoder->type == INTEL_OUTPUT_UNKNOWN)
+                        intel_hdmi_info(m, intel_connector);
+                break;
+        default:
+                break;
         }
 
         seq_printf(m, "\tmodes:\n");
@@ -2974,7 +3033,7 @@ static void intel_connector_info(struct seq_file *m,
 
 static bool cursor_active(struct drm_device *dev, int pipe)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         u32 state;
 
         if (IS_845G(dev) || IS_I865G(dev))
@@ -2987,7 +3046,7 @@ static bool cursor_active(struct drm_device *dev, int pipe)
 
 static bool cursor_position(struct drm_device *dev, int pipe, int *x, int *y)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         u32 pos;
 
         pos = I915_READ(CURPOS(pipe));
@@ -3108,7 +3167,7 @@ static int i915_display_info(struct seq_file *m, void *unused)
 {
         struct drm_info_node *node = m->private;
         struct drm_device *dev = node->minor->dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         struct intel_crtc *crtc;
         struct drm_connector *connector;
 
@@ -3163,13 +3222,13 @@ static int i915_semaphore_status(struct seq_file *m, void *unused)
 {
         struct drm_info_node *node = (struct drm_info_node *) m->private;
         struct drm_device *dev = node->minor->dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         struct intel_engine_cs *engine;
         int num_rings = hweight32(INTEL_INFO(dev)->ring_mask);
         enum intel_engine_id id;
         int j, ret;
 
-        if (!i915_semaphore_is_enabled(dev)) {
+        if (!i915_semaphore_is_enabled(dev_priv)) {
                 seq_puts(m, "Semaphores are disabled\n");
                 return 0;
         }
@@ -3236,7 +3295,7 @@ static int i915_shared_dplls_info(struct seq_file *m, void *unused)
 {
         struct drm_info_node *node = (struct drm_info_node *) m->private;
         struct drm_device *dev = node->minor->dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         int i;
 
         drm_modeset_lock_all(dev);
@@ -3266,7 +3325,7 @@ static int i915_wa_registers(struct seq_file *m, void *unused)
         struct intel_engine_cs *engine;
         struct drm_info_node *node = (struct drm_info_node *) m->private;
         struct drm_device *dev = node->minor->dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         struct i915_workarounds *workarounds = &dev_priv->workarounds;
         enum intel_engine_id id;
 
@@ -3304,7 +3363,7 @@ static int i915_ddb_info(struct seq_file *m, void *unused)
 {
         struct drm_info_node *node = m->private;
         struct drm_device *dev = node->minor->dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         struct skl_ddb_allocation *ddb;
         struct skl_ddb_entry *entry;
         enum pipe pipe;
@@ -3342,31 +3401,16 @@ static int i915_ddb_info(struct seq_file *m, void *unused)
 static void drrs_status_per_crtc(struct seq_file *m,
                 struct drm_device *dev, struct intel_crtc *intel_crtc)
 {
-        struct intel_encoder *intel_encoder;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         struct i915_drrs *drrs = &dev_priv->drrs;
         int vrefresh = 0;
+        struct drm_connector *connector;
 
-        for_each_encoder_on_crtc(dev, &intel_crtc->base, intel_encoder) {
-                /* Encoder connected on this CRTC */
-                switch (intel_encoder->type) {
-                case INTEL_OUTPUT_EDP:
-                        seq_puts(m, "eDP:\n");
-                        break;
-                case INTEL_OUTPUT_DSI:
-                        seq_puts(m, "DSI:\n");
-                        break;
-                case INTEL_OUTPUT_HDMI:
-                        seq_puts(m, "HDMI:\n");
-                        break;
-                case INTEL_OUTPUT_DISPLAYPORT:
-                        seq_puts(m, "DP:\n");
-                        break;
-                default:
-                        seq_printf(m, "Other encoder (id=%d).\n",
-                                                intel_encoder->type);
-                        return;
-                }
+        drm_for_each_connector(connector, dev) {
+                if (connector->state->crtc != &intel_crtc->base)
+                        continue;
+
+                seq_printf(m, "%s:\n", connector->name);
         }
 
         if (dev_priv->vbt.drrs_type == STATIC_DRRS_SUPPORT)
@@ -3429,18 +3473,16 @@ static int i915_drrs_status(struct seq_file *m, void *unused)
         struct intel_crtc *intel_crtc;
         int active_crtc_cnt = 0;
 
+        drm_modeset_lock_all(dev);
         for_each_intel_crtc(dev, intel_crtc) {
-                drm_modeset_lock(&intel_crtc->base.mutex, NULL);
-
                 if (intel_crtc->base.state->active) {
                         active_crtc_cnt++;
                         seq_printf(m, "\nCRTC %d:  ", active_crtc_cnt);
 
                         drrs_status_per_crtc(m, dev, intel_crtc);
                 }
-
-                drm_modeset_unlock(&intel_crtc->base.mutex);
         }
+        drm_modeset_unlock_all(dev);
 
         if (!active_crtc_cnt)
                 seq_puts(m, "No active crtc found\n");
@@ -3458,17 +3500,23 @@ static int i915_dp_mst_info(struct seq_file *m, void *unused)
 {
         struct drm_info_node *node = (struct drm_info_node *) m->private;
         struct drm_device *dev = node->minor->dev;
-        struct drm_encoder *encoder;
         struct intel_encoder *intel_encoder;
         struct intel_digital_port *intel_dig_port;
+        struct drm_connector *connector;
+
         drm_modeset_lock_all(dev);
-        list_for_each_entry(encoder, &dev->mode_config.encoder_list, head) {
-                intel_encoder = to_intel_encoder(encoder);
-                if (intel_encoder->type != INTEL_OUTPUT_DISPLAYPORT)
+        drm_for_each_connector(connector, dev) {
+                if (connector->connector_type != DRM_MODE_CONNECTOR_DisplayPort)
                         continue;
-                intel_dig_port = enc_to_dig_port(encoder);
+
+                intel_encoder = intel_attached_encoder(connector);
+                if (!intel_encoder || intel_encoder->type == INTEL_OUTPUT_DP_MST)
+                        continue;
+
+                intel_dig_port = enc_to_dig_port(&intel_encoder->base);
                 if (!intel_dig_port->dp.can_mst)
                         continue;
+
                 seq_printf(m, "MST Source Port %c\n",
                            port_name(intel_dig_port->port));
                 drm_dp_mst_dump_topology(m, &intel_dig_port->dp.mst_mgr);
@@ -3480,7 +3528,7 @@ static int i915_dp_mst_info(struct seq_file *m, void *unused)
 static int i915_pipe_crc_open(struct inode *inode, struct file *filep)
 {
         struct pipe_crc_info *info = inode->i_private;
-        struct drm_i915_private *dev_priv = info->dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(info->dev);
         struct intel_pipe_crc *pipe_crc = &dev_priv->pipe_crc[info->pipe];
 
         if (info->pipe >= INTEL_INFO(info->dev)->num_pipes)
@@ -3504,7 +3552,7 @@ static int i915_pipe_crc_open(struct inode *inode, struct file *filep)
 static int i915_pipe_crc_release(struct inode *inode, struct file *filep)
 {
         struct pipe_crc_info *info = inode->i_private;
-        struct drm_i915_private *dev_priv = info->dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(info->dev);
         struct intel_pipe_crc *pipe_crc = &dev_priv->pipe_crc[info->pipe];
 
         spin_lock_irq(&pipe_crc->lock);
@@ -3532,7 +3580,7 @@ i915_pipe_crc_read(struct file *filep, char __user *user_buf, size_t count,
 {
         struct pipe_crc_info *info = filep->private_data;
         struct drm_device *dev = info->dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         struct intel_pipe_crc *pipe_crc = &dev_priv->pipe_crc[info->pipe];
         char buf[PIPE_CRC_BUFFER_LEN];
         int n_entries;
@@ -3665,7 +3713,7 @@ static const char *pipe_crc_source_name(enum intel_pipe_crc_source source)
 static int display_crc_ctl_show(struct seq_file *m, void *data)
 {
         struct drm_device *dev = m->private;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         int i;
 
         for (i = 0; i < I915_MAX_PIPES; i++)
@@ -3726,7 +3774,7 @@ static int i9xx_pipe_crc_auto_source(struct drm_device *dev, enum pipe pipe,
                 case INTEL_OUTPUT_TVOUT:
                         *source = INTEL_PIPE_CRC_SOURCE_TV;
                         break;
-                case INTEL_OUTPUT_DISPLAYPORT:
+                case INTEL_OUTPUT_DP:
                 case INTEL_OUTPUT_EDP:
                         dig_port = enc_to_dig_port(&encoder->base);
                         switch (dig_port->port) {
@@ -3759,7 +3807,7 @@ static int vlv_pipe_crc_ctl_reg(struct drm_device *dev,
                                 enum intel_pipe_crc_source *source,
                                 uint32_t *val)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         bool need_stable_symbols = false;
 
         if (*source == INTEL_PIPE_CRC_SOURCE_AUTO) {
@@ -3830,7 +3878,7 @@ static int i9xx_pipe_crc_ctl_reg(struct drm_device *dev,
                                  enum intel_pipe_crc_source *source,
                                  uint32_t *val)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         bool need_stable_symbols = false;
 
         if (*source == INTEL_PIPE_CRC_SOURCE_AUTO) {
@@ -3904,7 +3952,7 @@ static int i9xx_pipe_crc_ctl_reg(struct drm_device *dev,
 static void vlv_undo_pipe_scramble_reset(struct drm_device *dev,
                                          enum pipe pipe)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         uint32_t tmp = I915_READ(PORT_DFT2_G4X);
 
         switch (pipe) {
@@ -3929,7 +3977,7 @@ static void vlv_undo_pipe_scramble_reset(struct drm_device *dev,
 static void g4x_undo_pipe_scramble_reset(struct drm_device *dev,
                                          enum pipe pipe)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         uint32_t tmp = I915_READ(PORT_DFT2_G4X);
 
         if (pipe == PIPE_A)
@@ -3972,7 +4020,7 @@ static int ilk_pipe_crc_ctl_reg(enum intel_pipe_crc_source *source,
 
 static void hsw_trans_edp_pipe_A_crc_wa(struct drm_device *dev, bool enable)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         struct intel_crtc *crtc =
                 to_intel_crtc(dev_priv->pipe_to_crtc_mapping[PIPE_A]);
         struct intel_crtc_state *pipe_config;
@@ -4040,7 +4088,7 @@ static int ivb_pipe_crc_ctl_reg(struct drm_device *dev,
 static int pipe_crc_set_source(struct drm_device *dev, enum pipe pipe,
                                enum intel_pipe_crc_source source)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         struct intel_pipe_crc *pipe_crc = &dev_priv->pipe_crc[pipe];
         struct intel_crtc *crtc = to_intel_crtc(intel_get_crtc_for_pipe(dev,
                                                                         pipe));
@@ -4547,7 +4595,7 @@ static void wm_latency_show(struct seq_file *m, const uint16_t wm[8])
 static int pri_wm_latency_show(struct seq_file *m, void *data)
 {
         struct drm_device *dev = m->private;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         const uint16_t *latencies;
 
         if (INTEL_INFO(dev)->gen >= 9)
@@ -4563,7 +4611,7 @@ static int pri_wm_latency_show(struct seq_file *m, void *data)
 static int spr_wm_latency_show(struct seq_file *m, void *data)
 {
         struct drm_device *dev = m->private;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         const uint16_t *latencies;
 
         if (INTEL_INFO(dev)->gen >= 9)
@@ -4579,7 +4627,7 @@ static int spr_wm_latency_show(struct seq_file *m, void *data)
 static int cur_wm_latency_show(struct seq_file *m, void *data)
 {
         struct drm_device *dev = m->private;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         const uint16_t *latencies;
 
         if (INTEL_INFO(dev)->gen >= 9)
@@ -4670,7 +4718,7 @@ static ssize_t pri_wm_latency_write(struct file *file, const char __user *ubuf,
 {
         struct seq_file *m = file->private_data;
         struct drm_device *dev = m->private;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         uint16_t *latencies;
 
         if (INTEL_INFO(dev)->gen >= 9)
@@ -4686,7 +4734,7 @@ static ssize_t spr_wm_latency_write(struct file *file, const char __user *ubuf,
 {
         struct seq_file *m = file->private_data;
         struct drm_device *dev = m->private;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         uint16_t *latencies;
 
         if (INTEL_INFO(dev)->gen >= 9)
@@ -4702,7 +4750,7 @@ static ssize_t cur_wm_latency_write(struct file *file, const char __user *ubuf,
 {
         struct seq_file *m = file->private_data;
         struct drm_device *dev = m->private;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         uint16_t *latencies;
 
         if (INTEL_INFO(dev)->gen >= 9)
@@ -4744,7 +4792,7 @@ static int
 i915_wedged_get(void *data, u64 *val)
 {
         struct drm_device *dev = data;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
 
         *val = i915_terminally_wedged(&dev_priv->gpu_error);
 
@@ -4755,7 +4803,7 @@ static int
 i915_wedged_set(void *data, u64 val)
 {
         struct drm_device *dev = data;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
 
         /*
          * There is no safeguard against this debugfs entry colliding
@@ -4770,7 +4818,7 @@ i915_wedged_set(void *data, u64 val)
 
         intel_runtime_pm_get(dev_priv);
 
-        i915_handle_error(dev, val,
+        i915_handle_error(dev_priv, val,
                           "Manually setting wedged to %llu", val);
 
         intel_runtime_pm_put(dev_priv);
@@ -4783,44 +4831,10 @@ DEFINE_SIMPLE_ATTRIBUTE(i915_wedged_fops,
                         "%llu\n");
 
 static int
-i915_ring_stop_get(void *data, u64 *val)
-{
-        struct drm_device *dev = data;
-        struct drm_i915_private *dev_priv = dev->dev_private;
-
-        *val = dev_priv->gpu_error.stop_rings;
-
-        return 0;
-}
-
-static int
-i915_ring_stop_set(void *data, u64 val)
-{
-        struct drm_device *dev = data;
-        struct drm_i915_private *dev_priv = dev->dev_private;
-        int ret;
-
-        DRM_DEBUG_DRIVER("Stopping rings 0x%08llx\n", val);
-
-        ret = mutex_lock_interruptible(&dev->struct_mutex);
-        if (ret)
-                return ret;
-
-        dev_priv->gpu_error.stop_rings = val;
-        mutex_unlock(&dev->struct_mutex);
-
-        return 0;
-}
-
-DEFINE_SIMPLE_ATTRIBUTE(i915_ring_stop_fops,
-                        i915_ring_stop_get, i915_ring_stop_set,
-                        "0x%08llx\n");
-
-static int
 i915_ring_missed_irq_get(void *data, u64 *val)
 {
         struct drm_device *dev = data;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
 
         *val = dev_priv->gpu_error.missed_irq_rings;
         return 0;
@@ -4830,7 +4844,7 @@ static int
 i915_ring_missed_irq_set(void *data, u64 val)
 {
         struct drm_device *dev = data;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         int ret;
 
         /* Lock against concurrent debugfs callers */
@@ -4851,7 +4865,7 @@ static int
 i915_ring_test_irq_get(void *data, u64 *val)
 {
         struct drm_device *dev = data;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
 
         *val = dev_priv->gpu_error.test_irq_rings;
 
@@ -4862,18 +4876,11 @@ static int
 i915_ring_test_irq_set(void *data, u64 val)
 {
         struct drm_device *dev = data;
-        struct drm_i915_private *dev_priv = dev->dev_private;
-        int ret;
+        struct drm_i915_private *dev_priv = to_i915(dev);
 
+        val &= INTEL_INFO(dev_priv)->ring_mask;
         DRM_DEBUG_DRIVER("Masking interrupts on rings 0x%08llx\n", val);
-
-        /* Lock against concurrent debugfs callers */
-        ret = mutex_lock_interruptible(&dev->struct_mutex);
-        if (ret)
-                return ret;
-
         dev_priv->gpu_error.test_irq_rings = val;
-        mutex_unlock(&dev->struct_mutex);
 
         return 0;
 }
@@ -4902,7 +4909,7 @@ static int
 i915_drop_caches_set(void *data, u64 val)
 {
         struct drm_device *dev = data;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         int ret;
 
         DRM_DEBUG("Dropping caches: 0x%08llx\n", val);
@@ -4914,13 +4921,13 @@ i915_drop_caches_set(void *data, u64 val)
                 return ret;
 
         if (val & DROP_ACTIVE) {
-                ret = i915_gpu_idle(dev);
+                ret = i915_gem_wait_for_idle(dev_priv);
                 if (ret)
                         goto unlock;
         }
 
         if (val & (DROP_RETIRE | DROP_ACTIVE))
-                i915_gem_retire_requests(dev);
+                i915_gem_retire_requests(dev_priv);
 
         if (val & DROP_BOUND)
                 i915_gem_shrink(dev_priv, LONG_MAX, I915_SHRINK_BOUND);
@@ -4942,7 +4949,7 @@ static int
 i915_max_freq_get(void *data, u64 *val)
 {
         struct drm_device *dev = data;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         int ret;
 
         if (INTEL_INFO(dev)->gen < 6)
@@ -4964,7 +4971,7 @@ static int
 i915_max_freq_set(void *data, u64 val)
 {
         struct drm_device *dev = data;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         u32 hw_max, hw_min;
         int ret;
 
@@ -4994,7 +5001,7 @@ i915_max_freq_set(void *data, u64 val)
 
         dev_priv->rps.max_freq_softlimit = val;
 
-        intel_set_rps(dev, val);
+        intel_set_rps(dev_priv, val);
 
         mutex_unlock(&dev_priv->rps.hw_lock);
 
@@ -5009,7 +5016,7 @@ static int
 i915_min_freq_get(void *data, u64 *val)
 {
         struct drm_device *dev = data;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         int ret;
 
         if (INTEL_INFO(dev)->gen < 6)
@@ -5031,7 +5038,7 @@ static int
 i915_min_freq_set(void *data, u64 val)
 {
         struct drm_device *dev = data;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         u32 hw_max, hw_min;
         int ret;
 
@@ -5061,7 +5068,7 @@ i915_min_freq_set(void *data, u64 val)
 
         dev_priv->rps.min_freq_softlimit = val;
 
-        intel_set_rps(dev, val);
+        intel_set_rps(dev_priv, val);
 
         mutex_unlock(&dev_priv->rps.hw_lock);
 
@@ -5076,7 +5083,7 @@ static int
 i915_cache_sharing_get(void *data, u64 *val)
 {
         struct drm_device *dev = data;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         u32 snpcr;
         int ret;
 
@@ -5091,7 +5098,7 @@ i915_cache_sharing_get(void *data, u64 *val)
         snpcr = I915_READ(GEN6_MBCUNIT_SNPCR);
 
         intel_runtime_pm_put(dev_priv);
-        mutex_unlock(&dev_priv->dev->struct_mutex);
+        mutex_unlock(&dev_priv->drm.struct_mutex);
 
         *val = (snpcr & GEN6_MBC_SNPCR_MASK) >> GEN6_MBC_SNPCR_SHIFT;
 
@@ -5102,7 +5109,7 @@ static int
 i915_cache_sharing_set(void *data, u64 val)
 {
         struct drm_device *dev = data;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         u32 snpcr;
 
         if (!(IS_GEN6(dev) || IS_GEN7(dev)))
@@ -5139,7 +5146,7 @@ struct sseu_dev_status {
 static void cherryview_sseu_device_status(struct drm_device *dev,
                                           struct sseu_dev_status *stat)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         int ss_max = 2;
         int ss;
         u32 sig1[ss_max], sig2[ss_max];
@@ -5171,7 +5178,7 @@ static void cherryview_sseu_device_status(struct drm_device *dev,
 static void gen9_sseu_device_status(struct drm_device *dev,
                                     struct sseu_dev_status *stat)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         int s_max = 3, ss_max = 4;
         int s, ss;
         u32 s_reg[s_max], eu_reg[2*s_max], eu_mask[2];
@@ -5236,7 +5243,7 @@ static void gen9_sseu_device_status(struct drm_device *dev,
 static void broadwell_sseu_device_status(struct drm_device *dev,
                                          struct sseu_dev_status *stat)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         int s;
         u32 slice_info = I915_READ(GEN8_GT_SLICE_INFO);
 
@@ -5278,6 +5285,10 @@ static int i915_sseu_status(struct seq_file *m, void *unused)
                    INTEL_INFO(dev)->eu_total);
         seq_printf(m, "  Available EU Per Subslice: %u\n",
                    INTEL_INFO(dev)->eu_per_subslice);
+        seq_printf(m, "  Has Pooled EU: %s\n", yesno(HAS_POOLED_EU(dev)));
+        if (HAS_POOLED_EU(dev))
+                seq_printf(m, "  Min EU in pool: %u\n",
+                           INTEL_INFO(dev)->min_eu_in_pool);
         seq_printf(m, "  Has Slice Power Gating: %s\n",
                    yesno(INTEL_INFO(dev)->has_slice_pg));
         seq_printf(m, "  Has Subslice Power Gating: %s\n",
@@ -5311,7 +5322,7 @@ static int i915_sseu_status(struct seq_file *m, void *unused)
 static int i915_forcewake_open(struct inode *inode, struct file *file)
 {
         struct drm_device *dev = inode->i_private;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
 
         if (INTEL_INFO(dev)->gen < 6)
                 return 0;
@@ -5325,7 +5336,7 @@ static int i915_forcewake_open(struct inode *inode, struct file *file)
 static int i915_forcewake_release(struct inode *inode, struct file *file)
 {
         struct drm_device *dev = inode->i_private;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
 
         if (INTEL_INFO(dev)->gen < 6)
                 return 0;
@@ -5441,7 +5452,6 @@ static const struct i915_debugfs_files {
         {"i915_max_freq", &i915_max_freq_fops},
         {"i915_min_freq", &i915_min_freq_fops},
         {"i915_cache_sharing", &i915_cache_sharing_fops},
-        {"i915_ring_stop", &i915_ring_stop_fops},
         {"i915_ring_missed_irq", &i915_ring_missed_irq_fops},
         {"i915_ring_test_irq", &i915_ring_test_irq_fops},
         {"i915_gem_drop_caches", &i915_drop_caches_fops},
@@ -5459,7 +5469,7 @@ static const struct i915_debugfs_files {
 
 void intel_display_crc_init(struct drm_device *dev)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         enum pipe pipe;
 
         for_each_pipe(dev_priv, pipe) {
@@ -5471,8 +5481,9 @@ void intel_display_crc_init(struct drm_device *dev)
         }
 }
 
-int i915_debugfs_init(struct drm_minor *minor)
+int i915_debugfs_register(struct drm_i915_private *dev_priv)
 {
+        struct drm_minor *minor = dev_priv->drm.primary;
         int ret, i;
 
         ret = i915_forcewake_create(minor->debugfs_root, minor);
@@ -5498,8 +5509,9 @@ int i915_debugfs_init(struct drm_minor *minor)
                                         minor->debugfs_root, minor);
 }
 
-void i915_debugfs_cleanup(struct drm_minor *minor)
+void i915_debugfs_unregister(struct drm_i915_private *dev_priv)
 {
+        struct drm_minor *minor = dev_priv->drm.primary;
         int i;
 
         drm_debugfs_remove_files(i915_debugfs_list,
diff --git a/drivers/gpu/drm/i915/i915_dma.c b/drivers/gpu/drm/i915/i915_dma.c
deleted file mode 100644
index b3198fcd0536..000000000000
--- a/drivers/gpu/drm/i915/i915_dma.c
+++ /dev/null
@@ -1,1587 +0,0 @@
-/* i915_dma.c -- DMA support for the I915 -*- linux-c -*-
- */
-/*
- * Copyright 2003 Tungsten Graphics, Inc., Cedar Park, Texas.
- * All Rights Reserved.
- *
- * Permission is hereby granted, free of charge, to any person obtaining a
- * copy of this software and associated documentation files (the
- * "Software"), to deal in the Software without restriction, including
- * without limitation the rights to use, copy, modify, merge, publish,
- * distribute, sub license, and/or sell copies of the Software, and to
- * permit persons to whom the Software is furnished to do so, subject to
- * the following conditions:
- *
- * The above copyright notice and this permission notice (including the
- * next paragraph) shall be included in all copies or substantial portions
- * of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
- * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
- * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT.
- * IN NO EVENT SHALL TUNGSTEN GRAPHICS AND/OR ITS SUPPLIERS BE LIABLE FOR
- * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
- * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
- * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
- *
- */
-
-#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
-
-#include <drm/drmP.h>
-#include <drm/drm_crtc_helper.h>
-#include <drm/drm_fb_helper.h>
-#include <drm/drm_legacy.h>
-#include "intel_drv.h"
-#include <drm/i915_drm.h>
-#include "i915_drv.h"
-#include "i915_vgpu.h"
-#include "i915_trace.h"
-#include <linux/pci.h>
-#include <linux/console.h>
-#include <linux/vt.h>
-#include <linux/vgaarb.h>
-#include <linux/acpi.h>
-#include <linux/pnp.h>
-#include <linux/vga_switcheroo.h>
-#include <linux/slab.h>
-#include <acpi/video.h>
-#include <linux/pm.h>
-#include <linux/pm_runtime.h>
-#include <linux/oom.h>
-
-static unsigned int i915_load_fail_count;
-
-bool __i915_inject_load_failure(const char *func, int line)
-{
-        if (i915_load_fail_count >= i915.inject_load_failure)
-                return false;
-
-        if (++i915_load_fail_count == i915.inject_load_failure) {
-                DRM_INFO("Injecting failure at checkpoint %u [%s:%d]\n",
-                         i915.inject_load_failure, func, line);
-                return true;
-        }
-
-        return false;
-}
-
-#define FDO_BUG_URL "https://bugs.freedesktop.org/enter_bug.cgi?product=DRI"
-#define FDO_BUG_MSG "Please file a bug at " FDO_BUG_URL " against DRM/Intel " \
-                    "providing the dmesg log by booting with drm.debug=0xf"
-
-void
-__i915_printk(struct drm_i915_private *dev_priv, const char *level,
-              const char *fmt, ...)
-{
-        static bool shown_bug_once;
-        struct device *dev = dev_priv->dev->dev;
-        bool is_error = level[1] <= KERN_ERR[1];
-        bool is_debug = level[1] == KERN_DEBUG[1];
-        struct va_format vaf;
-        va_list args;
-
-        if (is_debug && !(drm_debug & DRM_UT_DRIVER))
-                return;
-
-        va_start(args, fmt);
-
-        vaf.fmt = fmt;
-        vaf.va = &args;
-
-        dev_printk(level, dev, "[" DRM_NAME ":%ps] %pV",
-                   __builtin_return_address(0), &vaf);
-
-        if (is_error && !shown_bug_once) {
-                dev_notice(dev, "%s", FDO_BUG_MSG);
-                shown_bug_once = true;
-        }
-
-        va_end(args);
-}
-
-static bool i915_error_injected(struct drm_i915_private *dev_priv)
-{
-        return i915.inject_load_failure &&
-               i915_load_fail_count == i915.inject_load_failure;
-}
-
-#define i915_load_error(dev_priv, fmt, ...)                                  \
-        __i915_printk(dev_priv,                                              \
-                      i915_error_injected(dev_priv) ? KERN_DEBUG : KERN_ERR, \
-                      fmt, ##__VA_ARGS__)
-
-static int i915_getparam(struct drm_device *dev, void *data,
-                         struct drm_file *file_priv)
-{
-        struct drm_i915_private *dev_priv = dev->dev_private;
-        drm_i915_getparam_t *param = data;
-        int value;
-
-        switch (param->param) {
-        case I915_PARAM_IRQ_ACTIVE:
-        case I915_PARAM_ALLOW_BATCHBUFFER:
-        case I915_PARAM_LAST_DISPATCH:
-                /* Reject all old ums/dri params. */
-                return -ENODEV;
-        case I915_PARAM_CHIPSET_ID:
-                value = dev->pdev->device;
-                break;
-        case I915_PARAM_REVISION:
-                value = dev->pdev->revision;
-                break;
-        case I915_PARAM_HAS_GEM:
-                value = 1;
-                break;
-        case I915_PARAM_NUM_FENCES_AVAIL:
-                value = dev_priv->num_fence_regs;
-                break;
-        case I915_PARAM_HAS_OVERLAY:
-                value = dev_priv->overlay ? 1 : 0;
-                break;
-        case I915_PARAM_HAS_PAGEFLIPPING:
-                value = 1;
-                break;
-        case I915_PARAM_HAS_EXECBUF2:
-                /* depends on GEM */
-                value = 1;
-                break;
-        case I915_PARAM_HAS_BSD:
-                value = intel_engine_initialized(&dev_priv->engine[VCS]);
-                break;
-        case I915_PARAM_HAS_BLT:
-                value = intel_engine_initialized(&dev_priv->engine[BCS]);
-                break;
-        case I915_PARAM_HAS_VEBOX:
-                value = intel_engine_initialized(&dev_priv->engine[VECS]);
-                break;
-        case I915_PARAM_HAS_BSD2:
-                value = intel_engine_initialized(&dev_priv->engine[VCS2]);
-                break;
-        case I915_PARAM_HAS_RELAXED_FENCING:
-                value = 1;
-                break;
-        case I915_PARAM_HAS_COHERENT_RINGS:
-                value = 1;
-                break;
-        case I915_PARAM_HAS_EXEC_CONSTANTS:
-                value = INTEL_INFO(dev)->gen >= 4;
-                break;
-        case I915_PARAM_HAS_RELAXED_DELTA:
-                value = 1;
-                break;
-        case I915_PARAM_HAS_GEN7_SOL_RESET:
-                value = 1;
-                break;
-        case I915_PARAM_HAS_LLC:
-                value = HAS_LLC(dev);
-                break;
-        case I915_PARAM_HAS_WT:
-                value = HAS_WT(dev);
-                break;
-        case I915_PARAM_HAS_ALIASING_PPGTT:
-                value = USES_PPGTT(dev);
-                break;
-        case I915_PARAM_HAS_WAIT_TIMEOUT:
-                value = 1;
-                break;
-        case I915_PARAM_HAS_SEMAPHORES:
-                value = i915_semaphore_is_enabled(dev);
-                break;
-        case I915_PARAM_HAS_PRIME_VMAP_FLUSH:
-                value = 1;
-                break;
-        case I915_PARAM_HAS_SECURE_BATCHES:
-                value = capable(CAP_SYS_ADMIN);
-                break;
-        case I915_PARAM_HAS_PINNED_BATCHES:
-                value = 1;
-                break;
-        case I915_PARAM_HAS_EXEC_NO_RELOC:
-                value = 1;
-                break;
-        case I915_PARAM_HAS_EXEC_HANDLE_LUT:
-                value = 1;
-                break;
-        case I915_PARAM_CMD_PARSER_VERSION:
-                value = i915_cmd_parser_get_version();
-                break;
-        case I915_PARAM_HAS_COHERENT_PHYS_GTT:
-                value = 1;
-                break;
-        case I915_PARAM_MMAP_VERSION:
-                value = 1;
-                break;
-        case I915_PARAM_SUBSLICE_TOTAL:
-                value = INTEL_INFO(dev)->subslice_total;
-                if (!value)
-                        return -ENODEV;
-                break;
-        case I915_PARAM_EU_TOTAL:
-                value = INTEL_INFO(dev)->eu_total;
-                if (!value)
-                        return -ENODEV;
-                break;
-        case I915_PARAM_HAS_GPU_RESET:
-                value = i915.enable_hangcheck &&
-                        intel_has_gpu_reset(dev);
-                break;
-        case I915_PARAM_HAS_RESOURCE_STREAMER:
-                value = HAS_RESOURCE_STREAMER(dev);
-                break;
-        case I915_PARAM_HAS_EXEC_SOFTPIN:
-                value = 1;
-                break;
-        default:
-                DRM_DEBUG("Unknown parameter %d\n", param->param);
-                return -EINVAL;
-        }
-
-        if (copy_to_user(param->value, &value, sizeof(int))) {
-                DRM_ERROR("copy_to_user failed\n");
-                return -EFAULT;
-        }
-
-        return 0;
-}
-
-static int i915_get_bridge_dev(struct drm_device *dev)
-{
-        struct drm_i915_private *dev_priv = dev->dev_private;
-
-        dev_priv->bridge_dev = pci_get_bus_and_slot(0, PCI_DEVFN(0, 0));
-        if (!dev_priv->bridge_dev) {
-                DRM_ERROR("bridge device not found\n");
-                return -1;
-        }
-        return 0;
-}
-
-/* Allocate space for the MCH regs if needed, return nonzero on error */
-static int
-intel_alloc_mchbar_resource(struct drm_device *dev)
-{
-        struct drm_i915_private *dev_priv = dev->dev_private;
-        int reg = INTEL_INFO(dev)->gen >= 4 ? MCHBAR_I965 : MCHBAR_I915;
-        u32 temp_lo, temp_hi = 0;
-        u64 mchbar_addr;
-        int ret;
-
-        if (INTEL_INFO(dev)->gen >= 4)
-                pci_read_config_dword(dev_priv->bridge_dev, reg + 4, &temp_hi);
-        pci_read_config_dword(dev_priv->bridge_dev, reg, &temp_lo);
-        mchbar_addr = ((u64)temp_hi << 32) | temp_lo;
-
-        /* If ACPI doesn't have it, assume we need to allocate it ourselves */
-#ifdef CONFIG_PNP
-        if (mchbar_addr &&
-            pnp_range_reserved(mchbar_addr, mchbar_addr + MCHBAR_SIZE))
-                return 0;
-#endif
-
-        /* Get some space for it */
-        dev_priv->mch_res.name = "i915 MCHBAR";
-        dev_priv->mch_res.flags = IORESOURCE_MEM;
-        ret = pci_bus_alloc_resource(dev_priv->bridge_dev->bus,
-                                     &dev_priv->mch_res,
-                                     MCHBAR_SIZE, MCHBAR_SIZE,
-                                     PCIBIOS_MIN_MEM,
-                                     0, pcibios_align_resource,
-                                     dev_priv->bridge_dev);
-        if (ret) {
-                DRM_DEBUG_DRIVER("failed bus alloc: %d\n", ret);
-                dev_priv->mch_res.start = 0;
-                return ret;
-        }
-
-        if (INTEL_INFO(dev)->gen >= 4)
-                pci_write_config_dword(dev_priv->bridge_dev, reg + 4,
-                                       upper_32_bits(dev_priv->mch_res.start));
-
-        pci_write_config_dword(dev_priv->bridge_dev, reg,
-                               lower_32_bits(dev_priv->mch_res.start));
-        return 0;
-}
-
-/* Setup MCHBAR if possible, return true if we should disable it again */
-static void
-intel_setup_mchbar(struct drm_device *dev)
-{
-        struct drm_i915_private *dev_priv = dev->dev_private;
-        int mchbar_reg = INTEL_INFO(dev)->gen >= 4 ? MCHBAR_I965 : MCHBAR_I915;
-        u32 temp;
-        bool enabled;
-
-        if (IS_VALLEYVIEW(dev) || IS_CHERRYVIEW(dev))
-                return;
-
-        dev_priv->mchbar_need_disable = false;
-
-        if (IS_I915G(dev) || IS_I915GM(dev)) {
-                pci_read_config_dword(dev_priv->bridge_dev, DEVEN, &temp);
-                enabled = !!(temp & DEVEN_MCHBAR_EN);
-        } else {
-                pci_read_config_dword(dev_priv->bridge_dev, mchbar_reg, &temp);
-                enabled = temp & 1;
-        }
-
-        /* If it's already enabled, don't have to do anything */
-        if (enabled)
-                return;
-
-        if (intel_alloc_mchbar_resource(dev))
-                return;
-
-        dev_priv->mchbar_need_disable = true;
-
-        /* Space is allocated or reserved, so enable it. */
-        if (IS_I915G(dev) || IS_I915GM(dev)) {
-                pci_write_config_dword(dev_priv->bridge_dev, DEVEN,
-                                       temp | DEVEN_MCHBAR_EN);
-        } else {
-                pci_read_config_dword(dev_priv->bridge_dev, mchbar_reg, &temp);
-                pci_write_config_dword(dev_priv->bridge_dev, mchbar_reg, temp | 1);
-        }
-}
-
-static void
-intel_teardown_mchbar(struct drm_device *dev)
-{
-        struct drm_i915_private *dev_priv = dev->dev_private;
-        int mchbar_reg = INTEL_INFO(dev)->gen >= 4 ? MCHBAR_I965 : MCHBAR_I915;
-
-        if (dev_priv->mchbar_need_disable) {
-                if (IS_I915G(dev) || IS_I915GM(dev)) {
-                        u32 deven_val;
-
-                        pci_read_config_dword(dev_priv->bridge_dev, DEVEN,
-                                              &deven_val);
-                        deven_val &= ~DEVEN_MCHBAR_EN;
-                        pci_write_config_dword(dev_priv->bridge_dev, DEVEN,
-                                               deven_val);
-                } else {
-                        u32 mchbar_val;
-
-                        pci_read_config_dword(dev_priv->bridge_dev, mchbar_reg,
-                                              &mchbar_val);
-                        mchbar_val &= ~1;
-                        pci_write_config_dword(dev_priv->bridge_dev, mchbar_reg,
-                                               mchbar_val);
-                }
-        }
-
-        if (dev_priv->mch_res.start)
-                release_resource(&dev_priv->mch_res);
-}
-
-/* true = enable decode, false = disable decoder */
-static unsigned int i915_vga_set_decode(void *cookie, bool state)
-{
-        struct drm_device *dev = cookie;
-
-        intel_modeset_vga_set_state(dev, state);
-        if (state)
-                return VGA_RSRC_LEGACY_IO | VGA_RSRC_LEGACY_MEM |
-                       VGA_RSRC_NORMAL_IO | VGA_RSRC_NORMAL_MEM;
-        else
-                return VGA_RSRC_NORMAL_IO | VGA_RSRC_NORMAL_MEM;
-}
-
-static void i915_switcheroo_set_state(struct pci_dev *pdev, enum vga_switcheroo_state state)
-{
-        struct drm_device *dev = pci_get_drvdata(pdev);
-        pm_message_t pmm = { .event = PM_EVENT_SUSPEND };
-
-        if (state == VGA_SWITCHEROO_ON) {
-                pr_info("switched on\n");
-                dev->switch_power_state = DRM_SWITCH_POWER_CHANGING;
-                /* i915 resume handler doesn't set to D0 */
-                pci_set_power_state(dev->pdev, PCI_D0);
-                i915_resume_switcheroo(dev);
-                dev->switch_power_state = DRM_SWITCH_POWER_ON;
-        } else {
-                pr_info("switched off\n");
-                dev->switch_power_state = DRM_SWITCH_POWER_CHANGING;
-                i915_suspend_switcheroo(dev, pmm);
-                dev->switch_power_state = DRM_SWITCH_POWER_OFF;
-        }
-}
-
-static bool i915_switcheroo_can_switch(struct pci_dev *pdev)
-{
-        struct drm_device *dev = pci_get_drvdata(pdev);
-
-        /*
-         * FIXME: open_count is protected by drm_global_mutex but that would lead to
-         * locking inversion with the driver load path. And the access here is
-         * completely racy anyway. So don't bother with locking for now.
-         */
-        return dev->open_count == 0;
-}
-
-static const struct vga_switcheroo_client_ops i915_switcheroo_ops = {
-        .set_gpu_state = i915_switcheroo_set_state,
-        .reprobe = NULL,
-        .can_switch = i915_switcheroo_can_switch,
-};
-
-static int i915_load_modeset_init(struct drm_device *dev)
-{
-        struct drm_i915_private *dev_priv = dev->dev_private;
-        int ret;
-
-        if (i915_inject_load_failure())
-                return -ENODEV;
-
-        ret = intel_bios_init(dev_priv);
-        if (ret)
-                DRM_INFO("failed to find VBIOS tables\n");
-
-        /* If we have > 1 VGA cards, then we need to arbitrate access
-         * to the common VGA resources.
-         *
-         * If we are a secondary display controller (!PCI_DISPLAY_CLASS_VGA),
-         * then we do not take part in VGA arbitration and the
-         * vga_client_register() fails with -ENODEV.
-         */
-        ret = vga_client_register(dev->pdev, dev, NULL, i915_vga_set_decode);
-        if (ret && ret != -ENODEV)
-                goto out;
-
-        intel_register_dsm_handler();
-
-        ret = vga_switcheroo_register_client(dev->pdev, &i915_switcheroo_ops, false);
-        if (ret)
-                goto cleanup_vga_client;
-
-        intel_power_domains_init_hw(dev_priv, false);
-
-        intel_csr_ucode_init(dev_priv);
-
-        ret = intel_irq_install(dev_priv);
-        if (ret)
-                goto cleanup_csr;
-
-        intel_setup_gmbus(dev);
-
-        /* Important: The output setup functions called by modeset_init need
-         * working irqs for e.g. gmbus and dp aux transfers. */
-        intel_modeset_init(dev);
-
-        intel_guc_ucode_init(dev);
-
-        ret = i915_gem_init(dev);
-        if (ret)
-                goto cleanup_irq;
-
-        intel_modeset_gem_init(dev);
-
-        if (INTEL_INFO(dev)->num_pipes == 0)
-                return 0;
-
-        ret = intel_fbdev_init(dev);
-        if (ret)
-                goto cleanup_gem;
-
-        /* Only enable hotplug handling once the fbdev is fully set up. */
-        intel_hpd_init(dev_priv);
-
-        /*
-         * Some ports require correctly set-up hpd registers for detection to
-         * work properly (leading to ghost connected connector status), e.g. VGA
-         * on gm45.  Hence we can only set up the initial fbdev config after hpd
-         * irqs are fully enabled. Now we should scan for the initial config
-         * only once hotplug handling is enabled, but due to screwed-up locking
-         * around kms/fbdev init we can't protect the fdbev initial config
-         * scanning against hotplug events. Hence do this first and ignore the
-         * tiny window where we will loose hotplug notifactions.
-         */
-        intel_fbdev_initial_config_async(dev);
-
-        drm_kms_helper_poll_init(dev);
-
-        return 0;
-
-cleanup_gem:
-        mutex_lock(&dev->struct_mutex);
-        i915_gem_cleanup_engines(dev);
-        i915_gem_context_fini(dev);
-        mutex_unlock(&dev->struct_mutex);
-cleanup_irq:
-        intel_guc_ucode_fini(dev);
-        drm_irq_uninstall(dev);
-        intel_teardown_gmbus(dev);
-cleanup_csr:
-        intel_csr_ucode_fini(dev_priv);
-        intel_power_domains_fini(dev_priv);
-        vga_switcheroo_unregister_client(dev->pdev);
-cleanup_vga_client:
-        vga_client_register(dev->pdev, NULL, NULL, NULL);
-out:
-        return ret;
-}
-
-#if IS_ENABLED(CONFIG_FB)
-static int i915_kick_out_firmware_fb(struct drm_i915_private *dev_priv)
-{
-        struct apertures_struct *ap;
-        struct pci_dev *pdev = dev_priv->dev->pdev;
-        struct i915_ggtt *ggtt = &dev_priv->ggtt;
-        bool primary;
-        int ret;
-
-        ap = alloc_apertures(1);
-        if (!ap)
-                return -ENOMEM;
-
-        ap->ranges[0].base = ggtt->mappable_base;
-        ap->ranges[0].size = ggtt->mappable_end;
-
-        primary =
-                pdev->resource[PCI_ROM_RESOURCE].flags & IORESOURCE_ROM_SHADOW;
-
-        ret = remove_conflicting_framebuffers(ap, "inteldrmfb", primary);
-
-        kfree(ap);
-
-        return ret;
-}
-#else
-static int i915_kick_out_firmware_fb(struct drm_i915_private *dev_priv)
-{
-        return 0;
-}
-#endif
-
-#if !defined(CONFIG_VGA_CONSOLE)
-static int i915_kick_out_vgacon(struct drm_i915_private *dev_priv)
-{
-        return 0;
-}
-#elif !defined(CONFIG_DUMMY_CONSOLE)
-static int i915_kick_out_vgacon(struct drm_i915_private *dev_priv)
-{
-        return -ENODEV;
-}
-#else
-static int i915_kick_out_vgacon(struct drm_i915_private *dev_priv)
-{
-        int ret = 0;
-
-        DRM_INFO("Replacing VGA console driver\n");
-
-        console_lock();
-        if (con_is_bound(&vga_con))
-                ret = do_take_over_console(&dummy_con, 0, MAX_NR_CONSOLES - 1, 1);
-        if (ret == 0) {
-                ret = do_unregister_con_driver(&vga_con);
-
-                /* Ignore "already unregistered". */
-                if (ret == -ENODEV)
-                        ret = 0;
-        }
-        console_unlock();
-
-        return ret;
-}
-#endif
-
-static void i915_dump_device_info(struct drm_i915_private *dev_priv)
-{
-        const struct intel_device_info *info = &dev_priv->info;
-
-#define PRINT_S(name) "%s"
-#define SEP_EMPTY
-#define PRINT_FLAG(name) info->name ? #name "," : ""
-#define SEP_COMMA ,
-        DRM_DEBUG_DRIVER("i915 device info: gen=%i, pciid=0x%04x rev=0x%02x flags="
-                         DEV_INFO_FOR_EACH_FLAG(PRINT_S, SEP_EMPTY),
-                         info->gen,
-                         dev_priv->dev->pdev->device,
-                         dev_priv->dev->pdev->revision,
-                         DEV_INFO_FOR_EACH_FLAG(PRINT_FLAG, SEP_COMMA));
-#undef PRINT_S
-#undef SEP_EMPTY
-#undef PRINT_FLAG
-#undef SEP_COMMA
-}
-
-static void cherryview_sseu_info_init(struct drm_device *dev)
-{
-        struct drm_i915_private *dev_priv = dev->dev_private;
-        struct intel_device_info *info;
-        u32 fuse, eu_dis;
-
-        info = (struct intel_device_info *)&dev_priv->info;
-        fuse = I915_READ(CHV_FUSE_GT);
-
-        info->slice_total = 1;
-
-        if (!(fuse & CHV_FGT_DISABLE_SS0)) {
-                info->subslice_per_slice++;
-                eu_dis = fuse & (CHV_FGT_EU_DIS_SS0_R0_MASK |
-                                 CHV_FGT_EU_DIS_SS0_R1_MASK);
-                info->eu_total += 8 - hweight32(eu_dis);
-        }
-
-        if (!(fuse & CHV_FGT_DISABLE_SS1)) {
-                info->subslice_per_slice++;
-                eu_dis = fuse & (CHV_FGT_EU_DIS_SS1_R0_MASK |
-                                 CHV_FGT_EU_DIS_SS1_R1_MASK);
-                info->eu_total += 8 - hweight32(eu_dis);
-        }
-
-        info->subslice_total = info->subslice_per_slice;
-        /*
-         * CHV expected to always have a uniform distribution of EU
-         * across subslices.
-        */
-        info->eu_per_subslice = info->subslice_total ?
-                                info->eu_total / info->subslice_total :
-                                0;
-        /*
-         * CHV supports subslice power gating on devices with more than
-         * one subslice, and supports EU power gating on devices with
-         * more than one EU pair per subslice.
-        */
-        info->has_slice_pg = 0;
-        info->has_subslice_pg = (info->subslice_total > 1);
-        info->has_eu_pg = (info->eu_per_subslice > 2);
-}
-
-static void gen9_sseu_info_init(struct drm_device *dev)
-{
-        struct drm_i915_private *dev_priv = dev->dev_private;
-        struct intel_device_info *info;
-        int s_max = 3, ss_max = 4, eu_max = 8;
-        int s, ss;
-        u32 fuse2, s_enable, ss_disable, eu_disable;
-        u8 eu_mask = 0xff;
-
-        info = (struct intel_device_info *)&dev_priv->info;
-        fuse2 = I915_READ(GEN8_FUSE2);
-        s_enable = (fuse2 & GEN8_F2_S_ENA_MASK) >>
-                   GEN8_F2_S_ENA_SHIFT;
-        ss_disable = (fuse2 & GEN9_F2_SS_DIS_MASK) >>
-                     GEN9_F2_SS_DIS_SHIFT;
-
-        info->slice_total = hweight32(s_enable);
-        /*
-         * The subslice disable field is global, i.e. it applies
-         * to each of the enabled slices.
-        */
-        info->subslice_per_slice = ss_max - hweight32(ss_disable);
-        info->subslice_total = info->slice_total *
-                               info->subslice_per_slice;
-
-        /*
-         * Iterate through enabled slices and subslices to
-         * count the total enabled EU.
-        */
-        for (s = 0; s < s_max; s++) {
-                if (!(s_enable & (0x1 << s)))
-                        /* skip disabled slice */
-                        continue;
-
-                eu_disable = I915_READ(GEN9_EU_DISABLE(s));
-                for (ss = 0; ss < ss_max; ss++) {
-                        int eu_per_ss;
-
-                        if (ss_disable & (0x1 << ss))
-                                /* skip disabled subslice */
-                                continue;
-
-                        eu_per_ss = eu_max - hweight8((eu_disable >> (ss*8)) &
-                                                      eu_mask);
-
-                        /*
-                         * Record which subslice(s) has(have) 7 EUs. we
-                         * can tune the hash used to spread work among
-                         * subslices if they are unbalanced.
-                         */
-                        if (eu_per_ss == 7)
-                                info->subslice_7eu[s] |= 1 << ss;
-
-                        info->eu_total += eu_per_ss;
-                }
-        }
-
-        /*
-         * SKL is expected to always have a uniform distribution
-         * of EU across subslices with the exception that any one
-         * EU in any one subslice may be fused off for die
-         * recovery. BXT is expected to be perfectly uniform in EU
-         * distribution.
-        */
-        info->eu_per_subslice = info->subslice_total ?
-                                DIV_ROUND_UP(info->eu_total,
-                                             info->subslice_total) : 0;
-        /*
-         * SKL supports slice power gating on devices with more than
-         * one slice, and supports EU power gating on devices with
-         * more than one EU pair per subslice. BXT supports subslice
-         * power gating on devices with more than one subslice, and
-         * supports EU power gating on devices with more than one EU
-         * pair per subslice.
-        */
-        info->has_slice_pg = ((IS_SKYLAKE(dev) || IS_KABYLAKE(dev)) &&
-                               (info->slice_total > 1));
-        info->has_subslice_pg = (IS_BROXTON(dev) && (info->subslice_total > 1));
-        info->has_eu_pg = (info->eu_per_subslice > 2);
-}
-
-static void broadwell_sseu_info_init(struct drm_device *dev)
-{
-        struct drm_i915_private *dev_priv = dev->dev_private;
-        struct intel_device_info *info;
-        const int s_max = 3, ss_max = 3, eu_max = 8;
-        int s, ss;
-        u32 fuse2, eu_disable[s_max], s_enable, ss_disable;
-
-        fuse2 = I915_READ(GEN8_FUSE2);
-        s_enable = (fuse2 & GEN8_F2_S_ENA_MASK) >> GEN8_F2_S_ENA_SHIFT;
-        ss_disable = (fuse2 & GEN8_F2_SS_DIS_MASK) >> GEN8_F2_SS_DIS_SHIFT;
-
-        eu_disable[0] = I915_READ(GEN8_EU_DISABLE0) & GEN8_EU_DIS0_S0_MASK;
-        eu_disable[1] = (I915_READ(GEN8_EU_DISABLE0) >> GEN8_EU_DIS0_S1_SHIFT) |
-                        ((I915_READ(GEN8_EU_DISABLE1) & GEN8_EU_DIS1_S1_MASK) <<
-                         (32 - GEN8_EU_DIS0_S1_SHIFT));
-        eu_disable[2] = (I915_READ(GEN8_EU_DISABLE1) >> GEN8_EU_DIS1_S2_SHIFT) |
-                        ((I915_READ(GEN8_EU_DISABLE2) & GEN8_EU_DIS2_S2_MASK) <<
-                         (32 - GEN8_EU_DIS1_S2_SHIFT));
-
-
-        info = (struct intel_device_info *)&dev_priv->info;
-        info->slice_total = hweight32(s_enable);
-
-        /*
-         * The subslice disable field is global, i.e. it applies
-         * to each of the enabled slices.
-         */
-        info->subslice_per_slice = ss_max - hweight32(ss_disable);
-        info->subslice_total = info->slice_total * info->subslice_per_slice;
-
-        /*
-         * Iterate through enabled slices and subslices to
-         * count the total enabled EU.
-         */
-        for (s = 0; s < s_max; s++) {
-                if (!(s_enable & (0x1 << s)))
-                        /* skip disabled slice */
-                        continue;
-
-                for (ss = 0; ss < ss_max; ss++) {
-                        u32 n_disabled;
-
-                        if (ss_disable & (0x1 << ss))
-                                /* skip disabled subslice */
-                                continue;
-
-                        n_disabled = hweight8(eu_disable[s] >> (ss * eu_max));
-
-                        /*
-                         * Record which subslices have 7 EUs.
-                         */
-                        if (eu_max - n_disabled == 7)
-                                info->subslice_7eu[s] |= 1 << ss;
-
-                        info->eu_total += eu_max - n_disabled;
-                }
-        }
-
-        /*
-         * BDW is expected to always have a uniform distribution of EU across
-         * subslices with the exception that any one EU in any one subslice may
-         * be fused off for die recovery.
-         */
-        info->eu_per_subslice = info->subslice_total ?
-                DIV_ROUND_UP(info->eu_total, info->subslice_total) : 0;
-
-        /*
-         * BDW supports slice power gating on devices with more than
-         * one slice.
-         */
-        info->has_slice_pg = (info->slice_total > 1);
-        info->has_subslice_pg = 0;
-        info->has_eu_pg = 0;
-}
-
-/*
- * Determine various intel_device_info fields at runtime.
- *
- * Use it when either:
- *   - it's judged too laborious to fill n static structures with the limit
- *     when a simple if statement does the job,
- *   - run-time checks (eg read fuse/strap registers) are needed.
- *
- * This function needs to be called:
- *   - after the MMIO has been setup as we are reading registers,
- *   - after the PCH has been detected,
- *   - before the first usage of the fields it can tweak.
- */
-static void intel_device_info_runtime_init(struct drm_device *dev)
-{
-        struct drm_i915_private *dev_priv = dev->dev_private;
-        struct intel_device_info *info;
-        enum pipe pipe;
-
-        info = (struct intel_device_info *)&dev_priv->info;
-
-        /*
-         * Skylake and Broxton currently don't expose the topmost plane as its
-         * use is exclusive with the legacy cursor and we only want to expose
-         * one of those, not both. Until we can safely expose the topmost plane
-         * as a DRM_PLANE_TYPE_CURSOR with all the features exposed/supported,
-         * we don't expose the topmost plane at all to prevent ABI breakage
-         * down the line.
-         */
-        if (IS_BROXTON(dev)) {
-                info->num_sprites[PIPE_A] = 2;
-                info->num_sprites[PIPE_B] = 2;
-                info->num_sprites[PIPE_C] = 1;
-        } else if (IS_VALLEYVIEW(dev) || IS_CHERRYVIEW(dev))
-                for_each_pipe(dev_priv, pipe)
-                        info->num_sprites[pipe] = 2;
-        else
-                for_each_pipe(dev_priv, pipe)
-                        info->num_sprites[pipe] = 1;
-
-        if (i915.disable_display) {
-                DRM_INFO("Display disabled (module parameter)\n");
-                info->num_pipes = 0;
-        } else if (info->num_pipes > 0 &&
-                   (INTEL_INFO(dev)->gen == 7 || INTEL_INFO(dev)->gen == 8) &&
-                   HAS_PCH_SPLIT(dev)) {
-                u32 fuse_strap = I915_READ(FUSE_STRAP);
-                u32 sfuse_strap = I915_READ(SFUSE_STRAP);
-
-                /*
-                 * SFUSE_STRAP is supposed to have a bit signalling the display
-                 * is fused off. Unfortunately it seems that, at least in
-                 * certain cases, fused off display means that PCH display
-                 * reads don't land anywhere. In that case, we read 0s.
-                 *
-                 * On CPT/PPT, we can detect this case as SFUSE_STRAP_FUSE_LOCK
-                 * should be set when taking over after the firmware.
-                 */
-                if (fuse_strap & ILK_INTERNAL_DISPLAY_DISABLE ||
-                    sfuse_strap & SFUSE_STRAP_DISPLAY_DISABLED ||
-                    (dev_priv->pch_type == PCH_CPT &&
-                     !(sfuse_strap & SFUSE_STRAP_FUSE_LOCK))) {
-                        DRM_INFO("Display fused off, disabling\n");
-                        info->num_pipes = 0;
-                } else if (fuse_strap & IVB_PIPE_C_DISABLE) {
-                        DRM_INFO("PipeC fused off\n");
-                        info->num_pipes -= 1;
-                }
-        } else if (info->num_pipes > 0 && INTEL_INFO(dev)->gen == 9) {
-                u32 dfsm = I915_READ(SKL_DFSM);
-                u8 disabled_mask = 0;
-                bool invalid;
-                int num_bits;
-
-                if (dfsm & SKL_DFSM_PIPE_A_DISABLE)
-                        disabled_mask |= BIT(PIPE_A);
-                if (dfsm & SKL_DFSM_PIPE_B_DISABLE)
-                        disabled_mask |= BIT(PIPE_B);
-                if (dfsm & SKL_DFSM_PIPE_C_DISABLE)
-                        disabled_mask |= BIT(PIPE_C);
-
-                num_bits = hweight8(disabled_mask);
-
-                switch (disabled_mask) {
-                case BIT(PIPE_A):
-                case BIT(PIPE_B):
-                case BIT(PIPE_A) | BIT(PIPE_B):
-                case BIT(PIPE_A) | BIT(PIPE_C):
-                        invalid = true;
-                        break;
-                default:
-                        invalid = false;
-                }
-
-                if (num_bits > info->num_pipes || invalid)
-                        DRM_ERROR("invalid pipe fuse configuration: 0x%x\n",
-                                  disabled_mask);
-                else
-                        info->num_pipes -= num_bits;
-        }
-
-        /* Initialize slice/subslice/EU info */
-        if (IS_CHERRYVIEW(dev))
-                cherryview_sseu_info_init(dev);
-        else if (IS_BROADWELL(dev))
-                broadwell_sseu_info_init(dev);
-        else if (INTEL_INFO(dev)->gen >= 9)
-                gen9_sseu_info_init(dev);
-
-        /* Snooping is broken on BXT A stepping. */
-        info->has_snoop = !info->has_llc;
-        info->has_snoop &= !IS_BXT_REVID(dev, 0, BXT_REVID_A1);
-
-        DRM_DEBUG_DRIVER("slice total: %u\n", info->slice_total);
-        DRM_DEBUG_DRIVER("subslice total: %u\n", info->subslice_total);
-        DRM_DEBUG_DRIVER("subslice per slice: %u\n", info->subslice_per_slice);
-        DRM_DEBUG_DRIVER("EU total: %u\n", info->eu_total);
-        DRM_DEBUG_DRIVER("EU per subslice: %u\n", info->eu_per_subslice);
-        DRM_DEBUG_DRIVER("has slice power gating: %s\n",
-                         info->has_slice_pg ? "y" : "n");
-        DRM_DEBUG_DRIVER("has subslice power gating: %s\n",
-                         info->has_subslice_pg ? "y" : "n");
-        DRM_DEBUG_DRIVER("has EU power gating: %s\n",
-                         info->has_eu_pg ? "y" : "n");
-}
-
-static void intel_init_dpio(struct drm_i915_private *dev_priv)
-{
-        /*
-         * IOSF_PORT_DPIO is used for VLV x2 PHY (DP/HDMI B and C),
-         * CHV x1 PHY (DP/HDMI D)
-         * IOSF_PORT_DPIO_2 is used for CHV x2 PHY (DP/HDMI B and C)
-         */
-        if (IS_CHERRYVIEW(dev_priv)) {
-                DPIO_PHY_IOSF_PORT(DPIO_PHY0) = IOSF_PORT_DPIO_2;
-                DPIO_PHY_IOSF_PORT(DPIO_PHY1) = IOSF_PORT_DPIO;
-        } else if (IS_VALLEYVIEW(dev_priv)) {
-                DPIO_PHY_IOSF_PORT(DPIO_PHY0) = IOSF_PORT_DPIO;
-        }
-}
-
-static int i915_workqueues_init(struct drm_i915_private *dev_priv)
-{
-        /*
-         * The i915 workqueue is primarily used for batched retirement of
-         * requests (and thus managing bo) once the task has been completed
-         * by the GPU. i915_gem_retire_requests() is called directly when we
-         * need high-priority retirement, such as waiting for an explicit
-         * bo.
-         *
-         * It is also used for periodic low-priority events, such as
-         * idle-timers and recording error state.
-         *
-         * All tasks on the workqueue are expected to acquire the dev mutex
-         * so there is no point in running more than one instance of the
-         * workqueue at any time.  Use an ordered one.
-         */
-        dev_priv->wq = alloc_ordered_workqueue("i915", 0);
-        if (dev_priv->wq == NULL)
-                goto out_err;
-
-        dev_priv->hotplug.dp_wq = alloc_ordered_workqueue("i915-dp", 0);
-        if (dev_priv->hotplug.dp_wq == NULL)
-                goto out_free_wq;
-
-        dev_priv->gpu_error.hangcheck_wq =
-                alloc_ordered_workqueue("i915-hangcheck", 0);
-        if (dev_priv->gpu_error.hangcheck_wq == NULL)
-                goto out_free_dp_wq;
-
-        return 0;
-
-out_free_dp_wq:
-        destroy_workqueue(dev_priv->hotplug.dp_wq);
-out_free_wq:
-        destroy_workqueue(dev_priv->wq);
-out_err:
-        DRM_ERROR("Failed to allocate workqueues.\n");
-
-        return -ENOMEM;
-}
-
-static void i915_workqueues_cleanup(struct drm_i915_private *dev_priv)
-{
-        destroy_workqueue(dev_priv->gpu_error.hangcheck_wq);
-        destroy_workqueue(dev_priv->hotplug.dp_wq);
-        destroy_workqueue(dev_priv->wq);
-}
-
-/**
- * i915_driver_init_early - setup state not requiring device access
- * @dev_priv: device private
- *
- * Initialize everything that is a "SW-only" state, that is state not
- * requiring accessing the device or exposing the driver via kernel internal
- * or userspace interfaces. Example steps belonging here: lock initialization,
- * system memory allocation, setting up device specific attributes and
- * function hooks not requiring accessing the device.
- */
-static int i915_driver_init_early(struct drm_i915_private *dev_priv,
-                                  struct drm_device *dev,
-                                  struct intel_device_info *info)
-{
-        struct intel_device_info *device_info;
-        int ret = 0;
-
-        if (i915_inject_load_failure())
-                return -ENODEV;
-
-        /* Setup the write-once "constant" device info */
-        device_info = (struct intel_device_info *)&dev_priv->info;
-        memcpy(device_info, info, sizeof(dev_priv->info));
-        device_info->device_id = dev->pdev->device;
-
-        spin_lock_init(&dev_priv->irq_lock);
-        spin_lock_init(&dev_priv->gpu_error.lock);
-        mutex_init(&dev_priv->backlight_lock);
-        spin_lock_init(&dev_priv->uncore.lock);
-        spin_lock_init(&dev_priv->mm.object_stat_lock);
-        spin_lock_init(&dev_priv->mmio_flip_lock);
-        mutex_init(&dev_priv->sb_lock);
-        mutex_init(&dev_priv->modeset_restore_lock);
-        mutex_init(&dev_priv->av_mutex);
-        mutex_init(&dev_priv->wm.wm_mutex);
-        mutex_init(&dev_priv->pps_mutex);
-
-        ret = i915_workqueues_init(dev_priv);
-        if (ret < 0)
-                return ret;
-
-        /* This must be called before any calls to HAS_PCH_* */
-        intel_detect_pch(dev);
-
-        intel_pm_setup(dev);
-        intel_init_dpio(dev_priv);
-        intel_power_domains_init(dev_priv);
-        intel_irq_init(dev_priv);
-        intel_init_display_hooks(dev_priv);
-        intel_init_clock_gating_hooks(dev_priv);
-        intel_init_audio_hooks(dev_priv);
-        i915_gem_load_init(dev);
-
-        intel_display_crc_init(dev);
-
-        i915_dump_device_info(dev_priv);
-
-        /* Not all pre-production machines fall into this category, only the
-         * very first ones. Almost everything should work, except for maybe
-         * suspend/resume. And we don't implement workarounds that affect only
-         * pre-production machines. */
-        if (IS_HSW_EARLY_SDV(dev))
-                DRM_INFO("This is an early pre-production Haswell machine. "
-                         "It may not be fully functional.\n");
-
-        return 0;
-}
-
-/**
- * i915_driver_cleanup_early - cleanup the setup done in i915_driver_init_early()
- * @dev_priv: device private
- */
-static void i915_driver_cleanup_early(struct drm_i915_private *dev_priv)
-{
-        i915_gem_load_cleanup(dev_priv->dev);
-        i915_workqueues_cleanup(dev_priv);
-}
-
-static int i915_mmio_setup(struct drm_device *dev)
-{
-        struct drm_i915_private *dev_priv = to_i915(dev);
-        int mmio_bar;
-        int mmio_size;
-
-        mmio_bar = IS_GEN2(dev) ? 1 : 0;
-        /*
-         * Before gen4, the registers and the GTT are behind different BARs.
-         * However, from gen4 onwards, the registers and the GTT are shared
-         * in the same BAR, so we want to restrict this ioremap from
-         * clobbering the GTT which we want ioremap_wc instead. Fortunately,
-         * the register BAR remains the same size for all the earlier
-         * generations up to Ironlake.
-         */
-        if (INTEL_INFO(dev)->gen < 5)
-                mmio_size = 512 * 1024;
-        else
-                mmio_size = 2 * 1024 * 1024;
-        dev_priv->regs = pci_iomap(dev->pdev, mmio_bar, mmio_size);
-        if (dev_priv->regs == NULL) {
-                DRM_ERROR("failed to map registers\n");
-
-                return -EIO;
-        }
-
-        /* Try to make sure MCHBAR is enabled before poking at it */
-        intel_setup_mchbar(dev);
-
-        return 0;
-}
-
-static void i915_mmio_cleanup(struct drm_device *dev)
-{
-        struct drm_i915_private *dev_priv = to_i915(dev);
-
-        intel_teardown_mchbar(dev);
-        pci_iounmap(dev->pdev, dev_priv->regs);
-}
-
-/**
- * i915_driver_init_mmio - setup device MMIO
- * @dev_priv: device private
- *
- * Setup minimal device state necessary for MMIO accesses later in the
- * initialization sequence. The setup here should avoid any other device-wide
- * side effects or exposing the driver via kernel internal or user space
- * interfaces.
- */
-static int i915_driver_init_mmio(struct drm_i915_private *dev_priv)
-{
-        struct drm_device *dev = dev_priv->dev;
-        int ret;
-
-        if (i915_inject_load_failure())
-                return -ENODEV;
-
-        if (i915_get_bridge_dev(dev))
-                return -EIO;
-
-        ret = i915_mmio_setup(dev);
-        if (ret < 0)
-                goto put_bridge;
-
-        intel_uncore_init(dev);
-
-        return 0;
-
-put_bridge:
-        pci_dev_put(dev_priv->bridge_dev);
-
-        return ret;
-}
-
-/**
- * i915_driver_cleanup_mmio - cleanup the setup done in i915_driver_init_mmio()
- * @dev_priv: device private
- */
-static void i915_driver_cleanup_mmio(struct drm_i915_private *dev_priv)
-{
-        struct drm_device *dev = dev_priv->dev;
-
-        intel_uncore_fini(dev);
-        i915_mmio_cleanup(dev);
-        pci_dev_put(dev_priv->bridge_dev);
-}
-
-/**
- * i915_driver_init_hw - setup state requiring device access
- * @dev_priv: device private
- *
- * Setup state that requires accessing the device, but doesn't require
- * exposing the driver via kernel internal or userspace interfaces.
- */
-static int i915_driver_init_hw(struct drm_i915_private *dev_priv)
-{
-        struct drm_device *dev = dev_priv->dev;
-        struct i915_ggtt *ggtt = &dev_priv->ggtt;
-        uint32_t aperture_size;
-        int ret;
-
-        if (i915_inject_load_failure())
-                return -ENODEV;
-
-        intel_device_info_runtime_init(dev);
-
-        ret = i915_ggtt_init_hw(dev);
-        if (ret)
-                return ret;
-
-        ret = i915_ggtt_enable_hw(dev);
-        if (ret) {
-                DRM_ERROR("failed to enable GGTT\n");
-                goto out_ggtt;
-        }
-
-        /* WARNING: Apparently we must kick fbdev drivers before vgacon,
-         * otherwise the vga fbdev driver falls over. */
-        ret = i915_kick_out_firmware_fb(dev_priv);
-        if (ret) {
-                DRM_ERROR("failed to remove conflicting framebuffer drivers\n");
-                goto out_ggtt;
-        }
-
-        ret = i915_kick_out_vgacon(dev_priv);
-        if (ret) {
-                DRM_ERROR("failed to remove conflicting VGA console\n");
-                goto out_ggtt;
-        }
-
-        pci_set_master(dev->pdev);
-
-        /* overlay on gen2 is broken and can't address above 1G */
-        if (IS_GEN2(dev))
-                dma_set_coherent_mask(&dev->pdev->dev, DMA_BIT_MASK(30));
-
-        /* 965GM sometimes incorrectly writes to hardware status page (HWS)
-         * using 32bit addressing, overwriting memory if HWS is located
-         * above 4GB.
-         *
-         * The documentation also mentions an issue with undefined
-         * behaviour if any general state is accessed within a page above 4GB,
-         * which also needs to be handled carefully.
-         */
-        if (IS_BROADWATER(dev) || IS_CRESTLINE(dev))
-                dma_set_coherent_mask(&dev->pdev->dev, DMA_BIT_MASK(32));
-
-        aperture_size = ggtt->mappable_end;
-
-        ggtt->mappable =
-                io_mapping_create_wc(ggtt->mappable_base,
-                                     aperture_size);
-        if (!ggtt->mappable) {
-                ret = -EIO;
-                goto out_ggtt;
-        }
-
-        ggtt->mtrr = arch_phys_wc_add(ggtt->mappable_base,
-                                              aperture_size);
-
-        pm_qos_add_request(&dev_priv->pm_qos, PM_QOS_CPU_DMA_LATENCY,
-                           PM_QOS_DEFAULT_VALUE);
-
-        intel_uncore_sanitize(dev);
-
-        intel_opregion_setup(dev);
-
-        i915_gem_load_init_fences(dev_priv);
-
-        /* On the 945G/GM, the chipset reports the MSI capability on the
-         * integrated graphics even though the support isn't actually there
-         * according to the published specs.  It doesn't appear to function
-         * correctly in testing on 945G.
-         * This may be a side effect of MSI having been made available for PEG
-         * and the registers being closely associated.
-         *
-         * According to chipset errata, on the 965GM, MSI interrupts may
-         * be lost or delayed, but we use them anyways to avoid
-         * stuck interrupts on some machines.
-         */
-        if (!IS_I945G(dev) && !IS_I945GM(dev)) {
-                if (pci_enable_msi(dev->pdev) < 0)
-                        DRM_DEBUG_DRIVER("can't enable MSI");
-        }
-
-        return 0;
-
-out_ggtt:
-        i915_ggtt_cleanup_hw(dev);
-
-        return ret;
-}
-
-/**
- * i915_driver_cleanup_hw - cleanup the setup done in i915_driver_init_hw()
- * @dev_priv: device private
- */
-static void i915_driver_cleanup_hw(struct drm_i915_private *dev_priv)
-{
-        struct drm_device *dev = dev_priv->dev;
-        struct i915_ggtt *ggtt = &dev_priv->ggtt;
-
-        if (dev->pdev->msi_enabled)
-                pci_disable_msi(dev->pdev);
-
-        pm_qos_remove_request(&dev_priv->pm_qos);
-        arch_phys_wc_del(ggtt->mtrr);
-        io_mapping_free(ggtt->mappable);
-        i915_ggtt_cleanup_hw(dev);
-}
-
-/**
- * i915_driver_register - register the driver with the rest of the system
- * @dev_priv: device private
- *
- * Perform any steps necessary to make the driver available via kernel
- * internal or userspace interfaces.
- */
-static void i915_driver_register(struct drm_i915_private *dev_priv)
-{
-        struct drm_device *dev = dev_priv->dev;
-
-        i915_gem_shrinker_init(dev_priv);
-        /*
-         * Notify a valid surface after modesetting,
-         * when running inside a VM.
-         */
-        if (intel_vgpu_active(dev))
-                I915_WRITE(vgtif_reg(display_ready), VGT_DRV_DISPLAY_READY);
-
-        i915_setup_sysfs(dev);
-
-        if (INTEL_INFO(dev_priv)->num_pipes) {
-                /* Must be done after probing outputs */
-                intel_opregion_init(dev);
-                acpi_video_register();
-        }
-
-        if (IS_GEN5(dev_priv))
-                intel_gpu_ips_init(dev_priv);
-
-        i915_audio_component_init(dev_priv);
-}
-
-/**
- * i915_driver_unregister - cleanup the registration done in i915_driver_regiser()
- * @dev_priv: device private
- */
-static void i915_driver_unregister(struct drm_i915_private *dev_priv)
-{
-        i915_audio_component_cleanup(dev_priv);
-        intel_gpu_ips_teardown();
-        acpi_video_unregister();
-        intel_opregion_fini(dev_priv->dev);
-        i915_teardown_sysfs(dev_priv->dev);
-        i915_gem_shrinker_cleanup(dev_priv);
-}
-
-/**
- * i915_driver_load - setup chip and create an initial config
- * @dev: DRM device
- * @flags: startup flags
- *
- * The driver load routine has to do several things:
- *   - drive output discovery via intel_modeset_init()
- *   - initialize the memory manager
- *   - allocate initial config memory
- *   - setup the DRM framebuffer with the allocated memory
- */
-int i915_driver_load(struct drm_device *dev, unsigned long flags)
-{
-        struct drm_i915_private *dev_priv;
-        int ret = 0;
-
-        dev_priv = kzalloc(sizeof(*dev_priv), GFP_KERNEL);
-        if (dev_priv == NULL)
-                return -ENOMEM;
-
-        dev->dev_private = dev_priv;
-        /* Must be set before calling __i915_printk */
-        dev_priv->dev = dev;
-
-        ret = i915_driver_init_early(dev_priv, dev,
-                                     (struct intel_device_info *)flags);
-
-        if (ret < 0)
-                goto out_free_priv;
-
-        intel_runtime_pm_get(dev_priv);
-
-        ret = i915_driver_init_mmio(dev_priv);
-        if (ret < 0)
-                goto out_runtime_pm_put;
-
-        ret = i915_driver_init_hw(dev_priv);
-        if (ret < 0)
-                goto out_cleanup_mmio;
-
-        /*
-         * TODO: move the vblank init and parts of modeset init steps into one
-         * of the i915_driver_init_/i915_driver_register functions according
-         * to the role/effect of the given init step.
-         */
-        if (INTEL_INFO(dev)->num_pipes) {
-                ret = drm_vblank_init(dev, INTEL_INFO(dev)->num_pipes);
-                if (ret)
-                        goto out_cleanup_hw;
-        }
-
-        ret = i915_load_modeset_init(dev);
-        if (ret < 0)
-                goto out_cleanup_vblank;
-
-        i915_driver_register(dev_priv);
-
-        intel_runtime_pm_enable(dev_priv);
-
-        intel_runtime_pm_put(dev_priv);
-
-        return 0;
-
-out_cleanup_vblank:
-        drm_vblank_cleanup(dev);
-out_cleanup_hw:
-        i915_driver_cleanup_hw(dev_priv);
-out_cleanup_mmio:
-        i915_driver_cleanup_mmio(dev_priv);
-out_runtime_pm_put:
-        intel_runtime_pm_put(dev_priv);
-        i915_driver_cleanup_early(dev_priv);
-out_free_priv:
-        i915_load_error(dev_priv, "Device initialization failed (%d)\n", ret);
-
-        kfree(dev_priv);
-
-        return ret;
-}
-
-int i915_driver_unload(struct drm_device *dev)
-{
-        struct drm_i915_private *dev_priv = dev->dev_private;
-        int ret;
-
-        intel_fbdev_fini(dev);
-
-        ret = i915_gem_suspend(dev);
-        if (ret) {
-                DRM_ERROR("failed to idle hardware: %d\n", ret);
-                return ret;
-        }
-
-        intel_display_power_get(dev_priv, POWER_DOMAIN_INIT);
-
-        i915_driver_unregister(dev_priv);
-
-        drm_vblank_cleanup(dev);
-
-        intel_modeset_cleanup(dev);
-
-        /*
-         * free the memory space allocated for the child device
-         * config parsed from VBT
-         */
-        if (dev_priv->vbt.child_dev && dev_priv->vbt.child_dev_num) {
-                kfree(dev_priv->vbt.child_dev);
-                dev_priv->vbt.child_dev = NULL;
-                dev_priv->vbt.child_dev_num = 0;
-        }
-        kfree(dev_priv->vbt.sdvo_lvds_vbt_mode);
-        dev_priv->vbt.sdvo_lvds_vbt_mode = NULL;
-        kfree(dev_priv->vbt.lfp_lvds_vbt_mode);
-        dev_priv->vbt.lfp_lvds_vbt_mode = NULL;
-
-        vga_switcheroo_unregister_client(dev->pdev);
-        vga_client_register(dev->pdev, NULL, NULL, NULL);
-
-        intel_csr_ucode_fini(dev_priv);
-
-        /* Free error state after interrupts are fully disabled. */
-        cancel_delayed_work_sync(&dev_priv->gpu_error.hangcheck_work);
-        i915_destroy_error_state(dev);
-
-        /* Flush any outstanding unpin_work. */
-        flush_workqueue(dev_priv->wq);
-
-        intel_guc_ucode_fini(dev);
-        mutex_lock(&dev->struct_mutex);
-        i915_gem_cleanup_engines(dev);
-        i915_gem_context_fini(dev);
-        mutex_unlock(&dev->struct_mutex);
-        intel_fbc_cleanup_cfb(dev_priv);
-
-        intel_power_domains_fini(dev_priv);
-
-        i915_driver_cleanup_hw(dev_priv);
-        i915_driver_cleanup_mmio(dev_priv);
-
-        intel_display_power_put(dev_priv, POWER_DOMAIN_INIT);
-
-        i915_driver_cleanup_early(dev_priv);
-        kfree(dev_priv);
-
-        return 0;
-}
-
-int i915_driver_open(struct drm_device *dev, struct drm_file *file)
-{
-        int ret;
-
-        ret = i915_gem_open(dev, file);
-        if (ret)
-                return ret;
-
-        return 0;
-}
-
-/**
- * i915_driver_lastclose - clean up after all DRM clients have exited
- * @dev: DRM device
- *
- * Take care of cleaning up after all DRM clients have exited.  In the
- * mode setting case, we want to restore the kernel's initial mode (just
- * in case the last client left us in a bad state).
- *
- * Additionally, in the non-mode setting case, we'll tear down the GTT
- * and DMA structures, since the kernel won't be using them, and clea
- * up any GEM state.
- */
-void i915_driver_lastclose(struct drm_device *dev)
-{
-        intel_fbdev_restore_mode(dev);
-        vga_switcheroo_process_delayed_switch();
-}
-
-void i915_driver_preclose(struct drm_device *dev, struct drm_file *file)
-{
-        mutex_lock(&dev->struct_mutex);
-        i915_gem_context_close(dev, file);
-        i915_gem_release(dev, file);
-        mutex_unlock(&dev->struct_mutex);
-}
-
-void i915_driver_postclose(struct drm_device *dev, struct drm_file *file)
-{
-        struct drm_i915_file_private *file_priv = file->driver_priv;
-
-        kfree(file_priv);
-}
-
-static int
-i915_gem_reject_pin_ioctl(struct drm_device *dev, void *data,
-                          struct drm_file *file)
-{
-        return -ENODEV;
-}
-
-const struct drm_ioctl_desc i915_ioctls[] = {
-        DRM_IOCTL_DEF_DRV(I915_INIT, drm_noop, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
-        DRM_IOCTL_DEF_DRV(I915_FLUSH, drm_noop, DRM_AUTH),
-        DRM_IOCTL_DEF_DRV(I915_FLIP, drm_noop, DRM_AUTH),
-        DRM_IOCTL_DEF_DRV(I915_BATCHBUFFER, drm_noop, DRM_AUTH),
-        DRM_IOCTL_DEF_DRV(I915_IRQ_EMIT, drm_noop, DRM_AUTH),
-        DRM_IOCTL_DEF_DRV(I915_IRQ_WAIT, drm_noop, DRM_AUTH),
-        DRM_IOCTL_DEF_DRV(I915_GETPARAM, i915_getparam, DRM_AUTH|DRM_RENDER_ALLOW),
-        DRM_IOCTL_DEF_DRV(I915_SETPARAM, drm_noop, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
-        DRM_IOCTL_DEF_DRV(I915_ALLOC, drm_noop, DRM_AUTH),
-        DRM_IOCTL_DEF_DRV(I915_FREE, drm_noop, DRM_AUTH),
-        DRM_IOCTL_DEF_DRV(I915_INIT_HEAP, drm_noop, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
-        DRM_IOCTL_DEF_DRV(I915_CMDBUFFER, drm_noop, DRM_AUTH),
-        DRM_IOCTL_DEF_DRV(I915_DESTROY_HEAP,  drm_noop, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
-        DRM_IOCTL_DEF_DRV(I915_SET_VBLANK_PIPE,  drm_noop, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
-        DRM_IOCTL_DEF_DRV(I915_GET_VBLANK_PIPE,  drm_noop, DRM_AUTH),
-        DRM_IOCTL_DEF_DRV(I915_VBLANK_SWAP, drm_noop, DRM_AUTH),
-        DRM_IOCTL_DEF_DRV(I915_HWS_ADDR, drm_noop, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
-        DRM_IOCTL_DEF_DRV(I915_GEM_INIT, drm_noop, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
-        DRM_IOCTL_DEF_DRV(I915_GEM_EXECBUFFER, i915_gem_execbuffer, DRM_AUTH),
-        DRM_IOCTL_DEF_DRV(I915_GEM_EXECBUFFER2, i915_gem_execbuffer2, DRM_AUTH|DRM_RENDER_ALLOW),
-        DRM_IOCTL_DEF_DRV(I915_GEM_PIN, i915_gem_reject_pin_ioctl, DRM_AUTH|DRM_ROOT_ONLY),
-        DRM_IOCTL_DEF_DRV(I915_GEM_UNPIN, i915_gem_reject_pin_ioctl, DRM_AUTH|DRM_ROOT_ONLY),
-        DRM_IOCTL_DEF_DRV(I915_GEM_BUSY, i915_gem_busy_ioctl, DRM_AUTH|DRM_RENDER_ALLOW),
-        DRM_IOCTL_DEF_DRV(I915_GEM_SET_CACHING, i915_gem_set_caching_ioctl, DRM_RENDER_ALLOW),
-        DRM_IOCTL_DEF_DRV(I915_GEM_GET_CACHING, i915_gem_get_caching_ioctl, DRM_RENDER_ALLOW),
-        DRM_IOCTL_DEF_DRV(I915_GEM_THROTTLE, i915_gem_throttle_ioctl, DRM_AUTH|DRM_RENDER_ALLOW),
-        DRM_IOCTL_DEF_DRV(I915_GEM_ENTERVT, drm_noop, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
-        DRM_IOCTL_DEF_DRV(I915_GEM_LEAVEVT, drm_noop, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
-        DRM_IOCTL_DEF_DRV(I915_GEM_CREATE, i915_gem_create_ioctl, DRM_RENDER_ALLOW),
-        DRM_IOCTL_DEF_DRV(I915_GEM_PREAD, i915_gem_pread_ioctl, DRM_RENDER_ALLOW),
-        DRM_IOCTL_DEF_DRV(I915_GEM_PWRITE, i915_gem_pwrite_ioctl, DRM_RENDER_ALLOW),
-        DRM_IOCTL_DEF_DRV(I915_GEM_MMAP, i915_gem_mmap_ioctl, DRM_RENDER_ALLOW),
-        DRM_IOCTL_DEF_DRV(I915_GEM_MMAP_GTT, i915_gem_mmap_gtt_ioctl, DRM_RENDER_ALLOW),
-        DRM_IOCTL_DEF_DRV(I915_GEM_SET_DOMAIN, i915_gem_set_domain_ioctl, DRM_RENDER_ALLOW),
-        DRM_IOCTL_DEF_DRV(I915_GEM_SW_FINISH, i915_gem_sw_finish_ioctl, DRM_RENDER_ALLOW),
-        DRM_IOCTL_DEF_DRV(I915_GEM_SET_TILING, i915_gem_set_tiling, DRM_RENDER_ALLOW),
-        DRM_IOCTL_DEF_DRV(I915_GEM_GET_TILING, i915_gem_get_tiling, DRM_RENDER_ALLOW),
-        DRM_IOCTL_DEF_DRV(I915_GEM_GET_APERTURE, i915_gem_get_aperture_ioctl, DRM_RENDER_ALLOW),
-        DRM_IOCTL_DEF_DRV(I915_GET_PIPE_FROM_CRTC_ID, intel_get_pipe_from_crtc_id, 0),
-        DRM_IOCTL_DEF_DRV(I915_GEM_MADVISE, i915_gem_madvise_ioctl, DRM_RENDER_ALLOW),
-        DRM_IOCTL_DEF_DRV(I915_OVERLAY_PUT_IMAGE, intel_overlay_put_image, DRM_MASTER|DRM_CONTROL_ALLOW),
-        DRM_IOCTL_DEF_DRV(I915_OVERLAY_ATTRS, intel_overlay_attrs, DRM_MASTER|DRM_CONTROL_ALLOW),
-        DRM_IOCTL_DEF_DRV(I915_SET_SPRITE_COLORKEY, intel_sprite_set_colorkey, DRM_MASTER|DRM_CONTROL_ALLOW),
-        DRM_IOCTL_DEF_DRV(I915_GET_SPRITE_COLORKEY, drm_noop, DRM_MASTER|DRM_CONTROL_ALLOW),
-        DRM_IOCTL_DEF_DRV(I915_GEM_WAIT, i915_gem_wait_ioctl, DRM_AUTH|DRM_RENDER_ALLOW),
-        DRM_IOCTL_DEF_DRV(I915_GEM_CONTEXT_CREATE, i915_gem_context_create_ioctl, DRM_RENDER_ALLOW),
-        DRM_IOCTL_DEF_DRV(I915_GEM_CONTEXT_DESTROY, i915_gem_context_destroy_ioctl, DRM_RENDER_ALLOW),
-        DRM_IOCTL_DEF_DRV(I915_REG_READ, i915_reg_read_ioctl, DRM_RENDER_ALLOW),
-        DRM_IOCTL_DEF_DRV(I915_GET_RESET_STATS, i915_get_reset_stats_ioctl, DRM_RENDER_ALLOW),
-        DRM_IOCTL_DEF_DRV(I915_GEM_USERPTR, i915_gem_userptr_ioctl, DRM_RENDER_ALLOW),
-        DRM_IOCTL_DEF_DRV(I915_GEM_CONTEXT_GETPARAM, i915_gem_context_getparam_ioctl, DRM_RENDER_ALLOW),
-        DRM_IOCTL_DEF_DRV(I915_GEM_CONTEXT_SETPARAM, i915_gem_context_setparam_ioctl, DRM_RENDER_ALLOW),
-};
-
-int i915_max_ioctl = ARRAY_SIZE(i915_ioctls);
diff --git a/drivers/gpu/drm/i915/i915_drv.c b/drivers/gpu/drm/i915/i915_drv.c
index 85c4debf47e0..95ddd56b89f0 100644
--- a/drivers/gpu/drm/i915/i915_drv.c
+++ b/drivers/gpu/drm/i915/i915_drv.c
@@ -27,400 +27,92 @@
  *
  */
 
-#include <linux/device.h>
 #include <linux/acpi.h>
-#include <drm/drmP.h>
-#include <drm/i915_drm.h>
-#include "i915_drv.h"
-#include "i915_trace.h"
-#include "intel_drv.h"
-
-#include <linux/apple-gmux.h>
-#include <linux/console.h>
+#include <linux/device.h>
+#include <linux/oom.h>
 #include <linux/module.h>
+#include <linux/pci.h>
+#include <linux/pm.h>
 #include <linux/pm_runtime.h>
+#include <linux/pnp.h>
+#include <linux/slab.h>
 #include <linux/vgaarb.h>
 #include <linux/vga_switcheroo.h>
-#include <drm/drm_crtc_helper.h>
-
-static struct drm_driver driver;
-
-#define GEN_DEFAULT_PIPEOFFSETS \
-        .pipe_offsets = { PIPE_A_OFFSET, PIPE_B_OFFSET, \
-                          PIPE_C_OFFSET, PIPE_EDP_OFFSET }, \
-        .trans_offsets = { TRANSCODER_A_OFFSET, TRANSCODER_B_OFFSET, \
-                           TRANSCODER_C_OFFSET, TRANSCODER_EDP_OFFSET }, \
-        .palette_offsets = { PALETTE_A_OFFSET, PALETTE_B_OFFSET }
-
-#define GEN_CHV_PIPEOFFSETS \
-        .pipe_offsets = { PIPE_A_OFFSET, PIPE_B_OFFSET, \
-                          CHV_PIPE_C_OFFSET }, \
-        .trans_offsets = { TRANSCODER_A_OFFSET, TRANSCODER_B_OFFSET, \
-                           CHV_TRANSCODER_C_OFFSET, }, \
-        .palette_offsets = { PALETTE_A_OFFSET, PALETTE_B_OFFSET, \
-                             CHV_PALETTE_C_OFFSET }
-
-#define CURSOR_OFFSETS \
-        .cursor_offsets = { CURSOR_A_OFFSET, CURSOR_B_OFFSET, CHV_CURSOR_C_OFFSET }
-
-#define IVB_CURSOR_OFFSETS \
-        .cursor_offsets = { CURSOR_A_OFFSET, IVB_CURSOR_B_OFFSET, IVB_CURSOR_C_OFFSET }
-
-#define BDW_COLORS \
-        .color = { .degamma_lut_size = 512, .gamma_lut_size = 512 }
-#define CHV_COLORS \
-        .color = { .degamma_lut_size = 65, .gamma_lut_size = 257 }
-
-static const struct intel_device_info intel_i830_info = {
-        .gen = 2, .is_mobile = 1, .cursor_needs_physical = 1, .num_pipes = 2,
-        .has_overlay = 1, .overlay_needs_physical = 1,
-        .ring_mask = RENDER_RING,
-        GEN_DEFAULT_PIPEOFFSETS,
-        CURSOR_OFFSETS,
-};
-
-static const struct intel_device_info intel_845g_info = {
-        .gen = 2, .num_pipes = 1,
-        .has_overlay = 1, .overlay_needs_physical = 1,
-        .ring_mask = RENDER_RING,
-        GEN_DEFAULT_PIPEOFFSETS,
-        CURSOR_OFFSETS,
-};
-
-static const struct intel_device_info intel_i85x_info = {
-        .gen = 2, .is_i85x = 1, .is_mobile = 1, .num_pipes = 2,
-        .cursor_needs_physical = 1,
-        .has_overlay = 1, .overlay_needs_physical = 1,
-        .has_fbc = 1,
-        .ring_mask = RENDER_RING,
-        GEN_DEFAULT_PIPEOFFSETS,
-        CURSOR_OFFSETS,
-};
-
-static const struct intel_device_info intel_i865g_info = {
-        .gen = 2, .num_pipes = 1,
-        .has_overlay = 1, .overlay_needs_physical = 1,
-        .ring_mask = RENDER_RING,
-        GEN_DEFAULT_PIPEOFFSETS,
-        CURSOR_OFFSETS,
-};
-
-static const struct intel_device_info intel_i915g_info = {
-        .gen = 3, .is_i915g = 1, .cursor_needs_physical = 1, .num_pipes = 2,
-        .has_overlay = 1, .overlay_needs_physical = 1,
-        .ring_mask = RENDER_RING,
-        GEN_DEFAULT_PIPEOFFSETS,
-        CURSOR_OFFSETS,
-};
-static const struct intel_device_info intel_i915gm_info = {
-        .gen = 3, .is_mobile = 1, .num_pipes = 2,
-        .cursor_needs_physical = 1,
-        .has_overlay = 1, .overlay_needs_physical = 1,
-        .supports_tv = 1,
-        .has_fbc = 1,
-        .ring_mask = RENDER_RING,
-        GEN_DEFAULT_PIPEOFFSETS,
-        CURSOR_OFFSETS,
-};
-static const struct intel_device_info intel_i945g_info = {
-        .gen = 3, .has_hotplug = 1, .cursor_needs_physical = 1, .num_pipes = 2,
-        .has_overlay = 1, .overlay_needs_physical = 1,
-        .ring_mask = RENDER_RING,
-        GEN_DEFAULT_PIPEOFFSETS,
-        CURSOR_OFFSETS,
-};
-static const struct intel_device_info intel_i945gm_info = {
-        .gen = 3, .is_i945gm = 1, .is_mobile = 1, .num_pipes = 2,
-        .has_hotplug = 1, .cursor_needs_physical = 1,
-        .has_overlay = 1, .overlay_needs_physical = 1,
-        .supports_tv = 1,
-        .has_fbc = 1,
-        .ring_mask = RENDER_RING,
-        GEN_DEFAULT_PIPEOFFSETS,
-        CURSOR_OFFSETS,
-};
-
-static const struct intel_device_info intel_i965g_info = {
-        .gen = 4, .is_broadwater = 1, .num_pipes = 2,
-        .has_hotplug = 1,
-        .has_overlay = 1,
-        .ring_mask = RENDER_RING,
-        GEN_DEFAULT_PIPEOFFSETS,
-        CURSOR_OFFSETS,
-};
-
-static const struct intel_device_info intel_i965gm_info = {
-        .gen = 4, .is_crestline = 1, .num_pipes = 2,
-        .is_mobile = 1, .has_fbc = 1, .has_hotplug = 1,
-        .has_overlay = 1,
-        .supports_tv = 1,
-        .ring_mask = RENDER_RING,
-        GEN_DEFAULT_PIPEOFFSETS,
-        CURSOR_OFFSETS,
-};
-
-static const struct intel_device_info intel_g33_info = {
-        .gen = 3, .is_g33 = 1, .num_pipes = 2,
-        .need_gfx_hws = 1, .has_hotplug = 1,
-        .has_overlay = 1,
-        .ring_mask = RENDER_RING,
-        GEN_DEFAULT_PIPEOFFSETS,
-        CURSOR_OFFSETS,
-};
-
-static const struct intel_device_info intel_g45_info = {
-        .gen = 4, .is_g4x = 1, .need_gfx_hws = 1, .num_pipes = 2,
-        .has_pipe_cxsr = 1, .has_hotplug = 1,
-        .ring_mask = RENDER_RING | BSD_RING,
-        GEN_DEFAULT_PIPEOFFSETS,
-        CURSOR_OFFSETS,
-};
-
-static const struct intel_device_info intel_gm45_info = {
-        .gen = 4, .is_g4x = 1, .num_pipes = 2,
-        .is_mobile = 1, .need_gfx_hws = 1, .has_fbc = 1,
-        .has_pipe_cxsr = 1, .has_hotplug = 1,
-        .supports_tv = 1,
-        .ring_mask = RENDER_RING | BSD_RING,
-        GEN_DEFAULT_PIPEOFFSETS,
-        CURSOR_OFFSETS,
-};
-
-static const struct intel_device_info intel_pineview_info = {
-        .gen = 3, .is_g33 = 1, .is_pineview = 1, .is_mobile = 1, .num_pipes = 2,
-        .need_gfx_hws = 1, .has_hotplug = 1,
-        .has_overlay = 1,
-        GEN_DEFAULT_PIPEOFFSETS,
-        CURSOR_OFFSETS,
-};
-
-static const struct intel_device_info intel_ironlake_d_info = {
-        .gen = 5, .num_pipes = 2,
-        .need_gfx_hws = 1, .has_hotplug = 1,
-        .ring_mask = RENDER_RING | BSD_RING,
-        GEN_DEFAULT_PIPEOFFSETS,
-        CURSOR_OFFSETS,
-};
+#include <linux/vt.h>
+#include <acpi/video.h>
 
-static const struct intel_device_info intel_ironlake_m_info = {
-        .gen = 5, .is_mobile = 1, .num_pipes = 2,
-        .need_gfx_hws = 1, .has_hotplug = 1,
-        .has_fbc = 1,
-        .ring_mask = RENDER_RING | BSD_RING,
-        GEN_DEFAULT_PIPEOFFSETS,
-        CURSOR_OFFSETS,
-};
-
-static const struct intel_device_info intel_sandybridge_d_info = {
-        .gen = 6, .num_pipes = 2,
-        .need_gfx_hws = 1, .has_hotplug = 1,
-        .has_fbc = 1,
-        .ring_mask = RENDER_RING | BSD_RING | BLT_RING,
-        .has_llc = 1,
-        GEN_DEFAULT_PIPEOFFSETS,
-        CURSOR_OFFSETS,
-};
-
-static const struct intel_device_info intel_sandybridge_m_info = {
-        .gen = 6, .is_mobile = 1, .num_pipes = 2,
-        .need_gfx_hws = 1, .has_hotplug = 1,
-        .has_fbc = 1,
-        .ring_mask = RENDER_RING | BSD_RING | BLT_RING,
-        .has_llc = 1,
-        GEN_DEFAULT_PIPEOFFSETS,
-        CURSOR_OFFSETS,
-};
-
-#define GEN7_FEATURES  \
-        .gen = 7, .num_pipes = 3, \
-        .need_gfx_hws = 1, .has_hotplug = 1, \
-        .has_fbc = 1, \
-        .ring_mask = RENDER_RING | BSD_RING | BLT_RING, \
-        .has_llc = 1, \
-        GEN_DEFAULT_PIPEOFFSETS, \
-        IVB_CURSOR_OFFSETS
-
-static const struct intel_device_info intel_ivybridge_d_info = {
-        GEN7_FEATURES,
-        .is_ivybridge = 1,
-};
-
-static const struct intel_device_info intel_ivybridge_m_info = {
-        GEN7_FEATURES,
-        .is_ivybridge = 1,
-        .is_mobile = 1,
-};
-
-static const struct intel_device_info intel_ivybridge_q_info = {
-        GEN7_FEATURES,
-        .is_ivybridge = 1,
-        .num_pipes = 0, /* legal, last one wins */
-};
-
-#define VLV_FEATURES  \
-        .gen = 7, .num_pipes = 2, \
-        .need_gfx_hws = 1, .has_hotplug = 1, \
-        .ring_mask = RENDER_RING | BSD_RING | BLT_RING, \
-        .display_mmio_offset = VLV_DISPLAY_BASE, \
-        GEN_DEFAULT_PIPEOFFSETS, \
-        CURSOR_OFFSETS
-
-static const struct intel_device_info intel_valleyview_m_info = {
-        VLV_FEATURES,
-        .is_valleyview = 1,
-        .is_mobile = 1,
-};
+#include <drm/drmP.h>
+#include <drm/drm_crtc_helper.h>
+#include <drm/i915_drm.h>
 
-static const struct intel_device_info intel_valleyview_d_info = {
-        VLV_FEATURES,
-        .is_valleyview = 1,
-};
+#include "i915_drv.h"
+#include "i915_trace.h"
+#include "i915_vgpu.h"
+#include "intel_drv.h"
 
-#define HSW_FEATURES  \
-        GEN7_FEATURES, \
-        .ring_mask = RENDER_RING | BSD_RING | BLT_RING | VEBOX_RING, \
-        .has_ddi = 1, \
-        .has_fpga_dbg = 1
+static struct drm_driver driver;
 
-static const struct intel_device_info intel_haswell_d_info = {
-        HSW_FEATURES,
-        .is_haswell = 1,
-};
+static unsigned int i915_load_fail_count;
 
-static const struct intel_device_info intel_haswell_m_info = {
-        HSW_FEATURES,
-        .is_haswell = 1,
-        .is_mobile = 1,
-};
+bool __i915_inject_load_failure(const char *func, int line)
+{
+        if (i915_load_fail_count >= i915.inject_load_failure)
+                return false;
 
-#define BDW_FEATURES \
-        HSW_FEATURES, \
-        BDW_COLORS
+        if (++i915_load_fail_count == i915.inject_load_failure) {
+                DRM_INFO("Injecting failure at checkpoint %u [%s:%d]\n",
+                         i915.inject_load_failure, func, line);
+                return true;
+        }
 
-static const struct intel_device_info intel_broadwell_d_info = {
-        BDW_FEATURES,
-        .gen = 8,
-};
+        return false;
+}
 
-static const struct intel_device_info intel_broadwell_m_info = {
-        BDW_FEATURES,
-        .gen = 8, .is_mobile = 1,
-};
+#define FDO_BUG_URL "https://bugs.freedesktop.org/enter_bug.cgi?product=DRI"
+#define FDO_BUG_MSG "Please file a bug at " FDO_BUG_URL " against DRM/Intel " \
+                    "providing the dmesg log by booting with drm.debug=0xf"
 
-static const struct intel_device_info intel_broadwell_gt3d_info = {
-        BDW_FEATURES,
-        .gen = 8,
-        .ring_mask = RENDER_RING | BSD_RING | BLT_RING | VEBOX_RING | BSD2_RING,
-};
+void
+__i915_printk(struct drm_i915_private *dev_priv, const char *level,
+              const char *fmt, ...)
+{
+        static bool shown_bug_once;
+        struct device *dev = dev_priv->drm.dev;
+        bool is_error = level[1] <= KERN_ERR[1];
+        bool is_debug = level[1] == KERN_DEBUG[1];
+        struct va_format vaf;
+        va_list args;
+
+        if (is_debug && !(drm_debug & DRM_UT_DRIVER))
+                return;
 
-static const struct intel_device_info intel_broadwell_gt3m_info = {
-        BDW_FEATURES,
-        .gen = 8, .is_mobile = 1,
-        .ring_mask = RENDER_RING | BSD_RING | BLT_RING | VEBOX_RING | BSD2_RING,
-};
+        va_start(args, fmt);
 
-static const struct intel_device_info intel_cherryview_info = {
-        .gen = 8, .num_pipes = 3,
-        .need_gfx_hws = 1, .has_hotplug = 1,
-        .ring_mask = RENDER_RING | BSD_RING | BLT_RING | VEBOX_RING,
-        .is_cherryview = 1,
-        .display_mmio_offset = VLV_DISPLAY_BASE,
-        GEN_CHV_PIPEOFFSETS,
-        CURSOR_OFFSETS,
-        CHV_COLORS,
-};
+        vaf.fmt = fmt;
+        vaf.va = &args;
 
-static const struct intel_device_info intel_skylake_info = {
-        BDW_FEATURES,
-        .is_skylake = 1,
-        .gen = 9,
-};
+        dev_printk(level, dev, "[" DRM_NAME ":%ps] %pV",
+                   __builtin_return_address(0), &vaf);
 
-static const struct intel_device_info intel_skylake_gt3_info = {
-        BDW_FEATURES,
-        .is_skylake = 1,
-        .gen = 9,
-        .ring_mask = RENDER_RING | BSD_RING | BLT_RING | VEBOX_RING | BSD2_RING,
-};
+        if (is_error && !shown_bug_once) {
+                dev_notice(dev, "%s", FDO_BUG_MSG);
+                shown_bug_once = true;
+        }
 
-static const struct intel_device_info intel_broxton_info = {
-        .is_preliminary = 1,
-        .is_broxton = 1,
-        .gen = 9,
-        .need_gfx_hws = 1, .has_hotplug = 1,
-        .ring_mask = RENDER_RING | BSD_RING | BLT_RING | VEBOX_RING,
-        .num_pipes = 3,
-        .has_ddi = 1,
-        .has_fpga_dbg = 1,
-        .has_fbc = 1,
-        GEN_DEFAULT_PIPEOFFSETS,
-        IVB_CURSOR_OFFSETS,
-        BDW_COLORS,
-};
+        va_end(args);
+}
 
-static const struct intel_device_info intel_kabylake_info = {
-        BDW_FEATURES,
-        .is_kabylake = 1,
-        .gen = 9,
-};
+static bool i915_error_injected(struct drm_i915_private *dev_priv)
+{
+        return i915.inject_load_failure &&
+               i915_load_fail_count == i915.inject_load_failure;
+}
 
-static const struct intel_device_info intel_kabylake_gt3_info = {
-        BDW_FEATURES,
-        .is_kabylake = 1,
-        .gen = 9,
-        .ring_mask = RENDER_RING | BSD_RING | BLT_RING | VEBOX_RING | BSD2_RING,
-};
+#define i915_load_error(dev_priv, fmt, ...)                                  \
+        __i915_printk(dev_priv,                                              \
+                      i915_error_injected(dev_priv) ? KERN_DEBUG : KERN_ERR, \
+                      fmt, ##__VA_ARGS__)
 
-/*
- * Make sure any device matches here are from most specific to most
- * general.  For example, since the Quanta match is based on the subsystem
- * and subvendor IDs, we need it to come before the more general IVB
- * PCI ID matches, otherwise we'll use the wrong info struct above.
- */
-static const struct pci_device_id pciidlist[] = {
-        INTEL_I830_IDS(&intel_i830_info),
-        INTEL_I845G_IDS(&intel_845g_info),
-        INTEL_I85X_IDS(&intel_i85x_info),
-        INTEL_I865G_IDS(&intel_i865g_info),
-        INTEL_I915G_IDS(&intel_i915g_info),
-        INTEL_I915GM_IDS(&intel_i915gm_info),
-        INTEL_I945G_IDS(&intel_i945g_info),
-        INTEL_I945GM_IDS(&intel_i945gm_info),
-        INTEL_I965G_IDS(&intel_i965g_info),
-        INTEL_G33_IDS(&intel_g33_info),
-        INTEL_I965GM_IDS(&intel_i965gm_info),
-        INTEL_GM45_IDS(&intel_gm45_info),
-        INTEL_G45_IDS(&intel_g45_info),
-        INTEL_PINEVIEW_IDS(&intel_pineview_info),
-        INTEL_IRONLAKE_D_IDS(&intel_ironlake_d_info),
-        INTEL_IRONLAKE_M_IDS(&intel_ironlake_m_info),
-        INTEL_SNB_D_IDS(&intel_sandybridge_d_info),
-        INTEL_SNB_M_IDS(&intel_sandybridge_m_info),
-        INTEL_IVB_Q_IDS(&intel_ivybridge_q_info), /* must be first IVB */
-        INTEL_IVB_M_IDS(&intel_ivybridge_m_info),
-        INTEL_IVB_D_IDS(&intel_ivybridge_d_info),
-        INTEL_HSW_D_IDS(&intel_haswell_d_info),
-        INTEL_HSW_M_IDS(&intel_haswell_m_info),
-        INTEL_VLV_M_IDS(&intel_valleyview_m_info),
-        INTEL_VLV_D_IDS(&intel_valleyview_d_info),
-        INTEL_BDW_GT12M_IDS(&intel_broadwell_m_info),
-        INTEL_BDW_GT12D_IDS(&intel_broadwell_d_info),
-        INTEL_BDW_GT3M_IDS(&intel_broadwell_gt3m_info),
-        INTEL_BDW_GT3D_IDS(&intel_broadwell_gt3d_info),
-        INTEL_CHV_IDS(&intel_cherryview_info),
-        INTEL_SKL_GT1_IDS(&intel_skylake_info),
-        INTEL_SKL_GT2_IDS(&intel_skylake_info),
-        INTEL_SKL_GT3_IDS(&intel_skylake_gt3_info),
-        INTEL_SKL_GT4_IDS(&intel_skylake_gt3_info),
-        INTEL_BXT_IDS(&intel_broxton_info),
-        INTEL_KBL_GT1_IDS(&intel_kabylake_info),
-        INTEL_KBL_GT2_IDS(&intel_kabylake_info),
-        INTEL_KBL_GT3_IDS(&intel_kabylake_gt3_info),
-        INTEL_KBL_GT4_IDS(&intel_kabylake_gt3_info),
-        {0, 0, 0}
-};
-
-MODULE_DEVICE_TABLE(pci, pciidlist);
 
 static enum intel_pch intel_virt_detect_pch(struct drm_device *dev)
 {
@@ -450,9 +142,9 @@ static enum intel_pch intel_virt_detect_pch(struct drm_device *dev)
         return ret;
 }
 
-void intel_detect_pch(struct drm_device *dev)
+static void intel_detect_pch(struct drm_device *dev)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         struct pci_dev *pch = NULL;
 
         /* In all current cases, num_pipes is equivalent to the PCH_NOP setting
@@ -519,8 +211,10 @@ void intel_detect_pch(struct drm_device *dev)
                         } else if ((id == INTEL_PCH_P2X_DEVICE_ID_TYPE) ||
                                    (id == INTEL_PCH_P3X_DEVICE_ID_TYPE) ||
                                    ((id == INTEL_PCH_QEMU_DEVICE_ID_TYPE) &&
-                                    pch->subsystem_vendor == 0x1af4 &&
-                                    pch->subsystem_device == 0x1100)) {
+                                    pch->subsystem_vendor ==
+                                            PCI_SUBVENDOR_ID_REDHAT_QUMRANET &&
+                                    pch->subsystem_device ==
+                                            PCI_SUBDEVICE_ID_QEMU)) {
                                 dev_priv->pch_type = intel_virt_detect_pch(dev);
                         } else
                                 continue;
@@ -534,9 +228,9 @@ void intel_detect_pch(struct drm_device *dev)
         pci_dev_put(pch);
 }
 
-bool i915_semaphore_is_enabled(struct drm_device *dev)
+bool i915_semaphore_is_enabled(struct drm_i915_private *dev_priv)
 {
-        if (INTEL_INFO(dev)->gen < 6)
+        if (INTEL_GEN(dev_priv) < 6)
                 return false;
 
         if (i915.semaphores >= 0)
@@ -546,22 +240,1172 @@ bool i915_semaphore_is_enabled(struct drm_device *dev)
         if (i915.enable_execlists)
                 return false;
 
-        /* Until we get further testing... */
-        if (IS_GEN8(dev))
-                return false;
-
 #ifdef CONFIG_INTEL_IOMMU
         /* Enable semaphores on SNB when IO remapping is off */
-        if (INTEL_INFO(dev)->gen == 6 && intel_iommu_gfx_mapped)
+        if (IS_GEN6(dev_priv) && intel_iommu_gfx_mapped)
                 return false;
 #endif
 
         return true;
 }
 
+static int i915_getparam(struct drm_device *dev, void *data,
+                         struct drm_file *file_priv)
+{
+        struct drm_i915_private *dev_priv = to_i915(dev);
+        drm_i915_getparam_t *param = data;
+        int value;
+
+        switch (param->param) {
+        case I915_PARAM_IRQ_ACTIVE:
+        case I915_PARAM_ALLOW_BATCHBUFFER:
+        case I915_PARAM_LAST_DISPATCH:
+                /* Reject all old ums/dri params. */
+                return -ENODEV;
+        case I915_PARAM_CHIPSET_ID:
+                value = dev->pdev->device;
+                break;
+        case I915_PARAM_REVISION:
+                value = dev->pdev->revision;
+                break;
+        case I915_PARAM_HAS_GEM:
+                value = 1;
+                break;
+        case I915_PARAM_NUM_FENCES_AVAIL:
+                value = dev_priv->num_fence_regs;
+                break;
+        case I915_PARAM_HAS_OVERLAY:
+                value = dev_priv->overlay ? 1 : 0;
+                break;
+        case I915_PARAM_HAS_PAGEFLIPPING:
+                value = 1;
+                break;
+        case I915_PARAM_HAS_EXECBUF2:
+                /* depends on GEM */
+                value = 1;
+                break;
+        case I915_PARAM_HAS_BSD:
+                value = intel_engine_initialized(&dev_priv->engine[VCS]);
+                break;
+        case I915_PARAM_HAS_BLT:
+                value = intel_engine_initialized(&dev_priv->engine[BCS]);
+                break;
+        case I915_PARAM_HAS_VEBOX:
+                value = intel_engine_initialized(&dev_priv->engine[VECS]);
+                break;
+        case I915_PARAM_HAS_BSD2:
+                value = intel_engine_initialized(&dev_priv->engine[VCS2]);
+                break;
+        case I915_PARAM_HAS_RELAXED_FENCING:
+                value = 1;
+                break;
+        case I915_PARAM_HAS_COHERENT_RINGS:
+                value = 1;
+                break;
+        case I915_PARAM_HAS_EXEC_CONSTANTS:
+                value = INTEL_INFO(dev)->gen >= 4;
+                break;
+        case I915_PARAM_HAS_RELAXED_DELTA:
+                value = 1;
+                break;
+        case I915_PARAM_HAS_GEN7_SOL_RESET:
+                value = 1;
+                break;
+        case I915_PARAM_HAS_LLC:
+                value = HAS_LLC(dev);
+                break;
+        case I915_PARAM_HAS_WT:
+                value = HAS_WT(dev);
+                break;
+        case I915_PARAM_HAS_ALIASING_PPGTT:
+                value = USES_PPGTT(dev);
+                break;
+        case I915_PARAM_HAS_WAIT_TIMEOUT:
+                value = 1;
+                break;
+        case I915_PARAM_HAS_SEMAPHORES:
+                value = i915_semaphore_is_enabled(dev_priv);
+                break;
+        case I915_PARAM_HAS_PRIME_VMAP_FLUSH:
+                value = 1;
+                break;
+        case I915_PARAM_HAS_SECURE_BATCHES:
+                value = capable(CAP_SYS_ADMIN);
+                break;
+        case I915_PARAM_HAS_PINNED_BATCHES:
+                value = 1;
+                break;
+        case I915_PARAM_HAS_EXEC_NO_RELOC:
+                value = 1;
+                break;
+        case I915_PARAM_HAS_EXEC_HANDLE_LUT:
+                value = 1;
+                break;
+        case I915_PARAM_CMD_PARSER_VERSION:
+                value = i915_cmd_parser_get_version(dev_priv);
+                break;
+        case I915_PARAM_HAS_COHERENT_PHYS_GTT:
+                value = 1;
+                break;
+        case I915_PARAM_MMAP_VERSION:
+                value = 1;
+                break;
+        case I915_PARAM_SUBSLICE_TOTAL:
+                value = INTEL_INFO(dev)->subslice_total;
+                if (!value)
+                        return -ENODEV;
+                break;
+        case I915_PARAM_EU_TOTAL:
+                value = INTEL_INFO(dev)->eu_total;
+                if (!value)
+                        return -ENODEV;
+                break;
+        case I915_PARAM_HAS_GPU_RESET:
+                value = i915.enable_hangcheck && intel_has_gpu_reset(dev_priv);
+                break;
+        case I915_PARAM_HAS_RESOURCE_STREAMER:
+                value = HAS_RESOURCE_STREAMER(dev);
+                break;
+        case I915_PARAM_HAS_EXEC_SOFTPIN:
+                value = 1;
+                break;
+        case I915_PARAM_HAS_POOLED_EU:
+                value = HAS_POOLED_EU(dev);
+                break;
+        case I915_PARAM_MIN_EU_IN_POOL:
+                value = INTEL_INFO(dev)->min_eu_in_pool;
+                break;
+        default:
+                DRM_DEBUG("Unknown parameter %d\n", param->param);
+                return -EINVAL;
+        }
+
+        if (put_user(value, param->value))
+                return -EFAULT;
+
+        return 0;
+}
+
+static int i915_get_bridge_dev(struct drm_device *dev)
+{
+        struct drm_i915_private *dev_priv = to_i915(dev);
+
+        dev_priv->bridge_dev = pci_get_bus_and_slot(0, PCI_DEVFN(0, 0));
+        if (!dev_priv->bridge_dev) {
+                DRM_ERROR("bridge device not found\n");
+                return -1;
+        }
+        return 0;
+}
+
+/* Allocate space for the MCH regs if needed, return nonzero on error */
+static int
+intel_alloc_mchbar_resource(struct drm_device *dev)
+{
+        struct drm_i915_private *dev_priv = to_i915(dev);
+        int reg = INTEL_INFO(dev)->gen >= 4 ? MCHBAR_I965 : MCHBAR_I915;
+        u32 temp_lo, temp_hi = 0;
+        u64 mchbar_addr;
+        int ret;
+
+        if (INTEL_INFO(dev)->gen >= 4)
+                pci_read_config_dword(dev_priv->bridge_dev, reg + 4, &temp_hi);
+        pci_read_config_dword(dev_priv->bridge_dev, reg, &temp_lo);
+        mchbar_addr = ((u64)temp_hi << 32) | temp_lo;
+
+        /* If ACPI doesn't have it, assume we need to allocate it ourselves */
+#ifdef CONFIG_PNP
+        if (mchbar_addr &&
+            pnp_range_reserved(mchbar_addr, mchbar_addr + MCHBAR_SIZE))
+                return 0;
+#endif
+
+        /* Get some space for it */
+        dev_priv->mch_res.name = "i915 MCHBAR";
+        dev_priv->mch_res.flags = IORESOURCE_MEM;
+        ret = pci_bus_alloc_resource(dev_priv->bridge_dev->bus,
+                                     &dev_priv->mch_res,
+                                     MCHBAR_SIZE, MCHBAR_SIZE,
+                                     PCIBIOS_MIN_MEM,
+                                     0, pcibios_align_resource,
+                                     dev_priv->bridge_dev);
+        if (ret) {
+                DRM_DEBUG_DRIVER("failed bus alloc: %d\n", ret);
+                dev_priv->mch_res.start = 0;
+                return ret;
+        }
+
+        if (INTEL_INFO(dev)->gen >= 4)
+                pci_write_config_dword(dev_priv->bridge_dev, reg + 4,
+                                       upper_32_bits(dev_priv->mch_res.start));
+
+        pci_write_config_dword(dev_priv->bridge_dev, reg,
+                               lower_32_bits(dev_priv->mch_res.start));
+        return 0;
+}
+
+/* Setup MCHBAR if possible, return true if we should disable it again */
+static void
+intel_setup_mchbar(struct drm_device *dev)
+{
+        struct drm_i915_private *dev_priv = to_i915(dev);
+        int mchbar_reg = INTEL_INFO(dev)->gen >= 4 ? MCHBAR_I965 : MCHBAR_I915;
+        u32 temp;
+        bool enabled;
+
+        if (IS_VALLEYVIEW(dev) || IS_CHERRYVIEW(dev))
+                return;
+
+        dev_priv->mchbar_need_disable = false;
+
+        if (IS_I915G(dev) || IS_I915GM(dev)) {
+                pci_read_config_dword(dev_priv->bridge_dev, DEVEN, &temp);
+                enabled = !!(temp & DEVEN_MCHBAR_EN);
+        } else {
+                pci_read_config_dword(dev_priv->bridge_dev, mchbar_reg, &temp);
+                enabled = temp & 1;
+        }
+
+        /* If it's already enabled, don't have to do anything */
+        if (enabled)
+                return;
+
+        if (intel_alloc_mchbar_resource(dev))
+                return;
+
+        dev_priv->mchbar_need_disable = true;
+
+        /* Space is allocated or reserved, so enable it. */
+        if (IS_I915G(dev) || IS_I915GM(dev)) {
+                pci_write_config_dword(dev_priv->bridge_dev, DEVEN,
+                                       temp | DEVEN_MCHBAR_EN);
+        } else {
+                pci_read_config_dword(dev_priv->bridge_dev, mchbar_reg, &temp);
+                pci_write_config_dword(dev_priv->bridge_dev, mchbar_reg, temp | 1);
+        }
+}
+
+static void
+intel_teardown_mchbar(struct drm_device *dev)
+{
+        struct drm_i915_private *dev_priv = to_i915(dev);
+        int mchbar_reg = INTEL_INFO(dev)->gen >= 4 ? MCHBAR_I965 : MCHBAR_I915;
+
+        if (dev_priv->mchbar_need_disable) {
+                if (IS_I915G(dev) || IS_I915GM(dev)) {
+                        u32 deven_val;
+
+                        pci_read_config_dword(dev_priv->bridge_dev, DEVEN,
+                                              &deven_val);
+                        deven_val &= ~DEVEN_MCHBAR_EN;
+                        pci_write_config_dword(dev_priv->bridge_dev, DEVEN,
+                                               deven_val);
+                } else {
+                        u32 mchbar_val;
+
+                        pci_read_config_dword(dev_priv->bridge_dev, mchbar_reg,
+                                              &mchbar_val);
+                        mchbar_val &= ~1;
+                        pci_write_config_dword(dev_priv->bridge_dev, mchbar_reg,
+                                               mchbar_val);
+                }
+        }
+
+        if (dev_priv->mch_res.start)
+                release_resource(&dev_priv->mch_res);
+}
+
+/* true = enable decode, false = disable decoder */
+static unsigned int i915_vga_set_decode(void *cookie, bool state)
+{
+        struct drm_device *dev = cookie;
+
+        intel_modeset_vga_set_state(dev, state);
+        if (state)
+                return VGA_RSRC_LEGACY_IO | VGA_RSRC_LEGACY_MEM |
+                       VGA_RSRC_NORMAL_IO | VGA_RSRC_NORMAL_MEM;
+        else
+                return VGA_RSRC_NORMAL_IO | VGA_RSRC_NORMAL_MEM;
+}
+
+static void i915_switcheroo_set_state(struct pci_dev *pdev, enum vga_switcheroo_state state)
+{
+        struct drm_device *dev = pci_get_drvdata(pdev);
+        pm_message_t pmm = { .event = PM_EVENT_SUSPEND };
+
+        if (state == VGA_SWITCHEROO_ON) {
+                pr_info("switched on\n");
+                dev->switch_power_state = DRM_SWITCH_POWER_CHANGING;
+                /* i915 resume handler doesn't set to D0 */
+                pci_set_power_state(dev->pdev, PCI_D0);
+                i915_resume_switcheroo(dev);
+                dev->switch_power_state = DRM_SWITCH_POWER_ON;
+        } else {
+                pr_info("switched off\n");
+                dev->switch_power_state = DRM_SWITCH_POWER_CHANGING;
+                i915_suspend_switcheroo(dev, pmm);
+                dev->switch_power_state = DRM_SWITCH_POWER_OFF;
+        }
+}
+
+static bool i915_switcheroo_can_switch(struct pci_dev *pdev)
+{
+        struct drm_device *dev = pci_get_drvdata(pdev);
+
+        /*
+         * FIXME: open_count is protected by drm_global_mutex but that would lead to
+         * locking inversion with the driver load path. And the access here is
+         * completely racy anyway. So don't bother with locking for now.
+         */
+        return dev->open_count == 0;
+}
+
+static const struct vga_switcheroo_client_ops i915_switcheroo_ops = {
+        .set_gpu_state = i915_switcheroo_set_state,
+        .reprobe = NULL,
+        .can_switch = i915_switcheroo_can_switch,
+};
+
+static void i915_gem_fini(struct drm_device *dev)
+{
+        struct drm_i915_private *dev_priv = to_i915(dev);
+
+        /*
+         * Neither the BIOS, ourselves or any other kernel
+         * expects the system to be in execlists mode on startup,
+         * so we need to reset the GPU back to legacy mode. And the only
+         * known way to disable logical contexts is through a GPU reset.
+         *
+         * So in order to leave the system in a known default configuration,
+         * always reset the GPU upon unload. Afterwards we then clean up the
+         * GEM state tracking, flushing off the requests and leaving the
+         * system in a known idle state.
+         *
+         * Note that is of the upmost importance that the GPU is idle and
+         * all stray writes are flushed *before* we dismantle the backing
+         * storage for the pinned objects.
+         *
+         * However, since we are uncertain that reseting the GPU on older
+         * machines is a good idea, we don't - just in case it leaves the
+         * machine in an unusable condition.
+         */
+        if (HAS_HW_CONTEXTS(dev)) {
+                int reset = intel_gpu_reset(dev_priv, ALL_ENGINES);
+                WARN_ON(reset && reset != -ENODEV);
+        }
+
+        mutex_lock(&dev->struct_mutex);
+        i915_gem_reset(dev);
+        i915_gem_cleanup_engines(dev);
+        i915_gem_context_fini(dev);
+        mutex_unlock(&dev->struct_mutex);
+
+        WARN_ON(!list_empty(&to_i915(dev)->context_list));
+}
+
+static int i915_load_modeset_init(struct drm_device *dev)
+{
+        struct drm_i915_private *dev_priv = to_i915(dev);
+        int ret;
+
+        if (i915_inject_load_failure())
+                return -ENODEV;
+
+        ret = intel_bios_init(dev_priv);
+        if (ret)
+                DRM_INFO("failed to find VBIOS tables\n");
+
+        /* If we have > 1 VGA cards, then we need to arbitrate access
+         * to the common VGA resources.
+         *
+         * If we are a secondary display controller (!PCI_DISPLAY_CLASS_VGA),
+         * then we do not take part in VGA arbitration and the
+         * vga_client_register() fails with -ENODEV.
+         */
+        ret = vga_client_register(dev->pdev, dev, NULL, i915_vga_set_decode);
+        if (ret && ret != -ENODEV)
+                goto out;
+
+        intel_register_dsm_handler();
+
+        ret = vga_switcheroo_register_client(dev->pdev, &i915_switcheroo_ops, false);
+        if (ret)
+                goto cleanup_vga_client;
+
+        /* must happen before intel_power_domains_init_hw() on VLV/CHV */
+        intel_update_rawclk(dev_priv);
+
+        intel_power_domains_init_hw(dev_priv, false);
+
+        intel_csr_ucode_init(dev_priv);
+
+        ret = intel_irq_install(dev_priv);
+        if (ret)
+                goto cleanup_csr;
+
+        intel_setup_gmbus(dev);
+
+        /* Important: The output setup functions called by modeset_init need
+         * working irqs for e.g. gmbus and dp aux transfers. */
+        intel_modeset_init(dev);
+
+        intel_guc_init(dev);
+
+        ret = i915_gem_init(dev);
+        if (ret)
+                goto cleanup_irq;
+
+        intel_modeset_gem_init(dev);
+
+        if (INTEL_INFO(dev)->num_pipes == 0)
+                return 0;
+
+        ret = intel_fbdev_init(dev);
+        if (ret)
+                goto cleanup_gem;
+
+        /* Only enable hotplug handling once the fbdev is fully set up. */
+        intel_hpd_init(dev_priv);
+
+        drm_kms_helper_poll_init(dev);
+
+        return 0;
+
+cleanup_gem:
+        i915_gem_fini(dev);
+cleanup_irq:
+        intel_guc_fini(dev);
+        drm_irq_uninstall(dev);
+        intel_teardown_gmbus(dev);
+cleanup_csr:
+        intel_csr_ucode_fini(dev_priv);
+        intel_power_domains_fini(dev_priv);
+        vga_switcheroo_unregister_client(dev->pdev);
+cleanup_vga_client:
+        vga_client_register(dev->pdev, NULL, NULL, NULL);
+out:
+        return ret;
+}
+
+#if IS_ENABLED(CONFIG_FB)
+static int i915_kick_out_firmware_fb(struct drm_i915_private *dev_priv)
+{
+        struct apertures_struct *ap;
+        struct pci_dev *pdev = dev_priv->drm.pdev;
+        struct i915_ggtt *ggtt = &dev_priv->ggtt;
+        bool primary;
+        int ret;
+
+        ap = alloc_apertures(1);
+        if (!ap)
+                return -ENOMEM;
+
+        ap->ranges[0].base = ggtt->mappable_base;
+        ap->ranges[0].size = ggtt->mappable_end;
+
+        primary =
+                pdev->resource[PCI_ROM_RESOURCE].flags & IORESOURCE_ROM_SHADOW;
+
+        ret = remove_conflicting_framebuffers(ap, "inteldrmfb", primary);
+
+        kfree(ap);
+
+        return ret;
+}
+#else
+static int i915_kick_out_firmware_fb(struct drm_i915_private *dev_priv)
+{
+        return 0;
+}
+#endif
+
+#if !defined(CONFIG_VGA_CONSOLE)
+static int i915_kick_out_vgacon(struct drm_i915_private *dev_priv)
+{
+        return 0;
+}
+#elif !defined(CONFIG_DUMMY_CONSOLE)
+static int i915_kick_out_vgacon(struct drm_i915_private *dev_priv)
+{
+        return -ENODEV;
+}
+#else
+static int i915_kick_out_vgacon(struct drm_i915_private *dev_priv)
+{
+        int ret = 0;
+
+        DRM_INFO("Replacing VGA console driver\n");
+
+        console_lock();
+        if (con_is_bound(&vga_con))
+                ret = do_take_over_console(&dummy_con, 0, MAX_NR_CONSOLES - 1, 1);
+        if (ret == 0) {
+                ret = do_unregister_con_driver(&vga_con);
+
+                /* Ignore "already unregistered". */
+                if (ret == -ENODEV)
+                        ret = 0;
+        }
+        console_unlock();
+
+        return ret;
+}
+#endif
+
+static void intel_init_dpio(struct drm_i915_private *dev_priv)
+{
+        /*
+         * IOSF_PORT_DPIO is used for VLV x2 PHY (DP/HDMI B and C),
+         * CHV x1 PHY (DP/HDMI D)
+         * IOSF_PORT_DPIO_2 is used for CHV x2 PHY (DP/HDMI B and C)
+         */
+        if (IS_CHERRYVIEW(dev_priv)) {
+                DPIO_PHY_IOSF_PORT(DPIO_PHY0) = IOSF_PORT_DPIO_2;
+                DPIO_PHY_IOSF_PORT(DPIO_PHY1) = IOSF_PORT_DPIO;
+        } else if (IS_VALLEYVIEW(dev_priv)) {
+                DPIO_PHY_IOSF_PORT(DPIO_PHY0) = IOSF_PORT_DPIO;
+        }
+}
+
+static int i915_workqueues_init(struct drm_i915_private *dev_priv)
+{
+        /*
+         * The i915 workqueue is primarily used for batched retirement of
+         * requests (and thus managing bo) once the task has been completed
+         * by the GPU. i915_gem_retire_requests() is called directly when we
+         * need high-priority retirement, such as waiting for an explicit
+         * bo.
+         *
+         * It is also used for periodic low-priority events, such as
+         * idle-timers and recording error state.
+         *
+         * All tasks on the workqueue are expected to acquire the dev mutex
+         * so there is no point in running more than one instance of the
+         * workqueue at any time.  Use an ordered one.
+         */
+        dev_priv->wq = alloc_ordered_workqueue("i915", 0);
+        if (dev_priv->wq == NULL)
+                goto out_err;
+
+        dev_priv->hotplug.dp_wq = alloc_ordered_workqueue("i915-dp", 0);
+        if (dev_priv->hotplug.dp_wq == NULL)
+                goto out_free_wq;
+
+        return 0;
+
+out_free_wq:
+        destroy_workqueue(dev_priv->wq);
+out_err:
+        DRM_ERROR("Failed to allocate workqueues.\n");
+
+        return -ENOMEM;
+}
+
+static void i915_workqueues_cleanup(struct drm_i915_private *dev_priv)
+{
+        destroy_workqueue(dev_priv->hotplug.dp_wq);
+        destroy_workqueue(dev_priv->wq);
+}
+
+/**
+ * i915_driver_init_early - setup state not requiring device access
+ * @dev_priv: device private
+ *
+ * Initialize everything that is a "SW-only" state, that is state not
+ * requiring accessing the device or exposing the driver via kernel internal
+ * or userspace interfaces. Example steps belonging here: lock initialization,
+ * system memory allocation, setting up device specific attributes and
+ * function hooks not requiring accessing the device.
+ */
+static int i915_driver_init_early(struct drm_i915_private *dev_priv,
+                                  const struct pci_device_id *ent)
+{
+        const struct intel_device_info *match_info =
+                (struct intel_device_info *)ent->driver_data;
+        struct intel_device_info *device_info;
+        int ret = 0;
+
+        if (i915_inject_load_failure())
+                return -ENODEV;
+
+        /* Setup the write-once "constant" device info */
+        device_info = mkwrite_device_info(dev_priv);
+        memcpy(device_info, match_info, sizeof(*device_info));
+        device_info->device_id = dev_priv->drm.pdev->device;
+
+        BUG_ON(device_info->gen > sizeof(device_info->gen_mask) * BITS_PER_BYTE);
+        device_info->gen_mask = BIT(device_info->gen - 1);
+
+        spin_lock_init(&dev_priv->irq_lock);
+        spin_lock_init(&dev_priv->gpu_error.lock);
+        mutex_init(&dev_priv->backlight_lock);
+        spin_lock_init(&dev_priv->uncore.lock);
+        spin_lock_init(&dev_priv->mm.object_stat_lock);
+        spin_lock_init(&dev_priv->mmio_flip_lock);
+        mutex_init(&dev_priv->sb_lock);
+        mutex_init(&dev_priv->modeset_restore_lock);
+        mutex_init(&dev_priv->av_mutex);
+        mutex_init(&dev_priv->wm.wm_mutex);
+        mutex_init(&dev_priv->pps_mutex);
+
+        ret = i915_workqueues_init(dev_priv);
+        if (ret < 0)
+                return ret;
+
+        ret = intel_gvt_init(dev_priv);
+        if (ret < 0)
+                goto err_workqueues;
+
+        /* This must be called before any calls to HAS_PCH_* */
+        intel_detect_pch(&dev_priv->drm);
+
+        intel_pm_setup(&dev_priv->drm);
+        intel_init_dpio(dev_priv);
+        intel_power_domains_init(dev_priv);
+        intel_irq_init(dev_priv);
+        intel_init_display_hooks(dev_priv);
+        intel_init_clock_gating_hooks(dev_priv);
+        intel_init_audio_hooks(dev_priv);
+        i915_gem_load_init(&dev_priv->drm);
+
+        intel_display_crc_init(&dev_priv->drm);
+
+        intel_device_info_dump(dev_priv);
+
+        /* Not all pre-production machines fall into this category, only the
+         * very first ones. Almost everything should work, except for maybe
+         * suspend/resume. And we don't implement workarounds that affect only
+         * pre-production machines. */
+        if (IS_HSW_EARLY_SDV(dev_priv))
+                DRM_INFO("This is an early pre-production Haswell machine. "
+                         "It may not be fully functional.\n");
+
+        return 0;
+
+err_workqueues:
+        i915_workqueues_cleanup(dev_priv);
+        return ret;
+}
+
+/**
+ * i915_driver_cleanup_early - cleanup the setup done in i915_driver_init_early()
+ * @dev_priv: device private
+ */
+static void i915_driver_cleanup_early(struct drm_i915_private *dev_priv)
+{
+        i915_gem_load_cleanup(&dev_priv->drm);
+        i915_workqueues_cleanup(dev_priv);
+}
+
+static int i915_mmio_setup(struct drm_device *dev)
+{
+        struct drm_i915_private *dev_priv = to_i915(dev);
+        int mmio_bar;
+        int mmio_size;
+
+        mmio_bar = IS_GEN2(dev) ? 1 : 0;
+        /*
+         * Before gen4, the registers and the GTT are behind different BARs.
+         * However, from gen4 onwards, the registers and the GTT are shared
+         * in the same BAR, so we want to restrict this ioremap from
+         * clobbering the GTT which we want ioremap_wc instead. Fortunately,
+         * the register BAR remains the same size for all the earlier
+         * generations up to Ironlake.
+         */
+        if (INTEL_INFO(dev)->gen < 5)
+                mmio_size = 512 * 1024;
+        else
+                mmio_size = 2 * 1024 * 1024;
+        dev_priv->regs = pci_iomap(dev->pdev, mmio_bar, mmio_size);
+        if (dev_priv->regs == NULL) {
+                DRM_ERROR("failed to map registers\n");
+
+                return -EIO;
+        }
+
+        /* Try to make sure MCHBAR is enabled before poking at it */
+        intel_setup_mchbar(dev);
+
+        return 0;
+}
+
+static void i915_mmio_cleanup(struct drm_device *dev)
+{
+        struct drm_i915_private *dev_priv = to_i915(dev);
+
+        intel_teardown_mchbar(dev);
+        pci_iounmap(dev->pdev, dev_priv->regs);
+}
+
+/**
+ * i915_driver_init_mmio - setup device MMIO
+ * @dev_priv: device private
+ *
+ * Setup minimal device state necessary for MMIO accesses later in the
+ * initialization sequence. The setup here should avoid any other device-wide
+ * side effects or exposing the driver via kernel internal or user space
+ * interfaces.
+ */
+static int i915_driver_init_mmio(struct drm_i915_private *dev_priv)
+{
+        struct drm_device *dev = &dev_priv->drm;
+        int ret;
+
+        if (i915_inject_load_failure())
+                return -ENODEV;
+
+        if (i915_get_bridge_dev(dev))
+                return -EIO;
+
+        ret = i915_mmio_setup(dev);
+        if (ret < 0)
+                goto put_bridge;
+
+        intel_uncore_init(dev_priv);
+
+        return 0;
+
+put_bridge:
+        pci_dev_put(dev_priv->bridge_dev);
+
+        return ret;
+}
+
+/**
+ * i915_driver_cleanup_mmio - cleanup the setup done in i915_driver_init_mmio()
+ * @dev_priv: device private
+ */
+static void i915_driver_cleanup_mmio(struct drm_i915_private *dev_priv)
+{
+        struct drm_device *dev = &dev_priv->drm;
+
+        intel_uncore_fini(dev_priv);
+        i915_mmio_cleanup(dev);
+        pci_dev_put(dev_priv->bridge_dev);
+}
+
+static void intel_sanitize_options(struct drm_i915_private *dev_priv)
+{
+        i915.enable_execlists =
+                intel_sanitize_enable_execlists(dev_priv,
+                                                i915.enable_execlists);
+
+        /*
+         * i915.enable_ppgtt is read-only, so do an early pass to validate the
+         * user's requested state against the hardware/driver capabilities.  We
+         * do this now so that we can print out any log messages once rather
+         * than every time we check intel_enable_ppgtt().
+         */
+        i915.enable_ppgtt =
+                intel_sanitize_enable_ppgtt(dev_priv, i915.enable_ppgtt);
+        DRM_DEBUG_DRIVER("ppgtt mode: %i\n", i915.enable_ppgtt);
+}
+
+/**
+ * i915_driver_init_hw - setup state requiring device access
+ * @dev_priv: device private
+ *
+ * Setup state that requires accessing the device, but doesn't require
+ * exposing the driver via kernel internal or userspace interfaces.
+ */
+static int i915_driver_init_hw(struct drm_i915_private *dev_priv)
+{
+        struct drm_device *dev = &dev_priv->drm;
+        struct i915_ggtt *ggtt = &dev_priv->ggtt;
+        uint32_t aperture_size;
+        int ret;
+
+        if (i915_inject_load_failure())
+                return -ENODEV;
+
+        intel_device_info_runtime_init(dev_priv);
+
+        intel_sanitize_options(dev_priv);
+
+        ret = i915_ggtt_init_hw(dev);
+        if (ret)
+                return ret;
+
+        ret = i915_ggtt_enable_hw(dev);
+        if (ret) {
+                DRM_ERROR("failed to enable GGTT\n");
+                goto out_ggtt;
+        }
+
+        /* WARNING: Apparently we must kick fbdev drivers before vgacon,
+         * otherwise the vga fbdev driver falls over. */
+        ret = i915_kick_out_firmware_fb(dev_priv);
+        if (ret) {
+                DRM_ERROR("failed to remove conflicting framebuffer drivers\n");
+                goto out_ggtt;
+        }
+
+        ret = i915_kick_out_vgacon(dev_priv);
+        if (ret) {
+                DRM_ERROR("failed to remove conflicting VGA console\n");
+                goto out_ggtt;
+        }
+
+        pci_set_master(dev->pdev);
+
+        /* overlay on gen2 is broken and can't address above 1G */
+        if (IS_GEN2(dev)) {
+                ret = dma_set_coherent_mask(&dev->pdev->dev, DMA_BIT_MASK(30));
+                if (ret) {
+                        DRM_ERROR("failed to set DMA mask\n");
+
+                        goto out_ggtt;
+                }
+        }
+
+
+        /* 965GM sometimes incorrectly writes to hardware status page (HWS)
+         * using 32bit addressing, overwriting memory if HWS is located
+         * above 4GB.
+         *
+         * The documentation also mentions an issue with undefined
+         * behaviour if any general state is accessed within a page above 4GB,
+         * which also needs to be handled carefully.
+         */
+        if (IS_BROADWATER(dev) || IS_CRESTLINE(dev)) {
+                ret = dma_set_coherent_mask(&dev->pdev->dev, DMA_BIT_MASK(32));
+
+                if (ret) {
+                        DRM_ERROR("failed to set DMA mask\n");
+
+                        goto out_ggtt;
+                }
+        }
+
+        aperture_size = ggtt->mappable_end;
+
+        ggtt->mappable =
+                io_mapping_create_wc(ggtt->mappable_base,
+                                     aperture_size);
+        if (!ggtt->mappable) {
+                ret = -EIO;
+                goto out_ggtt;
+        }
+
+        ggtt->mtrr = arch_phys_wc_add(ggtt->mappable_base,
+                                              aperture_size);
+
+        pm_qos_add_request(&dev_priv->pm_qos, PM_QOS_CPU_DMA_LATENCY,
+                           PM_QOS_DEFAULT_VALUE);
+
+        intel_uncore_sanitize(dev_priv);
+
+        intel_opregion_setup(dev_priv);
+
+        i915_gem_load_init_fences(dev_priv);
+
+        /* On the 945G/GM, the chipset reports the MSI capability on the
+         * integrated graphics even though the support isn't actually there
+         * according to the published specs.  It doesn't appear to function
+         * correctly in testing on 945G.
+         * This may be a side effect of MSI having been made available for PEG
+         * and the registers being closely associated.
+         *
+         * According to chipset errata, on the 965GM, MSI interrupts may
+         * be lost or delayed, but we use them anyways to avoid
+         * stuck interrupts on some machines.
+         */
+        if (!IS_I945G(dev) && !IS_I945GM(dev)) {
+                if (pci_enable_msi(dev->pdev) < 0)
+                        DRM_DEBUG_DRIVER("can't enable MSI");
+        }
+
+        return 0;
+
+out_ggtt:
+        i915_ggtt_cleanup_hw(dev);
+
+        return ret;
+}
+
+/**
+ * i915_driver_cleanup_hw - cleanup the setup done in i915_driver_init_hw()
+ * @dev_priv: device private
+ */
+static void i915_driver_cleanup_hw(struct drm_i915_private *dev_priv)
+{
+        struct drm_device *dev = &dev_priv->drm;
+        struct i915_ggtt *ggtt = &dev_priv->ggtt;
+
+        if (dev->pdev->msi_enabled)
+                pci_disable_msi(dev->pdev);
+
+        pm_qos_remove_request(&dev_priv->pm_qos);
+        arch_phys_wc_del(ggtt->mtrr);
+        io_mapping_free(ggtt->mappable);
+        i915_ggtt_cleanup_hw(dev);
+}
+
+/**
+ * i915_driver_register - register the driver with the rest of the system
+ * @dev_priv: device private
+ *
+ * Perform any steps necessary to make the driver available via kernel
+ * internal or userspace interfaces.
+ */
+static void i915_driver_register(struct drm_i915_private *dev_priv)
+{
+        struct drm_device *dev = &dev_priv->drm;
+
+        i915_gem_shrinker_init(dev_priv);
+
+        /*
+         * Notify a valid surface after modesetting,
+         * when running inside a VM.
+         */
+        if (intel_vgpu_active(dev_priv))
+                I915_WRITE(vgtif_reg(display_ready), VGT_DRV_DISPLAY_READY);
+
+        /* Reveal our presence to userspace */
+        if (drm_dev_register(dev, 0) == 0) {
+                i915_debugfs_register(dev_priv);
+                i915_setup_sysfs(dev);
+        } else
+                DRM_ERROR("Failed to register driver for userspace access!\n");
+
+        if (INTEL_INFO(dev_priv)->num_pipes) {
+                /* Must be done after probing outputs */
+                intel_opregion_register(dev_priv);
+                acpi_video_register();
+        }
+
+        if (IS_GEN5(dev_priv))
+                intel_gpu_ips_init(dev_priv);
+
+        i915_audio_component_init(dev_priv);
+
+        /*
+         * Some ports require correctly set-up hpd registers for detection to
+         * work properly (leading to ghost connected connector status), e.g. VGA
+         * on gm45.  Hence we can only set up the initial fbdev config after hpd
+         * irqs are fully enabled. We do it last so that the async config
+         * cannot run before the connectors are registered.
+         */
+        intel_fbdev_initial_config_async(dev);
+}
+
+/**
+ * i915_driver_unregister - cleanup the registration done in i915_driver_regiser()
+ * @dev_priv: device private
+ */
+static void i915_driver_unregister(struct drm_i915_private *dev_priv)
+{
+        i915_audio_component_cleanup(dev_priv);
+
+        intel_gpu_ips_teardown();
+        acpi_video_unregister();
+        intel_opregion_unregister(dev_priv);
+
+        i915_teardown_sysfs(&dev_priv->drm);
+        i915_debugfs_unregister(dev_priv);
+        drm_dev_unregister(&dev_priv->drm);
+
+        i915_gem_shrinker_cleanup(dev_priv);
+}
+
+/**
+ * i915_driver_load - setup chip and create an initial config
+ * @dev: DRM device
+ * @flags: startup flags
+ *
+ * The driver load routine has to do several things:
+ *   - drive output discovery via intel_modeset_init()
+ *   - initialize the memory manager
+ *   - allocate initial config memory
+ *   - setup the DRM framebuffer with the allocated memory
+ */
+int i915_driver_load(struct pci_dev *pdev, const struct pci_device_id *ent)
+{
+        struct drm_i915_private *dev_priv;
+        int ret;
+
+        if (i915.nuclear_pageflip)
+                driver.driver_features |= DRIVER_ATOMIC;
+
+        ret = -ENOMEM;
+        dev_priv = kzalloc(sizeof(*dev_priv), GFP_KERNEL);
+        if (dev_priv)
+                ret = drm_dev_init(&dev_priv->drm, &driver, &pdev->dev);
+        if (ret) {
+                dev_printk(KERN_ERR, &pdev->dev,
+                           "[" DRM_NAME ":%s] allocation failed\n", __func__);
+                kfree(dev_priv);
+                return ret;
+        }
+
+        dev_priv->drm.pdev = pdev;
+        dev_priv->drm.dev_private = dev_priv;
+
+        ret = pci_enable_device(pdev);
+        if (ret)
+                goto out_free_priv;
+
+        pci_set_drvdata(pdev, &dev_priv->drm);
+
+        ret = i915_driver_init_early(dev_priv, ent);
+        if (ret < 0)
+                goto out_pci_disable;
+
+        intel_runtime_pm_get(dev_priv);
+
+        ret = i915_driver_init_mmio(dev_priv);
+        if (ret < 0)
+                goto out_runtime_pm_put;
+
+        ret = i915_driver_init_hw(dev_priv);
+        if (ret < 0)
+                goto out_cleanup_mmio;
+
+        /*
+         * TODO: move the vblank init and parts of modeset init steps into one
+         * of the i915_driver_init_/i915_driver_register functions according
+         * to the role/effect of the given init step.
+         */
+        if (INTEL_INFO(dev_priv)->num_pipes) {
+                ret = drm_vblank_init(&dev_priv->drm,
+                                      INTEL_INFO(dev_priv)->num_pipes);
+                if (ret)
+                        goto out_cleanup_hw;
+        }
+
+        ret = i915_load_modeset_init(&dev_priv->drm);
+        if (ret < 0)
+                goto out_cleanup_vblank;
+
+        i915_driver_register(dev_priv);
+
+        intel_runtime_pm_enable(dev_priv);
+
+        intel_runtime_pm_put(dev_priv);
+
+        return 0;
+
+out_cleanup_vblank:
+        drm_vblank_cleanup(&dev_priv->drm);
+out_cleanup_hw:
+        i915_driver_cleanup_hw(dev_priv);
+out_cleanup_mmio:
+        i915_driver_cleanup_mmio(dev_priv);
+out_runtime_pm_put:
+        intel_runtime_pm_put(dev_priv);
+        i915_driver_cleanup_early(dev_priv);
+out_pci_disable:
+        pci_disable_device(pdev);
+out_free_priv:
+        i915_load_error(dev_priv, "Device initialization failed (%d)\n", ret);
+        drm_dev_unref(&dev_priv->drm);
+        return ret;
+}
+
+void i915_driver_unload(struct drm_device *dev)
+{
+        struct drm_i915_private *dev_priv = to_i915(dev);
+
+        intel_fbdev_fini(dev);
+
+        if (i915_gem_suspend(dev))
+                DRM_ERROR("failed to idle hardware; continuing to unload!\n");
+
+        intel_display_power_get(dev_priv, POWER_DOMAIN_INIT);
+
+        i915_driver_unregister(dev_priv);
+
+        drm_vblank_cleanup(dev);
+
+        intel_modeset_cleanup(dev);
+
+        /*
+         * free the memory space allocated for the child device
+         * config parsed from VBT
+         */
+        if (dev_priv->vbt.child_dev && dev_priv->vbt.child_dev_num) {
+                kfree(dev_priv->vbt.child_dev);
+                dev_priv->vbt.child_dev = NULL;
+                dev_priv->vbt.child_dev_num = 0;
+        }
+        kfree(dev_priv->vbt.sdvo_lvds_vbt_mode);
+        dev_priv->vbt.sdvo_lvds_vbt_mode = NULL;
+        kfree(dev_priv->vbt.lfp_lvds_vbt_mode);
+        dev_priv->vbt.lfp_lvds_vbt_mode = NULL;
+
+        vga_switcheroo_unregister_client(dev->pdev);
+        vga_client_register(dev->pdev, NULL, NULL, NULL);
+
+        intel_csr_ucode_fini(dev_priv);
+
+        /* Free error state after interrupts are fully disabled. */
+        cancel_delayed_work_sync(&dev_priv->gpu_error.hangcheck_work);
+        i915_destroy_error_state(dev);
+
+        /* Flush any outstanding unpin_work. */
+        flush_workqueue(dev_priv->wq);
+
+        intel_guc_fini(dev);
+        i915_gem_fini(dev);
+        intel_fbc_cleanup_cfb(dev_priv);
+
+        intel_power_domains_fini(dev_priv);
+
+        i915_driver_cleanup_hw(dev_priv);
+        i915_driver_cleanup_mmio(dev_priv);
+
+        intel_display_power_put(dev_priv, POWER_DOMAIN_INIT);
+
+        i915_driver_cleanup_early(dev_priv);
+}
+
+static int i915_driver_open(struct drm_device *dev, struct drm_file *file)
+{
+        int ret;
+
+        ret = i915_gem_open(dev, file);
+        if (ret)
+                return ret;
+
+        return 0;
+}
+
+/**
+ * i915_driver_lastclose - clean up after all DRM clients have exited
+ * @dev: DRM device
+ *
+ * Take care of cleaning up after all DRM clients have exited.  In the
+ * mode setting case, we want to restore the kernel's initial mode (just
+ * in case the last client left us in a bad state).
+ *
+ * Additionally, in the non-mode setting case, we'll tear down the GTT
+ * and DMA structures, since the kernel won't be using them, and clea
+ * up any GEM state.
+ */
+static void i915_driver_lastclose(struct drm_device *dev)
+{
+        intel_fbdev_restore_mode(dev);
+        vga_switcheroo_process_delayed_switch();
+}
+
+static void i915_driver_preclose(struct drm_device *dev, struct drm_file *file)
+{
+        mutex_lock(&dev->struct_mutex);
+        i915_gem_context_close(dev, file);
+        i915_gem_release(dev, file);
+        mutex_unlock(&dev->struct_mutex);
+}
+
+static void i915_driver_postclose(struct drm_device *dev, struct drm_file *file)
+{
+        struct drm_i915_file_private *file_priv = file->driver_priv;
+
+        kfree(file_priv);
+}
+
 static void intel_suspend_encoders(struct drm_i915_private *dev_priv)
 {
-        struct drm_device *dev = dev_priv->dev;
+        struct drm_device *dev = &dev_priv->drm;
         struct intel_encoder *encoder;
 
         drm_modeset_lock_all(dev);
@@ -586,7 +1430,7 @@ static bool suspend_to_idle(struct drm_i915_private *dev_priv)
 
 static int i915_drm_suspend(struct drm_device *dev)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         pci_power_t opregion_target_state;
         int error;
 
@@ -614,7 +1458,7 @@ static int i915_drm_suspend(struct drm_device *dev)
 
         intel_guc_suspend(dev);
 
-        intel_suspend_gt_powersave(dev);
+        intel_suspend_gt_powersave(dev_priv);
 
         intel_display_suspend(dev);
 
@@ -632,10 +1476,10 @@ static int i915_drm_suspend(struct drm_device *dev)
         i915_save_state(dev);
 
         opregion_target_state = suspend_to_idle(dev_priv) ? PCI_D1 : PCI_D3cold;
-        intel_opregion_notify_adapter(dev, opregion_target_state);
+        intel_opregion_notify_adapter(dev_priv, opregion_target_state);
 
-        intel_uncore_forcewake_reset(dev, false);
-        intel_opregion_fini(dev);
+        intel_uncore_forcewake_reset(dev_priv, false);
+        intel_opregion_unregister(dev_priv);
 
         intel_fbdev_set_suspend(dev, FBINFO_STATE_SUSPENDED, true);
 
@@ -653,7 +1497,7 @@ out:
 
 static int i915_drm_suspend_late(struct drm_device *drm_dev, bool hibernation)
 {
-        struct drm_i915_private *dev_priv = drm_dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(drm_dev);
         bool fw_csr;
         int ret;
 
@@ -715,7 +1559,7 @@ int i915_suspend_switcheroo(struct drm_device *dev, pm_message_t state)
 {
         int error;
 
-        if (!dev || !dev->dev_private) {
+        if (!dev) {
                 DRM_ERROR("dev: %p\n", dev);
                 DRM_ERROR("DRM not initialized, aborting suspend.\n");
                 return -ENODEV;
@@ -737,7 +1581,7 @@ int i915_suspend_switcheroo(struct drm_device *dev, pm_message_t state)
 
 static int i915_drm_resume(struct drm_device *dev)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         int ret;
 
         disable_rpm_wakeref_asserts(dev_priv);
@@ -753,7 +1597,7 @@ static int i915_drm_resume(struct drm_device *dev)
         mutex_unlock(&dev->struct_mutex);
 
         i915_restore_state(dev);
-        intel_opregion_setup(dev);
+        intel_opregion_setup(dev_priv);
 
         intel_init_pch_refclk(dev);
         drm_mode_config_reset(dev);
@@ -771,7 +1615,7 @@ static int i915_drm_resume(struct drm_device *dev)
         mutex_lock(&dev->struct_mutex);
         if (i915_gem_init_hw(dev)) {
                 DRM_ERROR("failed to re-initialize GPU, declaring wedged!\n");
-                        atomic_or(I915_WEDGED, &dev_priv->gpu_error.reset_counter);
+                atomic_or(I915_WEDGED, &dev_priv->gpu_error.reset_counter);
         }
         mutex_unlock(&dev->struct_mutex);
 
@@ -781,7 +1625,7 @@ static int i915_drm_resume(struct drm_device *dev)
 
         spin_lock_irq(&dev_priv->irq_lock);
         if (dev_priv->display.hpd_irq_setup)
-                dev_priv->display.hpd_irq_setup(dev);
+                dev_priv->display.hpd_irq_setup(dev_priv);
         spin_unlock_irq(&dev_priv->irq_lock);
 
         intel_dp_mst_resume(dev);
@@ -798,7 +1642,7 @@ static int i915_drm_resume(struct drm_device *dev)
         /* Config may have changed between suspend and resume */
         drm_helper_hpd_irq_event(dev);
 
-        intel_opregion_init(dev);
+        intel_opregion_register(dev_priv);
 
         intel_fbdev_set_suspend(dev, FBINFO_STATE_RUNNING, false);
 
@@ -806,7 +1650,7 @@ static int i915_drm_resume(struct drm_device *dev)
         dev_priv->modeset_restore = MODESET_DONE;
         mutex_unlock(&dev_priv->modeset_restore_lock);
 
-        intel_opregion_notify_adapter(dev, PCI_D0);
+        intel_opregion_notify_adapter(dev_priv, PCI_D0);
 
         drm_kms_helper_poll_enable(dev);
 
@@ -817,7 +1661,7 @@ static int i915_drm_resume(struct drm_device *dev)
 
 static int i915_drm_resume_early(struct drm_device *dev)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         int ret;
 
         /*
@@ -874,9 +1718,9 @@ static int i915_drm_resume_early(struct drm_device *dev)
                 DRM_ERROR("Resume prepare failed: %d, continuing anyway\n",
                           ret);
 
-        intel_uncore_early_sanitize(dev, true);
+        intel_uncore_early_sanitize(dev_priv, true);
 
-        if (IS_BROXTON(dev)) {
+        if (IS_BROXTON(dev_priv)) {
                 if (!dev_priv->suspended_to_idle)
                         gen9_sanitize_dc_state(dev_priv);
                 bxt_disable_dc9(dev_priv);
@@ -884,7 +1728,7 @@ static int i915_drm_resume_early(struct drm_device *dev)
                 hsw_disable_pc8(dev_priv);
         }
 
-        intel_uncore_sanitize(dev);
+        intel_uncore_sanitize(dev_priv);
 
         if (IS_BROXTON(dev_priv) ||
             !(dev_priv->suspended_to_idle && dev_priv->csr.dmc_payload))
@@ -927,14 +1771,14 @@ int i915_resume_switcheroo(struct drm_device *dev)
  *   - re-init interrupt state
  *   - re-init display
  */
-int i915_reset(struct drm_device *dev)
+int i915_reset(struct drm_i915_private *dev_priv)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_device *dev = &dev_priv->drm;
         struct i915_gpu_error *error = &dev_priv->gpu_error;
         unsigned reset_counter;
         int ret;
 
-        intel_reset_gt_powersave(dev);
+        intel_reset_gt_powersave(dev_priv);
 
         mutex_lock(&dev->struct_mutex);
 
@@ -948,24 +1792,11 @@ int i915_reset(struct drm_device *dev)
                 goto error;
         }
 
-        i915_gem_reset(dev);
-
-        ret = intel_gpu_reset(dev, ALL_ENGINES);
-
-        /* Also reset the gpu hangman. */
-        if (error->stop_rings != 0) {
-                DRM_INFO("Simulated gpu hang, resetting stop_rings\n");
-                error->stop_rings = 0;
-                if (ret == -ENODEV) {
-                        DRM_INFO("Reset not implemented, but ignoring "
-                                 "error for simulated gpu hangs\n");
-                        ret = 0;
-                }
-        }
+        pr_notice("drm/i915: Resetting chip after gpu hang\n");
 
-        if (i915_stop_ring_allow_warn(dev_priv))
-                pr_notice("drm/i915: Resetting chip after gpu hang\n");
+        i915_gem_reset(dev);
 
+        ret = intel_gpu_reset(dev_priv, ALL_ENGINES);
         if (ret) {
                 if (ret != -ENODEV)
                         DRM_ERROR("Failed to reset chip: %i\n", ret);
@@ -1005,7 +1836,7 @@ int i915_reset(struct drm_device *dev)
          * of re-init after reset.
          */
         if (INTEL_INFO(dev)->gen > 5)
-                intel_enable_gt_powersave(dev);
+                intel_enable_gt_powersave(dev_priv);
 
         return 0;
 
@@ -1015,51 +1846,12 @@ error:
         return ret;
 }
 
-static int i915_pci_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
-{
-        struct intel_device_info *intel_info =
-                (struct intel_device_info *) ent->driver_data;
-
-        if (IS_PRELIMINARY_HW(intel_info) && !i915.preliminary_hw_support) {
-                DRM_INFO("This hardware requires preliminary hardware support.\n"
-                         "See CONFIG_DRM_I915_PRELIMINARY_HW_SUPPORT, and/or modparam preliminary_hw_support\n");
-                return -ENODEV;
-        }
-
-        /* Only bind to function 0 of the device. Early generations
-         * used function 1 as a placeholder for multi-head. This causes
-         * us confusion instead, especially on the systems where both
-         * functions have the same PCI-ID!
-         */
-        if (PCI_FUNC(pdev->devfn))
-                return -ENODEV;
-
-        /*
-         * apple-gmux is needed on dual GPU MacBook Pro
-         * to probe the panel if we're the inactive GPU.
-         */
-        if (IS_ENABLED(CONFIG_VGA_ARB) && IS_ENABLED(CONFIG_VGA_SWITCHEROO) &&
-            apple_gmux_present() && pdev != vga_default_device() &&
-            !vga_switcheroo_handler_flags())
-                return -EPROBE_DEFER;
-
-        return drm_get_pci_dev(pdev, ent, &driver);
-}
-
-static void
-i915_pci_remove(struct pci_dev *pdev)
-{
-        struct drm_device *dev = pci_get_drvdata(pdev);
-
-        drm_put_dev(dev);
-}
-
 static int i915_pm_suspend(struct device *dev)
 {
         struct pci_dev *pdev = to_pci_dev(dev);
         struct drm_device *drm_dev = pci_get_drvdata(pdev);
 
-        if (!drm_dev || !drm_dev->dev_private) {
+        if (!drm_dev) {
                 dev_err(dev, "DRM not initialized, aborting suspend.\n");
                 return -ENODEV;
         }
@@ -1072,7 +1864,7 @@ static int i915_pm_suspend(struct device *dev)
 
 static int i915_pm_suspend_late(struct device *dev)
 {
-        struct drm_device *drm_dev = dev_to_i915(dev)->dev;
+        struct drm_device *drm_dev = &dev_to_i915(dev)->drm;
 
         /*
          * We have a suspend ordering issue with the snd-hda driver also
@@ -1091,7 +1883,7 @@ static int i915_pm_suspend_late(struct device *dev)
 
 static int i915_pm_poweroff_late(struct device *dev)
 {
-        struct drm_device *drm_dev = dev_to_i915(dev)->dev;
+        struct drm_device *drm_dev = &dev_to_i915(dev)->drm;
 
         if (drm_dev->switch_power_state == DRM_SWITCH_POWER_OFF)
                 return 0;
@@ -1101,7 +1893,7 @@ static int i915_pm_poweroff_late(struct device *dev)
 
 static int i915_pm_resume_early(struct device *dev)
 {
-        struct drm_device *drm_dev = dev_to_i915(dev)->dev;
+        struct drm_device *drm_dev = &dev_to_i915(dev)->drm;
 
         if (drm_dev->switch_power_state == DRM_SWITCH_POWER_OFF)
                 return 0;
@@ -1111,7 +1903,7 @@ static int i915_pm_resume_early(struct device *dev)
 
 static int i915_pm_resume(struct device *dev)
 {
-        struct drm_device *drm_dev = dev_to_i915(dev)->dev;
+        struct drm_device *drm_dev = &dev_to_i915(dev)->drm;
 
         if (drm_dev->switch_power_state == DRM_SWITCH_POWER_OFF)
                 return 0;
@@ -1119,6 +1911,49 @@ static int i915_pm_resume(struct device *dev)
         return i915_drm_resume(drm_dev);
 }
 
+/* freeze: before creating the hibernation_image */
+static int i915_pm_freeze(struct device *dev)
+{
+        return i915_pm_suspend(dev);
+}
+
+static int i915_pm_freeze_late(struct device *dev)
+{
+        int ret;
+
+        ret = i915_pm_suspend_late(dev);
+        if (ret)
+                return ret;
+
+        ret = i915_gem_freeze_late(dev_to_i915(dev));
+        if (ret)
+                return ret;
+
+        return 0;
+}
+
+/* thaw: called after creating the hibernation image, but before turning off. */
+static int i915_pm_thaw_early(struct device *dev)
+{
+        return i915_pm_resume_early(dev);
+}
+
+static int i915_pm_thaw(struct device *dev)
+{
+        return i915_pm_resume(dev);
+}
+
+/* restore: called after loading the hibernation image. */
+static int i915_pm_restore_early(struct device *dev)
+{
+        return i915_pm_resume_early(dev);
+}
+
+static int i915_pm_restore(struct device *dev)
+{
+        return i915_pm_resume(dev);
+}
+
 /*
  * Save all Gunit registers that may be lost after a D3 and a subsequent
  * S0i[R123] transition. The list of registers needing a save/restore is
@@ -1318,8 +2153,6 @@ int vlv_force_gfx_clock(struct drm_i915_private *dev_priv, bool force_on)
         u32 val;
         int err;
 
-#define COND (I915_READ(VLV_GTLC_SURVIVABILITY_REG) & VLV_GFX_CLK_STATUS_BIT)
-
         val = I915_READ(VLV_GTLC_SURVIVABILITY_REG);
         val &= ~VLV_GFX_CLK_FORCE_ON_BIT;
         if (force_on)
@@ -1329,13 +2162,16 @@ int vlv_force_gfx_clock(struct drm_i915_private *dev_priv, bool force_on)
         if (!force_on)
                 return 0;
 
-        err = wait_for(COND, 20);
+        err = intel_wait_for_register(dev_priv,
+                                      VLV_GTLC_SURVIVABILITY_REG,
+                                      VLV_GFX_CLK_STATUS_BIT,
+                                      VLV_GFX_CLK_STATUS_BIT,
+                                      20);
         if (err)
                 DRM_ERROR("timeout waiting for GFX clock force-on (%08x)\n",
                           I915_READ(VLV_GTLC_SURVIVABILITY_REG));
 
         return err;
-#undef COND
 }
 
 static int vlv_allow_gt_wake(struct drm_i915_private *dev_priv, bool allow)
@@ -1350,13 +2186,15 @@ static int vlv_allow_gt_wake(struct drm_i915_private *dev_priv, bool allow)
         I915_WRITE(VLV_GTLC_WAKE_CTRL, val);
         POSTING_READ(VLV_GTLC_WAKE_CTRL);
 
-#define COND (!!(I915_READ(VLV_GTLC_PW_STATUS) & VLV_GTLC_ALLOWWAKEACK) == \
-              allow)
-        err = wait_for(COND, 1);
+        err = intel_wait_for_register(dev_priv,
+                                      VLV_GTLC_PW_STATUS,
+                                      VLV_GTLC_ALLOWWAKEACK,
+                                      allow,
+                                      1);
         if (err)
                 DRM_ERROR("timeout disabling GT waking\n");
+
         return err;
-#undef COND
 }
 
 static int vlv_wait_for_gt_wells(struct drm_i915_private *dev_priv,
@@ -1368,8 +2206,7 @@ static int vlv_wait_for_gt_wells(struct drm_i915_private *dev_priv,
 
         mask = VLV_GTLC_PW_MEDIA_STATUS_MASK | VLV_GTLC_PW_RENDER_STATUS_MASK;
         val = wait_for_on ? mask : 0;
-#define COND ((I915_READ(VLV_GTLC_PW_STATUS) & mask) == val)
-        if (COND)
+        if ((I915_READ(VLV_GTLC_PW_STATUS) & mask) == val)
                 return 0;
 
         DRM_DEBUG_KMS("waiting for GT wells to go %s (%08x)\n",
@@ -1380,13 +2217,14 @@ static int vlv_wait_for_gt_wells(struct drm_i915_private *dev_priv,
          * RC6 transitioning can be delayed up to 2 msec (see
          * valleyview_enable_rps), use 3 msec for safety.
          */
-        err = wait_for(COND, 3);
+        err = intel_wait_for_register(dev_priv,
+                                      VLV_GTLC_PW_STATUS, mask, val,
+                                      3);
         if (err)
                 DRM_ERROR("timeout waiting for GT wells to go %s\n",
                           onoff(wait_for_on));
 
         return err;
-#undef COND
 }
 
 static void vlv_check_no_gt_access(struct drm_i915_private *dev_priv)
@@ -1443,7 +2281,7 @@ err1:
 static int vlv_resume_prepare(struct drm_i915_private *dev_priv,
                                 bool rpm_resume)
 {
-        struct drm_device *dev = dev_priv->dev;
+        struct drm_device *dev = &dev_priv->drm;
         int err;
         int ret;
 
@@ -1479,10 +2317,10 @@ static int intel_runtime_suspend(struct device *device)
 {
         struct pci_dev *pdev = to_pci_dev(device);
         struct drm_device *dev = pci_get_drvdata(pdev);
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         int ret;
 
-        if (WARN_ON_ONCE(!(dev_priv->rps.enabled && intel_enable_rc6(dev))))
+        if (WARN_ON_ONCE(!(dev_priv->rps.enabled && intel_enable_rc6())))
                 return -ENODEV;
 
         if (WARN_ON_ONCE(!HAS_RUNTIME_PM(dev)))
@@ -1517,11 +2355,8 @@ static int intel_runtime_suspend(struct device *device)
         i915_gem_release_all_mmaps(dev_priv);
         mutex_unlock(&dev->struct_mutex);
 
-        cancel_delayed_work_sync(&dev_priv->gpu_error.hangcheck_work);
-
         intel_guc_suspend(dev);
 
-        intel_suspend_gt_powersave(dev);
         intel_runtime_pm_disable_interrupts(dev_priv);
 
         ret = 0;
@@ -1543,7 +2378,7 @@ static int intel_runtime_suspend(struct device *device)
                 return ret;
         }
 
-        intel_uncore_forcewake_reset(dev, false);
+        intel_uncore_forcewake_reset(dev_priv, false);
 
         enable_rpm_wakeref_asserts(dev_priv);
         WARN_ON_ONCE(atomic_read(&dev_priv->pm.wakeref_count));
@@ -1557,14 +2392,14 @@ static int intel_runtime_suspend(struct device *device)
          * FIXME: We really should find a document that references the arguments
          * used below!
          */
-        if (IS_BROADWELL(dev)) {
+        if (IS_BROADWELL(dev_priv)) {
                 /*
                  * On Broadwell, if we use PCI_D1 the PCH DDI ports will stop
                  * being detected, and the call we do at intel_runtime_resume()
                  * won't be able to restore them. Since PCI_D3hot matches the
                  * actual specification and appears to be working, use it.
                  */
-                intel_opregion_notify_adapter(dev, PCI_D3hot);
+                intel_opregion_notify_adapter(dev_priv, PCI_D3hot);
         } else {
                 /*
                  * current versions of firmware which depend on this opregion
@@ -1573,11 +2408,14 @@ static int intel_runtime_suspend(struct device *device)
                  * to distinguish it from notifications that might be sent via
                  * the suspend path.
                  */
-                intel_opregion_notify_adapter(dev, PCI_D1);
+                intel_opregion_notify_adapter(dev_priv, PCI_D1);
         }
 
         assert_forcewakes_inactive(dev_priv);
 
+        if (!IS_VALLEYVIEW(dev_priv) || !IS_CHERRYVIEW(dev_priv))
+                intel_hpd_poll_init(dev_priv);
+
         DRM_DEBUG_KMS("Device suspended\n");
         return 0;
 }
@@ -1586,7 +2424,7 @@ static int intel_runtime_resume(struct device *device)
 {
         struct pci_dev *pdev = to_pci_dev(device);
         struct drm_device *dev = pci_get_drvdata(pdev);
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         int ret = 0;
 
         if (WARN_ON_ONCE(!HAS_RUNTIME_PM(dev)))
@@ -1597,7 +2435,7 @@ static int intel_runtime_resume(struct device *device)
         WARN_ON_ONCE(atomic_read(&dev_priv->pm.wakeref_count));
         disable_rpm_wakeref_asserts(dev_priv);
 
-        intel_opregion_notify_adapter(dev, PCI_D0);
+        intel_opregion_notify_adapter(dev_priv, PCI_D0);
         dev_priv->pm.suspended = false;
         if (intel_uncore_unclaimed_mmio(dev_priv))
                 DRM_DEBUG_DRIVER("Unclaimed access during suspend, bios?\n");
@@ -1624,7 +2462,7 @@ static int intel_runtime_resume(struct device *device)
          * we can do is to hope that things will still work (and disable RPM).
          */
         i915_gem_init_swizzling(dev);
-        gen6_update_ring_freq(dev);
+        gen6_update_ring_freq(dev_priv);
 
         intel_runtime_pm_enable_interrupts(dev_priv);
 
@@ -1636,8 +2474,6 @@ static int intel_runtime_resume(struct device *device)
         if (!IS_VALLEYVIEW(dev_priv) && !IS_CHERRYVIEW(dev_priv))
                 intel_hpd_init(dev_priv);
 
-        intel_enable_gt_powersave(dev);
-
         enable_rpm_wakeref_asserts(dev_priv);
 
         if (ret)
@@ -1648,7 +2484,7 @@ static int intel_runtime_resume(struct device *device)
         return ret;
 }
 
-static const struct dev_pm_ops i915_pm_ops = {
+const struct dev_pm_ops i915_pm_ops = {
         /*
          * S0ix (via system suspend) and S3 event handlers [PMSG_SUSPEND,
          * PMSG_RESUME]
@@ -1673,14 +2509,14 @@ static const struct dev_pm_ops i915_pm_ops = {
          * @restore, @restore_early : called after rebooting and restoring the
          *                            hibernation image [PMSG_RESTORE]
          */
-        .freeze = i915_pm_suspend,
-        .freeze_late = i915_pm_suspend_late,
-        .thaw_early = i915_pm_resume_early,
-        .thaw = i915_pm_resume,
+        .freeze = i915_pm_freeze,
+        .freeze_late = i915_pm_freeze_late,
+        .thaw_early = i915_pm_thaw_early,
+        .thaw = i915_pm_thaw,
         .poweroff = i915_pm_suspend,
         .poweroff_late = i915_pm_poweroff_late,
-        .restore_early = i915_pm_resume_early,
-        .restore = i915_pm_resume,
+        .restore_early = i915_pm_restore_early,
+        .restore = i915_pm_restore,
 
         /* S0ix (via runtime suspend) event handlers */
         .runtime_suspend = intel_runtime_suspend,
@@ -1707,6 +2543,68 @@ static const struct file_operations i915_driver_fops = {
         .llseek = noop_llseek,
 };
 
+static int
+i915_gem_reject_pin_ioctl(struct drm_device *dev, void *data,
+                          struct drm_file *file)
+{
+        return -ENODEV;
+}
+
+static const struct drm_ioctl_desc i915_ioctls[] = {
+        DRM_IOCTL_DEF_DRV(I915_INIT, drm_noop, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
+        DRM_IOCTL_DEF_DRV(I915_FLUSH, drm_noop, DRM_AUTH),
+        DRM_IOCTL_DEF_DRV(I915_FLIP, drm_noop, DRM_AUTH),
+        DRM_IOCTL_DEF_DRV(I915_BATCHBUFFER, drm_noop, DRM_AUTH),
+        DRM_IOCTL_DEF_DRV(I915_IRQ_EMIT, drm_noop, DRM_AUTH),
+        DRM_IOCTL_DEF_DRV(I915_IRQ_WAIT, drm_noop, DRM_AUTH),
+        DRM_IOCTL_DEF_DRV(I915_GETPARAM, i915_getparam, DRM_AUTH|DRM_RENDER_ALLOW),
+        DRM_IOCTL_DEF_DRV(I915_SETPARAM, drm_noop, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
+        DRM_IOCTL_DEF_DRV(I915_ALLOC, drm_noop, DRM_AUTH),
+        DRM_IOCTL_DEF_DRV(I915_FREE, drm_noop, DRM_AUTH),
+        DRM_IOCTL_DEF_DRV(I915_INIT_HEAP, drm_noop, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
+        DRM_IOCTL_DEF_DRV(I915_CMDBUFFER, drm_noop, DRM_AUTH),
+        DRM_IOCTL_DEF_DRV(I915_DESTROY_HEAP,  drm_noop, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
+        DRM_IOCTL_DEF_DRV(I915_SET_VBLANK_PIPE,  drm_noop, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
+        DRM_IOCTL_DEF_DRV(I915_GET_VBLANK_PIPE,  drm_noop, DRM_AUTH),
+        DRM_IOCTL_DEF_DRV(I915_VBLANK_SWAP, drm_noop, DRM_AUTH),
+        DRM_IOCTL_DEF_DRV(I915_HWS_ADDR, drm_noop, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
+        DRM_IOCTL_DEF_DRV(I915_GEM_INIT, drm_noop, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
+        DRM_IOCTL_DEF_DRV(I915_GEM_EXECBUFFER, i915_gem_execbuffer, DRM_AUTH),
+        DRM_IOCTL_DEF_DRV(I915_GEM_EXECBUFFER2, i915_gem_execbuffer2, DRM_AUTH|DRM_RENDER_ALLOW),
+        DRM_IOCTL_DEF_DRV(I915_GEM_PIN, i915_gem_reject_pin_ioctl, DRM_AUTH|DRM_ROOT_ONLY),
+        DRM_IOCTL_DEF_DRV(I915_GEM_UNPIN, i915_gem_reject_pin_ioctl, DRM_AUTH|DRM_ROOT_ONLY),
+        DRM_IOCTL_DEF_DRV(I915_GEM_BUSY, i915_gem_busy_ioctl, DRM_AUTH|DRM_RENDER_ALLOW),
+        DRM_IOCTL_DEF_DRV(I915_GEM_SET_CACHING, i915_gem_set_caching_ioctl, DRM_RENDER_ALLOW),
+        DRM_IOCTL_DEF_DRV(I915_GEM_GET_CACHING, i915_gem_get_caching_ioctl, DRM_RENDER_ALLOW),
+        DRM_IOCTL_DEF_DRV(I915_GEM_THROTTLE, i915_gem_throttle_ioctl, DRM_AUTH|DRM_RENDER_ALLOW),
+        DRM_IOCTL_DEF_DRV(I915_GEM_ENTERVT, drm_noop, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
+        DRM_IOCTL_DEF_DRV(I915_GEM_LEAVEVT, drm_noop, DRM_AUTH|DRM_MASTER|DRM_ROOT_ONLY),
+        DRM_IOCTL_DEF_DRV(I915_GEM_CREATE, i915_gem_create_ioctl, DRM_RENDER_ALLOW),
+        DRM_IOCTL_DEF_DRV(I915_GEM_PREAD, i915_gem_pread_ioctl, DRM_RENDER_ALLOW),
+        DRM_IOCTL_DEF_DRV(I915_GEM_PWRITE, i915_gem_pwrite_ioctl, DRM_RENDER_ALLOW),
+        DRM_IOCTL_DEF_DRV(I915_GEM_MMAP, i915_gem_mmap_ioctl, DRM_RENDER_ALLOW),
+        DRM_IOCTL_DEF_DRV(I915_GEM_MMAP_GTT, i915_gem_mmap_gtt_ioctl, DRM_RENDER_ALLOW),
+        DRM_IOCTL_DEF_DRV(I915_GEM_SET_DOMAIN, i915_gem_set_domain_ioctl, DRM_RENDER_ALLOW),
+        DRM_IOCTL_DEF_DRV(I915_GEM_SW_FINISH, i915_gem_sw_finish_ioctl, DRM_RENDER_ALLOW),
+        DRM_IOCTL_DEF_DRV(I915_GEM_SET_TILING, i915_gem_set_tiling, DRM_RENDER_ALLOW),
+        DRM_IOCTL_DEF_DRV(I915_GEM_GET_TILING, i915_gem_get_tiling, DRM_RENDER_ALLOW),
+        DRM_IOCTL_DEF_DRV(I915_GEM_GET_APERTURE, i915_gem_get_aperture_ioctl, DRM_RENDER_ALLOW),
+        DRM_IOCTL_DEF_DRV(I915_GET_PIPE_FROM_CRTC_ID, intel_get_pipe_from_crtc_id, 0),
+        DRM_IOCTL_DEF_DRV(I915_GEM_MADVISE, i915_gem_madvise_ioctl, DRM_RENDER_ALLOW),
+        DRM_IOCTL_DEF_DRV(I915_OVERLAY_PUT_IMAGE, intel_overlay_put_image_ioctl, DRM_MASTER|DRM_CONTROL_ALLOW),
+        DRM_IOCTL_DEF_DRV(I915_OVERLAY_ATTRS, intel_overlay_attrs_ioctl, DRM_MASTER|DRM_CONTROL_ALLOW),
+        DRM_IOCTL_DEF_DRV(I915_SET_SPRITE_COLORKEY, intel_sprite_set_colorkey, DRM_MASTER|DRM_CONTROL_ALLOW),
+        DRM_IOCTL_DEF_DRV(I915_GET_SPRITE_COLORKEY, drm_noop, DRM_MASTER|DRM_CONTROL_ALLOW),
+        DRM_IOCTL_DEF_DRV(I915_GEM_WAIT, i915_gem_wait_ioctl, DRM_AUTH|DRM_RENDER_ALLOW),
+        DRM_IOCTL_DEF_DRV(I915_GEM_CONTEXT_CREATE, i915_gem_context_create_ioctl, DRM_RENDER_ALLOW),
+        DRM_IOCTL_DEF_DRV(I915_GEM_CONTEXT_DESTROY, i915_gem_context_destroy_ioctl, DRM_RENDER_ALLOW),
+        DRM_IOCTL_DEF_DRV(I915_REG_READ, i915_reg_read_ioctl, DRM_RENDER_ALLOW),
+        DRM_IOCTL_DEF_DRV(I915_GET_RESET_STATS, i915_gem_context_reset_stats_ioctl, DRM_RENDER_ALLOW),
+        DRM_IOCTL_DEF_DRV(I915_GEM_USERPTR, i915_gem_userptr_ioctl, DRM_RENDER_ALLOW),
+        DRM_IOCTL_DEF_DRV(I915_GEM_CONTEXT_GETPARAM, i915_gem_context_getparam_ioctl, DRM_RENDER_ALLOW),
+        DRM_IOCTL_DEF_DRV(I915_GEM_CONTEXT_SETPARAM, i915_gem_context_setparam_ioctl, DRM_RENDER_ALLOW),
+};
+
 static struct drm_driver driver = {
         /* Don't use MTRRs here; the Xserver or userspace app should
          * deal with them for Intel hardware.
@@ -1714,18 +2612,12 @@ static struct drm_driver driver = {
         .driver_features =
             DRIVER_HAVE_IRQ | DRIVER_IRQ_SHARED | DRIVER_GEM | DRIVER_PRIME |
             DRIVER_RENDER | DRIVER_MODESET,
-        .load = i915_driver_load,
-        .unload = i915_driver_unload,
         .open = i915_driver_open,
         .lastclose = i915_driver_lastclose,
         .preclose = i915_driver_preclose,
         .postclose = i915_driver_postclose,
         .set_busid = drm_pci_set_busid,
 
-#if defined(CONFIG_DEBUG_FS)
-        .debugfs_init = i915_debugfs_init,
-        .debugfs_cleanup = i915_debugfs_cleanup,
-#endif
         .gem_free_object = i915_gem_free_object,
         .gem_vm_ops = &i915_gem_vm_ops,
 
@@ -1738,6 +2630,7 @@ static struct drm_driver driver = {
         .dumb_map_offset = i915_gem_mmap_gtt,
         .dumb_destroy = drm_gem_dumb_destroy,
         .ioctls = i915_ioctls,
+        .num_ioctls = ARRAY_SIZE(i915_ioctls),
         .fops = &i915_driver_fops,
         .name = DRIVER_NAME,
         .desc = DRIVER_DESC,
@@ -1746,56 +2639,3 @@ static struct drm_driver driver = {
         .minor = DRIVER_MINOR,
         .patchlevel = DRIVER_PATCHLEVEL,
 };
-
-static struct pci_driver i915_pci_driver = {
-        .name = DRIVER_NAME,
-        .id_table = pciidlist,
-        .probe = i915_pci_probe,
-        .remove = i915_pci_remove,
-        .driver.pm = &i915_pm_ops,
-};
-
-static int __init i915_init(void)
-{
-        driver.num_ioctls = i915_max_ioctl;
-
-        /*
-         * Enable KMS by default, unless explicitly overriden by
-         * either the i915.modeset prarameter or by the
-         * vga_text_mode_force boot option.
-         */
-
-        if (i915.modeset == 0)
-                driver.driver_features &= ~DRIVER_MODESET;
-
-        if (vgacon_text_force() && i915.modeset == -1)
-                driver.driver_features &= ~DRIVER_MODESET;
-
-        if (!(driver.driver_features & DRIVER_MODESET)) {
-                /* Silently fail loading to not upset userspace. */
-                DRM_DEBUG_DRIVER("KMS and UMS disabled.\n");
-                return 0;
-        }
-
-        if (i915.nuclear_pageflip)
-                driver.driver_features |= DRIVER_ATOMIC;
-
-        return drm_pci_init(&driver, &i915_pci_driver);
-}
-
-static void __exit i915_exit(void)
-{
-        if (!(driver.driver_features & DRIVER_MODESET))
-                return; /* Never loaded a driver. */
-
-        drm_pci_exit(&driver, &i915_pci_driver);
-}
-
-module_init(i915_init);
-module_exit(i915_exit);
-
-MODULE_AUTHOR("Tungsten Graphics, Inc.");
-MODULE_AUTHOR("Intel Corporation");
-
-MODULE_DESCRIPTION(DRIVER_DESC);
-MODULE_LICENSE("GPL and additional rights");
diff --git a/drivers/gpu/drm/i915/i915_drv.h b/drivers/gpu/drm/i915/i915_drv.h
index bc3f2e6842e7..915a3d0acff3 100644
--- a/drivers/gpu/drm/i915/i915_drv.h
+++ b/drivers/gpu/drm/i915/i915_drv.h
@@ -47,6 +47,7 @@
 #include <drm/intel-gtt.h>
 #include <drm/drm_legacy.h> /* for struct drm_dma_handle */
 #include <drm/drm_gem.h>
+#include <drm/drm_auth.h>
 
 #include "i915_params.h"
 #include "i915_reg.h"
@@ -61,12 +62,14 @@
 #include "i915_gem_gtt.h"
 #include "i915_gem_render_state.h"
 
+#include "intel_gvt.h"
+
 /* General customization:
  */
 
 #define DRIVER_NAME             "i915"
 #define DRIVER_DESC             "Intel Graphics"
-#define DRIVER_DATE             "20160425"
+#define DRIVER_DATE             "20160711"
 
 #undef WARN_ON
 /* Many gcc seem to no see through this and fall over :( */
@@ -281,6 +284,9 @@ struct i915_hotplug {
         u32 short_port_mask;
         struct work_struct dig_port_work;
 
+        struct work_struct poll_init_work;
+        bool poll_enabled;
+
         /*
          * if we get a HPD irq from DP and a HPD irq from non-DP
          * the non-DP HPD could block the workqueue on a mode config
@@ -317,21 +323,36 @@ struct i915_hotplug {
                 for_each_if ((__ports_mask) & (1 << (__port)))
 
 #define for_each_crtc(dev, crtc) \
-        list_for_each_entry(crtc, &dev->mode_config.crtc_list, head)
+        list_for_each_entry(crtc, &(dev)->mode_config.crtc_list, head)
 
 #define for_each_intel_plane(dev, intel_plane) \
         list_for_each_entry(intel_plane,                        \
-                            &dev->mode_config.plane_list,       \
+                            &(dev)->mode_config.plane_list,     \
                             base.head)
 
+#define for_each_intel_plane_mask(dev, intel_plane, plane_mask)         \
+        list_for_each_entry(intel_plane,                                \
+                            &(dev)->mode_config.plane_list,             \
+                            base.head)                                  \
+                for_each_if ((plane_mask) &                             \
+                             (1 << drm_plane_index(&intel_plane->base)))
+
 #define for_each_intel_plane_on_crtc(dev, intel_crtc, intel_plane)      \
         list_for_each_entry(intel_plane,                                \
                             &(dev)->mode_config.plane_list,             \
                             base.head)                                  \
                 for_each_if ((intel_plane)->pipe == (intel_crtc)->pipe)
 
-#define for_each_intel_crtc(dev, intel_crtc) \
-        list_for_each_entry(intel_crtc, &dev->mode_config.crtc_list, base.head)
+#define for_each_intel_crtc(dev, intel_crtc)                            \
+        list_for_each_entry(intel_crtc,                                 \
+                            &(dev)->mode_config.crtc_list,              \
+                            base.head)
+
+#define for_each_intel_crtc_mask(dev, intel_crtc, crtc_mask)            \
+        list_for_each_entry(intel_crtc,                                 \
+                            &(dev)->mode_config.crtc_list,              \
+                            base.head)                                  \
+                for_each_if ((crtc_mask) & (1 << drm_crtc_index(&intel_crtc->base)))
 
 #define for_each_intel_encoder(dev, intel_encoder)              \
         list_for_each_entry(intel_encoder,                      \
@@ -340,7 +361,7 @@ struct i915_hotplug {
 
 #define for_each_intel_connector(dev, intel_connector)          \
         list_for_each_entry(intel_connector,                    \
-                            &dev->mode_config.connector_list,   \
+                            &(dev)->mode_config.connector_list, \
                             base.head)
 
 #define for_each_encoder_on_crtc(dev, __crtc, intel_encoder) \
@@ -462,6 +483,7 @@ struct drm_i915_error_state {
         struct timeval time;
 
         char error_msg[128];
+        bool simulated;
         int iommu;
         u32 reset_count;
         u32 suspend_count;
@@ -493,6 +515,7 @@ struct drm_i915_error_state {
                 bool valid;
                 /* Software tracked state */
                 bool waiting;
+                int num_waiters;
                 int hangcheck_score;
                 enum intel_ring_hangcheck_action hangcheck_action;
                 int num_requests;
@@ -538,6 +561,12 @@ struct drm_i915_error_state {
                         u32 tail;
                 } *requests;
 
+                struct drm_i915_error_waiter {
+                        char comm[TASK_COMM_LEN];
+                        pid_t pid;
+                        u32 seqno;
+                } *waiters;
+
                 struct {
                         u32 gfx_mode;
                         union {
@@ -588,6 +617,7 @@ struct drm_i915_display_funcs {
                                        struct intel_crtc_state *newstate);
         void (*initial_watermarks)(struct intel_crtc_state *cstate);
         void (*optimize_watermarks)(struct intel_crtc_state *cstate);
+        int (*compute_global_watermarks)(struct drm_atomic_state *state);
         void (*update_wm)(struct drm_crtc *crtc);
         int (*modeset_calc_cdclk)(struct drm_atomic_state *state);
         void (*modeset_commit_cdclk)(struct drm_atomic_state *state);
@@ -612,7 +642,7 @@ struct drm_i915_display_funcs {
                           struct drm_i915_gem_object *obj,
                           struct drm_i915_gem_request *req,
                           uint32_t flags);
-        void (*hpd_irq_setup)(struct drm_device *dev);
+        void (*hpd_irq_setup)(struct drm_i915_private *dev_priv);
         /* clock updates for mode set */
         /* cursor updates */
         /* render clock increase/decrease */
@@ -735,6 +765,7 @@ struct intel_csr {
         func(is_valleyview) sep \
         func(is_cherryview) sep \
         func(is_haswell) sep \
+        func(is_broadwell) sep \
         func(is_skylake) sep \
         func(is_broxton) sep \
         func(is_kabylake) sep \
@@ -749,7 +780,8 @@ struct intel_csr {
         func(has_llc) sep \
         func(has_snoop) sep \
         func(has_ddi) sep \
-        func(has_fpga_dbg)
+        func(has_fpga_dbg) sep \
+        func(has_pooled_eu)
 
 #define DEFINE_FLAG(name) u8 name:1
 #define SEP_SEMICOLON ;
@@ -757,9 +789,10 @@ struct intel_csr {
 struct intel_device_info {
         u32 display_mmio_offset;
         u16 device_id;
-        u8 num_pipes:3;
+        u8 num_pipes;
         u8 num_sprites[I915_MAX_PIPES];
         u8 gen;
+        u16 gen_mask;
         u8 ring_mask; /* Rings supported by the HW */
         DEV_INFO_FOR_EACH_FLAG(DEFINE_FLAG, SEP_SEMICOLON);
         /* Register offsets for the various display pipes and transcoders */
@@ -774,6 +807,7 @@ struct intel_device_info {
         u8 subslice_per_slice;
         u8 eu_total;
         u8 eu_per_subslice;
+        u8 min_eu_in_pool;
         /* For each slice, which subslice(s) has(have) 7 EUs (bitfield)? */
         u8 subslice_7eu[3];
         u8 has_slice_pg:1;
@@ -821,9 +855,8 @@ struct i915_ctx_hang_stats {
 /* This must match up with the value previously used for execbuf2.rsvd1. */
 #define DEFAULT_CONTEXT_HANDLE 0
 
-#define CONTEXT_NO_ZEROMAP (1<<0)
 /**
- * struct intel_context - as the name implies, represents a context.
+ * struct i915_gem_context - as the name implies, represents a context.
  * @ref: reference count.
  * @user_handle: userspace tracking identity for this context.
  * @remap_slice: l3 row remapping information.
@@ -841,33 +874,40 @@ struct i915_ctx_hang_stats {
  * Contexts are memory images used by the hardware to store copies of their
  * internal state.
  */
-struct intel_context {
+struct i915_gem_context {
         struct kref ref;
-        int user_handle;
-        uint8_t remap_slice;
         struct drm_i915_private *i915;
-        int flags;
         struct drm_i915_file_private *file_priv;
-        struct i915_ctx_hang_stats hang_stats;
         struct i915_hw_ppgtt *ppgtt;
 
-        /* Legacy ring buffer submission */
-        struct {
-                struct drm_i915_gem_object *rcs_state;
-                bool initialized;
-        } legacy_hw_ctx;
+        struct i915_ctx_hang_stats hang_stats;
 
-        /* Execlists */
-        struct {
+        /* Unique identifier for this context, used by the hw for tracking */
+        unsigned long flags;
+#define CONTEXT_NO_ZEROMAP              BIT(0)
+#define CONTEXT_NO_ERROR_CAPTURE        BIT(1)
+        unsigned hw_id;
+        u32 user_handle;
+
+        u32 ggtt_alignment;
+
+        struct intel_context {
                 struct drm_i915_gem_object *state;
                 struct intel_ringbuffer *ringbuf;
-                int pin_count;
                 struct i915_vma *lrc_vma;
-                u64 lrc_desc;
                 uint32_t *lrc_reg_state;
+                u64 lrc_desc;
+                int pin_count;
+                bool initialised;
         } engine[I915_NUM_ENGINES];
+        u32 ring_size;
+        u32 desc_template;
+        struct atomic_notifier_head status_notifier;
+        bool execlists_force_single_submission;
 
         struct list_head link;
+
+        u8 remap_slice;
 };
 
 enum fb_op_origin {
@@ -1116,6 +1156,8 @@ struct intel_gen6_power_mgmt {
         bool interrupts_enabled;
         u32 pm_iir;
 
+        u32 pm_intr_keep;
+
         /* Frequencies are stored in potentially platform dependent multiples.
          * In other words, *_freq needs to be multiplied by X to be interesting.
          * Soft limits are those which are used for the dynamic reclocking done
@@ -1283,37 +1325,11 @@ struct i915_gem_mm {
         struct list_head fence_list;
 
         /**
-         * We leave the user IRQ off as much as possible,
-         * but this means that requests will finish and never
-         * be retired once the system goes idle. Set a timer to
-         * fire periodically while the ring is running. When it
-         * fires, go retire requests.
-         */
-        struct delayed_work retire_work;
-
-        /**
-         * When we detect an idle GPU, we want to turn on
-         * powersaving features. So once we see that there
-         * are no more requests outstanding and no more
-         * arrive within a small period of time, we fire
-         * off the idle_work.
-         */
-        struct delayed_work idle_work;
-
-        /**
          * Are we in a non-interruptible section of code like
          * modesetting?
          */
         bool interruptible;
 
-        /**
-         * Is the GPU currently considered idle, or busy executing userspace
-         * requests?  Whilst idle, we attempt to power down the hardware and
-         * display clocks. In order to reduce the effect on performance, there
-         * is a slight delay before we do so.
-         */
-        bool busy;
-
         /* the indicator for dispatch video commands on two BSD rings */
         unsigned int bsd_ring_dispatch_index;
 
@@ -1350,7 +1366,6 @@ struct i915_gpu_error {
         /* Hang gpu twice in this window and your context gets banned */
 #define DRM_I915_CTX_BAN_PERIOD DIV_ROUND_UP(8*DRM_I915_HANGCHECK_PERIOD, 1000)
 
-        struct workqueue_struct *hangcheck_wq;
         struct delayed_work hangcheck_work;
 
         /* For reset and error_state handling. */
@@ -1387,20 +1402,19 @@ struct i915_gpu_error {
 #define I915_WEDGED                     (1 << 31)
 
         /**
+         * Waitqueue to signal when a hang is detected. Used to for waiters
+         * to release the struct_mutex for the reset to procede.
+         */
+        wait_queue_head_t wait_queue;
+
+        /**
          * Waitqueue to signal when the reset has completed. Used by clients
          * that wait for dev_priv->mm.wedged to settle.
          */
         wait_queue_head_t reset_queue;
 
-        /* Userspace knobs for gpu hang simulation;
-         * combines both a ring mask, and extra flags
-         */
-        u32 stop_rings;
-#define I915_STOP_RING_ALLOW_BAN       (1 << 31)
-#define I915_STOP_RING_ALLOW_WARN      (1 << 30)
-
         /* For missed irq/seqno simulation. */
-        unsigned int test_irq_rings;
+        unsigned long test_irq_rings;
 };
 
 enum modeset_restore {
@@ -1489,6 +1503,7 @@ struct intel_vbt_data {
                 bool present;
                 bool active_low_pwm;
                 u8 min_brightness;      /* min_brightness/255 of max */
+                enum intel_backlight_type type;
         } backlight;
 
         /* MIPI DSI */
@@ -1581,7 +1596,7 @@ struct skl_ddb_allocation {
 };
 
 struct skl_wm_values {
-        bool dirty[I915_MAX_PIPES];
+        unsigned dirty_pipes;
         struct skl_ddb_allocation ddb;
         uint32_t wm_linetime[I915_MAX_PIPES];
         uint32_t plane[I915_MAX_PIPES][I915_MAX_PLANES][8];
@@ -1698,7 +1713,7 @@ struct i915_execbuffer_params {
         uint64_t                        batch_obj_vm_offset;
         struct intel_engine_cs *engine;
         struct drm_i915_gem_object      *batch_obj;
-        struct intel_context            *ctx;
+        struct i915_gem_context            *ctx;
         struct drm_i915_gem_request     *request;
 };
 
@@ -1710,7 +1725,8 @@ struct intel_wm_config {
 };
 
 struct drm_i915_private {
-        struct drm_device *dev;
+        struct drm_device drm;
+
         struct kmem_cache *objects;
         struct kmem_cache *vmas;
         struct kmem_cache *requests;
@@ -1725,6 +1741,8 @@ struct drm_i915_private {
 
         struct i915_virtual_gpu vgpu;
 
+        struct intel_gvt gvt;
+
         struct intel_guc guc;
 
         struct intel_csr csr;
@@ -1748,6 +1766,7 @@ struct drm_i915_private {
         wait_queue_head_t gmbus_wait_queue;
 
         struct pci_dev *bridge_dev;
+        struct i915_gem_context *kernel_context;
         struct intel_engine_cs engine[I915_NUM_ENGINES];
         struct drm_i915_gem_object *semaphore_obj;
         uint32_t last_seqno, next_seqno;
@@ -1803,13 +1822,17 @@ struct drm_i915_private {
         int num_fence_regs; /* 8 on pre-965, 16 otherwise */
 
         unsigned int fsb_freq, mem_freq, is_ddr3;
-        unsigned int skl_boot_cdclk;
+        unsigned int skl_preferred_vco_freq;
         unsigned int cdclk_freq, max_cdclk_freq, atomic_cdclk_freq;
         unsigned int max_dotclk_freq;
         unsigned int rawclk_freq;
         unsigned int hpll_freq;
         unsigned int czclk_freq;
 
+        struct {
+                unsigned int vco, ref;
+        } cdclk_pll;
+
         /**
          * wq - Driver workqueue for GEM.
          *
@@ -1839,6 +1862,13 @@ struct drm_i915_private {
         DECLARE_HASHTABLE(mm_structs, 7);
         struct mutex mm_lock;
 
+        /* The hw wants to have a stable context identifier for the lifetime
+         * of the context (for OA, PASID, faults, etc). This is limited
+         * in execlists to 21 bits.
+         */
+        struct ida context_hw_ida;
+#define MAX_CONTEXT_HW_ID (1<<21) /* exclusive */
+
         /* Kernel Modesetting */
 
         struct drm_crtc *plane_to_crtc_mapping[I915_MAX_PIPES];
@@ -1951,9 +1981,6 @@ struct drm_i915_private {
                  */
                 uint16_t skl_latency[8];
 
-                /* Committed wm config */
-                struct intel_wm_config config;
-
                 /*
                  * The skl_wm_values structure is a bit too big for stack
                  * allocation, so we keep the staging struct where we store
@@ -1976,6 +2003,13 @@ struct drm_i915_private {
                  * cstate->wm.need_postvbl_update.
                  */
                 struct mutex wm_mutex;
+
+                /*
+                 * Set during HW readout of watermarks/DDB.  Some platforms
+                 * need to know when we're still using BIOS-provided values
+                 * (which we don't fully trust).
+                 */
+                bool distrust_bios_wm;
         } wm;
 
         struct i915_runtime_pm pm;
@@ -1988,9 +2022,35 @@ struct drm_i915_private {
                 int (*init_engines)(struct drm_device *dev);
                 void (*cleanup_engine)(struct intel_engine_cs *engine);
                 void (*stop_engine)(struct intel_engine_cs *engine);
-        } gt;
 
-        struct intel_context *kernel_context;
+                /**
+                 * Is the GPU currently considered idle, or busy executing
+                 * userspace requests? Whilst idle, we allow runtime power
+                 * management to power down the hardware and display clocks.
+                 * In order to reduce the effect on performance, there
+                 * is a slight delay before we do so.
+                 */
+                unsigned int active_engines;
+                bool awake;
+
+                /**
+                 * We leave the user IRQ off as much as possible,
+                 * but this means that requests will finish and never
+                 * be retired once the system goes idle. Set a timer to
+                 * fire periodically while the ring is running. When it
+                 * fires, go retire requests.
+                 */
+                struct delayed_work retire_work;
+
+                /**
+                 * When we detect an idle GPU, we want to turn on
+                 * powersaving features. So once we see that there
+                 * are no more requests outstanding and no more
+                 * arrive within a small period of time, we fire
+                 * off the idle_work.
+                 */
+                struct delayed_work idle_work;
+        } gt;
 
         /* perform PHY state sanity checks? */
         bool chv_phy_assert[2];
@@ -2005,7 +2065,7 @@ struct drm_i915_private {
 
 static inline struct drm_i915_private *to_i915(const struct drm_device *dev)
 {
-        return dev->dev_private;
+        return container_of(dev, struct drm_i915_private, drm);
 }
 
 static inline struct drm_i915_private *dev_to_i915(struct device *dev)
@@ -2176,6 +2236,7 @@ struct drm_i915_gem_object {
 
         unsigned int frontbuffer_bits:INTEL_FRONTBUFFER_BITS;
 
+        unsigned int has_wc_mmap;
         unsigned int pin_display;
 
         struct sg_table *pages;
@@ -2228,9 +2289,81 @@ struct drm_i915_gem_object {
 };
 #define to_intel_bo(x) container_of(x, struct drm_i915_gem_object, base)
 
-void i915_gem_track_fb(struct drm_i915_gem_object *old,
-                       struct drm_i915_gem_object *new,
-                       unsigned frontbuffer_bits);
+static inline bool
+i915_gem_object_has_struct_page(const struct drm_i915_gem_object *obj)
+{
+        return obj->ops->flags & I915_GEM_OBJECT_HAS_STRUCT_PAGE;
+}
+
+/*
+ * Optimised SGL iterator for GEM objects
+ */
+static __always_inline struct sgt_iter {
+        struct scatterlist *sgp;
+        union {
+                unsigned long pfn;
+                dma_addr_t dma;
+        };
+        unsigned int curr;
+        unsigned int max;
+} __sgt_iter(struct scatterlist *sgl, bool dma) {
+        struct sgt_iter s = { .sgp = sgl };
+
+        if (s.sgp) {
+                s.max = s.curr = s.sgp->offset;
+                s.max += s.sgp->length;
+                if (dma)
+                        s.dma = sg_dma_address(s.sgp);
+                else
+                        s.pfn = page_to_pfn(sg_page(s.sgp));
+        }
+
+        return s;
+}
+
+/**
+ * __sg_next - return the next scatterlist entry in a list
+ * @sg:         The current sg entry
+ *
+ * Description:
+ *   If the entry is the last, return NULL; otherwise, step to the next
+ *   element in the array (@sg@+1). If that's a chain pointer, follow it;
+ *   otherwise just return the pointer to the current element.
+ **/
+static inline struct scatterlist *__sg_next(struct scatterlist *sg)
+{
+#ifdef CONFIG_DEBUG_SG
+        BUG_ON(sg->sg_magic != SG_MAGIC);
+#endif
+        return sg_is_last(sg) ? NULL :
+                likely(!sg_is_chain(++sg)) ? sg :
+                sg_chain_ptr(sg);
+}
+
+/**
+ * for_each_sgt_dma - iterate over the DMA addresses of the given sg_table
+ * @__dmap:     DMA address (output)
+ * @__iter:     'struct sgt_iter' (iterator state, internal)
+ * @__sgt:      sg_table to iterate over (input)
+ */
+#define for_each_sgt_dma(__dmap, __iter, __sgt)                         \
+        for ((__iter) = __sgt_iter((__sgt)->sgl, true);                 \
+             ((__dmap) = (__iter).dma + (__iter).curr);                 \
+             (((__iter).curr += PAGE_SIZE) < (__iter).max) ||           \
+             ((__iter) = __sgt_iter(__sg_next((__iter).sgp), true), 0))
+
+/**
+ * for_each_sgt_page - iterate over the pages of the given sg_table
+ * @__pp:       page pointer (output)
+ * @__iter:     'struct sgt_iter' (iterator state, internal)
+ * @__sgt:      sg_table to iterate over (input)
+ */
+#define for_each_sgt_page(__pp, __iter, __sgt)                          \
+        for ((__iter) = __sgt_iter((__sgt)->sgl, false);                \
+             ((__pp) = (__iter).pfn == 0 ? NULL :                       \
+              pfn_to_page((__iter).pfn + ((__iter).curr >> PAGE_SHIFT))); \
+             (((__iter).curr += PAGE_SIZE) < (__iter).max) ||           \
+             ((__iter) = __sgt_iter(__sg_next((__iter).sgp), false), 0))
 
 /**
  * Request queue structure.
@@ -2252,7 +2385,7 @@ struct drm_i915_gem_request {
         /** On Which ring this request was generated */
         struct drm_i915_private *i915;
         struct intel_engine_cs *engine;
-        unsigned reset_counter;
+        struct intel_signal_node signaling;
 
          /** GEM sequence number associated with the previous request,
           * when the HWS breadcrumb is equal to this the GPU is processing
@@ -2279,6 +2412,9 @@ struct drm_i915_gem_request {
         /** Position in the ringbuffer of the end of the whole request */
         u32 tail;
 
+        /** Preallocate space in the ringbuffer for the emitting the request */
+        u32 reserved_space;
+
         /**
          * Context and ring buffer related to this request
          * Contexts are refcounted, so when this request is associated with a
@@ -2289,9 +2425,20 @@ struct drm_i915_gem_request {
          * i915_gem_request_free() will then decrement the refcount on the
          * context.
          */
-        struct intel_context *ctx;
+        struct i915_gem_context *ctx;
         struct intel_ringbuffer *ringbuf;
 
+        /**
+         * Context related to the previous request.
+         * As the contexts are accessed by the hardware until the switch is
+         * completed to a new context, the hardware may still be writing
+         * to the context object after the breadcrumb is visible. We must
+         * not unpin/unbind/prune that object whilst still active and so
+         * we keep the previous context pinned until the following (this)
+         * request is retired.
+         */
+        struct i915_gem_context *previous_context;
+
         /** Batch buffer related to this request if any (used for
             error state dump only) */
         struct drm_i915_gem_object *batch_obj;
@@ -2328,11 +2475,13 @@ struct drm_i915_gem_request {
         /** Execlists no. of times this request has been sent to the ELSP */
         int elsp_submitted;
 
+        /** Execlists context hardware id. */
+        unsigned ctx_hw_id;
 };
 
 struct drm_i915_gem_request * __must_check
 i915_gem_request_alloc(struct intel_engine_cs *engine,
-                       struct intel_context *ctx);
+                       struct i915_gem_context *ctx);
 void i915_gem_request_free(struct kref *req_ref);
 int i915_gem_request_add_to_client(struct drm_i915_gem_request *req,
                                    struct drm_file *file);
@@ -2360,23 +2509,9 @@ i915_gem_request_reference(struct drm_i915_gem_request *req)
 static inline void
 i915_gem_request_unreference(struct drm_i915_gem_request *req)
 {
-        WARN_ON(!mutex_is_locked(&req->engine->dev->struct_mutex));
         kref_put(&req->ref, i915_gem_request_free);
 }
 
-static inline void
-i915_gem_request_unreference__unlocked(struct drm_i915_gem_request *req)
-{
-        struct drm_device *dev;
-
-        if (!req)
-                return;
-
-        dev = req->engine->dev;
-        if (kref_put_mutex(&req->ref, i915_gem_request_free, &dev->struct_mutex))
-                mutex_unlock(&dev->struct_mutex);
-}
-
 static inline void i915_gem_request_assign(struct drm_i915_gem_request **pdst,
                                            struct drm_i915_gem_request *src)
 {
@@ -2504,9 +2639,29 @@ struct drm_i915_cmd_table {
 #define INTEL_INFO(p)   (&__I915__(p)->info)
 #define INTEL_GEN(p)    (INTEL_INFO(p)->gen)
 #define INTEL_DEVID(p)  (INTEL_INFO(p)->device_id)
-#define INTEL_REVID(p)  (__I915__(p)->dev->pdev->revision)
 
 #define REVID_FOREVER           0xff
+#define INTEL_REVID(p)  (__I915__(p)->drm.pdev->revision)
+
+#define GEN_FOREVER (0)
+/*
+ * Returns true if Gen is in inclusive range [Start, End].
+ *
+ * Use GEN_FOREVER for unbound start and or end.
+ */
+#define IS_GEN(p, s, e) ({ \
+        unsigned int __s = (s), __e = (e); \
+        BUILD_BUG_ON(!__builtin_constant_p(s)); \
+        BUILD_BUG_ON(!__builtin_constant_p(e)); \
+        if ((__s) != GEN_FOREVER) \
+                __s = (s) - 1; \
+        if ((__e) == GEN_FOREVER) \
+                __e = BITS_PER_LONG - 1; \
+        else \
+                __e = (e) - 1; \
+        !!(INTEL_INFO(p)->gen_mask & GENMASK((__e), (__s))); \
+})
+
 /*
  * Return true if revision is in range [since,until] inclusive.
  *
@@ -2539,7 +2694,7 @@ struct drm_i915_cmd_table {
 #define IS_VALLEYVIEW(dev)      (INTEL_INFO(dev)->is_valleyview)
 #define IS_CHERRYVIEW(dev)      (INTEL_INFO(dev)->is_cherryview)
 #define IS_HASWELL(dev) (INTEL_INFO(dev)->is_haswell)
-#define IS_BROADWELL(dev)       (!INTEL_INFO(dev)->is_cherryview && IS_GEN8(dev))
+#define IS_BROADWELL(dev)       (INTEL_INFO(dev)->is_broadwell)
 #define IS_SKYLAKE(dev) (INTEL_INFO(dev)->is_skylake)
 #define IS_BROXTON(dev)         (INTEL_INFO(dev)->is_broxton)
 #define IS_KABYLAKE(dev)        (INTEL_INFO(dev)->is_kabylake)
@@ -2591,6 +2746,8 @@ struct drm_i915_cmd_table {
 #define SKL_REVID_D0            0x3
 #define SKL_REVID_E0            0x4
 #define SKL_REVID_F0            0x5
+#define SKL_REVID_G0            0x6
+#define SKL_REVID_H0            0x7
 
 #define IS_SKL_REVID(p, since, until) (IS_SKYLAKE(p) && IS_REVID(p, since, until))
 
@@ -2616,29 +2773,34 @@ struct drm_i915_cmd_table {
  * have their own (e.g. HAS_PCH_SPLIT for ILK+ display, IS_foo for particular
  * chips, etc.).
  */
-#define IS_GEN2(dev)    (INTEL_INFO(dev)->gen == 2)
-#define IS_GEN3(dev)    (INTEL_INFO(dev)->gen == 3)
-#define IS_GEN4(dev)    (INTEL_INFO(dev)->gen == 4)
-#define IS_GEN5(dev)    (INTEL_INFO(dev)->gen == 5)
-#define IS_GEN6(dev)    (INTEL_INFO(dev)->gen == 6)
-#define IS_GEN7(dev)    (INTEL_INFO(dev)->gen == 7)
-#define IS_GEN8(dev)    (INTEL_INFO(dev)->gen == 8)
-#define IS_GEN9(dev)    (INTEL_INFO(dev)->gen == 9)
-
-#define RENDER_RING             (1<<RCS)
-#define BSD_RING                (1<<VCS)
-#define BLT_RING                (1<<BCS)
-#define VEBOX_RING              (1<<VECS)
-#define BSD2_RING               (1<<VCS2)
-#define ALL_ENGINES             (~0)
-
-#define HAS_BSD(dev)            (INTEL_INFO(dev)->ring_mask & BSD_RING)
-#define HAS_BSD2(dev)           (INTEL_INFO(dev)->ring_mask & BSD2_RING)
-#define HAS_BLT(dev)            (INTEL_INFO(dev)->ring_mask & BLT_RING)
-#define HAS_VEBOX(dev)          (INTEL_INFO(dev)->ring_mask & VEBOX_RING)
+#define IS_GEN2(dev)    (!!(INTEL_INFO(dev)->gen_mask & BIT(1)))
+#define IS_GEN3(dev)    (!!(INTEL_INFO(dev)->gen_mask & BIT(2)))
+#define IS_GEN4(dev)    (!!(INTEL_INFO(dev)->gen_mask & BIT(3)))
+#define IS_GEN5(dev)    (!!(INTEL_INFO(dev)->gen_mask & BIT(4)))
+#define IS_GEN6(dev)    (!!(INTEL_INFO(dev)->gen_mask & BIT(5)))
+#define IS_GEN7(dev)    (!!(INTEL_INFO(dev)->gen_mask & BIT(6)))
+#define IS_GEN8(dev)    (!!(INTEL_INFO(dev)->gen_mask & BIT(7)))
+#define IS_GEN9(dev)    (!!(INTEL_INFO(dev)->gen_mask & BIT(8)))
+
+#define ENGINE_MASK(id) BIT(id)
+#define RENDER_RING     ENGINE_MASK(RCS)
+#define BSD_RING        ENGINE_MASK(VCS)
+#define BLT_RING        ENGINE_MASK(BCS)
+#define VEBOX_RING      ENGINE_MASK(VECS)
+#define BSD2_RING       ENGINE_MASK(VCS2)
+#define ALL_ENGINES     (~0)
+
+#define HAS_ENGINE(dev_priv, id) \
+        (!!(INTEL_INFO(dev_priv)->ring_mask & ENGINE_MASK(id)))
+
+#define HAS_BSD(dev_priv)       HAS_ENGINE(dev_priv, VCS)
+#define HAS_BSD2(dev_priv)      HAS_ENGINE(dev_priv, VCS2)
+#define HAS_BLT(dev_priv)       HAS_ENGINE(dev_priv, BCS)
+#define HAS_VEBOX(dev_priv)     HAS_ENGINE(dev_priv, VECS)
+
 #define HAS_LLC(dev)            (INTEL_INFO(dev)->has_llc)
 #define HAS_SNOOP(dev)          (INTEL_INFO(dev)->has_snoop)
-#define HAS_EDRAM(dev)          (__I915__(dev)->edram_cap & EDRAM_ENABLED)
+#define HAS_EDRAM(dev)          (!!(__I915__(dev)->edram_cap & EDRAM_ENABLED))
 #define HAS_WT(dev)             ((IS_HASWELL(dev) || IS_BROADWELL(dev)) && \
                                  HAS_EDRAM(dev))
 #define I915_NEED_GFX_HWS(dev)  (INTEL_INFO(dev)->need_gfx_hws)
@@ -2656,9 +2818,10 @@ struct drm_i915_cmd_table {
 #define HAS_BROKEN_CS_TLB(dev)          (IS_I830(dev) || IS_845G(dev))
 
 /* WaRsDisableCoarsePowerGating:skl,bxt */
-#define NEEDS_WaRsDisableCoarsePowerGating(dev) (IS_BXT_REVID(dev, 0, BXT_REVID_A1) || \
-                                                 IS_SKL_GT3(dev) || \
-                                                 IS_SKL_GT4(dev))
+#define NEEDS_WaRsDisableCoarsePowerGating(dev_priv) \
+        (IS_BXT_REVID(dev_priv, 0, BXT_REVID_A1) || \
+         IS_SKL_GT3(dev_priv) || \
+         IS_SKL_GT4(dev_priv))
 
 /*
  * dp aux and gmbus irq on gen4 seems to be able to generate legacy interrupts
@@ -2696,12 +2859,18 @@ struct drm_i915_cmd_table {
                                  IS_CHERRYVIEW(dev) || IS_SKYLAKE(dev) || \
                                  IS_KABYLAKE(dev) || IS_BROXTON(dev))
 #define HAS_RC6(dev)            (INTEL_INFO(dev)->gen >= 6)
-#define HAS_RC6p(dev)           (INTEL_INFO(dev)->gen == 6 || IS_IVYBRIDGE(dev))
+#define HAS_RC6p(dev)           (IS_GEN6(dev) || IS_IVYBRIDGE(dev))
 
 #define HAS_CSR(dev)    (IS_GEN9(dev))
 
-#define HAS_GUC_UCODE(dev)      (IS_GEN9(dev) && !IS_KABYLAKE(dev))
-#define HAS_GUC_SCHED(dev)      (IS_GEN9(dev) && !IS_KABYLAKE(dev))
+/*
+ * For now, anything with a GuC requires uCode loading, and then supports
+ * command submission once loaded. But these are logically independent
+ * properties, so we have separate macros to test them.
+ */
+#define HAS_GUC(dev)            (IS_GEN9(dev))
+#define HAS_GUC_UCODE(dev)      (HAS_GUC(dev))
+#define HAS_GUC_SCHED(dev)      (HAS_GUC(dev))
 
 #define HAS_RESOURCE_STREAMER(dev) (IS_HASWELL(dev) || \
                                     INTEL_INFO(dev)->gen >= 8)
@@ -2710,6 +2879,8 @@ struct drm_i915_cmd_table {
                                  !IS_VALLEYVIEW(dev) && !IS_CHERRYVIEW(dev) && \
                                  !IS_BROXTON(dev))
 
+#define HAS_POOLED_EU(dev)      (INTEL_INFO(dev)->has_pooled_eu)
+
 #define INTEL_PCH_DEVICE_ID_MASK                0xff00
 #define INTEL_PCH_IBX_DEVICE_ID_TYPE            0x3b00
 #define INTEL_PCH_CPT_DEVICE_ID_TYPE            0x1c00
@@ -2746,13 +2917,22 @@ struct drm_i915_cmd_table {
 
 #include "i915_trace.h"
 
-extern const struct drm_ioctl_desc i915_ioctls[];
-extern int i915_max_ioctl;
+static inline bool intel_scanout_needs_vtd_wa(struct drm_i915_private *dev_priv)
+{
+#ifdef CONFIG_INTEL_IOMMU
+        if (INTEL_GEN(dev_priv) >= 6 && intel_iommu_gfx_mapped)
+                return true;
+#endif
+        return false;
+}
 
 extern int i915_suspend_switcheroo(struct drm_device *dev, pm_message_t state);
 extern int i915_resume_switcheroo(struct drm_device *dev);
 
-/* i915_dma.c */
+int intel_sanitize_enable_ppgtt(struct drm_i915_private *dev_priv,
+                                int enable_ppgtt);
+
+/* i915_drv.c */
 void __printf(3, 4)
 __i915_printk(struct drm_i915_private *dev_priv, const char *level,
               const char *fmt, ...);
@@ -2760,21 +2940,13 @@ __i915_printk(struct drm_i915_private *dev_priv, const char *level,
 #define i915_report_error(dev_priv, fmt, ...)                              \
         __i915_printk(dev_priv, KERN_ERR, fmt, ##__VA_ARGS__)
 
-extern int i915_driver_load(struct drm_device *, unsigned long flags);
-extern int i915_driver_unload(struct drm_device *);
-extern int i915_driver_open(struct drm_device *dev, struct drm_file *file);
-extern void i915_driver_lastclose(struct drm_device * dev);
-extern void i915_driver_preclose(struct drm_device *dev,
-                                 struct drm_file *file);
-extern void i915_driver_postclose(struct drm_device *dev,
-                                  struct drm_file *file);
 #ifdef CONFIG_COMPAT
 extern long i915_compat_ioctl(struct file *filp, unsigned int cmd,
                               unsigned long arg);
 #endif
-extern int intel_gpu_reset(struct drm_device *dev, u32 engine_mask);
-extern bool intel_has_gpu_reset(struct drm_device *dev);
-extern int i915_reset(struct drm_device *dev);
+extern int intel_gpu_reset(struct drm_i915_private *dev_priv, u32 engine_mask);
+extern bool intel_has_gpu_reset(struct drm_i915_private *dev_priv);
+extern int i915_reset(struct drm_i915_private *dev_priv);
 extern int intel_guc_reset(struct drm_i915_private *dev_priv);
 extern void intel_engine_init_hangcheck(struct intel_engine_cs *engine);
 extern unsigned long i915_chipset_val(struct drm_i915_private *dev_priv);
@@ -2784,30 +2956,51 @@ extern void i915_update_gfx_val(struct drm_i915_private *dev_priv);
 int vlv_force_gfx_clock(struct drm_i915_private *dev_priv, bool on);
 
 /* intel_hotplug.c */
-void intel_hpd_irq_handler(struct drm_device *dev, u32 pin_mask, u32 long_mask);
+void intel_hpd_irq_handler(struct drm_i915_private *dev_priv,
+                           u32 pin_mask, u32 long_mask);
 void intel_hpd_init(struct drm_i915_private *dev_priv);
 void intel_hpd_init_work(struct drm_i915_private *dev_priv);
 void intel_hpd_cancel_work(struct drm_i915_private *dev_priv);
 bool intel_hpd_pin_to_port(enum hpd_pin pin, enum port *port);
+bool intel_hpd_disable(struct drm_i915_private *dev_priv, enum hpd_pin pin);
+void intel_hpd_enable(struct drm_i915_private *dev_priv, enum hpd_pin pin);
 
 /* i915_irq.c */
-void i915_queue_hangcheck(struct drm_device *dev);
+static inline void i915_queue_hangcheck(struct drm_i915_private *dev_priv)
+{
+        unsigned long delay;
+
+        if (unlikely(!i915.enable_hangcheck))
+                return;
+
+        /* Don't continually defer the hangcheck so that it is always run at
+         * least once after work has been scheduled on any ring. Otherwise,
+         * we will ignore a hung ring if a second ring is kept busy.
+         */
+
+        delay = round_jiffies_up_relative(DRM_I915_HANGCHECK_JIFFIES);
+        queue_delayed_work(system_long_wq,
+                           &dev_priv->gpu_error.hangcheck_work, delay);
+}
+
 __printf(3, 4)
-void i915_handle_error(struct drm_device *dev, u32 engine_mask,
+void i915_handle_error(struct drm_i915_private *dev_priv,
+                       u32 engine_mask,
                        const char *fmt, ...);
 
 extern void intel_irq_init(struct drm_i915_private *dev_priv);
 int intel_irq_install(struct drm_i915_private *dev_priv);
 void intel_irq_uninstall(struct drm_i915_private *dev_priv);
 
-extern void intel_uncore_sanitize(struct drm_device *dev);
-extern void intel_uncore_early_sanitize(struct drm_device *dev,
+extern void intel_uncore_sanitize(struct drm_i915_private *dev_priv);
+extern void intel_uncore_early_sanitize(struct drm_i915_private *dev_priv,
                                         bool restore_forcewake);
-extern void intel_uncore_init(struct drm_device *dev);
+extern void intel_uncore_init(struct drm_i915_private *dev_priv);
 extern bool intel_uncore_unclaimed_mmio(struct drm_i915_private *dev_priv);
 extern bool intel_uncore_arm_unclaimed_mmio_detection(struct drm_i915_private *dev_priv);
-extern void intel_uncore_fini(struct drm_device *dev);
-extern void intel_uncore_forcewake_reset(struct drm_device *dev, bool restore);
+extern void intel_uncore_fini(struct drm_i915_private *dev_priv);
+extern void intel_uncore_forcewake_reset(struct drm_i915_private *dev_priv,
+                                         bool restore);
 const char *intel_uncore_forcewake_domain_to_str(const enum forcewake_domain_id id);
 void intel_uncore_forcewake_get(struct drm_i915_private *dev_priv,
                                 enum forcewake_domains domains);
@@ -2823,9 +3016,26 @@ void intel_uncore_forcewake_put__locked(struct drm_i915_private *dev_priv,
 u64 intel_uncore_edram_size(struct drm_i915_private *dev_priv);
 
 void assert_forcewakes_inactive(struct drm_i915_private *dev_priv);
-static inline bool intel_vgpu_active(struct drm_device *dev)
+
+int intel_wait_for_register(struct drm_i915_private *dev_priv,
+                            i915_reg_t reg,
+                            const u32 mask,
+                            const u32 value,
+                            const unsigned long timeout_ms);
+int intel_wait_for_register_fw(struct drm_i915_private *dev_priv,
+                               i915_reg_t reg,
+                               const u32 mask,
+                               const u32 value,
+                               const unsigned long timeout_ms);
+
+static inline bool intel_gvt_active(struct drm_i915_private *dev_priv)
+{
+        return dev_priv->gvt.initialized;
+}
+
+static inline bool intel_vgpu_active(struct drm_i915_private *dev_priv)
 {
-        return to_i915(dev)->vgpu.active;
+        return dev_priv->vgpu.active;
 }
 
 void
@@ -2882,7 +3092,6 @@ ibx_disable_display_interrupt(struct drm_i915_private *dev_priv, uint32_t bits)
         ibx_display_interrupt_update(dev_priv, bits, 0);
 }
 
-
 /* i915_gem.c */
 int i915_gem_create_ioctl(struct drm_device *dev, void *data,
                           struct drm_file *file_priv);
@@ -2921,7 +3130,7 @@ int i915_gem_set_tiling(struct drm_device *dev, void *data,
                         struct drm_file *file_priv);
 int i915_gem_get_tiling(struct drm_device *dev, void *data,
                         struct drm_file *file_priv);
-int i915_gem_init_userptr(struct drm_device *dev);
+void i915_gem_init_userptr(struct drm_i915_private *dev_priv);
 int i915_gem_userptr_ioctl(struct drm_device *dev, void *data,
                            struct drm_file *file);
 int i915_gem_get_aperture_ioctl(struct drm_device *dev, void *data,
@@ -2931,11 +3140,13 @@ int i915_gem_wait_ioctl(struct drm_device *dev, void *data,
 void i915_gem_load_init(struct drm_device *dev);
 void i915_gem_load_cleanup(struct drm_device *dev);
 void i915_gem_load_init_fences(struct drm_i915_private *dev_priv);
+int i915_gem_freeze_late(struct drm_i915_private *dev_priv);
+
 void *i915_gem_object_alloc(struct drm_device *dev);
 void i915_gem_object_free(struct drm_i915_gem_object *obj);
 void i915_gem_object_init(struct drm_i915_gem_object *obj,
                          const struct drm_i915_gem_object_ops *ops);
-struct drm_i915_gem_object *i915_gem_alloc_object(struct drm_device *dev,
+struct drm_i915_gem_object *i915_gem_object_create(struct drm_device *dev,
                                                   size_t size);
 struct drm_i915_gem_object *i915_gem_object_create_from_data(
                 struct drm_device *dev, const void *data, size_t size);
@@ -2990,6 +3201,23 @@ static inline int __sg_page_count(struct scatterlist *sg)
 struct page *
 i915_gem_object_get_dirty_page(struct drm_i915_gem_object *obj, int n);
 
+static inline dma_addr_t
+i915_gem_object_get_dma_address(struct drm_i915_gem_object *obj, int n)
+{
+        if (n < obj->get_page.last) {
+                obj->get_page.sg = obj->pages->sgl;
+                obj->get_page.last = 0;
+        }
+
+        while (obj->get_page.last + __sg_page_count(obj->get_page.sg) <= n) {
+                obj->get_page.last += __sg_page_count(obj->get_page.sg++);
+                if (unlikely(sg_is_chain(obj->get_page.sg)))
+                        obj->get_page.sg = sg_chain_ptr(obj->get_page.sg);
+        }
+
+        return sg_dma_address(obj->get_page.sg) + ((n - obj->get_page.last) << PAGE_SHIFT);
+}
+
 static inline struct page *
 i915_gem_object_get_page(struct drm_i915_gem_object *obj, int n)
 {
@@ -3066,6 +3294,11 @@ int i915_gem_dumb_create(struct drm_file *file_priv,
                          struct drm_mode_create_dumb *args);
 int i915_gem_mmap_gtt(struct drm_file *file_priv, struct drm_device *dev,
                       uint32_t handle, uint64_t *offset);
+
+void i915_gem_track_fb(struct drm_i915_gem_object *old,
+                       struct drm_i915_gem_object *new,
+                       unsigned frontbuffer_bits);
+
 /**
  * Returns true if seq1 is later than seq2.
  */
@@ -3075,31 +3308,34 @@ i915_seqno_passed(uint32_t seq1, uint32_t seq2)
         return (int32_t)(seq1 - seq2) >= 0;
 }
 
-static inline bool i915_gem_request_started(struct drm_i915_gem_request *req,
-                                           bool lazy_coherency)
+static inline bool i915_gem_request_started(const struct drm_i915_gem_request *req)
 {
-        if (!lazy_coherency && req->engine->irq_seqno_barrier)
-                req->engine->irq_seqno_barrier(req->engine);
-        return i915_seqno_passed(req->engine->get_seqno(req->engine),
+        return i915_seqno_passed(intel_engine_get_seqno(req->engine),
                                  req->previous_seqno);
 }
 
-static inline bool i915_gem_request_completed(struct drm_i915_gem_request *req,
-                                              bool lazy_coherency)
+static inline bool i915_gem_request_completed(const struct drm_i915_gem_request *req)
 {
-        if (!lazy_coherency && req->engine->irq_seqno_barrier)
-                req->engine->irq_seqno_barrier(req->engine);
-        return i915_seqno_passed(req->engine->get_seqno(req->engine),
+        return i915_seqno_passed(intel_engine_get_seqno(req->engine),
                                  req->seqno);
 }
 
-int __must_check i915_gem_get_seqno(struct drm_device *dev, u32 *seqno);
+bool __i915_spin_request(const struct drm_i915_gem_request *request,
+                         int state, unsigned long timeout_us);
+static inline bool i915_spin_request(const struct drm_i915_gem_request *request,
+                                     int state, unsigned long timeout_us)
+{
+        return (i915_gem_request_started(request) &&
+                __i915_spin_request(request, state, timeout_us));
+}
+
+int __must_check i915_gem_get_seqno(struct drm_i915_private *dev_priv, u32 *seqno);
 int __must_check i915_gem_set_seqno(struct drm_device *dev, u32 seqno);
 
 struct drm_i915_gem_request *
 i915_gem_find_active_request(struct intel_engine_cs *engine);
 
-bool i915_gem_retire_requests(struct drm_device *dev);
+void i915_gem_retire_requests(struct drm_i915_private *dev_priv);
 void i915_gem_retire_requests_ring(struct intel_engine_cs *engine);
 
 static inline u32 i915_reset_counter(struct i915_gpu_error *error)
@@ -3142,27 +3378,14 @@ static inline u32 i915_reset_count(struct i915_gpu_error *error)
         return ((i915_reset_counter(error) & ~I915_WEDGED) + 1) / 2;
 }
 
-static inline bool i915_stop_ring_allow_ban(struct drm_i915_private *dev_priv)
-{
-        return dev_priv->gpu_error.stop_rings == 0 ||
-                dev_priv->gpu_error.stop_rings & I915_STOP_RING_ALLOW_BAN;
-}
-
-static inline bool i915_stop_ring_allow_warn(struct drm_i915_private *dev_priv)
-{
-        return dev_priv->gpu_error.stop_rings == 0 ||
-                dev_priv->gpu_error.stop_rings & I915_STOP_RING_ALLOW_WARN;
-}
-
 void i915_gem_reset(struct drm_device *dev);
 bool i915_gem_clflush_object(struct drm_i915_gem_object *obj, bool force);
 int __must_check i915_gem_init(struct drm_device *dev);
 int i915_gem_init_engines(struct drm_device *dev);
 int __must_check i915_gem_init_hw(struct drm_device *dev);
-int i915_gem_l3_remap(struct drm_i915_gem_request *req, int slice);
 void i915_gem_init_swizzling(struct drm_device *dev);
 void i915_gem_cleanup_engines(struct drm_device *dev);
-int __must_check i915_gpu_idle(struct drm_device *dev);
+int __must_check i915_gem_wait_for_idle(struct drm_i915_private *dev_priv);
 int __must_check i915_gem_suspend(struct drm_device *dev);
 void __i915_add_request(struct drm_i915_gem_request *req,
                         struct drm_i915_gem_object *batch_obj,
@@ -3227,8 +3450,6 @@ bool i915_gem_obj_ggtt_bound_view(struct drm_i915_gem_object *o,
 bool i915_gem_obj_bound(struct drm_i915_gem_object *o,
                         struct i915_address_space *vm);
 
-unsigned long i915_gem_obj_size(struct drm_i915_gem_object *o,
-                                struct i915_address_space *vm);
 struct i915_vma *
 i915_gem_obj_to_vma(struct drm_i915_gem_object *obj,
                     struct i915_address_space *vm);
@@ -3263,14 +3484,8 @@ static inline bool i915_gem_obj_ggtt_bound(struct drm_i915_gem_object *obj)
         return i915_gem_obj_ggtt_bound_view(obj, &i915_ggtt_view_normal);
 }
 
-static inline unsigned long
-i915_gem_obj_ggtt_size(struct drm_i915_gem_object *obj)
-{
-        struct drm_i915_private *dev_priv = to_i915(obj->base.dev);
-        struct i915_ggtt *ggtt = &dev_priv->ggtt;
-
-        return i915_gem_obj_size(obj, &ggtt->base);
-}
+unsigned long
+i915_gem_obj_ggtt_size(struct drm_i915_gem_object *obj);
 
 static inline int __must_check
 i915_gem_obj_ggtt_pin(struct drm_i915_gem_object *obj,
@@ -3284,12 +3499,6 @@ i915_gem_obj_ggtt_pin(struct drm_i915_gem_object *obj,
                                    alignment, flags | PIN_GLOBAL);
 }
 
-static inline int
-i915_gem_object_ggtt_unbind(struct drm_i915_gem_object *obj)
-{
-        return i915_vma_unbind(i915_gem_obj_to_ggtt(obj));
-}
-
 void i915_gem_object_ggtt_unpin_view(struct drm_i915_gem_object *obj,
                                      const struct i915_ggtt_view *view);
 static inline void
@@ -3313,28 +3522,44 @@ void i915_gem_object_save_bit_17_swizzle(struct drm_i915_gem_object *obj);
 
 /* i915_gem_context.c */
 int __must_check i915_gem_context_init(struct drm_device *dev);
+void i915_gem_context_lost(struct drm_i915_private *dev_priv);
 void i915_gem_context_fini(struct drm_device *dev);
 void i915_gem_context_reset(struct drm_device *dev);
 int i915_gem_context_open(struct drm_device *dev, struct drm_file *file);
-int i915_gem_context_enable(struct drm_i915_gem_request *req);
 void i915_gem_context_close(struct drm_device *dev, struct drm_file *file);
 int i915_switch_context(struct drm_i915_gem_request *req);
-struct intel_context *
-i915_gem_context_get(struct drm_i915_file_private *file_priv, u32 id);
 void i915_gem_context_free(struct kref *ctx_ref);
 struct drm_i915_gem_object *
 i915_gem_alloc_context_obj(struct drm_device *dev, size_t size);
-static inline void i915_gem_context_reference(struct intel_context *ctx)
+struct i915_gem_context *
+i915_gem_context_create_gvt(struct drm_device *dev);
+
+static inline struct i915_gem_context *
+i915_gem_context_lookup(struct drm_i915_file_private *file_priv, u32 id)
+{
+        struct i915_gem_context *ctx;
+
+        lockdep_assert_held(&file_priv->dev_priv->drm.struct_mutex);
+
+        ctx = idr_find(&file_priv->context_idr, id);
+        if (!ctx)
+                return ERR_PTR(-ENOENT);
+
+        return ctx;
+}
+
+static inline void i915_gem_context_reference(struct i915_gem_context *ctx)
 {
         kref_get(&ctx->ref);
 }
 
-static inline void i915_gem_context_unreference(struct intel_context *ctx)
+static inline void i915_gem_context_unreference(struct i915_gem_context *ctx)
 {
+        lockdep_assert_held(&ctx->i915->drm.struct_mutex);
         kref_put(&ctx->ref, i915_gem_context_free);
 }
 
-static inline bool i915_gem_context_is_default(const struct intel_context *c)
+static inline bool i915_gem_context_is_default(const struct i915_gem_context *c)
 {
         return c->user_handle == DEFAULT_CONTEXT_HANDLE;
 }
@@ -3347,6 +3572,8 @@ int i915_gem_context_getparam_ioctl(struct drm_device *dev, void *data,
                                     struct drm_file *file_priv);
 int i915_gem_context_setparam_ioctl(struct drm_device *dev, void *data,
                                     struct drm_file *file_priv);
+int i915_gem_context_reset_stats_ioctl(struct drm_device *dev, void *data,
+                                       struct drm_file *file);
 
 /* i915_gem_evict.c */
 int __must_check i915_gem_evict_something(struct drm_device *dev,
@@ -3361,9 +3588,9 @@ int __must_check i915_gem_evict_for_vma(struct i915_vma *target);
 int i915_gem_evict_vm(struct i915_address_space *vm, bool do_idle);
 
 /* belongs in i915_gem_gtt.h */
-static inline void i915_gem_chipset_flush(struct drm_device *dev)
+static inline void i915_gem_chipset_flush(struct drm_i915_private *dev_priv)
 {
-        if (INTEL_INFO(dev)->gen < 6)
+        if (INTEL_GEN(dev_priv) < 6)
                 intel_gtt_chipset_flush();
 }
 
@@ -3404,7 +3631,7 @@ void i915_gem_shrinker_cleanup(struct drm_i915_private *dev_priv);
 /* i915_gem_tiling.c */
 static inline bool i915_gem_object_needs_bit17_swizzle(struct drm_i915_gem_object *obj)
 {
-        struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(obj->base.dev);
 
         return dev_priv->mm.bit_6_swizzle_x == I915_BIT_6_SWIZZLE_9_10_17 &&
                 obj->tiling_mode != I915_TILING_NONE;
@@ -3418,12 +3645,14 @@ int i915_verify_lists(struct drm_device *dev);
 #endif
 
 /* i915_debugfs.c */
-int i915_debugfs_init(struct drm_minor *minor);
-void i915_debugfs_cleanup(struct drm_minor *minor);
 #ifdef CONFIG_DEBUG_FS
+int i915_debugfs_register(struct drm_i915_private *dev_priv);
+void i915_debugfs_unregister(struct drm_i915_private *dev_priv);
 int i915_debugfs_connector_add(struct drm_connector *connector);
 void intel_display_crc_init(struct drm_device *dev);
 #else
+static inline int i915_debugfs_register(struct drm_i915_private *) {return 0;}
+static inline void i915_debugfs_unregister(struct drm_i915_private *) {}
 static inline int i915_debugfs_connector_add(struct drm_connector *connector)
 { return 0; }
 static inline void intel_display_crc_init(struct drm_device *dev) {}
@@ -3442,18 +3671,19 @@ static inline void i915_error_state_buf_release(
 {
         kfree(eb->buf);
 }
-void i915_capture_error_state(struct drm_device *dev, u32 engine_mask,
+void i915_capture_error_state(struct drm_i915_private *dev_priv,
+                              u32 engine_mask,
                               const char *error_msg);
 void i915_error_state_get(struct drm_device *dev,
                           struct i915_error_state_file_priv *error_priv);
 void i915_error_state_put(struct i915_error_state_file_priv *error_priv);
 void i915_destroy_error_state(struct drm_device *dev);
 
-void i915_get_extra_instdone(struct drm_device *dev, uint32_t *instdone);
+void i915_get_extra_instdone(struct drm_i915_private *dev_priv, uint32_t *instdone);
 const char *i915_cache_level_str(struct drm_i915_private *i915, int type);
 
 /* i915_cmd_parser.c */
-int i915_cmd_parser_get_version(void);
+int i915_cmd_parser_get_version(struct drm_i915_private *dev_priv);
 int i915_cmd_parser_init_ring(struct intel_engine_cs *engine);
 void i915_cmd_parser_fini_ring(struct intel_engine_cs *engine);
 bool i915_needs_cmd_parser(struct intel_engine_cs *engine);
@@ -3502,31 +3732,33 @@ bool intel_bios_is_port_hpd_inverted(struct drm_i915_private *dev_priv,
 
 /* intel_opregion.c */
 #ifdef CONFIG_ACPI
-extern int intel_opregion_setup(struct drm_device *dev);
-extern void intel_opregion_init(struct drm_device *dev);
-extern void intel_opregion_fini(struct drm_device *dev);
-extern void intel_opregion_asle_intr(struct drm_device *dev);
+extern int intel_opregion_setup(struct drm_i915_private *dev_priv);
+extern void intel_opregion_register(struct drm_i915_private *dev_priv);
+extern void intel_opregion_unregister(struct drm_i915_private *dev_priv);
+extern void intel_opregion_asle_intr(struct drm_i915_private *dev_priv);
 extern int intel_opregion_notify_encoder(struct intel_encoder *intel_encoder,
                                          bool enable);
-extern int intel_opregion_notify_adapter(struct drm_device *dev,
+extern int intel_opregion_notify_adapter(struct drm_i915_private *dev_priv,
                                          pci_power_t state);
-extern int intel_opregion_get_panel_type(struct drm_device *dev);
+extern int intel_opregion_get_panel_type(struct drm_i915_private *dev_priv);
 #else
-static inline int intel_opregion_setup(struct drm_device *dev) { return 0; }
-static inline void intel_opregion_init(struct drm_device *dev) { return; }
-static inline void intel_opregion_fini(struct drm_device *dev) { return; }
-static inline void intel_opregion_asle_intr(struct drm_device *dev) { return; }
+static inline int intel_opregion_setup(struct drm_i915_private *dev) { return 0; }
+static inline void intel_opregion_register(struct drm_i915_private *dev_priv) { }
+static inline void intel_opregion_unregister(struct drm_i915_private *dev_priv) { }
+static inline void intel_opregion_asle_intr(struct drm_i915_private *dev_priv)
+{
+}
 static inline int
 intel_opregion_notify_encoder(struct intel_encoder *intel_encoder, bool enable)
 {
         return 0;
 }
 static inline int
-intel_opregion_notify_adapter(struct drm_device *dev, pci_power_t state)
+intel_opregion_notify_adapter(struct drm_i915_private *dev, pci_power_t state)
 {
         return 0;
 }
-static inline int intel_opregion_get_panel_type(struct drm_device *dev)
+static inline int intel_opregion_get_panel_type(struct drm_i915_private *dev)
 {
         return -ENODEV;
 }
@@ -3541,36 +3773,45 @@ static inline void intel_register_dsm_handler(void) { return; }
 static inline void intel_unregister_dsm_handler(void) { return; }
 #endif /* CONFIG_ACPI */
 
+/* intel_device_info.c */
+static inline struct intel_device_info *
+mkwrite_device_info(struct drm_i915_private *dev_priv)
+{
+        return (struct intel_device_info *)&dev_priv->info;
+}
+
+void intel_device_info_runtime_init(struct drm_i915_private *dev_priv);
+void intel_device_info_dump(struct drm_i915_private *dev_priv);
+
 /* modesetting */
 extern void intel_modeset_init_hw(struct drm_device *dev);
 extern void intel_modeset_init(struct drm_device *dev);
 extern void intel_modeset_gem_init(struct drm_device *dev);
 extern void intel_modeset_cleanup(struct drm_device *dev);
-extern void intel_connector_unregister(struct intel_connector *);
+extern int intel_connector_register(struct drm_connector *);
+extern void intel_connector_unregister(struct drm_connector *);
 extern int intel_modeset_vga_set_state(struct drm_device *dev, bool state);
 extern void intel_display_resume(struct drm_device *dev);
 extern void i915_redisable_vga(struct drm_device *dev);
 extern void i915_redisable_vga_power_on(struct drm_device *dev);
-extern bool ironlake_set_drps(struct drm_device *dev, u8 val);
+extern bool ironlake_set_drps(struct drm_i915_private *dev_priv, u8 val);
 extern void intel_init_pch_refclk(struct drm_device *dev);
-extern void intel_set_rps(struct drm_device *dev, u8 val);
+extern void intel_set_rps(struct drm_i915_private *dev_priv, u8 val);
 extern void intel_set_memory_cxsr(struct drm_i915_private *dev_priv,
                                   bool enable);
-extern void intel_detect_pch(struct drm_device *dev);
-extern int intel_enable_rc6(const struct drm_device *dev);
 
-extern bool i915_semaphore_is_enabled(struct drm_device *dev);
+extern bool i915_semaphore_is_enabled(struct drm_i915_private *dev_priv);
 int i915_reg_read_ioctl(struct drm_device *dev, void *data,
                         struct drm_file *file);
-int i915_get_reset_stats_ioctl(struct drm_device *dev, void *data,
-                               struct drm_file *file);
 
 /* overlay */
-extern struct intel_overlay_error_state *intel_overlay_capture_error_state(struct drm_device *dev);
+extern struct intel_overlay_error_state *
+intel_overlay_capture_error_state(struct drm_i915_private *dev_priv);
 extern void intel_overlay_print_error_state(struct drm_i915_error_state_buf *e,
                                             struct intel_overlay_error_state *error);
 
-extern struct intel_display_error_state *intel_display_capture_error_state(struct drm_device *dev);
+extern struct intel_display_error_state *
+intel_display_capture_error_state(struct drm_i915_private *dev_priv);
 extern void intel_display_print_error_state(struct drm_i915_error_state_buf *e,
                                             struct drm_device *dev,
                                             struct intel_display_error_state *error);
@@ -3599,6 +3840,24 @@ void intel_sbi_write(struct drm_i915_private *dev_priv, u16 reg, u32 value,
 u32 vlv_flisdsi_read(struct drm_i915_private *dev_priv, u32 reg);
 void vlv_flisdsi_write(struct drm_i915_private *dev_priv, u32 reg, u32 val);
 
+/* intel_dpio_phy.c */
+void chv_set_phy_signal_level(struct intel_encoder *encoder,
+                              u32 deemph_reg_value, u32 margin_reg_value,
+                              bool uniq_trans_scale);
+void chv_data_lane_soft_reset(struct intel_encoder *encoder,
+                              bool reset);
+void chv_phy_pre_pll_enable(struct intel_encoder *encoder);
+void chv_phy_pre_encoder_enable(struct intel_encoder *encoder);
+void chv_phy_release_cl2_override(struct intel_encoder *encoder);
+void chv_phy_post_pll_disable(struct intel_encoder *encoder);
+
+void vlv_set_phy_signal_level(struct intel_encoder *encoder,
+                              u32 demph_reg_value, u32 preemph_reg_value,
+                              u32 uniqtranscale_reg_value, u32 tx3_demph);
+void vlv_phy_pre_pll_enable(struct intel_encoder *encoder);
+void vlv_phy_pre_encoder_enable(struct intel_encoder *encoder);
+void vlv_phy_reset_lanes(struct intel_encoder *encoder);
+
 int intel_gpu_freq(struct drm_i915_private *dev_priv, int val);
 int intel_freq_opcode(struct drm_i915_private *dev_priv, int val);
 
@@ -3672,6 +3931,7 @@ __raw_write(64, q)
  */
 #define I915_READ_FW(reg__) __raw_i915_read32(dev_priv, (reg__))
 #define I915_WRITE_FW(reg__, val__) __raw_i915_write32(dev_priv, (reg__), (val__))
+#define I915_WRITE64_FW(reg__, val__) __raw_i915_write64(dev_priv, (reg__), (val__))
 #define POSTING_READ_FW(reg__) (void)I915_READ_FW(reg__)
 
 /* "Broadcast RGB" property */
@@ -3735,12 +3995,80 @@ wait_remaining_ms_from_jiffies(unsigned long timestamp_jiffies, int to_wait_ms)
                             schedule_timeout_uninterruptible(remaining_jiffies);
         }
 }
-
-static inline void i915_trace_irq_get(struct intel_engine_cs *engine,
-                                      struct drm_i915_gem_request *req)
+static inline bool __i915_request_irq_complete(struct drm_i915_gem_request *req)
 {
-        if (engine->trace_irq_req == NULL && engine->irq_get(engine))
-                i915_gem_request_assign(&engine->trace_irq_req, req);
+        struct intel_engine_cs *engine = req->engine;
+
+        /* Before we do the heavier coherent read of the seqno,
+         * check the value (hopefully) in the CPU cacheline.
+         */
+        if (i915_gem_request_completed(req))
+                return true;
+
+        /* Ensure our read of the seqno is coherent so that we
+         * do not "miss an interrupt" (i.e. if this is the last
+         * request and the seqno write from the GPU is not visible
+         * by the time the interrupt fires, we will see that the
+         * request is incomplete and go back to sleep awaiting
+         * another interrupt that will never come.)
+         *
+         * Strictly, we only need to do this once after an interrupt,
+         * but it is easier and safer to do it every time the waiter
+         * is woken.
+         */
+        if (engine->irq_seqno_barrier &&
+            READ_ONCE(engine->breadcrumbs.irq_seqno_bh) == current &&
+            cmpxchg_relaxed(&engine->breadcrumbs.irq_posted, 1, 0)) {
+                struct task_struct *tsk;
+
+                /* The ordering of irq_posted versus applying the barrier
+                 * is crucial. The clearing of the current irq_posted must
+                 * be visible before we perform the barrier operation,
+                 * such that if a subsequent interrupt arrives, irq_posted
+                 * is reasserted and our task rewoken (which causes us to
+                 * do another __i915_request_irq_complete() immediately
+                 * and reapply the barrier). Conversely, if the clear
+                 * occurs after the barrier, then an interrupt that arrived
+                 * whilst we waited on the barrier would not trigger a
+                 * barrier on the next pass, and the read may not see the
+                 * seqno update.
+                 */
+                engine->irq_seqno_barrier(engine);
+
+                /* If we consume the irq, but we are no longer the bottom-half,
+                 * the real bottom-half may not have serialised their own
+                 * seqno check with the irq-barrier (i.e. may have inspected
+                 * the seqno before we believe it coherent since they see
+                 * irq_posted == false but we are still running).
+                 */
+                rcu_read_lock();
+                tsk = READ_ONCE(engine->breadcrumbs.irq_seqno_bh);
+                if (tsk && tsk != current)
+                        /* Note that if the bottom-half is changed as we
+                         * are sending the wake-up, the new bottom-half will
+                         * be woken by whomever made the change. We only have
+                         * to worry about when we steal the irq-posted for
+                         * ourself.
+                         */
+                        wake_up_process(tsk);
+                rcu_read_unlock();
+
+                if (i915_gem_request_completed(req))
+                        return true;
+        }
+
+        /* We need to check whether any gpu reset happened in between
+         * the request being submitted and now. If a reset has occurred,
+         * the seqno will have been advance past ours and our request
+         * is complete. If we are in the process of handling a reset,
+         * the request is effectively complete as the rendering will
+         * be discarded, but we need to return in order to drop the
+         * struct_mutex.
+         */
+        if (i915_reset_in_progress(&req->i915->gpu_error))
+                return true;
+
+        return false;
 }
 
 #endif
diff --git a/drivers/gpu/drm/i915/i915_gem.c b/drivers/gpu/drm/i915/i915_gem.c
index ed6117a0ee84..11681501d7b1 100644
--- a/drivers/gpu/drm/i915/i915_gem.c
+++ b/drivers/gpu/drm/i915/i915_gem.c
@@ -54,12 +54,33 @@ static bool cpu_cache_is_coherent(struct drm_device *dev,
 
 static bool cpu_write_needs_clflush(struct drm_i915_gem_object *obj)
 {
+        if (obj->base.write_domain == I915_GEM_DOMAIN_CPU)
+                return false;
+
         if (!cpu_cache_is_coherent(obj->base.dev, obj->cache_level))
                 return true;
 
         return obj->pin_display;
 }
 
+static int
+insert_mappable_node(struct drm_i915_private *i915,
+                     struct drm_mm_node *node, u32 size)
+{
+        memset(node, 0, sizeof(*node));
+        return drm_mm_insert_node_in_range_generic(&i915->ggtt.base.mm, node,
+                                                   size, 0, 0, 0,
+                                                   i915->ggtt.mappable_end,
+                                                   DRM_MM_SEARCH_DEFAULT,
+                                                   DRM_MM_CREATE_DEFAULT);
+}
+
+static void
+remove_mappable_node(struct drm_mm_node *node)
+{
+        drm_mm_remove_node(node);
+}
+
 /* some bookkeeping */
 static void i915_gem_info_add_obj(struct drm_i915_private *dev_priv,
                                   size_t size)
@@ -107,7 +128,7 @@ i915_gem_wait_for_error(struct i915_gpu_error *error)
 
 int i915_mutex_lock_interruptible(struct drm_device *dev)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         int ret;
 
         ret = i915_gem_wait_for_error(&dev_priv->gpu_error);
@@ -177,7 +198,7 @@ i915_gem_object_get_pages_phys(struct drm_i915_gem_object *obj)
                 vaddr += PAGE_SIZE;
         }
 
-        i915_gem_chipset_flush(obj->base.dev);
+        i915_gem_chipset_flush(to_i915(obj->base.dev));
 
         st = kmalloc(sizeof(*st), GFP_KERNEL);
         if (st == NULL)
@@ -347,7 +368,7 @@ i915_gem_phys_pwrite(struct drm_i915_gem_object *obj,
         }
 
         drm_clflush_virt_range(vaddr, args->size);
-        i915_gem_chipset_flush(dev);
+        i915_gem_chipset_flush(to_i915(dev));
 
 out:
         intel_fb_obj_flush(obj, false, ORIGIN_CPU);
@@ -356,13 +377,13 @@ out:
 
 void *i915_gem_object_alloc(struct drm_device *dev)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         return kmem_cache_zalloc(dev_priv->objects, GFP_KERNEL);
 }
 
 void i915_gem_object_free(struct drm_i915_gem_object *obj)
 {
-        struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(obj->base.dev);
         kmem_cache_free(dev_priv->objects, obj);
 }
 
@@ -381,9 +402,9 @@ i915_gem_create(struct drm_file *file,
                 return -EINVAL;
 
         /* Allocate the new object */
-        obj = i915_gem_alloc_object(dev, size);
-        if (obj == NULL)
-                return -ENOMEM;
+        obj = i915_gem_object_create(dev, size);
+        if (IS_ERR(obj))
+                return PTR_ERR(obj);
 
         ret = drm_gem_handle_create(file, &obj->base, &handle);
         /* drop reference from allocate - handle holds it now */
@@ -409,6 +430,9 @@ i915_gem_dumb_create(struct drm_file *file,
 
 /**
  * Creates a new mm object and returns a handle to it.
+ * @dev: drm device pointer
+ * @data: ioctl data blob
+ * @file: drm file pointer
  */
 int
 i915_gem_create_ioctl(struct drm_device *dev, void *data,
@@ -484,7 +508,7 @@ int i915_gem_obj_prepare_shmem_read(struct drm_i915_gem_object *obj,
 
         *needs_clflush = 0;
 
-        if (WARN_ON((obj->ops->flags & I915_GEM_OBJECT_HAS_STRUCT_PAGE) == 0))
+        if (WARN_ON(!i915_gem_object_has_struct_page(obj)))
                 return -EINVAL;
 
         if (!(obj->base.read_domains & I915_GEM_DOMAIN_CPU)) {
@@ -585,6 +609,142 @@ shmem_pread_slow(struct page *page, int shmem_page_offset, int page_length,
         return ret ? - EFAULT : 0;
 }
 
+static inline unsigned long
+slow_user_access(struct io_mapping *mapping,
+                 uint64_t page_base, int page_offset,
+                 char __user *user_data,
+                 unsigned long length, bool pwrite)
+{
+        void __iomem *ioaddr;
+        void *vaddr;
+        uint64_t unwritten;
+
+        ioaddr = io_mapping_map_wc(mapping, page_base, PAGE_SIZE);
+        /* We can use the cpu mem copy function because this is X86. */
+        vaddr = (void __force *)ioaddr + page_offset;
+        if (pwrite)
+                unwritten = __copy_from_user(vaddr, user_data, length);
+        else
+                unwritten = __copy_to_user(user_data, vaddr, length);
+
+        io_mapping_unmap(ioaddr);
+        return unwritten;
+}
+
+static int
+i915_gem_gtt_pread(struct drm_device *dev,
+                   struct drm_i915_gem_object *obj, uint64_t size,
+                   uint64_t data_offset, uint64_t data_ptr)
+{
+        struct drm_i915_private *dev_priv = to_i915(dev);
+        struct i915_ggtt *ggtt = &dev_priv->ggtt;
+        struct drm_mm_node node;
+        char __user *user_data;
+        uint64_t remain;
+        uint64_t offset;
+        int ret;
+
+        ret = i915_gem_obj_ggtt_pin(obj, 0, PIN_MAPPABLE);
+        if (ret) {
+                ret = insert_mappable_node(dev_priv, &node, PAGE_SIZE);
+                if (ret)
+                        goto out;
+
+                ret = i915_gem_object_get_pages(obj);
+                if (ret) {
+                        remove_mappable_node(&node);
+                        goto out;
+                }
+
+                i915_gem_object_pin_pages(obj);
+        } else {
+                node.start = i915_gem_obj_ggtt_offset(obj);
+                node.allocated = false;
+                ret = i915_gem_object_put_fence(obj);
+                if (ret)
+                        goto out_unpin;
+        }
+
+        ret = i915_gem_object_set_to_gtt_domain(obj, false);
+        if (ret)
+                goto out_unpin;
+
+        user_data = u64_to_user_ptr(data_ptr);
+        remain = size;
+        offset = data_offset;
+
+        mutex_unlock(&dev->struct_mutex);
+        if (likely(!i915.prefault_disable)) {
+                ret = fault_in_multipages_writeable(user_data, remain);
+                if (ret) {
+                        mutex_lock(&dev->struct_mutex);
+                        goto out_unpin;
+                }
+        }
+
+        while (remain > 0) {
+                /* Operation in this page
+                 *
+                 * page_base = page offset within aperture
+                 * page_offset = offset within page
+                 * page_length = bytes to copy for this page
+                 */
+                u32 page_base = node.start;
+                unsigned page_offset = offset_in_page(offset);
+                unsigned page_length = PAGE_SIZE - page_offset;
+                page_length = remain < page_length ? remain : page_length;
+                if (node.allocated) {
+                        wmb();
+                        ggtt->base.insert_page(&ggtt->base,
+                                               i915_gem_object_get_dma_address(obj, offset >> PAGE_SHIFT),
+                                               node.start,
+                                               I915_CACHE_NONE, 0);
+                        wmb();
+                } else {
+                        page_base += offset & PAGE_MASK;
+                }
+                /* This is a slow read/write as it tries to read from
+                 * and write to user memory which may result into page
+                 * faults, and so we cannot perform this under struct_mutex.
+                 */
+                if (slow_user_access(ggtt->mappable, page_base,
+                                     page_offset, user_data,
+                                     page_length, false)) {
+                        ret = -EFAULT;
+                        break;
+                }
+
+                remain -= page_length;
+                user_data += page_length;
+                offset += page_length;
+        }
+
+        mutex_lock(&dev->struct_mutex);
+        if (ret == 0 && (obj->base.read_domains & I915_GEM_DOMAIN_GTT) == 0) {
+                /* The user has modified the object whilst we tried
+                 * reading from it, and we now have no idea what domain
+                 * the pages should be in. As we have just been touching
+                 * them directly, flush everything back to the GTT
+                 * domain.
+                 */
+                ret = i915_gem_object_set_to_gtt_domain(obj, false);
+        }
+
+out_unpin:
+        if (node.allocated) {
+                wmb();
+                ggtt->base.clear_range(&ggtt->base,
+                                       node.start, node.size,
+                                       true);
+                i915_gem_object_unpin_pages(obj);
+                remove_mappable_node(&node);
+        } else {
+                i915_gem_object_ggtt_unpin(obj);
+        }
+out:
+        return ret;
+}
+
 static int
 i915_gem_shmem_pread(struct drm_device *dev,
                      struct drm_i915_gem_object *obj,
@@ -600,6 +760,9 @@ i915_gem_shmem_pread(struct drm_device *dev,
         int needs_clflush = 0;
         struct sg_page_iter sg_iter;
 
+        if (!i915_gem_object_has_struct_page(obj))
+                return -ENODEV;
+
         user_data = u64_to_user_ptr(args->data_ptr);
         remain = args->size;
 
@@ -672,6 +835,9 @@ out:
 
 /**
  * Reads data from the object referenced by handle.
+ * @dev: drm device pointer
+ * @data: ioctl data blob
+ * @file: drm file pointer
  *
  * On error, the contents of *data are undefined.
  */
@@ -708,18 +874,15 @@ i915_gem_pread_ioctl(struct drm_device *dev, void *data,
                 goto out;
         }
 
-        /* prime objects have no backing filp to GEM pread/pwrite
-         * pages from.
-         */
-        if (!obj->base.filp) {
-                ret = -EINVAL;
-                goto out;
-        }
-
         trace_i915_gem_object_pread(obj, args->offset, args->size);
 
         ret = i915_gem_shmem_pread(dev, obj, args, file);
 
+        /* pread for non shmem backed objects */
+        if (ret == -EFAULT || ret == -ENODEV)
+                ret = i915_gem_gtt_pread(dev, obj, args->size,
+                                        args->offset, args->data_ptr);
+
 out:
         drm_gem_object_unreference(&obj->base);
 unlock:
@@ -753,60 +916,99 @@ fast_user_write(struct io_mapping *mapping,
 /**
  * This is the fast pwrite path, where we copy the data directly from the
  * user into the GTT, uncached.
+ * @dev: drm device pointer
+ * @obj: i915 gem object
+ * @args: pwrite arguments structure
+ * @file: drm file pointer
  */
 static int
-i915_gem_gtt_pwrite_fast(struct drm_device *dev,
+i915_gem_gtt_pwrite_fast(struct drm_i915_private *i915,
                          struct drm_i915_gem_object *obj,
                          struct drm_i915_gem_pwrite *args,
                          struct drm_file *file)
 {
-        struct drm_i915_private *dev_priv = to_i915(dev);
-        struct i915_ggtt *ggtt = &dev_priv->ggtt;
-        ssize_t remain;
-        loff_t offset, page_base;
+        struct i915_ggtt *ggtt = &i915->ggtt;
+        struct drm_device *dev = obj->base.dev;
+        struct drm_mm_node node;
+        uint64_t remain, offset;
         char __user *user_data;
-        int page_offset, page_length, ret;
+        int ret;
+        bool hit_slow_path = false;
+
+        if (obj->tiling_mode != I915_TILING_NONE)
+                return -EFAULT;
 
         ret = i915_gem_obj_ggtt_pin(obj, 0, PIN_MAPPABLE | PIN_NONBLOCK);
-        if (ret)
-                goto out;
+        if (ret) {
+                ret = insert_mappable_node(i915, &node, PAGE_SIZE);
+                if (ret)
+                        goto out;
+
+                ret = i915_gem_object_get_pages(obj);
+                if (ret) {
+                        remove_mappable_node(&node);
+                        goto out;
+                }
+
+                i915_gem_object_pin_pages(obj);
+        } else {
+                node.start = i915_gem_obj_ggtt_offset(obj);
+                node.allocated = false;
+                ret = i915_gem_object_put_fence(obj);
+                if (ret)
+                        goto out_unpin;
+        }
 
         ret = i915_gem_object_set_to_gtt_domain(obj, true);
         if (ret)
                 goto out_unpin;
 
-        ret = i915_gem_object_put_fence(obj);
-        if (ret)
-                goto out_unpin;
+        intel_fb_obj_invalidate(obj, ORIGIN_GTT);
+        obj->dirty = true;
 
         user_data = u64_to_user_ptr(args->data_ptr);
+        offset = args->offset;
         remain = args->size;
-
-        offset = i915_gem_obj_ggtt_offset(obj) + args->offset;
-
-        intel_fb_obj_invalidate(obj, ORIGIN_GTT);
-
-        while (remain > 0) {
+        while (remain) {
                 /* Operation in this page
                  *
                  * page_base = page offset within aperture
                  * page_offset = offset within page
                  * page_length = bytes to copy for this page
                  */
-                page_base = offset & PAGE_MASK;
-                page_offset = offset_in_page(offset);
-                page_length = remain;
-                if ((page_offset + remain) > PAGE_SIZE)
-                        page_length = PAGE_SIZE - page_offset;
-
+                u32 page_base = node.start;
+                unsigned page_offset = offset_in_page(offset);
+                unsigned page_length = PAGE_SIZE - page_offset;
+                page_length = remain < page_length ? remain : page_length;
+                if (node.allocated) {
+                        wmb(); /* flush the write before we modify the GGTT */
+                        ggtt->base.insert_page(&ggtt->base,
+                                               i915_gem_object_get_dma_address(obj, offset >> PAGE_SHIFT),
+                                               node.start, I915_CACHE_NONE, 0);
+                        wmb(); /* flush modifications to the GGTT (insert_page) */
+                } else {
+                        page_base += offset & PAGE_MASK;
+                }
                 /* If we get a fault while copying data, then (presumably) our
                  * source page isn't available.  Return the error and we'll
                  * retry in the slow path.
+                 * If the object is non-shmem backed, we retry again with the
+                 * path that handles page fault.
                  */
                 if (fast_user_write(ggtt->mappable, page_base,
                                     page_offset, user_data, page_length)) {
-                        ret = -EFAULT;
-                        goto out_flush;
+                        hit_slow_path = true;
+                        mutex_unlock(&dev->struct_mutex);
+                        if (slow_user_access(ggtt->mappable,
+                                             page_base,
+                                             page_offset, user_data,
+                                             page_length, true)) {
+                                ret = -EFAULT;
+                                mutex_lock(&dev->struct_mutex);
+                                goto out_flush;
+                        }
+
+                        mutex_lock(&dev->struct_mutex);
                 }
 
                 remain -= page_length;
@@ -815,9 +1017,31 @@ i915_gem_gtt_pwrite_fast(struct drm_device *dev,
         }
 
 out_flush:
+        if (hit_slow_path) {
+                if (ret == 0 &&
+                    (obj->base.read_domains & I915_GEM_DOMAIN_GTT) == 0) {
+                        /* The user has modified the object whilst we tried
+                         * reading from it, and we now have no idea what domain
+                         * the pages should be in. As we have just been touching
+                         * them directly, flush everything back to the GTT
+                         * domain.
+                         */
+                        ret = i915_gem_object_set_to_gtt_domain(obj, false);
+                }
+        }
+
         intel_fb_obj_flush(obj, false, ORIGIN_GTT);
 out_unpin:
-        i915_gem_object_ggtt_unpin(obj);
+        if (node.allocated) {
+                wmb();
+                ggtt->base.clear_range(&ggtt->base,
+                                       node.start, node.size,
+                                       true);
+                i915_gem_object_unpin_pages(obj);
+                remove_mappable_node(&node);
+        } else {
+                i915_gem_object_ggtt_unpin(obj);
+        }
 out:
         return ret;
 }
@@ -1006,7 +1230,7 @@ out:
         }
 
         if (needs_clflush_after)
-                i915_gem_chipset_flush(dev);
+                i915_gem_chipset_flush(to_i915(dev));
         else
                 obj->cache_dirty = true;
 
@@ -1016,6 +1240,9 @@ out:
 
 /**
  * Writes data to the object referenced by handle.
+ * @dev: drm device
+ * @data: ioctl data blob
+ * @file: drm file
  *
  * On error, the contents of the buffer that were to be modified are undefined.
  */
@@ -1023,7 +1250,7 @@ int
 i915_gem_pwrite_ioctl(struct drm_device *dev, void *data,
                       struct drm_file *file)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         struct drm_i915_gem_pwrite *args = data;
         struct drm_i915_gem_object *obj;
         int ret;
@@ -1062,14 +1289,6 @@ i915_gem_pwrite_ioctl(struct drm_device *dev, void *data,
                 goto out;
         }
 
-        /* prime objects have no backing filp to GEM pread/pwrite
-         * pages from.
-         */
-        if (!obj->base.filp) {
-                ret = -EINVAL;
-                goto out;
-        }
-
         trace_i915_gem_object_pwrite(obj, args->offset, args->size);
 
         ret = -EFAULT;
@@ -1079,20 +1298,21 @@ i915_gem_pwrite_ioctl(struct drm_device *dev, void *data,
          * pread/pwrite currently are reading and writing from the CPU
          * perspective, requiring manual detiling by the client.
          */
-        if (obj->tiling_mode == I915_TILING_NONE &&
-            obj->base.write_domain != I915_GEM_DOMAIN_CPU &&
+        if (!i915_gem_object_has_struct_page(obj) ||
             cpu_write_needs_clflush(obj)) {
-                ret = i915_gem_gtt_pwrite_fast(dev, obj, args, file);
+                ret = i915_gem_gtt_pwrite_fast(dev_priv, obj, args, file);
                 /* Note that the gtt paths might fail with non-page-backed user
                  * pointers (e.g. gtt mappings when moving data between
                  * textures). Fallback to the shmem path in that case. */
         }
 
-        if (ret == -EFAULT || ret == -ENOSPC) {
+        if (ret == -EFAULT) {
                 if (obj->phys_handle)
                         ret = i915_gem_phys_pwrite(obj, args, file);
-                else
+                else if (i915_gem_object_has_struct_page(obj))
                         ret = i915_gem_shmem_pwrite(dev, obj, args, file);
+                else
+                        ret = -ENODEV;
         }
 
 out:
@@ -1123,17 +1343,6 @@ i915_gem_check_wedge(unsigned reset_counter, bool interruptible)
         return 0;
 }
 
-static void fake_irq(unsigned long data)
-{
-        wake_up_process((struct task_struct *)data);
-}
-
-static bool missed_irq(struct drm_i915_private *dev_priv,
-                       struct intel_engine_cs *engine)
-{
-        return test_bit(engine->id, &dev_priv->gpu_error.missed_irq_rings);
-}
-
 static unsigned long local_clock_us(unsigned *cpu)
 {
         unsigned long t;
@@ -1166,9 +1375,9 @@ static bool busywait_stop(unsigned long timeout, unsigned cpu)
         return this_cpu != cpu;
 }
 
-static int __i915_spin_request(struct drm_i915_gem_request *req, int state)
+bool __i915_spin_request(const struct drm_i915_gem_request *req,
+                         int state, unsigned long timeout_us)
 {
-        unsigned long timeout;
         unsigned cpu;
 
         /* When waiting for high frequency requests, e.g. during synchronous
@@ -1181,31 +1390,21 @@ static int __i915_spin_request(struct drm_i915_gem_request *req, int state)
          * takes to sleep on a request, on the order of a microsecond.
          */
 
-        if (req->engine->irq_refcount)
-                return -EBUSY;
-
-        /* Only spin if we know the GPU is processing this request */
-        if (!i915_gem_request_started(req, true))
-                return -EAGAIN;
-
-        timeout = local_clock_us(&cpu) + 5;
-        while (!need_resched()) {
-                if (i915_gem_request_completed(req, true))
-                        return 0;
+        timeout_us += local_clock_us(&cpu);
+        do {
+                if (i915_gem_request_completed(req))
+                        return true;
 
                 if (signal_pending_state(state, current))
                         break;
 
-                if (busywait_stop(timeout, cpu))
+                if (busywait_stop(timeout_us, cpu))
                         break;
 
                 cpu_relax_lowlatency();
-        }
+        } while (!need_resched());
 
-        if (i915_gem_request_completed(req, false))
-                return 0;
-
-        return -EAGAIN;
+        return false;
 }
 
 /**
@@ -1213,6 +1412,7 @@ static int __i915_spin_request(struct drm_i915_gem_request *req, int state)
  * @req: duh!
  * @interruptible: do an interruptible wait (normally yes)
  * @timeout: in - how long to wait (NULL forever); out - how much time remaining
+ * @rps: RPS client
  *
  * Note: It is of utmost importance that the passed in seqno and reset_counter
  * values have been read by the caller in an smp safe manner. Where read-side
@@ -1229,26 +1429,22 @@ int __i915_wait_request(struct drm_i915_gem_request *req,
                         s64 *timeout,
                         struct intel_rps_client *rps)
 {
-        struct intel_engine_cs *engine = i915_gem_request_get_engine(req);
-        struct drm_device *dev = engine->dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
-        const bool irq_test_in_progress =
-                ACCESS_ONCE(dev_priv->gpu_error.test_irq_rings) & intel_engine_flag(engine);
         int state = interruptible ? TASK_INTERRUPTIBLE : TASK_UNINTERRUPTIBLE;
-        DEFINE_WAIT(wait);
-        unsigned long timeout_expire;
+        DEFINE_WAIT(reset);
+        struct intel_wait wait;
+        unsigned long timeout_remain;
         s64 before = 0; /* Only to silence a compiler warning. */
-        int ret;
+        int ret = 0;
 
-        WARN(!intel_irqs_enabled(dev_priv), "IRQs disabled");
+        might_sleep();
 
         if (list_empty(&req->list))
                 return 0;
 
-        if (i915_gem_request_completed(req, true))
+        if (i915_gem_request_completed(req))
                 return 0;
 
-        timeout_expire = 0;
+        timeout_remain = MAX_SCHEDULE_TIMEOUT;
         if (timeout) {
                 if (WARN_ON(*timeout < 0))
                         return -EINVAL;
@@ -1256,7 +1452,7 @@ int __i915_wait_request(struct drm_i915_gem_request *req,
                 if (*timeout == 0)
                         return -ETIME;
 
-                timeout_expire = jiffies + nsecs_to_jiffies_timeout(*timeout);
+                timeout_remain = nsecs_to_jiffies_timeout(*timeout);
 
                 /*
                  * Record current time in case interrupted by signal, or wedged.
@@ -1264,75 +1460,76 @@ int __i915_wait_request(struct drm_i915_gem_request *req,
                 before = ktime_get_raw_ns();
         }
 
-        if (INTEL_INFO(dev_priv)->gen >= 6)
-                gen6_rps_boost(dev_priv, rps, req->emitted_jiffies);
-
         trace_i915_gem_request_wait_begin(req);
 
-        /* Optimistic spin for the next jiffie before touching IRQs */
-        ret = __i915_spin_request(req, state);
-        if (ret == 0)
-                goto out;
-
-        if (!irq_test_in_progress && WARN_ON(!engine->irq_get(engine))) {
-                ret = -ENODEV;
-                goto out;
-        }
+        /* This client is about to stall waiting for the GPU. In many cases
+         * this is undesirable and limits the throughput of the system, as
+         * many clients cannot continue processing user input/output whilst
+         * blocked. RPS autotuning may take tens of milliseconds to respond
+         * to the GPU load and thus incurs additional latency for the client.
+         * We can circumvent that by promoting the GPU frequency to maximum
+         * before we wait. This makes the GPU throttle up much more quickly
+         * (good for benchmarks and user experience, e.g. window animations),
+         * but at a cost of spending more power processing the workload
+         * (bad for battery). Not all clients even want their results
+         * immediately and for them we should just let the GPU select its own
+         * frequency to maximise efficiency. To prevent a single client from
+         * forcing the clocks too high for the whole system, we only allow
+         * each client to waitboost once in a busy period.
+         */
+        if (INTEL_INFO(req->i915)->gen >= 6)
+                gen6_rps_boost(req->i915, rps, req->emitted_jiffies);
 
-        for (;;) {
-                struct timer_list timer;
+        /* Optimistic spin for the next ~jiffie before touching IRQs */
+        if (i915_spin_request(req, state, 5))
+                goto complete;
 
-                prepare_to_wait(&engine->irq_queue, &wait, state);
+        set_current_state(state);
+        add_wait_queue(&req->i915->gpu_error.wait_queue, &reset);
 
-                /* We need to check whether any gpu reset happened in between
-                 * the request being submitted and now. If a reset has occurred,
-                 * the request is effectively complete (we either are in the
-                 * process of or have discarded the rendering and completely
-                 * reset the GPU. The results of the request are lost and we
-                 * are free to continue on with the original operation.
+        intel_wait_init(&wait, req->seqno);
+        if (intel_engine_add_wait(req->engine, &wait))
+                /* In order to check that we haven't missed the interrupt
+                 * as we enabled it, we need to kick ourselves to do a
+                 * coherent check on the seqno before we sleep.
                  */
-                if (req->reset_counter != i915_reset_counter(&dev_priv->gpu_error)) {
-                        ret = 0;
-                        break;
-                }
-
-                if (i915_gem_request_completed(req, false)) {
-                        ret = 0;
-                        break;
-                }
+                goto wakeup;
 
+        for (;;) {
                 if (signal_pending_state(state, current)) {
                         ret = -ERESTARTSYS;
                         break;
                 }
 
-                if (timeout && time_after_eq(jiffies, timeout_expire)) {
+                timeout_remain = io_schedule_timeout(timeout_remain);
+                if (timeout_remain == 0) {
                         ret = -ETIME;
                         break;
                 }
 
-                timer.function = NULL;
-                if (timeout || missed_irq(dev_priv, engine)) {
-                        unsigned long expire;
+                if (intel_wait_complete(&wait))
+                        break;
 
-                        setup_timer_on_stack(&timer, fake_irq, (unsigned long)current);
-                        expire = missed_irq(dev_priv, engine) ? jiffies + 1 : timeout_expire;
-                        mod_timer(&timer, expire);
-                }
+                set_current_state(state);
 
-                io_schedule();
+wakeup:
+                /* Carefully check if the request is complete, giving time
+                 * for the seqno to be visible following the interrupt.
+                 * We also have to check in case we are kicked by the GPU
+                 * reset in order to drop the struct_mutex.
+                 */
+                if (__i915_request_irq_complete(req))
+                        break;
 
-                if (timer.function) {
-                        del_singleshot_timer_sync(&timer);
-                        destroy_timer_on_stack(&timer);
-                }
+                /* Only spin if we know the GPU is processing this request */
+                if (i915_spin_request(req, state, 2))
+                        break;
         }
-        if (!irq_test_in_progress)
-                engine->irq_put(engine);
+        remove_wait_queue(&req->i915->gpu_error.wait_queue, &reset);
 
-        finish_wait(&engine->irq_queue, &wait);
-
-out:
+        intel_engine_remove_wait(req->engine, &wait);
+        __set_current_state(TASK_RUNNING);
+complete:
         trace_i915_gem_request_wait_end(req);
 
         if (timeout) {
@@ -1351,6 +1548,22 @@ out:
                         *timeout = 0;
         }
 
+        if (rps && req->seqno == req->engine->last_submitted_seqno) {
+                /* The GPU is now idle and this client has stalled.
+                 * Since no other client has submitted a request in the
+                 * meantime, assume that this client is the only one
+                 * supplying work to the GPU but is unable to keep that
+                 * work supplied because it is waiting. Since the GPU is
+                 * then never kept fully busy, RPS autoclocking will
+                 * keep the clocks relatively low, causing further delays.
+                 * Compensate by giving the synchronous client credit for
+                 * a waitboost next time.
+                 */
+                spin_lock(&req->i915->rps.client_lock);
+                list_del_init(&rps->link);
+                spin_unlock(&req->i915->rps.client_lock);
+        }
+
         return ret;
 }
 
@@ -1413,6 +1626,13 @@ static void i915_gem_request_retire(struct drm_i915_gem_request *request)
         list_del_init(&request->list);
         i915_gem_request_remove_from_client(request);
 
+        if (request->previous_context) {
+                if (i915.enable_execlists)
+                        intel_lr_context_unpin(request->previous_context,
+                                               request->engine);
+        }
+
+        i915_gem_context_unreference(request->ctx);
         i915_gem_request_unreference(request);
 }
 
@@ -1422,7 +1642,7 @@ __i915_gem_request_retire__upto(struct drm_i915_gem_request *req)
         struct intel_engine_cs *engine = req->engine;
         struct drm_i915_gem_request *tmp;
 
-        lockdep_assert_held(&engine->dev->struct_mutex);
+        lockdep_assert_held(&engine->i915->drm.struct_mutex);
 
         if (list_empty(&req->list))
                 return;
@@ -1440,6 +1660,7 @@ __i915_gem_request_retire__upto(struct drm_i915_gem_request *req)
 /**
  * Waits for a request to be signaled, and cleans up the
  * request and object lists appropriately for that event.
+ * @req: request to wait on
  */
 int
 i915_wait_request(struct drm_i915_gem_request *req)
@@ -1450,14 +1671,14 @@ i915_wait_request(struct drm_i915_gem_request *req)
 
         interruptible = dev_priv->mm.interruptible;
 
-        BUG_ON(!mutex_is_locked(&dev_priv->dev->struct_mutex));
+        BUG_ON(!mutex_is_locked(&dev_priv->drm.struct_mutex));
 
         ret = __i915_wait_request(req, interruptible, NULL, NULL);
         if (ret)
                 return ret;
 
         /* If the GPU hung, we want to keep the requests to find the guilty. */
-        if (req->reset_counter == i915_reset_counter(&dev_priv->gpu_error))
+        if (!i915_reset_in_progress(&dev_priv->gpu_error))
                 __i915_gem_request_retire__upto(req);
 
         return 0;
@@ -1466,6 +1687,8 @@ i915_wait_request(struct drm_i915_gem_request *req)
 /**
  * Ensures that all rendering to the object has completed and the object is
  * safe to unbind from the GTT or access from the CPU.
+ * @obj: i915 gem object
+ * @readonly: waiting for read access or write
  */
 int
 i915_gem_object_wait_rendering(struct drm_i915_gem_object *obj,
@@ -1516,7 +1739,7 @@ i915_gem_object_retire_request(struct drm_i915_gem_object *obj,
         else if (obj->last_write_req == req)
                 i915_gem_object_retire__write(obj);
 
-        if (req->reset_counter == i915_reset_counter(&req->i915->gpu_error))
+        if (!i915_reset_in_progress(&req->i915->gpu_error))
                 __i915_gem_request_retire__upto(req);
 }
 
@@ -1529,7 +1752,7 @@ i915_gem_object_wait_rendering__nonblocking(struct drm_i915_gem_object *obj,
                                             bool readonly)
 {
         struct drm_device *dev = obj->base.dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         struct drm_i915_gem_request *requests[I915_NUM_ENGINES];
         int ret, i, n = 0;
 
@@ -1580,9 +1803,19 @@ static struct intel_rps_client *to_rps_client(struct drm_file *file)
         return &fpriv->rps;
 }
 
+static enum fb_op_origin
+write_origin(struct drm_i915_gem_object *obj, unsigned domain)
+{
+        return domain == I915_GEM_DOMAIN_GTT && !obj->has_wc_mmap ?
+               ORIGIN_GTT : ORIGIN_CPU;
+}
+
 /**
  * Called when user space prepares to use an object with the CPU, either
  * through the mmap ioctl's mapping or a GTT mapping.
+ * @dev: drm device
+ * @data: ioctl data blob
+ * @file: drm file
  */
 int
 i915_gem_set_domain_ioctl(struct drm_device *dev, void *data,
@@ -1633,9 +1866,7 @@ i915_gem_set_domain_ioctl(struct drm_device *dev, void *data,
                 ret = i915_gem_object_set_to_cpu_domain(obj, write_domain != 0);
 
         if (write_domain != 0)
-                intel_fb_obj_invalidate(obj,
-                                        write_domain == I915_GEM_DOMAIN_GTT ?
-                                        ORIGIN_GTT : ORIGIN_CPU);
+                intel_fb_obj_invalidate(obj, write_origin(obj, write_domain));
 
 unref:
         drm_gem_object_unreference(&obj->base);
@@ -1646,6 +1877,9 @@ unlock:
 
 /**
  * Called when user space has done writes to this buffer
+ * @dev: drm device
+ * @data: ioctl data blob
+ * @file: drm file
  */
 int
 i915_gem_sw_finish_ioctl(struct drm_device *dev, void *data,
@@ -1676,8 +1910,11 @@ unlock:
 }
 
 /**
- * Maps the contents of an object, returning the address it is mapped
- * into.
+ * i915_gem_mmap_ioctl - Maps the contents of an object, returning the address
+ *                       it is mapped to.
+ * @dev: drm device
+ * @data: ioctl data blob
+ * @file: drm file
  *
  * While the mapping holds a reference on the contents of the object, it doesn't
  * imply a ref on the object itself.
@@ -1736,6 +1973,9 @@ i915_gem_mmap_ioctl(struct drm_device *dev, void *data,
                 else
                         addr = -ENOMEM;
                 up_write(&mm->mmap_sem);
+
+                /* This may race, but that's ok, it only gets set */
+                WRITE_ONCE(to_intel_bo(obj)->has_wc_mmap, true);
         }
         drm_gem_object_unreference_unlocked(obj);
         if (IS_ERR((void *)addr))
@@ -1982,7 +2222,7 @@ i915_gem_get_gtt_size(struct drm_device *dev, uint32_t size, int tiling_mode)
                 return size;
 
         /* Previous chips need a power-of-two fence region when tiling */
-        if (INTEL_INFO(dev)->gen == 3)
+        if (IS_GEN3(dev))
                 gtt_size = 1024*1024;
         else
                 gtt_size = 512*1024;
@@ -1995,7 +2235,10 @@ i915_gem_get_gtt_size(struct drm_device *dev, uint32_t size, int tiling_mode)
 
 /**
  * i915_gem_get_gtt_alignment - return required GTT alignment for an object
- * @obj: object to check
+ * @dev: drm device
+ * @size: object size
+ * @tiling_mode: tiling mode
+ * @fenced: is fenced alignemned required or not
  *
  * Return the required GTT alignment for an object, taking into account
  * potential fence register mapping.
@@ -2021,7 +2264,7 @@ i915_gem_get_gtt_alignment(struct drm_device *dev, uint32_t size,
 
 static int i915_gem_object_create_mmap_offset(struct drm_i915_gem_object *obj)
 {
-        struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(obj->base.dev);
         int ret;
 
         dev_priv->mm.shrinker_no_lock_stealing = true;
@@ -2162,7 +2405,8 @@ i915_gem_object_invalidate(struct drm_i915_gem_object *obj)
 static void
 i915_gem_object_put_pages_gtt(struct drm_i915_gem_object *obj)
 {
-        struct sg_page_iter sg_iter;
+        struct sgt_iter sgt_iter;
+        struct page *page;
         int ret;
 
         BUG_ON(obj->madv == __I915_MADV_PURGED);
@@ -2184,9 +2428,7 @@ i915_gem_object_put_pages_gtt(struct drm_i915_gem_object *obj)
         if (obj->madv == I915_MADV_DONTNEED)
                 obj->dirty = 0;
 
-        for_each_sg_page(obj->pages->sgl, &sg_iter, obj->pages->nents, 0) {
-                struct page *page = sg_page_iter_page(&sg_iter);
-
+        for_each_sgt_page(page, sgt_iter, obj->pages) {
                 if (obj->dirty)
                         set_page_dirty(page);
 
@@ -2238,12 +2480,12 @@ i915_gem_object_put_pages(struct drm_i915_gem_object *obj)
 static int
 i915_gem_object_get_pages_gtt(struct drm_i915_gem_object *obj)
 {
-        struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(obj->base.dev);
         int page_count, i;
         struct address_space *mapping;
         struct sg_table *st;
         struct scatterlist *sg;
-        struct sg_page_iter sg_iter;
+        struct sgt_iter sgt_iter;
         struct page *page;
         unsigned long last_pfn = 0;     /* suppress gcc warning */
         int ret;
@@ -2340,8 +2582,8 @@ i915_gem_object_get_pages_gtt(struct drm_i915_gem_object *obj)
 
 err_pages:
         sg_mark_end(sg);
-        for_each_sg_page(st->sgl, &sg_iter, st->nents, 0)
-                put_page(sg_page_iter_page(&sg_iter));
+        for_each_sgt_page(page, sgt_iter, st)
+                put_page(page);
         sg_free_table(st);
         kfree(st);
 
@@ -2369,7 +2611,7 @@ err_pages:
 int
 i915_gem_object_get_pages(struct drm_i915_gem_object *obj)
 {
-        struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(obj->base.dev);
         const struct drm_i915_gem_object_ops *ops = obj->ops;
         int ret;
 
@@ -2395,6 +2637,44 @@ i915_gem_object_get_pages(struct drm_i915_gem_object *obj)
         return 0;
 }
 
+/* The 'mapping' part of i915_gem_object_pin_map() below */
+static void *i915_gem_object_map(const struct drm_i915_gem_object *obj)
+{
+        unsigned long n_pages = obj->base.size >> PAGE_SHIFT;
+        struct sg_table *sgt = obj->pages;
+        struct sgt_iter sgt_iter;
+        struct page *page;
+        struct page *stack_pages[32];
+        struct page **pages = stack_pages;
+        unsigned long i = 0;
+        void *addr;
+
+        /* A single page can always be kmapped */
+        if (n_pages == 1)
+                return kmap(sg_page(sgt->sgl));
+
+        if (n_pages > ARRAY_SIZE(stack_pages)) {
+                /* Too big for stack -- allocate temporary array instead */
+                pages = drm_malloc_gfp(n_pages, sizeof(*pages), GFP_TEMPORARY);
+                if (!pages)
+                        return NULL;
+        }
+
+        for_each_sgt_page(page, sgt_iter, sgt)
+                pages[i++] = page;
+
+        /* Check that we have the expected number of pages */
+        GEM_BUG_ON(i != n_pages);
+
+        addr = vmap(pages, n_pages, 0, PAGE_KERNEL);
+
+        if (pages != stack_pages)
+                drm_free_large(pages);
+
+        return addr;
+}
+
+/* get, pin, and map the pages of the object into kernel space */
 void *i915_gem_object_pin_map(struct drm_i915_gem_object *obj)
 {
         int ret;
@@ -2407,29 +2687,9 @@ void *i915_gem_object_pin_map(struct drm_i915_gem_object *obj)
 
         i915_gem_object_pin_pages(obj);
 
-        if (obj->mapping == NULL) {
-                struct page **pages;
-
-                pages = NULL;
-                if (obj->base.size == PAGE_SIZE)
-                        obj->mapping = kmap(sg_page(obj->pages->sgl));
-                else
-                        pages = drm_malloc_gfp(obj->base.size >> PAGE_SHIFT,
-                                               sizeof(*pages),
-                                               GFP_TEMPORARY);
-                if (pages != NULL) {
-                        struct sg_page_iter sg_iter;
-                        int n;
-
-                        n = 0;
-                        for_each_sg_page(obj->pages->sgl, &sg_iter,
-                                         obj->pages->nents, 0)
-                                pages[n++] = sg_page_iter_page(&sg_iter);
-
-                        obj->mapping = vmap(pages, n, 0, PAGE_KERNEL);
-                        drm_free_large(pages);
-                }
-                if (obj->mapping == NULL) {
+        if (!obj->mapping) {
+                obj->mapping = i915_gem_object_map(obj);
+                if (!obj->mapping) {
                         i915_gem_object_unpin_pages(obj);
                         return ERR_PTR(-ENOMEM);
                 }
@@ -2502,9 +2762,8 @@ i915_gem_object_retire__read(struct drm_i915_gem_object *obj, int ring)
 }
 
 static int
-i915_gem_init_seqno(struct drm_device *dev, u32 seqno)
+i915_gem_init_seqno(struct drm_i915_private *dev_priv, u32 seqno)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
         struct intel_engine_cs *engine;
         int ret;
 
@@ -2514,7 +2773,14 @@ i915_gem_init_seqno(struct drm_device *dev, u32 seqno)
                 if (ret)
                         return ret;
         }
-        i915_gem_retire_requests(dev);
+        i915_gem_retire_requests(dev_priv);
+
+        /* If the seqno wraps around, we need to clear the breadcrumb rbtree */
+        if (!i915_seqno_passed(seqno, dev_priv->next_seqno)) {
+                while (intel_kick_waiters(dev_priv) ||
+                       intel_kick_signalers(dev_priv))
+                        yield();
+        }
 
         /* Finally reset hw state */
         for_each_engine(engine, dev_priv)
@@ -2525,7 +2791,7 @@ i915_gem_init_seqno(struct drm_device *dev, u32 seqno)
 
 int i915_gem_set_seqno(struct drm_device *dev, u32 seqno)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         int ret;
 
         if (seqno == 0)
@@ -2534,7 +2800,7 @@ int i915_gem_set_seqno(struct drm_device *dev, u32 seqno)
         /* HWS page needs to be set less than what we
          * will inject to ring
          */
-        ret = i915_gem_init_seqno(dev, seqno - 1);
+        ret = i915_gem_init_seqno(dev_priv, seqno - 1);
         if (ret)
                 return ret;
 
@@ -2550,13 +2816,11 @@ int i915_gem_set_seqno(struct drm_device *dev, u32 seqno)
 }
 
 int
-i915_gem_get_seqno(struct drm_device *dev, u32 *seqno)
+i915_gem_get_seqno(struct drm_i915_private *dev_priv, u32 *seqno)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
-
         /* reserve 0 for non-seqno */
         if (dev_priv->next_seqno == 0) {
-                int ret = i915_gem_init_seqno(dev, 0);
+                int ret = i915_gem_init_seqno(dev_priv, 0);
                 if (ret)
                         return ret;
 
@@ -2567,6 +2831,26 @@ i915_gem_get_seqno(struct drm_device *dev, u32 *seqno)
         return 0;
 }
 
+static void i915_gem_mark_busy(const struct intel_engine_cs *engine)
+{
+        struct drm_i915_private *dev_priv = engine->i915;
+
+        dev_priv->gt.active_engines |= intel_engine_flag(engine);
+        if (dev_priv->gt.awake)
+                return;
+
+        intel_runtime_pm_get_noresume(dev_priv);
+        dev_priv->gt.awake = true;
+
+        i915_update_gfx_val(dev_priv);
+        if (INTEL_GEN(dev_priv) >= 6)
+                gen6_rps_busy(dev_priv);
+
+        queue_delayed_work(dev_priv->wq,
+                           &dev_priv->gt.retire_work,
+                           round_jiffies_up_relative(HZ));
+}
+
 /*
  * NB: This function is not allowed to fail. Doing so would mean the the
  * request is not being tracked for completion but the work itself is
@@ -2577,16 +2861,15 @@ void __i915_add_request(struct drm_i915_gem_request *request,
                         bool flush_caches)
 {
         struct intel_engine_cs *engine;
-        struct drm_i915_private *dev_priv;
         struct intel_ringbuffer *ringbuf;
         u32 request_start;
+        u32 reserved_tail;
         int ret;
 
         if (WARN_ON(request == NULL))
                 return;
 
         engine = request->engine;
-        dev_priv = request->i915;
         ringbuf = request->ringbuf;
 
         /*
@@ -2594,9 +2877,10 @@ void __i915_add_request(struct drm_i915_gem_request *request,
          * should already have been reserved in the ring buffer. Let the ring
          * know that it is time to use that space up.
          */
-        intel_ring_reserved_space_use(ringbuf);
-
         request_start = intel_ring_get_tail(ringbuf);
+        reserved_tail = request->reserved_space;
+        request->reserved_space = 0;
+
         /*
          * Emit any outstanding flushes - execbuf can fail to emit the flush
          * after having emitted the batchbuffer command. Hence we need to fix
@@ -2651,56 +2935,42 @@ void __i915_add_request(struct drm_i915_gem_request *request,
         }
         /* Not allowed to fail! */
         WARN(ret, "emit|add_request failed: %d!\n", ret);
-
-        i915_queue_hangcheck(engine->dev);
-
-        queue_delayed_work(dev_priv->wq,
-                           &dev_priv->mm.retire_work,
-                           round_jiffies_up_relative(HZ));
-        intel_mark_busy(dev_priv->dev);
-
         /* Sanity check that the reserved size was large enough. */
-        intel_ring_reserved_space_end(ringbuf);
+        ret = intel_ring_get_tail(ringbuf) - request_start;
+        if (ret < 0)
+                ret += ringbuf->size;
+        WARN_ONCE(ret > reserved_tail,
+                  "Not enough space reserved (%d bytes) "
+                  "for adding the request (%d bytes)\n",
+                  reserved_tail, ret);
+
+        i915_gem_mark_busy(engine);
 }
 
-static bool i915_context_is_banned(struct drm_i915_private *dev_priv,
-                                   const struct intel_context *ctx)
+static bool i915_context_is_banned(const struct i915_gem_context *ctx)
 {
         unsigned long elapsed;
 
-        elapsed = get_seconds() - ctx->hang_stats.guilty_ts;
-
         if (ctx->hang_stats.banned)
                 return true;
 
+        elapsed = get_seconds() - ctx->hang_stats.guilty_ts;
         if (ctx->hang_stats.ban_period_seconds &&
             elapsed <= ctx->hang_stats.ban_period_seconds) {
-                if (!i915_gem_context_is_default(ctx)) {
-                        DRM_DEBUG("context hanging too fast, banning!\n");
-                        return true;
-                } else if (i915_stop_ring_allow_ban(dev_priv)) {
-                        if (i915_stop_ring_allow_warn(dev_priv))
-                                DRM_ERROR("gpu hanging too fast, banning!\n");
-                        return true;
-                }
+                DRM_DEBUG("context hanging too fast, banning!\n");
+                return true;
         }
 
         return false;
 }
 
-static void i915_set_reset_status(struct drm_i915_private *dev_priv,
-                                  struct intel_context *ctx,
+static void i915_set_reset_status(struct i915_gem_context *ctx,
                                   const bool guilty)
 {
-        struct i915_ctx_hang_stats *hs;
-
-        if (WARN_ON(!ctx))
-                return;
-
-        hs = &ctx->hang_stats;
+        struct i915_ctx_hang_stats *hs = &ctx->hang_stats;
 
         if (guilty) {
-                hs->banned = i915_context_is_banned(dev_priv, ctx);
+                hs->banned = i915_context_is_banned(ctx);
                 hs->batch_active++;
                 hs->guilty_ts = get_seconds();
         } else {
@@ -2712,27 +2982,15 @@ void i915_gem_request_free(struct kref *req_ref)
 {
         struct drm_i915_gem_request *req = container_of(req_ref,
                                                  typeof(*req), ref);
-        struct intel_context *ctx = req->ctx;
-
-        if (req->file_priv)
-                i915_gem_request_remove_from_client(req);
-
-        if (ctx) {
-                if (i915.enable_execlists && ctx != req->i915->kernel_context)
-                        intel_lr_context_unpin(ctx, req->engine);
-
-                i915_gem_context_unreference(ctx);
-        }
-
         kmem_cache_free(req->i915->requests, req);
 }
 
 static inline int
 __i915_gem_request_alloc(struct intel_engine_cs *engine,
-                         struct intel_context *ctx,
+                         struct i915_gem_context *ctx,
                          struct drm_i915_gem_request **req_out)
 {
-        struct drm_i915_private *dev_priv = to_i915(engine->dev);
+        struct drm_i915_private *dev_priv = engine->i915;
         unsigned reset_counter = i915_reset_counter(&dev_priv->gpu_error);
         struct drm_i915_gem_request *req;
         int ret;
@@ -2754,26 +3012,16 @@ __i915_gem_request_alloc(struct intel_engine_cs *engine,
         if (req == NULL)
                 return -ENOMEM;
 
-        ret = i915_gem_get_seqno(engine->dev, &req->seqno);
+        ret = i915_gem_get_seqno(engine->i915, &req->seqno);
         if (ret)
                 goto err;
 
         kref_init(&req->ref);
         req->i915 = dev_priv;
         req->engine = engine;
-        req->reset_counter = reset_counter;
         req->ctx  = ctx;
         i915_gem_context_reference(req->ctx);
 
-        if (i915.enable_execlists)
-                ret = intel_logical_ring_alloc_request_extras(req);
-        else
-                ret = intel_ring_alloc_request_extras(req);
-        if (ret) {
-                i915_gem_context_unreference(req->ctx);
-                goto err;
-        }
-
         /*
          * Reserve space in the ring buffer for all the commands required to
          * eventually emit this request. This is to guarantee that the
@@ -2781,24 +3029,20 @@ __i915_gem_request_alloc(struct intel_engine_cs *engine,
          * to be redone if the request is not actually submitted straight
          * away, e.g. because a GPU scheduler has deferred it.
          */
+        req->reserved_space = MIN_SPACE_FOR_ADD_REQUEST;
+
         if (i915.enable_execlists)
-                ret = intel_logical_ring_reserve_space(req);
+                ret = intel_logical_ring_alloc_request_extras(req);
         else
-                ret = intel_ring_reserve_space(req);
-        if (ret) {
-                /*
-                 * At this point, the request is fully allocated even if not
-                 * fully prepared. Thus it can be cleaned up using the proper
-                 * free code.
-                 */
-                intel_ring_reserved_space_cancel(req->ringbuf);
-                i915_gem_request_unreference(req);
-                return ret;
-        }
+                ret = intel_ring_alloc_request_extras(req);
+        if (ret)
+                goto err_ctx;
 
         *req_out = req;
         return 0;
 
+err_ctx:
+        i915_gem_context_unreference(ctx);
 err:
         kmem_cache_free(dev_priv->requests, req);
         return ret;
@@ -2818,13 +3062,13 @@ err:
  */
 struct drm_i915_gem_request *
 i915_gem_request_alloc(struct intel_engine_cs *engine,
-                       struct intel_context *ctx)
+                       struct i915_gem_context *ctx)
 {
         struct drm_i915_gem_request *req;
         int err;
 
         if (ctx == NULL)
-                ctx = to_i915(engine->dev)->kernel_context;
+                ctx = engine->i915->kernel_context;
         err = __i915_gem_request_alloc(engine, ctx, &req);
         return err ? ERR_PTR(err) : req;
 }
@@ -2834,8 +3078,16 @@ i915_gem_find_active_request(struct intel_engine_cs *engine)
 {
         struct drm_i915_gem_request *request;
 
+        /* We are called by the error capture and reset at a random
+         * point in time. In particular, note that neither is crucially
+         * ordered with an interrupt. After a hang, the GPU is dead and we
+         * assume that no more writes can happen (we waited long enough for
+         * all writes that were in transaction to be flushed) - adding an
+         * extra delay for a recent interrupt is pointless. Hence, we do
+         * not need an engine->irq_seqno_barrier() before the seqno reads.
+         */
         list_for_each_entry(request, &engine->request_list, list) {
-                if (i915_gem_request_completed(request, false))
+                if (i915_gem_request_completed(request))
                         continue;
 
                 return request;
@@ -2844,27 +3096,23 @@ i915_gem_find_active_request(struct intel_engine_cs *engine)
         return NULL;
 }
 
-static void i915_gem_reset_engine_status(struct drm_i915_private *dev_priv,
-                                       struct intel_engine_cs *engine)
+static void i915_gem_reset_engine_status(struct intel_engine_cs *engine)
 {
         struct drm_i915_gem_request *request;
         bool ring_hung;
 
         request = i915_gem_find_active_request(engine);
-
         if (request == NULL)
                 return;
 
         ring_hung = engine->hangcheck.score >= HANGCHECK_SCORE_RING_HUNG;
 
-        i915_set_reset_status(dev_priv, request->ctx, ring_hung);
-
+        i915_set_reset_status(request->ctx, ring_hung);
         list_for_each_entry_continue(request, &engine->request_list, list)
-                i915_set_reset_status(dev_priv, request->ctx, false);
+                i915_set_reset_status(request->ctx, false);
 }
 
-static void i915_gem_reset_engine_cleanup(struct drm_i915_private *dev_priv,
-                                        struct intel_engine_cs *engine)
+static void i915_gem_reset_engine_cleanup(struct intel_engine_cs *engine)
 {
         struct intel_ringbuffer *buffer;
 
@@ -2888,13 +3136,7 @@ static void i915_gem_reset_engine_cleanup(struct drm_i915_private *dev_priv,
                 /* Ensure irq handler finishes or is cancelled. */
                 tasklet_kill(&engine->irq_tasklet);
 
-                spin_lock_bh(&engine->execlist_lock);
-                /* list_splice_tail_init checks for empty lists */
-                list_splice_tail_init(&engine->execlist_queue,
-                                      &engine->execlist_retired_req_list);
-                spin_unlock_bh(&engine->execlist_lock);
-
-                intel_execlists_retire_requests(engine);
+                intel_execlists_cancel_requests(engine);
         }
 
         /*
@@ -2931,7 +3173,7 @@ static void i915_gem_reset_engine_cleanup(struct drm_i915_private *dev_priv,
 
 void i915_gem_reset(struct drm_device *dev)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         struct intel_engine_cs *engine;
 
         /*
@@ -2940,10 +3182,10 @@ void i915_gem_reset(struct drm_device *dev)
          * their reference to the objects, the inspection must be done first.
          */
         for_each_engine(engine, dev_priv)
-                i915_gem_reset_engine_status(dev_priv, engine);
+                i915_gem_reset_engine_status(engine);
 
         for_each_engine(engine, dev_priv)
-                i915_gem_reset_engine_cleanup(dev_priv, engine);
+                i915_gem_reset_engine_cleanup(engine);
 
         i915_gem_context_reset(dev);
 
@@ -2954,6 +3196,7 @@ void i915_gem_reset(struct drm_device *dev)
 
 /**
  * This function clears the request list as sequence numbers are passed.
+ * @engine: engine to retire requests on
  */
 void
 i915_gem_retire_requests_ring(struct intel_engine_cs *engine)
@@ -2972,7 +3215,7 @@ i915_gem_retire_requests_ring(struct intel_engine_cs *engine)
                                            struct drm_i915_gem_request,
                                            list);
 
-                if (!i915_gem_request_completed(request, true))
+                if (!i915_gem_request_completed(request))
                         break;
 
                 i915_gem_request_retire(request);
@@ -2995,58 +3238,52 @@ i915_gem_retire_requests_ring(struct intel_engine_cs *engine)
                 i915_gem_object_retire__read(obj, engine->id);
         }
 
-        if (unlikely(engine->trace_irq_req &&
-                     i915_gem_request_completed(engine->trace_irq_req, true))) {
-                engine->irq_put(engine);
-                i915_gem_request_assign(&engine->trace_irq_req, NULL);
-        }
-
         WARN_ON(i915_verify_lists(engine->dev));
 }
 
-bool
-i915_gem_retire_requests(struct drm_device *dev)
+void i915_gem_retire_requests(struct drm_i915_private *dev_priv)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
         struct intel_engine_cs *engine;
-        bool idle = true;
+
+        lockdep_assert_held(&dev_priv->drm.struct_mutex);
+
+        if (dev_priv->gt.active_engines == 0)
+                return;
+
+        GEM_BUG_ON(!dev_priv->gt.awake);
 
         for_each_engine(engine, dev_priv) {
                 i915_gem_retire_requests_ring(engine);
-                idle &= list_empty(&engine->request_list);
-                if (i915.enable_execlists) {
-                        spin_lock_bh(&engine->execlist_lock);
-                        idle &= list_empty(&engine->execlist_queue);
-                        spin_unlock_bh(&engine->execlist_lock);
-
-                        intel_execlists_retire_requests(engine);
-                }
+                if (list_empty(&engine->request_list))
+                        dev_priv->gt.active_engines &= ~intel_engine_flag(engine);
         }
 
-        if (idle)
-                mod_delayed_work(dev_priv->wq,
-                                   &dev_priv->mm.idle_work,
+        if (dev_priv->gt.active_engines == 0)
+                queue_delayed_work(dev_priv->wq,
+                                   &dev_priv->gt.idle_work,
                                    msecs_to_jiffies(100));
-
-        return idle;
 }
 
 static void
 i915_gem_retire_work_handler(struct work_struct *work)
 {
         struct drm_i915_private *dev_priv =
-                container_of(work, typeof(*dev_priv), mm.retire_work.work);
-        struct drm_device *dev = dev_priv->dev;
-        bool idle;
+                container_of(work, typeof(*dev_priv), gt.retire_work.work);
+        struct drm_device *dev = &dev_priv->drm;
 
         /* Come back later if the device is busy... */
-        idle = false;
         if (mutex_trylock(&dev->struct_mutex)) {
-                idle = i915_gem_retire_requests(dev);
+                i915_gem_retire_requests(dev_priv);
                 mutex_unlock(&dev->struct_mutex);
         }
-        if (!idle)
-                queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work,
+
+        /* Keep the retire handler running until we are finally idle.
+         * We do not need to do this test under locking as in the worst-case
+         * we queue the retire worker once too often.
+         */
+        if (READ_ONCE(dev_priv->gt.awake))
+                queue_delayed_work(dev_priv->wq,
+                                   &dev_priv->gt.retire_work,
                                    round_jiffies_up_relative(HZ));
 }
 
@@ -3054,25 +3291,55 @@ static void
 i915_gem_idle_work_handler(struct work_struct *work)
 {
         struct drm_i915_private *dev_priv =
-                container_of(work, typeof(*dev_priv), mm.idle_work.work);
-        struct drm_device *dev = dev_priv->dev;
+                container_of(work, typeof(*dev_priv), gt.idle_work.work);
+        struct drm_device *dev = &dev_priv->drm;
         struct intel_engine_cs *engine;
+        unsigned int stuck_engines;
+        bool rearm_hangcheck;
+
+        if (!READ_ONCE(dev_priv->gt.awake))
+                return;
+
+        if (READ_ONCE(dev_priv->gt.active_engines))
+                return;
+
+        rearm_hangcheck =
+                cancel_delayed_work_sync(&dev_priv->gpu_error.hangcheck_work);
+
+        if (!mutex_trylock(&dev->struct_mutex)) {
+                /* Currently busy, come back later */
+                mod_delayed_work(dev_priv->wq,
+                                 &dev_priv->gt.idle_work,
+                                 msecs_to_jiffies(50));
+                goto out_rearm;
+        }
+
+        if (dev_priv->gt.active_engines)
+                goto out_unlock;
 
         for_each_engine(engine, dev_priv)
-                if (!list_empty(&engine->request_list))
-                        return;
+                i915_gem_batch_pool_fini(&engine->batch_pool);
 
-        /* we probably should sync with hangcheck here, using cancel_work_sync.
-         * Also locking seems to be fubar here, engine->request_list is protected
-         * by dev->struct_mutex. */
+        GEM_BUG_ON(!dev_priv->gt.awake);
+        dev_priv->gt.awake = false;
+        rearm_hangcheck = false;
 
-        intel_mark_idle(dev);
+        stuck_engines = intel_kick_waiters(dev_priv);
+        if (unlikely(stuck_engines)) {
+                DRM_DEBUG_DRIVER("kicked stuck waiters...missed irq\n");
+                dev_priv->gpu_error.missed_irq_rings |= stuck_engines;
+        }
 
-        if (mutex_trylock(&dev->struct_mutex)) {
-                for_each_engine(engine, dev_priv)
-                        i915_gem_batch_pool_fini(&engine->batch_pool);
+        if (INTEL_GEN(dev_priv) >= 6)
+                gen6_rps_idle(dev_priv);
+        intel_runtime_pm_put(dev_priv);
+out_unlock:
+        mutex_unlock(&dev->struct_mutex);
 
-                mutex_unlock(&dev->struct_mutex);
+out_rearm:
+        if (rearm_hangcheck) {
+                GEM_BUG_ON(!dev_priv->gt.awake);
+                i915_queue_hangcheck(dev_priv);
         }
 }
 
@@ -3080,6 +3347,7 @@ i915_gem_idle_work_handler(struct work_struct *work)
  * Ensures that an object will eventually get non-busy by flushing any required
  * write domains, emitting any outstanding lazy request and retiring and
  * completed requests.
+ * @obj: object to flush
  */
 static int
 i915_gem_object_flush_active(struct drm_i915_gem_object *obj)
@@ -3096,14 +3364,8 @@ i915_gem_object_flush_active(struct drm_i915_gem_object *obj)
                 if (req == NULL)
                         continue;
 
-                if (list_empty(&req->list))
-                        goto retire;
-
-                if (i915_gem_request_completed(req, true)) {
-                        __i915_gem_request_retire__upto(req);
-retire:
+                if (i915_gem_request_completed(req))
                         i915_gem_object_retire__read(obj, i);
-                }
         }
 
         return 0;
@@ -3111,7 +3373,9 @@ retire:
 
 /**
  * i915_gem_wait_ioctl - implements DRM_IOCTL_I915_GEM_WAIT
- * @DRM_IOCTL_ARGS: standard ioctl arguments
+ * @dev: drm device pointer
+ * @data: ioctl data blob
+ * @file: drm file pointer
  *
  * Returns 0 if successful, else an error is returned with the remaining time in
  * the timeout parameter.
@@ -3185,7 +3449,7 @@ i915_gem_wait_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
                         ret = __i915_wait_request(req[i], true,
                                                   args->timeout_ns > 0 ? &args->timeout_ns : NULL,
                                                   to_rps_client(file));
-                i915_gem_request_unreference__unlocked(req[i]);
+                i915_gem_request_unreference(req[i]);
         }
         return ret;
 
@@ -3208,10 +3472,10 @@ __i915_gem_object_sync(struct drm_i915_gem_object *obj,
         if (to == from)
                 return 0;
 
-        if (i915_gem_request_completed(from_req, true))
+        if (i915_gem_request_completed(from_req))
                 return 0;
 
-        if (!i915_semaphore_is_enabled(obj->base.dev)) {
+        if (!i915_semaphore_is_enabled(to_i915(obj->base.dev))) {
                 struct drm_i915_private *i915 = to_i915(obj->base.dev);
                 ret = __i915_wait_request(from_req,
                                           i915->mm.interruptible,
@@ -3345,10 +3609,21 @@ static void i915_gem_object_finish_gtt(struct drm_i915_gem_object *obj)
                                             old_write_domain);
 }
 
+static void __i915_vma_iounmap(struct i915_vma *vma)
+{
+        GEM_BUG_ON(vma->pin_count);
+
+        if (vma->iomap == NULL)
+                return;
+
+        io_mapping_unmap(vma->iomap);
+        vma->iomap = NULL;
+}
+
 static int __i915_vma_unbind(struct i915_vma *vma, bool wait)
 {
         struct drm_i915_gem_object *obj = vma->obj;
-        struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(obj->base.dev);
         int ret;
 
         if (list_empty(&vma->obj_link))
@@ -3377,6 +3652,8 @@ static int __i915_vma_unbind(struct i915_vma *vma, bool wait)
                 ret = i915_gem_object_put_fence(obj);
                 if (ret)
                         return ret;
+
+                __i915_vma_iounmap(vma);
         }
 
         trace_i915_vma_unbind(vma);
@@ -3422,26 +3699,16 @@ int __i915_vma_unbind_no_wait(struct i915_vma *vma)
         return __i915_vma_unbind(vma, false);
 }
 
-int i915_gpu_idle(struct drm_device *dev)
+int i915_gem_wait_for_idle(struct drm_i915_private *dev_priv)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
         struct intel_engine_cs *engine;
         int ret;
 
-        /* Flush everything onto the inactive list. */
-        for_each_engine(engine, dev_priv) {
-                if (!i915.enable_execlists) {
-                        struct drm_i915_gem_request *req;
+        lockdep_assert_held(&dev_priv->drm.struct_mutex);
 
-                        req = i915_gem_request_alloc(engine, NULL);
-                        if (IS_ERR(req))
-                                return PTR_ERR(req);
-
-                        ret = i915_switch_context(req);
-                        i915_add_request_no_flush(req);
-                        if (ret)
-                                return ret;
-                }
+        for_each_engine(engine, dev_priv) {
+                if (engine->last_context == NULL)
+                        continue;
 
                 ret = intel_engine_idle(engine);
                 if (ret)
@@ -3488,6 +3755,11 @@ static bool i915_gem_valid_gtt_space(struct i915_vma *vma,
 /**
  * Finds free space in the GTT aperture and binds the object or a view of it
  * there.
+ * @obj: object to bind
+ * @vm: address space to bind into
+ * @ggtt_view: global gtt view if applicable
+ * @alignment: requested alignment
+ * @flags: mask of PIN_* flags to use
  */
 static struct i915_vma *
 i915_gem_object_bind_to_vm(struct drm_i915_gem_object *obj,
@@ -3731,7 +4003,7 @@ i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj)
                 return;
 
         if (i915_gem_clflush_object(obj, obj->pin_display))
-                i915_gem_chipset_flush(obj->base.dev);
+                i915_gem_chipset_flush(to_i915(obj->base.dev));
 
         old_write_domain = obj->base.write_domain;
         obj->base.write_domain = 0;
@@ -3745,6 +4017,8 @@ i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj)
 
 /**
  * Moves a single object to the GTT read, and possibly write domain.
+ * @obj: object to act on
+ * @write: ask for write access or read only
  *
  * This function returns when the move is complete, including waiting on
  * flushes to occur.
@@ -3816,6 +4090,8 @@ i915_gem_object_set_to_gtt_domain(struct drm_i915_gem_object *obj, bool write)
 
 /**
  * Changes the cache-level of an object across all VMA.
+ * @obj: object to act on
+ * @cache_level: new cache level to set for the object
  *
  * After this function returns, the object will be in the new cache-level
  * across all GTT and the contents of the backing storage will be coherent,
@@ -3925,11 +4201,9 @@ out:
          * object is now coherent at its new cache level (with respect
          * to the access domain).
          */
-        if (obj->cache_dirty &&
-            obj->base.write_domain != I915_GEM_DOMAIN_CPU &&
-            cpu_write_needs_clflush(obj)) {
+        if (obj->cache_dirty && cpu_write_needs_clflush(obj)) {
                 if (i915_gem_clflush_object(obj, true))
-                        i915_gem_chipset_flush(obj->base.dev);
+                        i915_gem_chipset_flush(to_i915(obj->base.dev));
         }
 
         return 0;
@@ -3967,7 +4241,7 @@ int i915_gem_get_caching_ioctl(struct drm_device *dev, void *data,
 int i915_gem_set_caching_ioctl(struct drm_device *dev, void *data,
                                struct drm_file *file)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         struct drm_i915_gem_caching *args = data;
         struct drm_i915_gem_object *obj;
         enum i915_cache_level level;
@@ -4097,6 +4371,8 @@ i915_gem_object_unpin_from_display_plane(struct drm_i915_gem_object *obj,
 
 /**
  * Moves a single object to the CPU read, and possibly write domain.
+ * @obj: object to act on
+ * @write: requesting write or read-only access
  *
  * This function returns when the move is complete, including waiting on
  * flushes to occur.
@@ -4159,7 +4435,7 @@ i915_gem_object_set_to_cpu_domain(struct drm_i915_gem_object *obj, bool write)
 static int
 i915_gem_ring_throttle(struct drm_device *dev, struct drm_file *file)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         struct drm_i915_file_private *file_priv = file->driver_priv;
         unsigned long recent_enough = jiffies - DRM_I915_THROTTLE_JIFFIES;
         struct drm_i915_gem_request *request, *target = NULL;
@@ -4195,10 +4471,7 @@ i915_gem_ring_throttle(struct drm_device *dev, struct drm_file *file)
                 return 0;
 
         ret = __i915_wait_request(target, true, NULL, NULL);
-        if (ret == 0)
-                queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, 0);
-
-        i915_gem_request_unreference__unlocked(target);
+        i915_gem_request_unreference(target);
 
         return ret;
 }
@@ -4256,7 +4529,7 @@ i915_gem_object_do_pin(struct drm_i915_gem_object *obj,
                        uint32_t alignment,
                        uint64_t flags)
 {
-        struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(obj->base.dev);
         struct i915_vma *vma;
         unsigned bound;
         int ret;
@@ -4420,7 +4693,7 @@ int
 i915_gem_madvise_ioctl(struct drm_device *dev, void *data,
                        struct drm_file *file_priv)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         struct drm_i915_gem_madvise *args = data;
         struct drm_i915_gem_object *obj;
         int ret;
@@ -4490,7 +4763,7 @@ void i915_gem_object_init(struct drm_i915_gem_object *obj,
         obj->fence_reg = I915_FENCE_REG_NONE;
         obj->madv = I915_MADV_WILLNEED;
 
-        i915_gem_info_add_obj(obj->base.dev->dev_private, obj->base.size);
+        i915_gem_info_add_obj(to_i915(obj->base.dev), obj->base.size);
 }
 
 static const struct drm_i915_gem_object_ops i915_gem_object_ops = {
@@ -4499,21 +4772,21 @@ static const struct drm_i915_gem_object_ops i915_gem_object_ops = {
         .put_pages = i915_gem_object_put_pages_gtt,
 };
 
-struct drm_i915_gem_object *i915_gem_alloc_object(struct drm_device *dev,
+struct drm_i915_gem_object *i915_gem_object_create(struct drm_device *dev,
                                                   size_t size)
 {
         struct drm_i915_gem_object *obj;
         struct address_space *mapping;
         gfp_t mask;
+        int ret;
 
         obj = i915_gem_object_alloc(dev);
         if (obj == NULL)
-                return NULL;
+                return ERR_PTR(-ENOMEM);
 
-        if (drm_gem_object_init(dev, &obj->base, size) != 0) {
-                i915_gem_object_free(obj);
-                return NULL;
-        }
+        ret = drm_gem_object_init(dev, &obj->base, size);
+        if (ret)
+                goto fail;
 
         mask = GFP_HIGHUSER | __GFP_RECLAIMABLE;
         if (IS_CRESTLINE(dev) || IS_BROADWATER(dev)) {
@@ -4550,6 +4823,11 @@ struct drm_i915_gem_object *i915_gem_alloc_object(struct drm_device *dev,
         trace_i915_gem_object_create(obj);
 
         return obj;
+
+fail:
+        i915_gem_object_free(obj);
+
+        return ERR_PTR(ret);
 }
 
 static bool discard_backing_storage(struct drm_i915_gem_object *obj)
@@ -4580,7 +4858,7 @@ void i915_gem_free_object(struct drm_gem_object *gem_obj)
 {
         struct drm_i915_gem_object *obj = to_intel_bo(gem_obj);
         struct drm_device *dev = obj->base.dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         struct i915_vma *vma, *next;
 
         intel_runtime_pm_get(dev_priv);
@@ -4655,16 +4933,12 @@ struct i915_vma *i915_gem_obj_to_vma(struct drm_i915_gem_object *obj,
 struct i915_vma *i915_gem_obj_to_ggtt_view(struct drm_i915_gem_object *obj,
                                            const struct i915_ggtt_view *view)
 {
-        struct drm_device *dev = obj->base.dev;
-        struct drm_i915_private *dev_priv = to_i915(dev);
-        struct i915_ggtt *ggtt = &dev_priv->ggtt;
         struct i915_vma *vma;
 
-        BUG_ON(!view);
+        GEM_BUG_ON(!view);
 
         list_for_each_entry(vma, &obj->vma_list, obj_link)
-                if (vma->vm == &ggtt->base &&
-                    i915_ggtt_view_equal(&vma->ggtt_view, view))
+                if (vma->is_ggtt && i915_ggtt_view_equal(&vma->ggtt_view, view))
                         return vma;
         return NULL;
 }
@@ -4688,7 +4962,7 @@ void i915_gem_vma_destroy(struct i915_vma *vma)
 static void
 i915_gem_stop_engines(struct drm_device *dev)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         struct intel_engine_cs *engine;
 
         for_each_engine(engine, dev_priv)
@@ -4698,27 +4972,28 @@ i915_gem_stop_engines(struct drm_device *dev)
 int
 i915_gem_suspend(struct drm_device *dev)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         int ret = 0;
 
         mutex_lock(&dev->struct_mutex);
-        ret = i915_gpu_idle(dev);
+        ret = i915_gem_wait_for_idle(dev_priv);
         if (ret)
                 goto err;
 
-        i915_gem_retire_requests(dev);
+        i915_gem_retire_requests(dev_priv);
 
         i915_gem_stop_engines(dev);
+        i915_gem_context_lost(dev_priv);
         mutex_unlock(&dev->struct_mutex);
 
         cancel_delayed_work_sync(&dev_priv->gpu_error.hangcheck_work);
-        cancel_delayed_work_sync(&dev_priv->mm.retire_work);
-        flush_delayed_work(&dev_priv->mm.idle_work);
+        cancel_delayed_work_sync(&dev_priv->gt.retire_work);
+        flush_delayed_work(&dev_priv->gt.idle_work);
 
         /* Assert that we sucessfully flushed all the work and
          * reset the GPU back to its idle, low power state.
          */
-        WARN_ON(dev_priv->mm.busy);
+        WARN_ON(dev_priv->gt.awake);
 
         return 0;
 
@@ -4727,40 +5002,9 @@ err:
         return ret;
 }
 
-int i915_gem_l3_remap(struct drm_i915_gem_request *req, int slice)
-{
-        struct intel_engine_cs *engine = req->engine;
-        struct drm_device *dev = engine->dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
-        u32 *remap_info = dev_priv->l3_parity.remap_info[slice];
-        int i, ret;
-
-        if (!HAS_L3_DPF(dev) || !remap_info)
-                return 0;
-
-        ret = intel_ring_begin(req, GEN7_L3LOG_SIZE / 4 * 3);
-        if (ret)
-                return ret;
-
-        /*
-         * Note: We do not worry about the concurrent register cacheline hang
-         * here because no other code should access these registers other than
-         * at initialization time.
-         */
-        for (i = 0; i < GEN7_L3LOG_SIZE / 4; i++) {
-                intel_ring_emit(engine, MI_LOAD_REGISTER_IMM(1));
-                intel_ring_emit_reg(engine, GEN7_L3LOG(slice, i));
-                intel_ring_emit(engine, remap_info[i]);
-        }
-
-        intel_ring_advance(engine);
-
-        return ret;
-}
-
 void i915_gem_init_swizzling(struct drm_device *dev)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
 
         if (INTEL_INFO(dev)->gen < 5 ||
             dev_priv->mm.bit_6_swizzle_x == I915_BIT_6_SWIZZLE_NONE)
@@ -4785,7 +5029,7 @@ void i915_gem_init_swizzling(struct drm_device *dev)
 
 static void init_unused_ring(struct drm_device *dev, u32 base)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
 
         I915_WRITE(RING_CTL(base), 0);
         I915_WRITE(RING_HEAD(base), 0);
@@ -4812,7 +5056,7 @@ static void init_unused_rings(struct drm_device *dev)
 
 int i915_gem_init_engines(struct drm_device *dev)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         int ret;
 
         ret = intel_init_render_ring_buffer(dev);
@@ -4860,9 +5104,9 @@ cleanup_render_ring:
 int
 i915_gem_init_hw(struct drm_device *dev)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         struct intel_engine_cs *engine;
-        int ret, j;
+        int ret;
 
         /* Double layer security blanket, see i915_gem_init() */
         intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
@@ -4914,59 +5158,10 @@ i915_gem_init_hw(struct drm_device *dev)
         intel_mocs_init_l3cc_table(dev);
 
         /* We can't enable contexts until all firmware is loaded */
-        if (HAS_GUC_UCODE(dev)) {
-                ret = intel_guc_ucode_load(dev);
-                if (ret) {
-                        DRM_ERROR("Failed to initialize GuC, error %d\n", ret);
-                        ret = -EIO;
-                        goto out;
-                }
-        }
-
-        /*
-         * Increment the next seqno by 0x100 so we have a visible break
-         * on re-initialisation
-         */
-        ret = i915_gem_set_seqno(dev, dev_priv->next_seqno+0x100);
+        ret = intel_guc_setup(dev);
         if (ret)
                 goto out;
 
-        /* Now it is safe to go back round and do everything else: */
-        for_each_engine(engine, dev_priv) {
-                struct drm_i915_gem_request *req;
-
-                req = i915_gem_request_alloc(engine, NULL);
-                if (IS_ERR(req)) {
-                        ret = PTR_ERR(req);
-                        break;
-                }
-
-                if (engine->id == RCS) {
-                        for (j = 0; j < NUM_L3_SLICES(dev); j++) {
-                                ret = i915_gem_l3_remap(req, j);
-                                if (ret)
-                                        goto err_request;
-                        }
-                }
-
-                ret = i915_ppgtt_init_ring(req);
-                if (ret)
-                        goto err_request;
-
-                ret = i915_gem_context_enable(req);
-                if (ret)
-                        goto err_request;
-
-err_request:
-                i915_add_request_no_flush(req);
-                if (ret) {
-                        DRM_ERROR("Failed to enable %s, error=%d\n",
-                                  engine->name, ret);
-                        i915_gem_cleanup_engines(dev);
-                        break;
-                }
-        }
-
 out:
         intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
         return ret;
@@ -4974,12 +5169,9 @@ out:
 
 int i915_gem_init(struct drm_device *dev)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         int ret;
 
-        i915.enable_execlists = intel_sanitize_enable_execlists(dev,
-                        i915.enable_execlists);
-
         mutex_lock(&dev->struct_mutex);
 
         if (!i915.enable_execlists) {
@@ -5002,10 +5194,7 @@ int i915_gem_init(struct drm_device *dev)
          */
         intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
 
-        ret = i915_gem_init_userptr(dev);
-        if (ret)
-                goto out_unlock;
-
+        i915_gem_init_userptr(dev_priv);
         i915_gem_init_ggtt(dev);
 
         ret = i915_gem_context_init(dev);
@@ -5037,19 +5226,11 @@ out_unlock:
 void
 i915_gem_cleanup_engines(struct drm_device *dev)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         struct intel_engine_cs *engine;
 
         for_each_engine(engine, dev_priv)
                 dev_priv->gt.cleanup_engine(engine);
-
-        if (i915.enable_execlists)
-                /*
-                 * Neither the BIOS, ourselves or any other kernel
-                 * expects the system to be in execlists mode on startup,
-                 * so we need to reset the GPU back to legacy mode.
-                 */
-                intel_gpu_reset(dev, ALL_ENGINES);
 }
 
 static void
@@ -5062,7 +5243,7 @@ init_engine_lists(struct intel_engine_cs *engine)
 void
 i915_gem_load_init_fences(struct drm_i915_private *dev_priv)
 {
-        struct drm_device *dev = dev_priv->dev;
+        struct drm_device *dev = &dev_priv->drm;
 
         if (INTEL_INFO(dev_priv)->gen >= 7 && !IS_VALLEYVIEW(dev_priv) &&
             !IS_CHERRYVIEW(dev_priv))
@@ -5073,7 +5254,7 @@ i915_gem_load_init_fences(struct drm_i915_private *dev_priv)
         else
                 dev_priv->num_fence_regs = 8;
 
-        if (intel_vgpu_active(dev))
+        if (intel_vgpu_active(dev_priv))
                 dev_priv->num_fence_regs =
                                 I915_READ(vgtif_reg(avail_rs.fence_num));
 
@@ -5086,7 +5267,7 @@ i915_gem_load_init_fences(struct drm_i915_private *dev_priv)
 void
 i915_gem_load_init(struct drm_device *dev)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         int i;
 
         dev_priv->objects =
@@ -5114,22 +5295,15 @@ i915_gem_load_init(struct drm_device *dev)
                 init_engine_lists(&dev_priv->engine[i]);
         for (i = 0; i < I915_MAX_NUM_FENCES; i++)
                 INIT_LIST_HEAD(&dev_priv->fence_regs[i].lru_list);
-        INIT_DELAYED_WORK(&dev_priv->mm.retire_work,
+        INIT_DELAYED_WORK(&dev_priv->gt.retire_work,
                           i915_gem_retire_work_handler);
-        INIT_DELAYED_WORK(&dev_priv->mm.idle_work,
+        INIT_DELAYED_WORK(&dev_priv->gt.idle_work,
                           i915_gem_idle_work_handler);
+        init_waitqueue_head(&dev_priv->gpu_error.wait_queue);
         init_waitqueue_head(&dev_priv->gpu_error.reset_queue);
 
         dev_priv->relative_constants_mode = I915_EXEC_CONSTANTS_REL_GENERAL;
 
-        /*
-         * Set initial sequence number for requests.
-         * Using this number allows the wraparound to happen early,
-         * catching any obvious problems.
-         */
-        dev_priv->next_seqno = ((u32)~0 - 0x1100);
-        dev_priv->last_seqno = ((u32)~0 - 0x1101);
-
         INIT_LIST_HEAD(&dev_priv->mm.fence_list);
 
         init_waitqueue_head(&dev_priv->pending_flip_queue);
@@ -5148,6 +5322,34 @@ void i915_gem_load_cleanup(struct drm_device *dev)
         kmem_cache_destroy(dev_priv->objects);
 }
 
+int i915_gem_freeze_late(struct drm_i915_private *dev_priv)
+{
+        struct drm_i915_gem_object *obj;
+
+        /* Called just before we write the hibernation image.
+         *
+         * We need to update the domain tracking to reflect that the CPU
+         * will be accessing all the pages to create and restore from the
+         * hibernation, and so upon restoration those pages will be in the
+         * CPU domain.
+         *
+         * To make sure the hibernation image contains the latest state,
+         * we update that state just before writing out the image.
+         */
+
+        list_for_each_entry(obj, &dev_priv->mm.unbound_list, global_list) {
+                obj->base.read_domains = I915_GEM_DOMAIN_CPU;
+                obj->base.write_domain = I915_GEM_DOMAIN_CPU;
+        }
+
+        list_for_each_entry(obj, &dev_priv->mm.bound_list, global_list) {
+                obj->base.read_domains = I915_GEM_DOMAIN_CPU;
+                obj->base.write_domain = I915_GEM_DOMAIN_CPU;
+        }
+
+        return 0;
+}
+
 void i915_gem_release(struct drm_device *dev, struct drm_file *file)
 {
         struct drm_i915_file_private *file_priv = file->driver_priv;
@@ -5187,7 +5389,7 @@ int i915_gem_open(struct drm_device *dev, struct drm_file *file)
                 return -ENOMEM;
 
         file->driver_priv = file_priv;
-        file_priv->dev_priv = dev->dev_private;
+        file_priv->dev_priv = to_i915(dev);
         file_priv->file = file;
         INIT_LIST_HEAD(&file_priv->rps.link);
 
@@ -5233,7 +5435,7 @@ void i915_gem_track_fb(struct drm_i915_gem_object *old,
 u64 i915_gem_obj_offset(struct drm_i915_gem_object *o,
                         struct i915_address_space *vm)
 {
-        struct drm_i915_private *dev_priv = o->base.dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(o->base.dev);
         struct i915_vma *vma;
 
         WARN_ON(vm == &dev_priv->mm.aliasing_ppgtt->base);
@@ -5254,13 +5456,10 @@ u64 i915_gem_obj_offset(struct drm_i915_gem_object *o,
 u64 i915_gem_obj_ggtt_offset_view(struct drm_i915_gem_object *o,
                                   const struct i915_ggtt_view *view)
 {
-        struct drm_i915_private *dev_priv = to_i915(o->base.dev);
-        struct i915_ggtt *ggtt = &dev_priv->ggtt;
         struct i915_vma *vma;
 
         list_for_each_entry(vma, &o->vma_list, obj_link)
-                if (vma->vm == &ggtt->base &&
-                    i915_ggtt_view_equal(&vma->ggtt_view, view))
+                if (vma->is_ggtt && i915_ggtt_view_equal(&vma->ggtt_view, view))
                         return vma->node.start;
 
         WARN(1, "global vma for this object not found. (view=%u)\n", view->type);
@@ -5286,12 +5485,10 @@ bool i915_gem_obj_bound(struct drm_i915_gem_object *o,
 bool i915_gem_obj_ggtt_bound_view(struct drm_i915_gem_object *o,
                                   const struct i915_ggtt_view *view)
 {
-        struct drm_i915_private *dev_priv = to_i915(o->base.dev);
-        struct i915_ggtt *ggtt = &dev_priv->ggtt;
         struct i915_vma *vma;
 
         list_for_each_entry(vma, &o->vma_list, obj_link)
-                if (vma->vm == &ggtt->base &&
+                if (vma->is_ggtt &&
                     i915_ggtt_view_equal(&vma->ggtt_view, view) &&
                     drm_mm_node_allocated(&vma->node))
                         return true;
@@ -5310,23 +5507,18 @@ bool i915_gem_obj_bound_any(struct drm_i915_gem_object *o)
         return false;
 }
 
-unsigned long i915_gem_obj_size(struct drm_i915_gem_object *o,
-                                struct i915_address_space *vm)
+unsigned long i915_gem_obj_ggtt_size(struct drm_i915_gem_object *o)
 {
-        struct drm_i915_private *dev_priv = o->base.dev->dev_private;
         struct i915_vma *vma;
 
-        WARN_ON(vm == &dev_priv->mm.aliasing_ppgtt->base);
-
-        BUG_ON(list_empty(&o->vma_list));
+        GEM_BUG_ON(list_empty(&o->vma_list));
 
         list_for_each_entry(vma, &o->vma_list, obj_link) {
                 if (vma->is_ggtt &&
-                    vma->ggtt_view.type != I915_GGTT_VIEW_NORMAL)
-                        continue;
-                if (vma->vm == vm)
+                    vma->ggtt_view.type == I915_GGTT_VIEW_NORMAL)
                         return vma->node.size;
         }
+
         return 0;
 }
 
@@ -5347,7 +5539,7 @@ i915_gem_object_get_dirty_page(struct drm_i915_gem_object *obj, int n)
         struct page *page;
 
         /* Only default objects have per-page dirty tracking */
-        if (WARN_ON((obj->ops->flags & I915_GEM_OBJECT_HAS_STRUCT_PAGE) == 0))
+        if (WARN_ON(!i915_gem_object_has_struct_page(obj)))
                 return NULL;
 
         page = i915_gem_object_get_page(obj, n);
@@ -5365,8 +5557,8 @@ i915_gem_object_create_from_data(struct drm_device *dev,
         size_t bytes;
         int ret;
 
-        obj = i915_gem_alloc_object(dev, round_up(size, PAGE_SIZE));
-        if (IS_ERR_OR_NULL(obj))
+        obj = i915_gem_object_create(dev, round_up(size, PAGE_SIZE));
+        if (IS_ERR(obj))
                 return obj;
 
         ret = i915_gem_object_set_to_cpu_domain(obj, true);
diff --git a/drivers/gpu/drm/i915/i915_gem_batch_pool.c b/drivers/gpu/drm/i915/i915_gem_batch_pool.c
index 7bf2f3f2968e..3752d5daa4b2 100644
--- a/drivers/gpu/drm/i915/i915_gem_batch_pool.c
+++ b/drivers/gpu/drm/i915/i915_gem_batch_pool.c
@@ -134,9 +134,9 @@ i915_gem_batch_pool_get(struct i915_gem_batch_pool *pool,
         if (obj == NULL) {
                 int ret;
 
-                obj = i915_gem_alloc_object(pool->dev, size);
-                if (obj == NULL)
-                        return ERR_PTR(-ENOMEM);
+                obj = i915_gem_object_create(pool->dev, size);
+                if (IS_ERR(obj))
+                        return obj;
 
                 ret = i915_gem_object_get_pages(obj);
                 if (ret)
diff --git a/drivers/gpu/drm/i915/i915_gem_context.c b/drivers/gpu/drm/i915/i915_gem_context.c
index e5acc3916f75..3c97f0e7a003 100644
--- a/drivers/gpu/drm/i915/i915_gem_context.c
+++ b/drivers/gpu/drm/i915/i915_gem_context.c
@@ -90,6 +90,8 @@
 #include "i915_drv.h"
 #include "i915_trace.h"
 
+#define ALL_L3_SLICES(dev) (1 << NUM_L3_SLICES(dev)) - 1
+
 /* This is a HW constraint. The value below is the largest known requirement
  * I've seen in a spec to date, and that was a workaround for a non-shipping
  * part. It should be safe to decrease this, but it's more future proof as is.
@@ -97,28 +99,27 @@
 #define GEN6_CONTEXT_ALIGN (64<<10)
 #define GEN7_CONTEXT_ALIGN 4096
 
-static size_t get_context_alignment(struct drm_device *dev)
+static size_t get_context_alignment(struct drm_i915_private *dev_priv)
 {
-        if (IS_GEN6(dev))
+        if (IS_GEN6(dev_priv))
                 return GEN6_CONTEXT_ALIGN;
 
         return GEN7_CONTEXT_ALIGN;
 }
 
-static int get_context_size(struct drm_device *dev)
+static int get_context_size(struct drm_i915_private *dev_priv)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
         int ret;
         u32 reg;
 
-        switch (INTEL_INFO(dev)->gen) {
+        switch (INTEL_GEN(dev_priv)) {
         case 6:
                 reg = I915_READ(CXT_SIZE);
                 ret = GEN6_CXT_TOTAL_SIZE(reg) * 64;
                 break;
         case 7:
                 reg = I915_READ(GEN7_CXT_SIZE);
-                if (IS_HASWELL(dev))
+                if (IS_HASWELL(dev_priv))
                         ret = HSW_CXT_TOTAL_SIZE;
                 else
                         ret = GEN7_CXT_TOTAL_SIZE(reg) * 64;
@@ -133,7 +134,7 @@ static int get_context_size(struct drm_device *dev)
         return ret;
 }
 
-static void i915_gem_context_clean(struct intel_context *ctx)
+static void i915_gem_context_clean(struct i915_gem_context *ctx)
 {
         struct i915_hw_ppgtt *ppgtt = ctx->ppgtt;
         struct i915_vma *vma, *next;
@@ -150,13 +151,12 @@ static void i915_gem_context_clean(struct intel_context *ctx)
 
 void i915_gem_context_free(struct kref *ctx_ref)
 {
-        struct intel_context *ctx = container_of(ctx_ref, typeof(*ctx), ref);
+        struct i915_gem_context *ctx = container_of(ctx_ref, typeof(*ctx), ref);
+        int i;
 
+        lockdep_assert_held(&ctx->i915->drm.struct_mutex);
         trace_i915_context_free(ctx);
 
-        if (i915.enable_execlists)
-                intel_lr_context_free(ctx);
-
         /*
          * This context is going away and we need to remove all VMAs still
          * around. This is to handle imported shared objects for which
@@ -166,9 +166,22 @@ void i915_gem_context_free(struct kref *ctx_ref)
 
         i915_ppgtt_put(ctx->ppgtt);
 
-        if (ctx->legacy_hw_ctx.rcs_state)
-                drm_gem_object_unreference(&ctx->legacy_hw_ctx.rcs_state->base);
+        for (i = 0; i < I915_NUM_ENGINES; i++) {
+                struct intel_context *ce = &ctx->engine[i];
+
+                if (!ce->state)
+                        continue;
+
+                WARN_ON(ce->pin_count);
+                if (ce->ringbuf)
+                        intel_ringbuffer_free(ce->ringbuf);
+
+                drm_gem_object_unreference(&ce->state->base);
+        }
+
         list_del(&ctx->link);
+
+        ida_simple_remove(&ctx->i915->context_hw_ida, ctx->hw_id);
         kfree(ctx);
 }
 
@@ -178,9 +191,11 @@ i915_gem_alloc_context_obj(struct drm_device *dev, size_t size)
         struct drm_i915_gem_object *obj;
         int ret;
 
-        obj = i915_gem_alloc_object(dev, size);
-        if (obj == NULL)
-                return ERR_PTR(-ENOMEM);
+        lockdep_assert_held(&dev->struct_mutex);
+
+        obj = i915_gem_object_create(dev, size);
+        if (IS_ERR(obj))
+                return obj;
 
         /*
          * Try to make the context utilize L3 as well as LLC.
@@ -209,22 +224,52 @@ i915_gem_alloc_context_obj(struct drm_device *dev, size_t size)
         return obj;
 }
 
-static struct intel_context *
+static int assign_hw_id(struct drm_i915_private *dev_priv, unsigned *out)
+{
+        int ret;
+
+        ret = ida_simple_get(&dev_priv->context_hw_ida,
+                             0, MAX_CONTEXT_HW_ID, GFP_KERNEL);
+        if (ret < 0) {
+                /* Contexts are only released when no longer active.
+                 * Flush any pending retires to hopefully release some
+                 * stale contexts and try again.
+                 */
+                i915_gem_retire_requests(dev_priv);
+                ret = ida_simple_get(&dev_priv->context_hw_ida,
+                                     0, MAX_CONTEXT_HW_ID, GFP_KERNEL);
+                if (ret < 0)
+                        return ret;
+        }
+
+        *out = ret;
+        return 0;
+}
+
+static struct i915_gem_context *
 __create_hw_context(struct drm_device *dev,
                     struct drm_i915_file_private *file_priv)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
-        struct intel_context *ctx;
+        struct drm_i915_private *dev_priv = to_i915(dev);
+        struct i915_gem_context *ctx;
         int ret;
 
         ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);
         if (ctx == NULL)
                 return ERR_PTR(-ENOMEM);
 
+        ret = assign_hw_id(dev_priv, &ctx->hw_id);
+        if (ret) {
+                kfree(ctx);
+                return ERR_PTR(ret);
+        }
+
         kref_init(&ctx->ref);
         list_add_tail(&ctx->link, &dev_priv->context_list);
         ctx->i915 = dev_priv;
 
+        ctx->ggtt_alignment = get_context_alignment(dev_priv);
+
         if (dev_priv->hw_context_size) {
                 struct drm_i915_gem_object *obj =
                                 i915_gem_alloc_context_obj(dev, dev_priv->hw_context_size);
@@ -232,7 +277,7 @@ __create_hw_context(struct drm_device *dev,
                         ret = PTR_ERR(obj);
                         goto err_out;
                 }
-                ctx->legacy_hw_ctx.rcs_state = obj;
+                ctx->engine[RCS].state = obj;
         }
 
         /* Default context will never have a file_priv */
@@ -249,9 +294,13 @@ __create_hw_context(struct drm_device *dev,
         /* NB: Mark all slices as needing a remap so that when the context first
          * loads it will restore whatever remap state already exists. If there
          * is no remap info, it will be a NOP. */
-        ctx->remap_slice = (1 << NUM_L3_SLICES(dev)) - 1;
+        ctx->remap_slice = ALL_L3_SLICES(dev_priv);
 
         ctx->hang_stats.ban_period_seconds = DRM_I915_CTX_BAN_PERIOD;
+        ctx->ring_size = 4 * PAGE_SIZE;
+        ctx->desc_template = GEN8_CTX_ADDRESSING_MODE(dev_priv) <<
+                             GEN8_CTX_ADDRESSING_MODE_SHIFT;
+        ATOMIC_INIT_NOTIFIER_HEAD(&ctx->status_notifier);
 
         return ctx;
 
@@ -265,44 +314,27 @@ err_out:
  * context state of the GPU for applications that don't utilize HW contexts, as
  * well as an idle case.
  */
-static struct intel_context *
+static struct i915_gem_context *
 i915_gem_create_context(struct drm_device *dev,
                         struct drm_i915_file_private *file_priv)
 {
-        const bool is_global_default_ctx = file_priv == NULL;
-        struct intel_context *ctx;
-        int ret = 0;
+        struct i915_gem_context *ctx;
 
-        BUG_ON(!mutex_is_locked(&dev->struct_mutex));
+        lockdep_assert_held(&dev->struct_mutex);
 
         ctx = __create_hw_context(dev, file_priv);
         if (IS_ERR(ctx))
                 return ctx;
 
-        if (is_global_default_ctx && ctx->legacy_hw_ctx.rcs_state) {
-                /* We may need to do things with the shrinker which
-                 * require us to immediately switch back to the default
-                 * context. This can cause a problem as pinning the
-                 * default context also requires GTT space which may not
-                 * be available. To avoid this we always pin the default
-                 * context.
-                 */
-                ret = i915_gem_obj_ggtt_pin(ctx->legacy_hw_ctx.rcs_state,
-                                            get_context_alignment(dev), 0);
-                if (ret) {
-                        DRM_DEBUG_DRIVER("Couldn't pin %d\n", ret);
-                        goto err_destroy;
-                }
-        }
-
         if (USES_FULL_PPGTT(dev)) {
                 struct i915_hw_ppgtt *ppgtt = i915_ppgtt_create(dev, file_priv);
 
-                if (IS_ERR_OR_NULL(ppgtt)) {
+                if (IS_ERR(ppgtt)) {
                         DRM_DEBUG_DRIVER("PPGTT setup failed (%ld)\n",
                                          PTR_ERR(ppgtt));
-                        ret = PTR_ERR(ppgtt);
-                        goto err_unpin;
+                        idr_remove(&file_priv->context_idr, ctx->user_handle);
+                        i915_gem_context_unreference(ctx);
+                        return ERR_CAST(ppgtt);
                 }
 
                 ctx->ppgtt = ppgtt;
@@ -311,76 +343,102 @@ i915_gem_create_context(struct drm_device *dev,
         trace_i915_context_create(ctx);
 
         return ctx;
+}
 
-err_unpin:
-        if (is_global_default_ctx && ctx->legacy_hw_ctx.rcs_state)
-                i915_gem_object_ggtt_unpin(ctx->legacy_hw_ctx.rcs_state);
-err_destroy:
-        idr_remove(&file_priv->context_idr, ctx->user_handle);
-        i915_gem_context_unreference(ctx);
-        return ERR_PTR(ret);
+/**
+ * i915_gem_context_create_gvt - create a GVT GEM context
+ * @dev: drm device *
+ *
+ * This function is used to create a GVT specific GEM context.
+ *
+ * Returns:
+ * pointer to i915_gem_context on success, error pointer if failed
+ *
+ */
+struct i915_gem_context *
+i915_gem_context_create_gvt(struct drm_device *dev)
+{
+        struct i915_gem_context *ctx;
+        int ret;
+
+        if (!IS_ENABLED(CONFIG_DRM_I915_GVT))
+                return ERR_PTR(-ENODEV);
+
+        ret = i915_mutex_lock_interruptible(dev);
+        if (ret)
+                return ERR_PTR(ret);
+
+        ctx = i915_gem_create_context(dev, NULL);
+        if (IS_ERR(ctx))
+                goto out;
+
+        ctx->execlists_force_single_submission = true;
+        ctx->ring_size = 512 * PAGE_SIZE; /* Max ring buffer size */
+out:
+        mutex_unlock(&dev->struct_mutex);
+        return ctx;
 }
 
-static void i915_gem_context_unpin(struct intel_context *ctx,
+static void i915_gem_context_unpin(struct i915_gem_context *ctx,
                                    struct intel_engine_cs *engine)
 {
         if (i915.enable_execlists) {
                 intel_lr_context_unpin(ctx, engine);
         } else {
-                if (engine->id == RCS && ctx->legacy_hw_ctx.rcs_state)
-                        i915_gem_object_ggtt_unpin(ctx->legacy_hw_ctx.rcs_state);
+                struct intel_context *ce = &ctx->engine[engine->id];
+
+                if (ce->state)
+                        i915_gem_object_ggtt_unpin(ce->state);
+
                 i915_gem_context_unreference(ctx);
         }
 }
 
 void i915_gem_context_reset(struct drm_device *dev)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
-        int i;
+        struct drm_i915_private *dev_priv = to_i915(dev);
+
+        lockdep_assert_held(&dev->struct_mutex);
 
         if (i915.enable_execlists) {
-                struct intel_context *ctx;
+                struct i915_gem_context *ctx;
 
                 list_for_each_entry(ctx, &dev_priv->context_list, link)
                         intel_lr_context_reset(dev_priv, ctx);
         }
 
-        for (i = 0; i < I915_NUM_ENGINES; i++) {
-                struct intel_engine_cs *engine = &dev_priv->engine[i];
-
-                if (engine->last_context) {
-                        i915_gem_context_unpin(engine->last_context, engine);
-                        engine->last_context = NULL;
-                }
-        }
-
-        /* Force the GPU state to be reinitialised on enabling */
-        dev_priv->kernel_context->legacy_hw_ctx.initialized = false;
+        i915_gem_context_lost(dev_priv);
 }
 
 int i915_gem_context_init(struct drm_device *dev)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
-        struct intel_context *ctx;
+        struct drm_i915_private *dev_priv = to_i915(dev);
+        struct i915_gem_context *ctx;
 
         /* Init should only be called once per module load. Eventually the
          * restriction on the context_disabled check can be loosened. */
         if (WARN_ON(dev_priv->kernel_context))
                 return 0;
 
-        if (intel_vgpu_active(dev) && HAS_LOGICAL_RING_CONTEXTS(dev)) {
+        if (intel_vgpu_active(dev_priv) &&
+            HAS_LOGICAL_RING_CONTEXTS(dev_priv)) {
                 if (!i915.enable_execlists) {
                         DRM_INFO("Only EXECLIST mode is supported in vgpu.\n");
                         return -EINVAL;
                 }
         }
 
+        /* Using the simple ida interface, the max is limited by sizeof(int) */
+        BUILD_BUG_ON(MAX_CONTEXT_HW_ID > INT_MAX);
+        ida_init(&dev_priv->context_hw_ida);
+
         if (i915.enable_execlists) {
                 /* NB: intentionally left blank. We will allocate our own
                  * backing objects as we need them, thank you very much */
                 dev_priv->hw_context_size = 0;
-        } else if (HAS_HW_CONTEXTS(dev)) {
-                dev_priv->hw_context_size = round_up(get_context_size(dev), 4096);
+        } else if (HAS_HW_CONTEXTS(dev_priv)) {
+                dev_priv->hw_context_size =
+                        round_up(get_context_size(dev_priv), 4096);
                 if (dev_priv->hw_context_size > (1<<20)) {
                         DRM_DEBUG_DRIVER("Disabling HW Contexts; invalid size %d\n",
                                          dev_priv->hw_context_size);
@@ -403,67 +461,60 @@ int i915_gem_context_init(struct drm_device *dev)
         return 0;
 }
 
-void i915_gem_context_fini(struct drm_device *dev)
+void i915_gem_context_lost(struct drm_i915_private *dev_priv)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
-        struct intel_context *dctx = dev_priv->kernel_context;
-        int i;
-
-        if (dctx->legacy_hw_ctx.rcs_state) {
-                /* The only known way to stop the gpu from accessing the hw context is
-                 * to reset it. Do this as the very last operation to avoid confusing
-                 * other code, leading to spurious errors. */
-                intel_gpu_reset(dev, ALL_ENGINES);
+        struct intel_engine_cs *engine;
 
-                /* When default context is created and switched to, base object refcount
-                 * will be 2 (+1 from object creation and +1 from do_switch()).
-                 * i915_gem_context_fini() will be called after gpu_idle() has switched
-                 * to default context. So we need to unreference the base object once
-                 * to offset the do_switch part, so that i915_gem_context_unreference()
-                 * can then free the base object correctly. */
-                WARN_ON(!dev_priv->engine[RCS].last_context);
-
-                i915_gem_object_ggtt_unpin(dctx->legacy_hw_ctx.rcs_state);
-        }
-
-        for (i = I915_NUM_ENGINES; --i >= 0;) {
-                struct intel_engine_cs *engine = &dev_priv->engine[i];
+        lockdep_assert_held(&dev_priv->drm.struct_mutex);
 
+        for_each_engine(engine, dev_priv) {
                 if (engine->last_context) {
                         i915_gem_context_unpin(engine->last_context, engine);
                         engine->last_context = NULL;
                 }
         }
 
-        i915_gem_context_unreference(dctx);
-        dev_priv->kernel_context = NULL;
-}
+        /* Force the GPU state to be restored on enabling */
+        if (!i915.enable_execlists) {
+                struct i915_gem_context *ctx;
 
-int i915_gem_context_enable(struct drm_i915_gem_request *req)
-{
-        struct intel_engine_cs *engine = req->engine;
-        int ret;
+                list_for_each_entry(ctx, &dev_priv->context_list, link) {
+                        if (!i915_gem_context_is_default(ctx))
+                                continue;
 
-        if (i915.enable_execlists) {
-                if (engine->init_context == NULL)
-                        return 0;
+                        for_each_engine(engine, dev_priv)
+                                ctx->engine[engine->id].initialised = false;
 
-                ret = engine->init_context(req);
-        } else
-                ret = i915_switch_context(req);
+                        ctx->remap_slice = ALL_L3_SLICES(dev_priv);
+                }
 
-        if (ret) {
-                DRM_ERROR("ring init context: %d\n", ret);
-                return ret;
+                for_each_engine(engine, dev_priv) {
+                        struct intel_context *kce =
+                                &dev_priv->kernel_context->engine[engine->id];
+
+                        kce->initialised = true;
+                }
         }
+}
 
-        return 0;
+void i915_gem_context_fini(struct drm_device *dev)
+{
+        struct drm_i915_private *dev_priv = to_i915(dev);
+        struct i915_gem_context *dctx = dev_priv->kernel_context;
+
+        lockdep_assert_held(&dev->struct_mutex);
+
+        i915_gem_context_unreference(dctx);
+        dev_priv->kernel_context = NULL;
+
+        ida_destroy(&dev_priv->context_hw_ida);
 }
 
 static int context_idr_cleanup(int id, void *p, void *data)
 {
-        struct intel_context *ctx = p;
+        struct i915_gem_context *ctx = p;
 
+        ctx->file_priv = ERR_PTR(-EBADF);
         i915_gem_context_unreference(ctx);
         return 0;
 }
@@ -471,7 +522,7 @@ static int context_idr_cleanup(int id, void *p, void *data)
 int i915_gem_context_open(struct drm_device *dev, struct drm_file *file)
 {
         struct drm_i915_file_private *file_priv = file->driver_priv;
-        struct intel_context *ctx;
+        struct i915_gem_context *ctx;
 
         idr_init(&file_priv->context_idr);
 
@@ -491,31 +542,22 @@ void i915_gem_context_close(struct drm_device *dev, struct drm_file *file)
 {
         struct drm_i915_file_private *file_priv = file->driver_priv;
 
+        lockdep_assert_held(&dev->struct_mutex);
+
         idr_for_each(&file_priv->context_idr, context_idr_cleanup, NULL);
         idr_destroy(&file_priv->context_idr);
 }
 
-struct intel_context *
-i915_gem_context_get(struct drm_i915_file_private *file_priv, u32 id)
-{
-        struct intel_context *ctx;
-
-        ctx = (struct intel_context *)idr_find(&file_priv->context_idr, id);
-        if (!ctx)
-                return ERR_PTR(-ENOENT);
-
-        return ctx;
-}
-
 static inline int
 mi_set_context(struct drm_i915_gem_request *req, u32 hw_flags)
 {
+        struct drm_i915_private *dev_priv = req->i915;
         struct intel_engine_cs *engine = req->engine;
         u32 flags = hw_flags | MI_MM_SPACE_GTT;
         const int num_rings =
                 /* Use an extended w/a on ivb+ if signalling from other rings */
-                i915_semaphore_is_enabled(engine->dev) ?
-                hweight32(INTEL_INFO(engine->dev)->ring_mask) - 1 :
+                i915_semaphore_is_enabled(dev_priv) ?
+                hweight32(INTEL_INFO(dev_priv)->ring_mask) - 1 :
                 0;
         int len, ret;
 
@@ -524,21 +566,21 @@ mi_set_context(struct drm_i915_gem_request *req, u32 hw_flags)
          * explicitly, so we rely on the value at ring init, stored in
          * itlb_before_ctx_switch.
          */
-        if (IS_GEN6(engine->dev)) {
+        if (IS_GEN6(dev_priv)) {
                 ret = engine->flush(req, I915_GEM_GPU_DOMAINS, 0);
                 if (ret)
                         return ret;
         }
 
         /* These flags are for resource streamer on HSW+ */
-        if (IS_HASWELL(engine->dev) || INTEL_INFO(engine->dev)->gen >= 8)
+        if (IS_HASWELL(dev_priv) || INTEL_GEN(dev_priv) >= 8)
                 flags |= (HSW_MI_RS_SAVE_STATE_EN | HSW_MI_RS_RESTORE_STATE_EN);
-        else if (INTEL_INFO(engine->dev)->gen < 8)
+        else if (INTEL_GEN(dev_priv) < 8)
                 flags |= (MI_SAVE_EXT_STATE_EN | MI_RESTORE_EXT_STATE_EN);
 
 
         len = 4;
-        if (INTEL_INFO(engine->dev)->gen >= 7)
+        if (INTEL_GEN(dev_priv) >= 7)
                 len += 2 + (num_rings ? 4*num_rings + 6 : 0);
 
         ret = intel_ring_begin(req, len);
@@ -546,14 +588,14 @@ mi_set_context(struct drm_i915_gem_request *req, u32 hw_flags)
                 return ret;
 
         /* WaProgramMiArbOnOffAroundMiSetContext:ivb,vlv,hsw,bdw,chv */
-        if (INTEL_INFO(engine->dev)->gen >= 7) {
+        if (INTEL_GEN(dev_priv) >= 7) {
                 intel_ring_emit(engine, MI_ARB_ON_OFF | MI_ARB_DISABLE);
                 if (num_rings) {
                         struct intel_engine_cs *signaller;
 
                         intel_ring_emit(engine,
                                         MI_LOAD_REGISTER_IMM(num_rings));
-                        for_each_engine(signaller, to_i915(engine->dev)) {
+                        for_each_engine(signaller, dev_priv) {
                                 if (signaller == engine)
                                         continue;
 
@@ -568,7 +610,7 @@ mi_set_context(struct drm_i915_gem_request *req, u32 hw_flags)
         intel_ring_emit(engine, MI_NOOP);
         intel_ring_emit(engine, MI_SET_CONTEXT);
         intel_ring_emit(engine,
-                        i915_gem_obj_ggtt_offset(req->ctx->legacy_hw_ctx.rcs_state) |
+                        i915_gem_obj_ggtt_offset(req->ctx->engine[RCS].state) |
                         flags);
         /*
          * w/a: MI_SET_CONTEXT must always be followed by MI_NOOP
@@ -576,14 +618,14 @@ mi_set_context(struct drm_i915_gem_request *req, u32 hw_flags)
          */
         intel_ring_emit(engine, MI_NOOP);
 
-        if (INTEL_INFO(engine->dev)->gen >= 7) {
+        if (INTEL_GEN(dev_priv) >= 7) {
                 if (num_rings) {
                         struct intel_engine_cs *signaller;
                         i915_reg_t last_reg = {}; /* keep gcc quiet */
 
                         intel_ring_emit(engine,
                                         MI_LOAD_REGISTER_IMM(num_rings));
-                        for_each_engine(signaller, to_i915(engine->dev)) {
+                        for_each_engine(signaller, dev_priv) {
                                 if (signaller == engine)
                                         continue;
 
@@ -609,45 +651,83 @@ mi_set_context(struct drm_i915_gem_request *req, u32 hw_flags)
         return ret;
 }
 
-static inline bool skip_rcs_switch(struct intel_engine_cs *engine,
-                                   struct intel_context *to)
+static int remap_l3(struct drm_i915_gem_request *req, int slice)
+{
+        u32 *remap_info = req->i915->l3_parity.remap_info[slice];
+        struct intel_engine_cs *engine = req->engine;
+        int i, ret;
+
+        if (!remap_info)
+                return 0;
+
+        ret = intel_ring_begin(req, GEN7_L3LOG_SIZE/4 * 2 + 2);
+        if (ret)
+                return ret;
+
+        /*
+         * Note: We do not worry about the concurrent register cacheline hang
+         * here because no other code should access these registers other than
+         * at initialization time.
+         */
+        intel_ring_emit(engine, MI_LOAD_REGISTER_IMM(GEN7_L3LOG_SIZE/4));
+        for (i = 0; i < GEN7_L3LOG_SIZE/4; i++) {
+                intel_ring_emit_reg(engine, GEN7_L3LOG(slice, i));
+                intel_ring_emit(engine, remap_info[i]);
+        }
+        intel_ring_emit(engine, MI_NOOP);
+        intel_ring_advance(engine);
+
+        return 0;
+}
+
+static inline bool skip_rcs_switch(struct i915_hw_ppgtt *ppgtt,
+                                   struct intel_engine_cs *engine,
+                                   struct i915_gem_context *to)
 {
         if (to->remap_slice)
                 return false;
 
-        if (!to->legacy_hw_ctx.initialized)
+        if (!to->engine[RCS].initialised)
                 return false;
 
-        if (to->ppgtt &&
-            !(intel_engine_flag(engine) & to->ppgtt->pd_dirty_rings))
+        if (ppgtt && (intel_engine_flag(engine) & ppgtt->pd_dirty_rings))
                 return false;
 
         return to == engine->last_context;
 }
 
 static bool
-needs_pd_load_pre(struct intel_engine_cs *engine, struct intel_context *to)
+needs_pd_load_pre(struct i915_hw_ppgtt *ppgtt,
+                  struct intel_engine_cs *engine,
+                  struct i915_gem_context *to)
 {
-        if (!to->ppgtt)
+        if (!ppgtt)
                 return false;
 
+        /* Always load the ppgtt on first use */
+        if (!engine->last_context)
+                return true;
+
+        /* Same context without new entries, skip */
         if (engine->last_context == to &&
-            !(intel_engine_flag(engine) & to->ppgtt->pd_dirty_rings))
+            !(intel_engine_flag(engine) & ppgtt->pd_dirty_rings))
                 return false;
 
         if (engine->id != RCS)
                 return true;
 
-        if (INTEL_INFO(engine->dev)->gen < 8)
+        if (INTEL_GEN(engine->i915) < 8)
                 return true;
 
         return false;
 }
 
 static bool
-needs_pd_load_post(struct intel_context *to, u32 hw_flags)
+needs_pd_load_post(struct i915_hw_ppgtt *ppgtt,
+                   struct i915_gem_context *to,
+                   u32 hw_flags)
 {
-        if (!to->ppgtt)
+        if (!ppgtt)
                 return false;
 
         if (!IS_GEN8(to->i915))
@@ -661,18 +741,19 @@ needs_pd_load_post(struct intel_context *to, u32 hw_flags)
 
 static int do_rcs_switch(struct drm_i915_gem_request *req)
 {
-        struct intel_context *to = req->ctx;
+        struct i915_gem_context *to = req->ctx;
         struct intel_engine_cs *engine = req->engine;
-        struct intel_context *from;
+        struct i915_hw_ppgtt *ppgtt = to->ppgtt ?: req->i915->mm.aliasing_ppgtt;
+        struct i915_gem_context *from;
         u32 hw_flags;
         int ret, i;
 
-        if (skip_rcs_switch(engine, to))
+        if (skip_rcs_switch(ppgtt, engine, to))
                 return 0;
 
         /* Trying to pin first makes error handling easier. */
-        ret = i915_gem_obj_ggtt_pin(to->legacy_hw_ctx.rcs_state,
-                                    get_context_alignment(engine->dev),
+        ret = i915_gem_obj_ggtt_pin(to->engine[RCS].state,
+                                    to->ggtt_alignment,
                                     0);
         if (ret)
                 return ret;
@@ -694,37 +775,32 @@ static int do_rcs_switch(struct drm_i915_gem_request *req)
          *
          * XXX: We need a real interface to do this instead of trickery.
          */
-        ret = i915_gem_object_set_to_gtt_domain(to->legacy_hw_ctx.rcs_state, false);
+        ret = i915_gem_object_set_to_gtt_domain(to->engine[RCS].state, false);
         if (ret)
                 goto unpin_out;
 
-        if (needs_pd_load_pre(engine, to)) {
+        if (needs_pd_load_pre(ppgtt, engine, to)) {
                 /* Older GENs and non render rings still want the load first,
                  * "PP_DCLV followed by PP_DIR_BASE register through Load
                  * Register Immediate commands in Ring Buffer before submitting
                  * a context."*/
                 trace_switch_mm(engine, to);
-                ret = to->ppgtt->switch_mm(to->ppgtt, req);
+                ret = ppgtt->switch_mm(ppgtt, req);
                 if (ret)
                         goto unpin_out;
         }
 
-        if (!to->legacy_hw_ctx.initialized || i915_gem_context_is_default(to))
+        if (!to->engine[RCS].initialised || i915_gem_context_is_default(to))
                 /* NB: If we inhibit the restore, the context is not allowed to
                  * die because future work may end up depending on valid address
                  * space. This means we must enforce that a page table load
                  * occur when this occurs. */
                 hw_flags = MI_RESTORE_INHIBIT;
-        else if (to->ppgtt &&
-                 intel_engine_flag(engine) & to->ppgtt->pd_dirty_rings)
+        else if (ppgtt && intel_engine_flag(engine) & ppgtt->pd_dirty_rings)
                 hw_flags = MI_FORCE_RESTORE;
         else
                 hw_flags = 0;
 
-        /* We should never emit switch_mm more than once */
-        WARN_ON(needs_pd_load_pre(engine, to) &&
-                needs_pd_load_post(to, hw_flags));
-
         if (to != from || (hw_flags & MI_FORCE_RESTORE)) {
                 ret = mi_set_context(req, hw_flags);
                 if (ret)
@@ -738,8 +814,8 @@ static int do_rcs_switch(struct drm_i915_gem_request *req)
          * MI_SET_CONTEXT instead of when the next seqno has completed.
          */
         if (from != NULL) {
-                from->legacy_hw_ctx.rcs_state->base.read_domains = I915_GEM_DOMAIN_INSTRUCTION;
-                i915_vma_move_to_active(i915_gem_obj_to_ggtt(from->legacy_hw_ctx.rcs_state), req);
+                from->engine[RCS].state->base.read_domains = I915_GEM_DOMAIN_INSTRUCTION;
+                i915_vma_move_to_active(i915_gem_obj_to_ggtt(from->engine[RCS].state), req);
                 /* As long as MI_SET_CONTEXT is serializing, ie. it flushes the
                  * whole damn pipeline, we don't need to explicitly mark the
                  * object dirty. The only exception is that the context must be
@@ -747,10 +823,10 @@ static int do_rcs_switch(struct drm_i915_gem_request *req)
                  * able to defer doing this until we know the object would be
                  * swapped, but there is no way to do that yet.
                  */
-                from->legacy_hw_ctx.rcs_state->dirty = 1;
+                from->engine[RCS].state->dirty = 1;
 
                 /* obj is kept alive until the next request by its active ref */
-                i915_gem_object_ggtt_unpin(from->legacy_hw_ctx.rcs_state);
+                i915_gem_object_ggtt_unpin(from->engine[RCS].state);
                 i915_gem_context_unreference(from);
         }
         i915_gem_context_reference(to);
@@ -759,9 +835,9 @@ static int do_rcs_switch(struct drm_i915_gem_request *req)
         /* GEN8 does *not* require an explicit reload if the PDPs have been
          * setup, and we do not wish to move them.
          */
-        if (needs_pd_load_post(to, hw_flags)) {
+        if (needs_pd_load_post(ppgtt, to, hw_flags)) {
                 trace_switch_mm(engine, to);
-                ret = to->ppgtt->switch_mm(to->ppgtt, req);
+                ret = ppgtt->switch_mm(ppgtt, req);
                 /* The hardware context switch is emitted, but we haven't
                  * actually changed the state - so it's probably safe to bail
                  * here. Still, let the user know something dangerous has
@@ -771,33 +847,33 @@ static int do_rcs_switch(struct drm_i915_gem_request *req)
                         return ret;
         }
 
-        if (to->ppgtt)
-                to->ppgtt->pd_dirty_rings &= ~intel_engine_flag(engine);
+        if (ppgtt)
+                ppgtt->pd_dirty_rings &= ~intel_engine_flag(engine);
 
         for (i = 0; i < MAX_L3_SLICES; i++) {
                 if (!(to->remap_slice & (1<<i)))
                         continue;
 
-                ret = i915_gem_l3_remap(req, i);
+                ret = remap_l3(req, i);
                 if (ret)
                         return ret;
 
                 to->remap_slice &= ~(1<<i);
         }
 
-        if (!to->legacy_hw_ctx.initialized) {
+        if (!to->engine[RCS].initialised) {
                 if (engine->init_context) {
                         ret = engine->init_context(req);
                         if (ret)
                                 return ret;
                 }
-                to->legacy_hw_ctx.initialized = true;
+                to->engine[RCS].initialised = true;
         }
 
         return 0;
 
 unpin_out:
-        i915_gem_object_ggtt_unpin(to->legacy_hw_ctx.rcs_state);
+        i915_gem_object_ggtt_unpin(to->engine[RCS].state);
         return ret;
 }
 
@@ -817,25 +893,24 @@ unpin_out:
 int i915_switch_context(struct drm_i915_gem_request *req)
 {
         struct intel_engine_cs *engine = req->engine;
-        struct drm_i915_private *dev_priv = req->i915;
 
         WARN_ON(i915.enable_execlists);
-        WARN_ON(!mutex_is_locked(&dev_priv->dev->struct_mutex));
+        lockdep_assert_held(&req->i915->drm.struct_mutex);
 
-        if (engine->id != RCS ||
-            req->ctx->legacy_hw_ctx.rcs_state == NULL) {
-                struct intel_context *to = req->ctx;
+        if (!req->ctx->engine[engine->id].state) {
+                struct i915_gem_context *to = req->ctx;
+                struct i915_hw_ppgtt *ppgtt =
+                        to->ppgtt ?: req->i915->mm.aliasing_ppgtt;
 
-                if (needs_pd_load_pre(engine, to)) {
+                if (needs_pd_load_pre(ppgtt, engine, to)) {
                         int ret;
 
                         trace_switch_mm(engine, to);
-                        ret = to->ppgtt->switch_mm(to->ppgtt, req);
+                        ret = ppgtt->switch_mm(ppgtt, req);
                         if (ret)
                                 return ret;
 
-                        /* Doing a PD load always reloads the page dirs */
-                        to->ppgtt->pd_dirty_rings &= ~intel_engine_flag(engine);
+                        ppgtt->pd_dirty_rings &= ~intel_engine_flag(engine);
                 }
 
                 if (to != engine->last_context) {
@@ -861,7 +936,7 @@ int i915_gem_context_create_ioctl(struct drm_device *dev, void *data,
 {
         struct drm_i915_gem_context_create *args = data;
         struct drm_i915_file_private *file_priv = file->driver_priv;
-        struct intel_context *ctx;
+        struct i915_gem_context *ctx;
         int ret;
 
         if (!contexts_enabled(dev))
@@ -890,7 +965,7 @@ int i915_gem_context_destroy_ioctl(struct drm_device *dev, void *data,
 {
         struct drm_i915_gem_context_destroy *args = data;
         struct drm_i915_file_private *file_priv = file->driver_priv;
-        struct intel_context *ctx;
+        struct i915_gem_context *ctx;
         int ret;
 
         if (args->pad != 0)
@@ -903,13 +978,13 @@ int i915_gem_context_destroy_ioctl(struct drm_device *dev, void *data,
         if (ret)
                 return ret;
 
-        ctx = i915_gem_context_get(file_priv, args->ctx_id);
+        ctx = i915_gem_context_lookup(file_priv, args->ctx_id);
         if (IS_ERR(ctx)) {
                 mutex_unlock(&dev->struct_mutex);
                 return PTR_ERR(ctx);
         }
 
-        idr_remove(&ctx->file_priv->context_idr, ctx->user_handle);
+        idr_remove(&file_priv->context_idr, ctx->user_handle);
         i915_gem_context_unreference(ctx);
         mutex_unlock(&dev->struct_mutex);
 
@@ -922,14 +997,14 @@ int i915_gem_context_getparam_ioctl(struct drm_device *dev, void *data,
 {
         struct drm_i915_file_private *file_priv = file->driver_priv;
         struct drm_i915_gem_context_param *args = data;
-        struct intel_context *ctx;
+        struct i915_gem_context *ctx;
         int ret;
 
         ret = i915_mutex_lock_interruptible(dev);
         if (ret)
                 return ret;
 
-        ctx = i915_gem_context_get(file_priv, args->ctx_id);
+        ctx = i915_gem_context_lookup(file_priv, args->ctx_id);
         if (IS_ERR(ctx)) {
                 mutex_unlock(&dev->struct_mutex);
                 return PTR_ERR(ctx);
@@ -951,6 +1026,9 @@ int i915_gem_context_getparam_ioctl(struct drm_device *dev, void *data,
                 else
                         args->value = to_i915(dev)->ggtt.base.total;
                 break;
+        case I915_CONTEXT_PARAM_NO_ERROR_CAPTURE:
+                args->value = !!(ctx->flags & CONTEXT_NO_ERROR_CAPTURE);
+                break;
         default:
                 ret = -EINVAL;
                 break;
@@ -965,14 +1043,14 @@ int i915_gem_context_setparam_ioctl(struct drm_device *dev, void *data,
 {
         struct drm_i915_file_private *file_priv = file->driver_priv;
         struct drm_i915_gem_context_param *args = data;
-        struct intel_context *ctx;
+        struct i915_gem_context *ctx;
         int ret;
 
         ret = i915_mutex_lock_interruptible(dev);
         if (ret)
                 return ret;
 
-        ctx = i915_gem_context_get(file_priv, args->ctx_id);
+        ctx = i915_gem_context_lookup(file_priv, args->ctx_id);
         if (IS_ERR(ctx)) {
                 mutex_unlock(&dev->struct_mutex);
                 return PTR_ERR(ctx);
@@ -996,6 +1074,16 @@ int i915_gem_context_setparam_ioctl(struct drm_device *dev, void *data,
                         ctx->flags |= args->value ? CONTEXT_NO_ZEROMAP : 0;
                 }
                 break;
+        case I915_CONTEXT_PARAM_NO_ERROR_CAPTURE:
+                if (args->size) {
+                        ret = -EINVAL;
+                } else {
+                        if (args->value)
+                                ctx->flags |= CONTEXT_NO_ERROR_CAPTURE;
+                        else
+                                ctx->flags &= ~CONTEXT_NO_ERROR_CAPTURE;
+                }
+                break;
         default:
                 ret = -EINVAL;
                 break;
@@ -1004,3 +1092,42 @@ int i915_gem_context_setparam_ioctl(struct drm_device *dev, void *data,
 
         return ret;
 }
+
+int i915_gem_context_reset_stats_ioctl(struct drm_device *dev,
+                                       void *data, struct drm_file *file)
+{
+        struct drm_i915_private *dev_priv = to_i915(dev);
+        struct drm_i915_reset_stats *args = data;
+        struct i915_ctx_hang_stats *hs;
+        struct i915_gem_context *ctx;
+        int ret;
+
+        if (args->flags || args->pad)
+                return -EINVAL;
+
+        if (args->ctx_id == DEFAULT_CONTEXT_HANDLE && !capable(CAP_SYS_ADMIN))
+                return -EPERM;
+
+        ret = i915_mutex_lock_interruptible(dev);
+        if (ret)
+                return ret;
+
+        ctx = i915_gem_context_lookup(file->driver_priv, args->ctx_id);
+        if (IS_ERR(ctx)) {
+                mutex_unlock(&dev->struct_mutex);
+                return PTR_ERR(ctx);
+        }
+        hs = &ctx->hang_stats;
+
+        if (capable(CAP_SYS_ADMIN))
+                args->reset_count = i915_reset_count(&dev_priv->gpu_error);
+        else
+                args->reset_count = 0;
+
+        args->batch_active = hs->batch_active;
+        args->batch_pending = hs->batch_pending;
+
+        mutex_unlock(&dev->struct_mutex);
+
+        return 0;
+}
diff --git a/drivers/gpu/drm/i915/i915_gem_dmabuf.h b/drivers/gpu/drm/i915/i915_gem_dmabuf.h
new file mode 100644
index 000000000000..91315557e421
--- /dev/null
+++ b/drivers/gpu/drm/i915/i915_gem_dmabuf.h
@@ -0,0 +1,45 @@
+/*
+ * Copyright 2016 Intel Corporation
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ *
+ */
+
+#ifndef _I915_GEM_DMABUF_H_
+#define _I915_GEM_DMABUF_H_
+
+#include <linux/dma-buf.h>
+
+static inline struct reservation_object *
+i915_gem_object_get_dmabuf_resv(struct drm_i915_gem_object *obj)
+{
+        struct dma_buf *dma_buf;
+
+        if (obj->base.dma_buf)
+                dma_buf = obj->base.dma_buf;
+        else if (obj->base.import_attach)
+                dma_buf = obj->base.import_attach->dmabuf;
+        else
+                return NULL;
+
+        return dma_buf->resv;
+}
+
+#endif
diff --git a/drivers/gpu/drm/i915/i915_gem_evict.c b/drivers/gpu/drm/i915/i915_gem_evict.c
index ea1f8d1bd228..3c1280ec7ff6 100644
--- a/drivers/gpu/drm/i915/i915_gem_evict.c
+++ b/drivers/gpu/drm/i915/i915_gem_evict.c
@@ -33,6 +33,37 @@
 #include "intel_drv.h"
 #include "i915_trace.h"
 
+static int switch_to_pinned_context(struct drm_i915_private *dev_priv)
+{
+        struct intel_engine_cs *engine;
+
+        if (i915.enable_execlists)
+                return 0;
+
+        for_each_engine(engine, dev_priv) {
+                struct drm_i915_gem_request *req;
+                int ret;
+
+                if (engine->last_context == NULL)
+                        continue;
+
+                if (engine->last_context == dev_priv->kernel_context)
+                        continue;
+
+                req = i915_gem_request_alloc(engine, dev_priv->kernel_context);
+                if (IS_ERR(req))
+                        return PTR_ERR(req);
+
+                ret = i915_switch_context(req);
+                i915_add_request_no_flush(req);
+                if (ret)
+                        return ret;
+        }
+
+        return 0;
+}
+
+
 static bool
 mark_free(struct i915_vma *vma, struct list_head *unwind)
 {
@@ -150,11 +181,19 @@ none:
 
         /* Only idle the GPU and repeat the search once */
         if (pass++ == 0) {
-                ret = i915_gpu_idle(dev);
+                struct drm_i915_private *dev_priv = to_i915(dev);
+
+                if (i915_is_ggtt(vm)) {
+                        ret = switch_to_pinned_context(dev_priv);
+                        if (ret)
+                                return ret;
+                }
+
+                ret = i915_gem_wait_for_idle(dev_priv);
                 if (ret)
                         return ret;
 
-                i915_gem_retire_requests(dev);
+                i915_gem_retire_requests(dev_priv);
                 goto search_again;
         }
 
@@ -261,11 +300,19 @@ int i915_gem_evict_vm(struct i915_address_space *vm, bool do_idle)
         trace_i915_gem_evict_vm(vm);
 
         if (do_idle) {
-                ret = i915_gpu_idle(vm->dev);
+                struct drm_i915_private *dev_priv = to_i915(vm->dev);
+
+                if (i915_is_ggtt(vm)) {
+                        ret = switch_to_pinned_context(dev_priv);
+                        if (ret)
+                                return ret;
+                }
+
+                ret = i915_gem_wait_for_idle(dev_priv);
                 if (ret)
                         return ret;
 
-                i915_gem_retire_requests(vm->dev);
+                i915_gem_retire_requests(dev_priv);
 
                 WARN_ON(!list_empty(&vm->active_list));
         }
diff --git a/drivers/gpu/drm/i915/i915_gem_execbuffer.c b/drivers/gpu/drm/i915/i915_gem_execbuffer.c
index 33df74d98269..1978633e7549 100644
--- a/drivers/gpu/drm/i915/i915_gem_execbuffer.c
+++ b/drivers/gpu/drm/i915/i915_gem_execbuffer.c
@@ -714,7 +714,7 @@ eb_vma_misplaced(struct i915_vma *vma)
 static int
 i915_gem_execbuffer_reserve(struct intel_engine_cs *engine,
                             struct list_head *vmas,
-                            struct intel_context *ctx,
+                            struct i915_gem_context *ctx,
                             bool *need_relocs)
 {
         struct drm_i915_gem_object *obj;
@@ -722,7 +722,7 @@ i915_gem_execbuffer_reserve(struct intel_engine_cs *engine,
         struct i915_address_space *vm;
         struct list_head ordered_vmas;
         struct list_head pinned_vmas;
-        bool has_fenced_gpu_access = INTEL_INFO(engine->dev)->gen < 4;
+        bool has_fenced_gpu_access = INTEL_GEN(engine->i915) < 4;
         int retry;
 
         i915_gem_retire_requests_ring(engine);
@@ -826,7 +826,7 @@ i915_gem_execbuffer_relocate_slow(struct drm_device *dev,
                                   struct intel_engine_cs *engine,
                                   struct eb_vmas *eb,
                                   struct drm_i915_gem_exec_object2 *exec,
-                                  struct intel_context *ctx)
+                                  struct i915_gem_context *ctx)
 {
         struct drm_i915_gem_relocation_entry *reloc;
         struct i915_address_space *vm;
@@ -963,7 +963,7 @@ i915_gem_execbuffer_move_to_gpu(struct drm_i915_gem_request *req,
         }
 
         if (flush_chipset)
-                i915_gem_chipset_flush(req->engine->dev);
+                i915_gem_chipset_flush(req->engine->i915);
 
         if (flush_domains & I915_GEM_DOMAIN_GTT)
                 wmb();
@@ -1063,17 +1063,17 @@ validate_exec_list(struct drm_device *dev,
         return 0;
 }
 
-static struct intel_context *
+static struct i915_gem_context *
 i915_gem_validate_context(struct drm_device *dev, struct drm_file *file,
                           struct intel_engine_cs *engine, const u32 ctx_id)
 {
-        struct intel_context *ctx = NULL;
+        struct i915_gem_context *ctx = NULL;
         struct i915_ctx_hang_stats *hs;
 
         if (engine->id != RCS && ctx_id != DEFAULT_CONTEXT_HANDLE)
                 return ERR_PTR(-EINVAL);
 
-        ctx = i915_gem_context_get(file->driver_priv, ctx_id);
+        ctx = i915_gem_context_lookup(file->driver_priv, ctx_id);
         if (IS_ERR(ctx))
                 return ctx;
 
@@ -1083,14 +1083,6 @@ i915_gem_validate_context(struct drm_device *dev, struct drm_file *file,
                 return ERR_PTR(-EIO);
         }
 
-        if (i915.enable_execlists && !ctx->engine[engine->id].state) {
-                int ret = intel_lr_context_deferred_alloc(ctx, engine);
-                if (ret) {
-                        DRM_DEBUG("Could not create LRC %u: %d\n", ctx_id, ret);
-                        return ERR_PTR(ret);
-                }
-        }
-
         return ctx;
 }
 
@@ -1125,7 +1117,7 @@ i915_gem_execbuffer_move_to_active(struct list_head *vmas,
                 if (entry->flags & EXEC_OBJECT_NEEDS_FENCE) {
                         i915_gem_request_assign(&obj->last_fenced_req, req);
                         if (entry->flags & __EXEC_OBJECT_HAS_FENCE) {
-                                struct drm_i915_private *dev_priv = to_i915(engine->dev);
+                                struct drm_i915_private *dev_priv = engine->i915;
                                 list_move_tail(&dev_priv->fence_regs[obj->fence_reg].lru_list,
                                                &dev_priv->mm.fence_list);
                         }
@@ -1150,7 +1142,7 @@ i915_reset_gen7_sol_offsets(struct drm_device *dev,
                             struct drm_i915_gem_request *req)
 {
         struct intel_engine_cs *engine = req->engine;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         int ret, i;
 
         if (!IS_GEN7(dev) || engine != &dev_priv->engine[RCS]) {
@@ -1233,7 +1225,7 @@ i915_gem_ringbuffer_submission(struct i915_execbuffer_params *params,
 {
         struct drm_device *dev = params->dev;
         struct intel_engine_cs *engine = params->engine;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         u64 exec_start, exec_len;
         int instp_mode;
         u32 instp_mask;
@@ -1336,10 +1328,10 @@ gen8_dispatch_bsd_ring(struct drm_i915_private *dev_priv, struct drm_file *file)
         /* Check whether the file_priv has already selected one ring. */
         if ((int)file_priv->bsd_ring < 0) {
                 /* If not, use the ping-pong mechanism to select one. */
-                mutex_lock(&dev_priv->dev->struct_mutex);
+                mutex_lock(&dev_priv->drm.struct_mutex);
                 file_priv->bsd_ring = dev_priv->mm.bsd_ring_dispatch_index;
                 dev_priv->mm.bsd_ring_dispatch_index ^= 1;
-                mutex_unlock(&dev_priv->dev->struct_mutex);
+                mutex_unlock(&dev_priv->drm.struct_mutex);
         }
 
         return file_priv->bsd_ring;
@@ -1436,7 +1428,7 @@ i915_gem_do_execbuffer(struct drm_device *dev, void *data,
         struct drm_i915_gem_object *batch_obj;
         struct drm_i915_gem_exec_object2 shadow_exec_entry;
         struct intel_engine_cs *engine;
-        struct intel_context *ctx;
+        struct i915_gem_context *ctx;
         struct i915_address_space *vm;
         struct i915_execbuffer_params params_master; /* XXX: will be removed later */
         struct i915_execbuffer_params *params = &params_master;
@@ -1454,7 +1446,7 @@ i915_gem_do_execbuffer(struct drm_device *dev, void *data,
 
         dispatch_flags = 0;
         if (args->flags & I915_EXEC_SECURE) {
-                if (!file->is_master || !capable(CAP_SYS_ADMIN))
+                if (!drm_is_current_master(file) || !capable(CAP_SYS_ADMIN))
                     return -EPERM;
 
                 dispatch_flags |= I915_DISPATCH_SECURE;
@@ -1485,6 +1477,12 @@ i915_gem_do_execbuffer(struct drm_device *dev, void *data,
                 dispatch_flags |= I915_DISPATCH_RS;
         }
 
+        /* Take a local wakeref for preparing to dispatch the execbuf as
+         * we expect to access the hardware fairly frequently in the
+         * process. Upon first dispatch, we acquire another prolonged
+         * wakeref that we hold until the GPU has been idle for at least
+         * 100ms.
+         */
         intel_runtime_pm_get(dev_priv);
 
         ret = i915_mutex_lock_interruptible(dev);
@@ -1561,7 +1559,7 @@ i915_gem_do_execbuffer(struct drm_device *dev, void *data,
                                                              batch_obj,
                                                              args->batch_start_offset,
                                                              args->batch_len,
-                                                             file->is_master);
+                                                             drm_is_current_master(file));
                 if (IS_ERR(parsed_batch_obj)) {
                         ret = PTR_ERR(parsed_batch_obj);
                         goto err;
diff --git a/drivers/gpu/drm/i915/i915_gem_fence.c b/drivers/gpu/drm/i915/i915_gem_fence.c
index a2b938ec01a7..251d7a95af89 100644
--- a/drivers/gpu/drm/i915/i915_gem_fence.c
+++ b/drivers/gpu/drm/i915/i915_gem_fence.c
@@ -58,7 +58,7 @@
 static void i965_write_fence_reg(struct drm_device *dev, int reg,
                                  struct drm_i915_gem_object *obj)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         i915_reg_t fence_reg_lo, fence_reg_hi;
         int fence_pitch_shift;
 
@@ -117,7 +117,7 @@ static void i965_write_fence_reg(struct drm_device *dev, int reg,
 static void i915_write_fence_reg(struct drm_device *dev, int reg,
                                  struct drm_i915_gem_object *obj)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         u32 val;
 
         if (obj) {
@@ -156,7 +156,7 @@ static void i915_write_fence_reg(struct drm_device *dev, int reg,
 static void i830_write_fence_reg(struct drm_device *dev, int reg,
                                 struct drm_i915_gem_object *obj)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         uint32_t val;
 
         if (obj) {
@@ -193,7 +193,7 @@ inline static bool i915_gem_object_needs_mb(struct drm_i915_gem_object *obj)
 static void i915_gem_write_fence(struct drm_device *dev, int reg,
                                  struct drm_i915_gem_object *obj)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
 
         /* Ensure that all CPU reads are completed before installing a fence
          * and all writes before removing the fence.
@@ -229,7 +229,7 @@ static void i915_gem_object_update_fence(struct drm_i915_gem_object *obj,
                                          struct drm_i915_fence_reg *fence,
                                          bool enable)
 {
-        struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(obj->base.dev);
         int reg = fence_number(dev_priv, fence);
 
         i915_gem_write_fence(obj->base.dev, reg, enable ? obj : NULL);
@@ -286,7 +286,7 @@ i915_gem_object_wait_fence(struct drm_i915_gem_object *obj)
 int
 i915_gem_object_put_fence(struct drm_i915_gem_object *obj)
 {
-        struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(obj->base.dev);
         struct drm_i915_fence_reg *fence;
         int ret;
 
@@ -311,7 +311,7 @@ i915_gem_object_put_fence(struct drm_i915_gem_object *obj)
 static struct drm_i915_fence_reg *
 i915_find_fence_reg(struct drm_device *dev)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         struct drm_i915_fence_reg *reg, *avail;
         int i;
 
@@ -367,7 +367,7 @@ int
 i915_gem_object_get_fence(struct drm_i915_gem_object *obj)
 {
         struct drm_device *dev = obj->base.dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         bool enable = obj->tiling_mode != I915_TILING_NONE;
         struct drm_i915_fence_reg *reg;
         int ret;
@@ -433,7 +433,7 @@ bool
 i915_gem_object_pin_fence(struct drm_i915_gem_object *obj)
 {
         if (obj->fence_reg != I915_FENCE_REG_NONE) {
-                struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
+                struct drm_i915_private *dev_priv = to_i915(obj->base.dev);
                 struct i915_vma *ggtt_vma = i915_gem_obj_to_ggtt(obj);
 
                 WARN_ON(!ggtt_vma ||
@@ -457,7 +457,7 @@ void
 i915_gem_object_unpin_fence(struct drm_i915_gem_object *obj)
 {
         if (obj->fence_reg != I915_FENCE_REG_NONE) {
-                struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
+                struct drm_i915_private *dev_priv = to_i915(obj->base.dev);
                 WARN_ON(dev_priv->fence_regs[obj->fence_reg].pin_count <= 0);
                 dev_priv->fence_regs[obj->fence_reg].pin_count--;
         }
@@ -472,7 +472,7 @@ i915_gem_object_unpin_fence(struct drm_i915_gem_object *obj)
  */
 void i915_gem_restore_fences(struct drm_device *dev)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         int i;
 
         for (i = 0; i < dev_priv->num_fence_regs; i++) {
@@ -549,7 +549,7 @@ void i915_gem_restore_fences(struct drm_device *dev)
 void
 i915_gem_detect_bit_6_swizzle(struct drm_device *dev)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         uint32_t swizzle_x = I915_BIT_6_SWIZZLE_UNKNOWN;
         uint32_t swizzle_y = I915_BIT_6_SWIZZLE_UNKNOWN;
 
@@ -745,15 +745,15 @@ i915_gem_swizzle_page(struct page *page)
 void
 i915_gem_object_do_bit_17_swizzle(struct drm_i915_gem_object *obj)
 {
-        struct sg_page_iter sg_iter;
+        struct sgt_iter sgt_iter;
+        struct page *page;
         int i;
 
         if (obj->bit_17 == NULL)
                 return;
 
         i = 0;
-        for_each_sg_page(obj->pages->sgl, &sg_iter, obj->pages->nents, 0) {
-                struct page *page = sg_page_iter_page(&sg_iter);
+        for_each_sgt_page(page, sgt_iter, obj->pages) {
                 char new_bit_17 = page_to_phys(page) >> 17;
                 if ((new_bit_17 & 0x1) !=
                     (test_bit(i, obj->bit_17) != 0)) {
@@ -775,7 +775,8 @@ i915_gem_object_do_bit_17_swizzle(struct drm_i915_gem_object *obj)
 void
 i915_gem_object_save_bit_17_swizzle(struct drm_i915_gem_object *obj)
 {
-        struct sg_page_iter sg_iter;
+        struct sgt_iter sgt_iter;
+        struct page *page;
         int page_count = obj->base.size >> PAGE_SHIFT;
         int i;
 
@@ -790,8 +791,9 @@ i915_gem_object_save_bit_17_swizzle(struct drm_i915_gem_object *obj)
         }
 
         i = 0;
-        for_each_sg_page(obj->pages->sgl, &sg_iter, obj->pages->nents, 0) {
-                if (page_to_phys(sg_page_iter_page(&sg_iter)) & (1 << 17))
+
+        for_each_sgt_page(page, sgt_iter, obj->pages) {
+                if (page_to_phys(page) & (1 << 17))
                         __set_bit(i, obj->bit_17);
                 else
                         __clear_bit(i, obj->bit_17);
diff --git a/drivers/gpu/drm/i915/i915_gem_gtt.c b/drivers/gpu/drm/i915/i915_gem_gtt.c
index 92acdff9dad3..10f1e32767e6 100644
--- a/drivers/gpu/drm/i915/i915_gem_gtt.c
+++ b/drivers/gpu/drm/i915/i915_gem_gtt.c
@@ -93,6 +93,13 @@
  *
  */
 
+static inline struct i915_ggtt *
+i915_vm_to_ggtt(struct i915_address_space *vm)
+{
+        GEM_BUG_ON(!i915_is_ggtt(vm));
+        return container_of(vm, struct i915_ggtt, base);
+}
+
 static int
 i915_get_ggtt_vma_pages(struct i915_vma *vma);
 
@@ -103,25 +110,29 @@ const struct i915_ggtt_view i915_ggtt_view_rotated = {
         .type = I915_GGTT_VIEW_ROTATED,
 };
 
-static int sanitize_enable_ppgtt(struct drm_device *dev, int enable_ppgtt)
+int intel_sanitize_enable_ppgtt(struct drm_i915_private *dev_priv,
+                                int enable_ppgtt)
 {
         bool has_aliasing_ppgtt;
         bool has_full_ppgtt;
         bool has_full_48bit_ppgtt;
 
-        has_aliasing_ppgtt = INTEL_INFO(dev)->gen >= 6;
-        has_full_ppgtt = INTEL_INFO(dev)->gen >= 7;
-        has_full_48bit_ppgtt = IS_BROADWELL(dev) || INTEL_INFO(dev)->gen >= 9;
+        has_aliasing_ppgtt = INTEL_GEN(dev_priv) >= 6;
+        has_full_ppgtt = INTEL_GEN(dev_priv) >= 7;
+        has_full_48bit_ppgtt =
+                IS_BROADWELL(dev_priv) || INTEL_GEN(dev_priv) >= 9;
 
-        if (intel_vgpu_active(dev))
+        if (intel_vgpu_active(dev_priv))
                 has_full_ppgtt = false; /* emulation is too hard */
 
+        if (!has_aliasing_ppgtt)
+                return 0;
+
         /*
          * We don't allow disabling PPGTT for gen9+ as it's a requirement for
          * execlists, the sole mechanism available to submit work.
          */
-        if (INTEL_INFO(dev)->gen < 9 &&
-            (enable_ppgtt == 0 || !has_aliasing_ppgtt))
+        if (enable_ppgtt == 0 && INTEL_GEN(dev_priv) < 9)
                 return 0;
 
         if (enable_ppgtt == 1)
@@ -135,19 +146,19 @@ static int sanitize_enable_ppgtt(struct drm_device *dev, int enable_ppgtt)
 
 #ifdef CONFIG_INTEL_IOMMU
         /* Disable ppgtt on SNB if VT-d is on. */
-        if (INTEL_INFO(dev)->gen == 6 && intel_iommu_gfx_mapped) {
+        if (IS_GEN6(dev_priv) && intel_iommu_gfx_mapped) {
                 DRM_INFO("Disabling PPGTT because VT-d is on\n");
                 return 0;
         }
 #endif
 
         /* Early VLV doesn't have this */
-        if (IS_VALLEYVIEW(dev) && dev->pdev->revision < 0xb) {
+        if (IS_VALLEYVIEW(dev_priv) && dev_priv->drm.pdev->revision < 0xb) {
                 DRM_DEBUG_DRIVER("disabling PPGTT on pre-B3 step VLV\n");
                 return 0;
         }
 
-        if (INTEL_INFO(dev)->gen >= 8 && i915.enable_execlists)
+        if (INTEL_GEN(dev_priv) >= 8 && i915.enable_execlists)
                 return has_full_48bit_ppgtt ? 3 : 2;
         else
                 return has_aliasing_ppgtt ? 1 : 0;
@@ -866,6 +877,7 @@ static void gen8_free_page_tables(struct drm_device *dev,
 static int gen8_init_scratch(struct i915_address_space *vm)
 {
         struct drm_device *dev = vm->dev;
+        int ret;
 
         vm->scratch_page = alloc_scratch_page(dev);
         if (IS_ERR(vm->scratch_page))
@@ -873,24 +885,21 @@ static int gen8_init_scratch(struct i915_address_space *vm)
 
         vm->scratch_pt = alloc_pt(dev);
         if (IS_ERR(vm->scratch_pt)) {
-                free_scratch_page(dev, vm->scratch_page);
-                return PTR_ERR(vm->scratch_pt);
+                ret = PTR_ERR(vm->scratch_pt);
+                goto free_scratch_page;
         }
 
         vm->scratch_pd = alloc_pd(dev);
         if (IS_ERR(vm->scratch_pd)) {
-                free_pt(dev, vm->scratch_pt);
-                free_scratch_page(dev, vm->scratch_page);
-                return PTR_ERR(vm->scratch_pd);
+                ret = PTR_ERR(vm->scratch_pd);
+                goto free_pt;
         }
 
         if (USES_FULL_48BIT_PPGTT(dev)) {
                 vm->scratch_pdp = alloc_pdp(dev);
                 if (IS_ERR(vm->scratch_pdp)) {
-                        free_pd(dev, vm->scratch_pd);
-                        free_pt(dev, vm->scratch_pt);
-                        free_scratch_page(dev, vm->scratch_page);
-                        return PTR_ERR(vm->scratch_pdp);
+                        ret = PTR_ERR(vm->scratch_pdp);
+                        goto free_pd;
                 }
         }
 
@@ -900,6 +909,15 @@ static int gen8_init_scratch(struct i915_address_space *vm)
                 gen8_initialize_pdp(vm, vm->scratch_pdp);
 
         return 0;
+
+free_pd:
+        free_pd(dev, vm->scratch_pd);
+free_pt:
+        free_pt(dev, vm->scratch_pt);
+free_scratch_page:
+        free_scratch_page(dev, vm->scratch_page);
+
+        return ret;
 }
 
 static int gen8_ppgtt_notify_vgt(struct i915_hw_ppgtt *ppgtt, bool create)
@@ -978,7 +996,7 @@ static void gen8_ppgtt_cleanup(struct i915_address_space *vm)
 {
         struct i915_hw_ppgtt *ppgtt = i915_vm_to_ppgtt(vm);
 
-        if (intel_vgpu_active(vm->dev))
+        if (intel_vgpu_active(to_i915(vm->dev)))
                 gen8_ppgtt_notify_vgt(ppgtt, false);
 
         if (!USES_FULL_48BIT_PPGTT(ppgtt->base.dev))
@@ -1529,14 +1547,14 @@ static int gen8_ppgtt_init(struct i915_hw_ppgtt *ppgtt)
                                                               0, 0,
                                                               GEN8_PML4E_SHIFT);
 
-                if (intel_vgpu_active(ppgtt->base.dev)) {
+                if (intel_vgpu_active(to_i915(ppgtt->base.dev))) {
                         ret = gen8_preallocate_top_level_pdps(ppgtt);
                         if (ret)
                                 goto free_scratch;
                 }
         }
 
-        if (intel_vgpu_active(ppgtt->base.dev))
+        if (intel_vgpu_active(to_i915(ppgtt->base.dev)))
                 gen8_ppgtt_notify_vgt(ppgtt, true);
 
         return 0;
@@ -1552,13 +1570,13 @@ static void gen6_dump_ppgtt(struct i915_hw_ppgtt *ppgtt, struct seq_file *m)
         struct i915_page_table *unused;
         gen6_pte_t scratch_pte;
         uint32_t pd_entry;
-        uint32_t  pte, pde, temp;
+        uint32_t  pte, pde;
         uint32_t start = ppgtt->base.start, length = ppgtt->base.total;
 
         scratch_pte = vm->pte_encode(px_dma(vm->scratch_page),
                                      I915_CACHE_LLC, true, 0);
 
-        gen6_for_each_pde(unused, &ppgtt->pd, start, length, temp, pde) {
+        gen6_for_each_pde(unused, &ppgtt->pd, start, length, pde) {
                 u32 expected;
                 gen6_pte_t *pt_vaddr;
                 const dma_addr_t pt_addr = px_dma(ppgtt->pd.page_table[pde]);
@@ -1622,9 +1640,9 @@ static void gen6_write_page_range(struct drm_i915_private *dev_priv,
 {
         struct i915_ggtt *ggtt = &dev_priv->ggtt;
         struct i915_page_table *pt;
-        uint32_t pde, temp;
+        uint32_t pde;
 
-        gen6_for_each_pde(pt, pd, start, length, temp, pde)
+        gen6_for_each_pde(pt, pd, start, length, pde)
                 gen6_write_pde(pd, pde, pt);
 
         /* Make sure write is complete before other code can use this page
@@ -1665,17 +1683,6 @@ static int hsw_mm_switch(struct i915_hw_ppgtt *ppgtt,
         return 0;
 }
 
-static int vgpu_mm_switch(struct i915_hw_ppgtt *ppgtt,
-                          struct drm_i915_gem_request *req)
-{
-        struct intel_engine_cs *engine = req->engine;
-        struct drm_i915_private *dev_priv = to_i915(ppgtt->base.dev);
-
-        I915_WRITE(RING_PP_DIR_DCLV(engine), PP_DIR_DCLV_2G);
-        I915_WRITE(RING_PP_DIR_BASE(engine), get_pd_offset(ppgtt));
-        return 0;
-}
-
 static int gen7_mm_switch(struct i915_hw_ppgtt *ppgtt,
                           struct drm_i915_gem_request *req)
 {
@@ -1713,21 +1720,16 @@ static int gen6_mm_switch(struct i915_hw_ppgtt *ppgtt,
                           struct drm_i915_gem_request *req)
 {
         struct intel_engine_cs *engine = req->engine;
-        struct drm_device *dev = ppgtt->base.dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
-
+        struct drm_i915_private *dev_priv = req->i915;
 
         I915_WRITE(RING_PP_DIR_DCLV(engine), PP_DIR_DCLV_2G);
         I915_WRITE(RING_PP_DIR_BASE(engine), get_pd_offset(ppgtt));
-
-        POSTING_READ(RING_PP_DIR_DCLV(engine));
-
         return 0;
 }
 
 static void gen8_ppgtt_enable(struct drm_device *dev)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         struct intel_engine_cs *engine;
 
         for_each_engine(engine, dev_priv) {
@@ -1739,7 +1741,7 @@ static void gen8_ppgtt_enable(struct drm_device *dev)
 
 static void gen7_ppgtt_enable(struct drm_device *dev)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         struct intel_engine_cs *engine;
         uint32_t ecochk, ecobits;
 
@@ -1764,7 +1766,7 @@ static void gen7_ppgtt_enable(struct drm_device *dev)
 
 static void gen6_ppgtt_enable(struct drm_device *dev)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         uint32_t ecochk, gab_ctl, ecobits;
 
         ecobits = I915_READ(GAC_ECO_BITS);
@@ -1821,20 +1823,19 @@ static void gen6_ppgtt_insert_entries(struct i915_address_space *vm,
                                       enum i915_cache_level cache_level, u32 flags)
 {
         struct i915_hw_ppgtt *ppgtt = i915_vm_to_ppgtt(vm);
-        gen6_pte_t *pt_vaddr;
         unsigned first_entry = start >> PAGE_SHIFT;
         unsigned act_pt = first_entry / GEN6_PTES;
         unsigned act_pte = first_entry % GEN6_PTES;
-        struct sg_page_iter sg_iter;
+        gen6_pte_t *pt_vaddr = NULL;
+        struct sgt_iter sgt_iter;
+        dma_addr_t addr;
 
-        pt_vaddr = NULL;
-        for_each_sg_page(pages->sgl, &sg_iter, pages->nents, 0) {
+        for_each_sgt_dma(addr, sgt_iter, pages) {
                 if (pt_vaddr == NULL)
                         pt_vaddr = kmap_px(ppgtt->pd.page_table[act_pt]);
 
                 pt_vaddr[act_pte] =
-                        vm->pte_encode(sg_page_iter_dma_address(&sg_iter),
-                                       cache_level, true, flags);
+                        vm->pte_encode(addr, cache_level, true, flags);
 
                 if (++act_pte == GEN6_PTES) {
                         kunmap_px(ppgtt, pt_vaddr);
@@ -1843,6 +1844,7 @@ static void gen6_ppgtt_insert_entries(struct i915_address_space *vm,
                         act_pte = 0;
                 }
         }
+
         if (pt_vaddr)
                 kunmap_px(ppgtt, pt_vaddr);
 }
@@ -1857,7 +1859,7 @@ static int gen6_alloc_va_range(struct i915_address_space *vm,
         struct i915_hw_ppgtt *ppgtt = i915_vm_to_ppgtt(vm);
         struct i915_page_table *pt;
         uint32_t start, length, start_save, length_save;
-        uint32_t pde, temp;
+        uint32_t pde;
         int ret;
 
         if (WARN_ON(start_in + length_in > ppgtt->base.total))
@@ -1873,7 +1875,7 @@ static int gen6_alloc_va_range(struct i915_address_space *vm,
          * need allocation. The second stage marks use ptes within the page
          * tables.
          */
-        gen6_for_each_pde(pt, &ppgtt->pd, start, length, temp, pde) {
+        gen6_for_each_pde(pt, &ppgtt->pd, start, length, pde) {
                 if (pt != vm->scratch_pt) {
                         WARN_ON(bitmap_empty(pt->used_ptes, GEN6_PTES));
                         continue;
@@ -1898,7 +1900,7 @@ static int gen6_alloc_va_range(struct i915_address_space *vm,
         start = start_save;
         length = length_save;
 
-        gen6_for_each_pde(pt, &ppgtt->pd, start, length, temp, pde) {
+        gen6_for_each_pde(pt, &ppgtt->pd, start, length, pde) {
                 DECLARE_BITMAP(tmp_bitmap, GEN6_PTES);
 
                 bitmap_zero(tmp_bitmap, GEN6_PTES);
@@ -1967,15 +1969,16 @@ static void gen6_free_scratch(struct i915_address_space *vm)
 static void gen6_ppgtt_cleanup(struct i915_address_space *vm)
 {
         struct i915_hw_ppgtt *ppgtt = i915_vm_to_ppgtt(vm);
+        struct i915_page_directory *pd = &ppgtt->pd;
+        struct drm_device *dev = vm->dev;
         struct i915_page_table *pt;
         uint32_t pde;
 
         drm_mm_remove_node(&ppgtt->node);
 
-        gen6_for_all_pdes(pt, ppgtt, pde) {
+        gen6_for_all_pdes(pt, pd, pde)
                 if (pt != vm->scratch_pt)
-                        free_pt(ppgtt->base.dev, pt);
-        }
+                        free_pt(dev, pt);
 
         gen6_free_scratch(vm);
 }
@@ -2041,9 +2044,9 @@ static void gen6_scratch_va_range(struct i915_hw_ppgtt *ppgtt,
                                   uint64_t start, uint64_t length)
 {
         struct i915_page_table *unused;
-        uint32_t pde, temp;
+        uint32_t pde;
 
-        gen6_for_each_pde(unused, &ppgtt->pd, start, length, temp, pde)
+        gen6_for_each_pde(unused, &ppgtt->pd, start, length, pde)
                 ppgtt->pd.page_table[pde] = ppgtt->base.scratch_pt;
 }
 
@@ -2055,18 +2058,15 @@ static int gen6_ppgtt_init(struct i915_hw_ppgtt *ppgtt)
         int ret;
 
         ppgtt->base.pte_encode = ggtt->base.pte_encode;
-        if (IS_GEN6(dev)) {
+        if (intel_vgpu_active(dev_priv) || IS_GEN6(dev))
                 ppgtt->switch_mm = gen6_mm_switch;
-        } else if (IS_HASWELL(dev)) {
+        else if (IS_HASWELL(dev))
                 ppgtt->switch_mm = hsw_mm_switch;
-        } else if (IS_GEN7(dev)) {
+        else if (IS_GEN7(dev))
                 ppgtt->switch_mm = gen7_mm_switch;
-        } else
+        else
                 BUG();
 
-        if (intel_vgpu_active(dev))
-                ppgtt->switch_mm = vgpu_mm_switch;
-
         ret = gen6_ppgtt_alloc(ppgtt);
         if (ret)
                 return ret;
@@ -2115,7 +2115,7 @@ static void i915_address_space_init(struct i915_address_space *vm,
                                     struct drm_i915_private *dev_priv)
 {
         drm_mm_init(&vm->mm, vm->start, vm->total);
-        vm->dev = dev_priv->dev;
+        vm->dev = &dev_priv->drm;
         INIT_LIST_HEAD(&vm->active_list);
         INIT_LIST_HEAD(&vm->inactive_list);
         list_add_tail(&vm->global_link, &dev_priv->vm_list);
@@ -2123,7 +2123,7 @@ static void i915_address_space_init(struct i915_address_space *vm,
 
 static void gtt_write_workarounds(struct drm_device *dev)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
 
         /* This function is for gtt related workarounds. This function is
          * called on driver load and after a GPU reset, so you can place
@@ -2140,9 +2140,9 @@ static void gtt_write_workarounds(struct drm_device *dev)
                 I915_WRITE(GEN8_L3_LRA_1_GPGPU, GEN9_L3_LRA_1_GPGPU_DEFAULT_VALUE_BXT);
 }
 
-int i915_ppgtt_init(struct drm_device *dev, struct i915_hw_ppgtt *ppgtt)
+static int i915_ppgtt_init(struct drm_device *dev, struct i915_hw_ppgtt *ppgtt)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         int ret = 0;
 
         ret = __hw_ppgtt_init(dev, ppgtt);
@@ -2179,20 +2179,6 @@ int i915_ppgtt_init_hw(struct drm_device *dev)
         return 0;
 }
 
-int i915_ppgtt_init_ring(struct drm_i915_gem_request *req)
-{
-        struct drm_i915_private *dev_priv = req->i915;
-        struct i915_hw_ppgtt *ppgtt = dev_priv->mm.aliasing_ppgtt;
-
-        if (i915.enable_execlists)
-                return 0;
-
-        if (!ppgtt)
-                return 0;
-
-        return ppgtt->switch_mm(ppgtt, req);
-}
-
 struct i915_hw_ppgtt *
 i915_ppgtt_create(struct drm_device *dev, struct drm_i915_file_private *fpriv)
 {
@@ -2257,8 +2243,8 @@ static bool do_idling(struct drm_i915_private *dev_priv)
 
         if (unlikely(ggtt->do_idle_maps)) {
                 dev_priv->mm.interruptible = false;
-                if (i915_gpu_idle(dev_priv->dev)) {
-                        DRM_ERROR("Couldn't idle GPU\n");
+                if (i915_gem_wait_for_idle(dev_priv)) {
+                        DRM_ERROR("Failed to wait for idle; VT'd may hang.\n");
                         /* Wait a bit, in hopes it avoids the hang */
                         udelay(10);
                 }
@@ -2275,12 +2261,11 @@ static void undo_idling(struct drm_i915_private *dev_priv, bool interruptible)
                 dev_priv->mm.interruptible = interruptible;
 }
 
-void i915_check_and_clear_faults(struct drm_device *dev)
+void i915_check_and_clear_faults(struct drm_i915_private *dev_priv)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
         struct intel_engine_cs *engine;
 
-        if (INTEL_INFO(dev)->gen < 6)
+        if (INTEL_INFO(dev_priv)->gen < 6)
                 return;
 
         for_each_engine(engine, dev_priv) {
@@ -2324,7 +2309,7 @@ void i915_gem_suspend_gtt_mappings(struct drm_device *dev)
         if (INTEL_INFO(dev)->gen < 6)
                 return;
 
-        i915_check_and_clear_faults(dev);
+        i915_check_and_clear_faults(dev_priv);
 
         ggtt->base.clear_range(&ggtt->base, ggtt->base.start, ggtt->base.total,
                              true);
@@ -2352,29 +2337,49 @@ static void gen8_set_pte(void __iomem *addr, gen8_pte_t pte)
 #endif
 }
 
+static void gen8_ggtt_insert_page(struct i915_address_space *vm,
+                                  dma_addr_t addr,
+                                  uint64_t offset,
+                                  enum i915_cache_level level,
+                                  u32 unused)
+{
+        struct drm_i915_private *dev_priv = to_i915(vm->dev);
+        gen8_pte_t __iomem *pte =
+                (gen8_pte_t __iomem *)dev_priv->ggtt.gsm +
+                (offset >> PAGE_SHIFT);
+        int rpm_atomic_seq;
+
+        rpm_atomic_seq = assert_rpm_atomic_begin(dev_priv);
+
+        gen8_set_pte(pte, gen8_pte_encode(addr, level, true));
+
+        I915_WRITE(GFX_FLSH_CNTL_GEN6, GFX_FLSH_CNTL_EN);
+        POSTING_READ(GFX_FLSH_CNTL_GEN6);
+
+        assert_rpm_atomic_end(dev_priv, rpm_atomic_seq);
+}
+
 static void gen8_ggtt_insert_entries(struct i915_address_space *vm,
                                      struct sg_table *st,
                                      uint64_t start,
                                      enum i915_cache_level level, u32 unused)
 {
         struct drm_i915_private *dev_priv = to_i915(vm->dev);
-        struct i915_ggtt *ggtt = &dev_priv->ggtt;
-        unsigned first_entry = start >> PAGE_SHIFT;
-        gen8_pte_t __iomem *gtt_entries =
-                (gen8_pte_t __iomem *)ggtt->gsm + first_entry;
-        int i = 0;
-        struct sg_page_iter sg_iter;
-        dma_addr_t addr = 0; /* shut up gcc */
+        struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm);
+        struct sgt_iter sgt_iter;
+        gen8_pte_t __iomem *gtt_entries;
+        gen8_pte_t gtt_entry;
+        dma_addr_t addr;
         int rpm_atomic_seq;
+        int i = 0;
 
         rpm_atomic_seq = assert_rpm_atomic_begin(dev_priv);
 
-        for_each_sg_page(st->sgl, &sg_iter, st->nents, 0) {
-                addr = sg_dma_address(sg_iter.sg) +
-                        (sg_iter.sg_pgoffset << PAGE_SHIFT);
-                gen8_set_pte(&gtt_entries[i],
-                             gen8_pte_encode(addr, level, true));
-                i++;
+        gtt_entries = (gen8_pte_t __iomem *)ggtt->gsm + (start >> PAGE_SHIFT);
+
+        for_each_sgt_dma(addr, sgt_iter, st) {
+                gtt_entry = gen8_pte_encode(addr, level, true);
+                gen8_set_pte(&gtt_entries[i++], gtt_entry);
         }
 
         /*
@@ -2385,8 +2390,7 @@ static void gen8_ggtt_insert_entries(struct i915_address_space *vm,
          * hardware should work, we must keep this posting read for paranoia.
          */
         if (i != 0)
-                WARN_ON(readq(&gtt_entries[i-1])
-                        != gen8_pte_encode(addr, level, true));
+                WARN_ON(readq(&gtt_entries[i-1]) != gtt_entry);
 
         /* This next bit makes the above posting read even more important. We
          * want to flush the TLBs only after we're certain all the PTE updates
@@ -2424,6 +2428,28 @@ static void gen8_ggtt_insert_entries__BKL(struct i915_address_space *vm,
         stop_machine(gen8_ggtt_insert_entries__cb, &arg, NULL);
 }
 
+static void gen6_ggtt_insert_page(struct i915_address_space *vm,
+                                  dma_addr_t addr,
+                                  uint64_t offset,
+                                  enum i915_cache_level level,
+                                  u32 flags)
+{
+        struct drm_i915_private *dev_priv = to_i915(vm->dev);
+        gen6_pte_t __iomem *pte =
+                (gen6_pte_t __iomem *)dev_priv->ggtt.gsm +
+                (offset >> PAGE_SHIFT);
+        int rpm_atomic_seq;
+
+        rpm_atomic_seq = assert_rpm_atomic_begin(dev_priv);
+
+        iowrite32(vm->pte_encode(addr, level, true, flags), pte);
+
+        I915_WRITE(GFX_FLSH_CNTL_GEN6, GFX_FLSH_CNTL_EN);
+        POSTING_READ(GFX_FLSH_CNTL_GEN6);
+
+        assert_rpm_atomic_end(dev_priv, rpm_atomic_seq);
+}
+
 /*
  * Binds an object into the global gtt with the specified cache level. The object
  * will be accessible to the GPU via commands whose operands reference offsets
@@ -2436,21 +2462,21 @@ static void gen6_ggtt_insert_entries(struct i915_address_space *vm,
                                      enum i915_cache_level level, u32 flags)
 {
         struct drm_i915_private *dev_priv = to_i915(vm->dev);
-        struct i915_ggtt *ggtt = &dev_priv->ggtt;
-        unsigned first_entry = start >> PAGE_SHIFT;
-        gen6_pte_t __iomem *gtt_entries =
-                (gen6_pte_t __iomem *)ggtt->gsm + first_entry;
-        int i = 0;
-        struct sg_page_iter sg_iter;
-        dma_addr_t addr = 0;
+        struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm);
+        struct sgt_iter sgt_iter;
+        gen6_pte_t __iomem *gtt_entries;
+        gen6_pte_t gtt_entry;
+        dma_addr_t addr;
         int rpm_atomic_seq;
+        int i = 0;
 
         rpm_atomic_seq = assert_rpm_atomic_begin(dev_priv);
 
-        for_each_sg_page(st->sgl, &sg_iter, st->nents, 0) {
-                addr = sg_page_iter_dma_address(&sg_iter);
-                iowrite32(vm->pte_encode(addr, level, true, flags), &gtt_entries[i]);
-                i++;
+        gtt_entries = (gen6_pte_t __iomem *)ggtt->gsm + (start >> PAGE_SHIFT);
+
+        for_each_sgt_dma(addr, sgt_iter, st) {
+                gtt_entry = vm->pte_encode(addr, level, true, flags);
+                iowrite32(gtt_entry, &gtt_entries[i++]);
         }
 
         /* XXX: This serves as a posting read to make sure that the PTE has
@@ -2459,10 +2485,8 @@ static void gen6_ggtt_insert_entries(struct i915_address_space *vm,
          * of NUMA access patterns. Therefore, even with the way we assume
          * hardware should work, we must keep this posting read for paranoia.
          */
-        if (i != 0) {
-                unsigned long gtt = readl(&gtt_entries[i-1]);
-                WARN_ON(gtt != vm->pte_encode(addr, level, true, flags));
-        }
+        if (i != 0)
+                WARN_ON(readl(&gtt_entries[i-1]) != gtt_entry);
 
         /* This next bit makes the above posting read even more important. We
          * want to flush the TLBs only after we're certain all the PTE updates
@@ -2474,13 +2498,20 @@ static void gen6_ggtt_insert_entries(struct i915_address_space *vm,
         assert_rpm_atomic_end(dev_priv, rpm_atomic_seq);
 }
 
+static void nop_clear_range(struct i915_address_space *vm,
+                            uint64_t start,
+                            uint64_t length,
+                            bool use_scratch)
+{
+}
+
 static void gen8_ggtt_clear_range(struct i915_address_space *vm,
                                   uint64_t start,
                                   uint64_t length,
                                   bool use_scratch)
 {
         struct drm_i915_private *dev_priv = to_i915(vm->dev);
-        struct i915_ggtt *ggtt = &dev_priv->ggtt;
+        struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm);
         unsigned first_entry = start >> PAGE_SHIFT;
         unsigned num_entries = length >> PAGE_SHIFT;
         gen8_pte_t scratch_pte, __iomem *gtt_base =
@@ -2512,7 +2543,7 @@ static void gen6_ggtt_clear_range(struct i915_address_space *vm,
                                   bool use_scratch)
 {
         struct drm_i915_private *dev_priv = to_i915(vm->dev);
-        struct i915_ggtt *ggtt = &dev_priv->ggtt;
+        struct i915_ggtt *ggtt = i915_vm_to_ggtt(vm);
         unsigned first_entry = start >> PAGE_SHIFT;
         unsigned num_entries = length >> PAGE_SHIFT;
         gen6_pte_t scratch_pte, __iomem *gtt_base =
@@ -2538,12 +2569,30 @@ static void gen6_ggtt_clear_range(struct i915_address_space *vm,
         assert_rpm_atomic_end(dev_priv, rpm_atomic_seq);
 }
 
+static void i915_ggtt_insert_page(struct i915_address_space *vm,
+                                  dma_addr_t addr,
+                                  uint64_t offset,
+                                  enum i915_cache_level cache_level,
+                                  u32 unused)
+{
+        struct drm_i915_private *dev_priv = to_i915(vm->dev);
+        unsigned int flags = (cache_level == I915_CACHE_NONE) ?
+                AGP_USER_MEMORY : AGP_USER_CACHED_MEMORY;
+        int rpm_atomic_seq;
+
+        rpm_atomic_seq = assert_rpm_atomic_begin(dev_priv);
+
+        intel_gtt_insert_page(addr, offset >> PAGE_SHIFT, flags);
+
+        assert_rpm_atomic_end(dev_priv, rpm_atomic_seq);
+}
+
 static void i915_ggtt_insert_entries(struct i915_address_space *vm,
                                      struct sg_table *pages,
                                      uint64_t start,
                                      enum i915_cache_level cache_level, u32 unused)
 {
-        struct drm_i915_private *dev_priv = vm->dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(vm->dev);
         unsigned int flags = (cache_level == I915_CACHE_NONE) ?
                 AGP_USER_MEMORY : AGP_USER_CACHED_MEMORY;
         int rpm_atomic_seq;
@@ -2561,7 +2610,7 @@ static void i915_ggtt_clear_range(struct i915_address_space *vm,
                                   uint64_t length,
                                   bool unused)
 {
-        struct drm_i915_private *dev_priv = vm->dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(vm->dev);
         unsigned first_entry = start >> PAGE_SHIFT;
         unsigned num_entries = length >> PAGE_SHIFT;
         int rpm_atomic_seq;
@@ -2642,7 +2691,7 @@ static int aliasing_gtt_bind_vma(struct i915_vma *vma,
 static void ggtt_unbind_vma(struct i915_vma *vma)
 {
         struct drm_device *dev = vma->vm->dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         struct drm_i915_gem_object *obj = vma->obj;
         const uint64_t size = min_t(uint64_t,
                                     obj->base.size,
@@ -2668,7 +2717,7 @@ static void ggtt_unbind_vma(struct i915_vma *vma)
 void i915_gem_gtt_finish_object(struct drm_i915_gem_object *obj)
 {
         struct drm_device *dev = obj->base.dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         bool interruptible;
 
         interruptible = do_idling(dev_priv);
@@ -2727,11 +2776,9 @@ static int i915_gem_setup_global_gtt(struct drm_device *dev,
         i915_address_space_init(&ggtt->base, dev_priv);
         ggtt->base.total += PAGE_SIZE;
 
-        if (intel_vgpu_active(dev)) {
-                ret = intel_vgt_balloon(dev);
-                if (ret)
-                        return ret;
-        }
+        ret = intel_vgt_balloon(dev_priv);
+        if (ret)
+                return ret;
 
         if (!HAS_LLC(dev))
                 ggtt->base.mm.color_adjust = i915_gtt_color_adjust;
@@ -2831,8 +2878,7 @@ void i915_ggtt_cleanup_hw(struct drm_device *dev)
         i915_gem_cleanup_stolen(dev);
 
         if (drm_mm_initialized(&ggtt->base.mm)) {
-                if (intel_vgpu_active(dev))
-                        intel_vgt_deballoon();
+                intel_vgt_deballoon(dev_priv);
 
                 drm_mm_takedown(&ggtt->base.mm);
                 list_del(&ggtt->base.global_link);
@@ -3069,13 +3115,16 @@ static int gen8_gmch_probe(struct i915_ggtt *ggtt)
 
         ret = ggtt_probe_common(dev, ggtt->size);
 
-        ggtt->base.clear_range = gen8_ggtt_clear_range;
-        if (IS_CHERRYVIEW(dev_priv))
-                ggtt->base.insert_entries = gen8_ggtt_insert_entries__BKL;
-        else
-                ggtt->base.insert_entries = gen8_ggtt_insert_entries;
         ggtt->base.bind_vma = ggtt_bind_vma;
         ggtt->base.unbind_vma = ggtt_unbind_vma;
+        ggtt->base.insert_page = gen8_ggtt_insert_page;
+        ggtt->base.clear_range = nop_clear_range;
+        if (!USES_FULL_PPGTT(dev_priv) || intel_scanout_needs_vtd_wa(dev_priv))
+                ggtt->base.clear_range = gen8_ggtt_clear_range;
+
+        ggtt->base.insert_entries = gen8_ggtt_insert_entries;
+        if (IS_CHERRYVIEW(dev_priv))
+                ggtt->base.insert_entries = gen8_ggtt_insert_entries__BKL;
 
         return ret;
 }
@@ -3108,6 +3157,7 @@ static int gen6_gmch_probe(struct i915_ggtt *ggtt)
         ret = ggtt_probe_common(dev, ggtt->size);
 
         ggtt->base.clear_range = gen6_ggtt_clear_range;
+        ggtt->base.insert_page = gen6_ggtt_insert_page;
         ggtt->base.insert_entries = gen6_ggtt_insert_entries;
         ggtt->base.bind_vma = ggtt_bind_vma;
         ggtt->base.unbind_vma = ggtt_unbind_vma;
@@ -3129,7 +3179,7 @@ static int i915_gmch_probe(struct i915_ggtt *ggtt)
         struct drm_i915_private *dev_priv = to_i915(dev);
         int ret;
 
-        ret = intel_gmch_probe(dev_priv->bridge_dev, dev_priv->dev->pdev, NULL);
+        ret = intel_gmch_probe(dev_priv->bridge_dev, dev_priv->drm.pdev, NULL);
         if (!ret) {
                 DRM_ERROR("failed to set up gmch\n");
                 return -EIO;
@@ -3138,7 +3188,8 @@ static int i915_gmch_probe(struct i915_ggtt *ggtt)
         intel_gtt_get(&ggtt->base.total, &ggtt->stolen_size,
                       &ggtt->mappable_base, &ggtt->mappable_end);
 
-        ggtt->do_idle_maps = needs_idle_maps(dev_priv->dev);
+        ggtt->do_idle_maps = needs_idle_maps(&dev_priv->drm);
+        ggtt->base.insert_page = i915_ggtt_insert_page;
         ggtt->base.insert_entries = i915_ggtt_insert_entries;
         ggtt->base.clear_range = i915_ggtt_clear_range;
         ggtt->base.bind_vma = ggtt_bind_vma;
@@ -3219,14 +3270,6 @@ int i915_ggtt_init_hw(struct drm_device *dev)
         if (intel_iommu_gfx_mapped)
                 DRM_INFO("VT-d active for gfx access\n");
 #endif
-        /*
-         * i915.enable_ppgtt is read-only, so do an early pass to validate the
-         * user's requested state against the hardware/driver capabilities.  We
-         * do this now so that we can print out any log messages once rather
-         * than every time we check intel_enable_ppgtt().
-         */
-        i915.enable_ppgtt = sanitize_enable_ppgtt(dev, i915.enable_ppgtt);
-        DRM_DEBUG_DRIVER("ppgtt mode: %i\n", i915.enable_ppgtt);
 
         return 0;
 
@@ -3250,9 +3293,8 @@ void i915_gem_restore_gtt_mappings(struct drm_device *dev)
         struct i915_ggtt *ggtt = &dev_priv->ggtt;
         struct drm_i915_gem_object *obj;
         struct i915_vma *vma;
-        bool flush;
 
-        i915_check_and_clear_faults(dev);
+        i915_check_and_clear_faults(dev_priv);
 
         /* First fill our portion of the GTT with scratch pages */
         ggtt->base.clear_range(&ggtt->base, ggtt->base.start, ggtt->base.total,
@@ -3260,19 +3302,16 @@ void i915_gem_restore_gtt_mappings(struct drm_device *dev)
 
         /* Cache flush objects bound into GGTT and rebind them. */
         list_for_each_entry(obj, &dev_priv->mm.bound_list, global_list) {
-                flush = false;
                 list_for_each_entry(vma, &obj->vma_list, obj_link) {
                         if (vma->vm != &ggtt->base)
                                 continue;
 
                         WARN_ON(i915_vma_bind(vma, obj->cache_level,
                                               PIN_UPDATE));
-
-                        flush = true;
                 }
 
-                if (flush)
-                        i915_gem_clflush_object(obj, obj->pin_display);
+                if (obj->pin_display)
+                        WARN_ON(i915_gem_object_set_to_gtt_domain(obj, false));
         }
 
         if (INTEL_INFO(dev)->gen >= 8) {
@@ -3398,9 +3437,11 @@ static struct sg_table *
 intel_rotate_fb_obj_pages(struct intel_rotation_info *rot_info,
                           struct drm_i915_gem_object *obj)
 {
+        const size_t n_pages = obj->base.size / PAGE_SIZE;
         unsigned int size_pages = rot_info->plane[0].width * rot_info->plane[0].height;
         unsigned int size_pages_uv;
-        struct sg_page_iter sg_iter;
+        struct sgt_iter sgt_iter;
+        dma_addr_t dma_addr;
         unsigned long i;
         dma_addr_t *page_addr_list;
         struct sg_table *st;
@@ -3409,7 +3450,7 @@ intel_rotate_fb_obj_pages(struct intel_rotation_info *rot_info,
         int ret = -ENOMEM;
 
         /* Allocate a temporary list of source pages for random access. */
-        page_addr_list = drm_malloc_gfp(obj->base.size / PAGE_SIZE,
+        page_addr_list = drm_malloc_gfp(n_pages,
                                         sizeof(dma_addr_t),
                                         GFP_TEMPORARY);
         if (!page_addr_list)
@@ -3432,11 +3473,10 @@ intel_rotate_fb_obj_pages(struct intel_rotation_info *rot_info,
 
         /* Populate source page list from the object. */
         i = 0;
-        for_each_sg_page(obj->pages->sgl, &sg_iter, obj->pages->nents, 0) {
-                page_addr_list[i] = sg_page_iter_dma_address(&sg_iter);
-                i++;
-        }
+        for_each_sgt_dma(dma_addr, sgt_iter, obj->pages)
+                page_addr_list[i++] = dma_addr;
 
+        GEM_BUG_ON(i != n_pages);
         st->nents = 0;
         sg = st->sgl;
 
@@ -3634,3 +3674,29 @@ i915_ggtt_view_size(struct drm_i915_gem_object *obj,
                 return obj->base.size;
         }
 }
+
+void __iomem *i915_vma_pin_iomap(struct i915_vma *vma)
+{
+        void __iomem *ptr;
+
+        lockdep_assert_held(&vma->vm->dev->struct_mutex);
+        if (WARN_ON(!vma->obj->map_and_fenceable))
+                return ERR_PTR(-ENODEV);
+
+        GEM_BUG_ON(!vma->is_ggtt);
+        GEM_BUG_ON((vma->bound & GLOBAL_BIND) == 0);
+
+        ptr = vma->iomap;
+        if (ptr == NULL) {
+                ptr = io_mapping_map_wc(i915_vm_to_ggtt(vma->vm)->mappable,
+                                        vma->node.start,
+                                        vma->node.size);
+                if (ptr == NULL)
+                        return ERR_PTR(-ENOMEM);
+
+                vma->iomap = ptr;
+        }
+
+        vma->pin_count++;
+        return ptr;
+}
diff --git a/drivers/gpu/drm/i915/i915_gem_gtt.h b/drivers/gpu/drm/i915/i915_gem_gtt.h
index 0008543d55f6..aa5f31d1c2ed 100644
--- a/drivers/gpu/drm/i915/i915_gem_gtt.h
+++ b/drivers/gpu/drm/i915/i915_gem_gtt.h
@@ -34,6 +34,8 @@
 #ifndef __I915_GEM_GTT_H__
 #define __I915_GEM_GTT_H__
 
+#include <linux/io-mapping.h>
+
 struct drm_i915_file_private;
 
 typedef uint32_t gen6_pte_t;
@@ -175,6 +177,7 @@ struct i915_vma {
         struct drm_mm_node node;
         struct drm_i915_gem_object *obj;
         struct i915_address_space *vm;
+        void __iomem *iomap;
 
         /** Flags and address space this VMA is bound to */
 #define GLOBAL_BIND     (1<<0)
@@ -316,6 +319,11 @@ struct i915_address_space {
                             uint64_t start,
                             uint64_t length,
                             bool use_scratch);
+        void (*insert_page)(struct i915_address_space *vm,
+                            dma_addr_t addr,
+                            uint64_t offset,
+                            enum i915_cache_level cache_level,
+                            u32 flags);
         void (*insert_entries)(struct i915_address_space *vm,
                                struct sg_table *st,
                                uint64_t start,
@@ -382,27 +390,27 @@ struct i915_hw_ppgtt {
         void (*debug_dump)(struct i915_hw_ppgtt *ppgtt, struct seq_file *m);
 };
 
-/* For each pde iterates over every pde between from start until start + length.
- * If start, and start+length are not perfectly divisible, the macro will round
- * down, and up as needed. The macro modifies pde, start, and length. Dev is
- * only used to differentiate shift values. Temp is temp.  On gen6/7, start = 0,
- * and length = 2G effectively iterates over every PDE in the system.
- *
- * XXX: temp is not actually needed, but it saves doing the ALIGN operation.
+/*
+ * gen6_for_each_pde() iterates over every pde from start until start+length.
+ * If start and start+length are not perfectly divisible, the macro will round
+ * down and up as needed. Start=0 and length=2G effectively iterates over
+ * every PDE in the system. The macro modifies ALL its parameters except 'pd',
+ * so each of the other parameters should preferably be a simple variable, or
+ * at most an lvalue with no side-effects!
  */
-#define gen6_for_each_pde(pt, pd, start, length, temp, iter) \
-        for (iter = gen6_pde_index(start); \
-             length > 0 && iter < I915_PDES ? \
-                        (pt = (pd)->page_table[iter]), 1 : 0; \
-             iter++, \
-             temp = ALIGN(start+1, 1 << GEN6_PDE_SHIFT) - start, \
-             temp = min_t(unsigned, temp, length), \
-             start += temp, length -= temp)
-
-#define gen6_for_all_pdes(pt, ppgtt, iter)  \
-        for (iter = 0;          \
-             pt = ppgtt->pd.page_table[iter], iter < I915_PDES; \
-             iter++)
+#define gen6_for_each_pde(pt, pd, start, length, iter)                  \
+        for (iter = gen6_pde_index(start);                              \
+             length > 0 && iter < I915_PDES &&                          \
+                (pt = (pd)->page_table[iter], true);                    \
+             ({ u32 temp = ALIGN(start+1, 1 << GEN6_PDE_SHIFT);         \
+                    temp = min(temp - start, length);                   \
+                    start += temp, length -= temp; }), ++iter)
+
+#define gen6_for_all_pdes(pt, pd, iter)                                 \
+        for (iter = 0;                                                  \
+             iter < I915_PDES &&                                        \
+                (pt = (pd)->page_table[iter], true);                    \
+             ++iter)
 
 static inline uint32_t i915_pte_index(uint64_t address, uint32_t pde_shift)
 {
@@ -518,9 +526,7 @@ int i915_ggtt_enable_hw(struct drm_device *dev);
 void i915_gem_init_ggtt(struct drm_device *dev);
 void i915_ggtt_cleanup_hw(struct drm_device *dev);
 
-int i915_ppgtt_init(struct drm_device *dev, struct i915_hw_ppgtt *ppgtt);
 int i915_ppgtt_init_hw(struct drm_device *dev);
-int i915_ppgtt_init_ring(struct drm_i915_gem_request *req);
 void i915_ppgtt_release(struct kref *kref);
 struct i915_hw_ppgtt *i915_ppgtt_create(struct drm_device *dev,
                                         struct drm_i915_file_private *fpriv);
@@ -535,7 +541,7 @@ static inline void i915_ppgtt_put(struct i915_hw_ppgtt *ppgtt)
                 kref_put(&ppgtt->ref, i915_ppgtt_release);
 }
 
-void i915_check_and_clear_faults(struct drm_device *dev);
+void i915_check_and_clear_faults(struct drm_i915_private *dev_priv);
 void i915_gem_suspend_gtt_mappings(struct drm_device *dev);
 void i915_gem_restore_gtt_mappings(struct drm_device *dev);
 
@@ -560,4 +566,36 @@ size_t
 i915_ggtt_view_size(struct drm_i915_gem_object *obj,
                     const struct i915_ggtt_view *view);
 
+/**
+ * i915_vma_pin_iomap - calls ioremap_wc to map the GGTT VMA via the aperture
+ * @vma: VMA to iomap
+ *
+ * The passed in VMA has to be pinned in the global GTT mappable region.
+ * An extra pinning of the VMA is acquired for the return iomapping,
+ * the caller must call i915_vma_unpin_iomap to relinquish the pinning
+ * after the iomapping is no longer required.
+ *
+ * Callers must hold the struct_mutex.
+ *
+ * Returns a valid iomapped pointer or ERR_PTR.
+ */
+void __iomem *i915_vma_pin_iomap(struct i915_vma *vma);
+
+/**
+ * i915_vma_unpin_iomap - unpins the mapping returned from i915_vma_iomap
+ * @vma: VMA to unpin
+ *
+ * Unpins the previously iomapped VMA from i915_vma_pin_iomap().
+ *
+ * Callers must hold the struct_mutex. This function is only valid to be
+ * called on a VMA previously iomapped by the caller with i915_vma_pin_iomap().
+ */
+static inline void i915_vma_unpin_iomap(struct i915_vma *vma)
+{
+        lockdep_assert_held(&vma->vm->dev->struct_mutex);
+        GEM_BUG_ON(vma->pin_count == 0);
+        GEM_BUG_ON(vma->iomap == NULL);
+        vma->pin_count--;
+}
+
 #endif
diff --git a/drivers/gpu/drm/i915/i915_gem_render_state.c b/drivers/gpu/drm/i915/i915_gem_render_state.c
index 71611bf21fca..f75bbd67a13a 100644
--- a/drivers/gpu/drm/i915/i915_gem_render_state.c
+++ b/drivers/gpu/drm/i915/i915_gem_render_state.c
@@ -29,7 +29,7 @@
 #include "intel_renderstate.h"
 
 static const struct intel_renderstate_rodata *
-render_state_get_rodata(struct drm_device *dev, const int gen)
+render_state_get_rodata(const int gen)
 {
         switch (gen) {
         case 6:
@@ -45,21 +45,22 @@ render_state_get_rodata(struct drm_device *dev, const int gen)
         return NULL;
 }
 
-static int render_state_init(struct render_state *so, struct drm_device *dev)
+static int render_state_init(struct render_state *so,
+                             struct drm_i915_private *dev_priv)
 {
         int ret;
 
-        so->gen = INTEL_INFO(dev)->gen;
-        so->rodata = render_state_get_rodata(dev, so->gen);
+        so->gen = INTEL_GEN(dev_priv);
+        so->rodata = render_state_get_rodata(so->gen);
         if (so->rodata == NULL)
                 return 0;
 
         if (so->rodata->batch_items * 4 > 4096)
                 return -EINVAL;
 
-        so->obj = i915_gem_alloc_object(dev, 4096);
-        if (so->obj == NULL)
-                return -ENOMEM;
+        so->obj = i915_gem_object_create(&dev_priv->drm, 4096);
+        if (IS_ERR(so->obj))
+                return PTR_ERR(so->obj);
 
         ret = i915_gem_obj_ggtt_pin(so->obj, 4096, 0);
         if (ret)
@@ -93,6 +94,7 @@ free_gem:
 
 static int render_state_setup(struct render_state *so)
 {
+        struct drm_device *dev = so->obj->base.dev;
         const struct intel_renderstate_rodata *rodata = so->rodata;
         unsigned int i = 0, reloc_index = 0;
         struct page *page;
@@ -134,6 +136,33 @@ static int render_state_setup(struct render_state *so)
 
         so->aux_batch_offset = i * sizeof(u32);
 
+        if (HAS_POOLED_EU(dev)) {
+                /*
+                 * We always program 3x6 pool config but depending upon which
+                 * subslice is disabled HW drops down to appropriate config
+                 * shown below.
+                 *
+                 * In the below table 2x6 config always refers to
+                 * fused-down version, native 2x6 is not available and can
+                 * be ignored
+                 *
+                 * SNo  subslices config                eu pool configuration
+                 * -----------------------------------------------------------
+                 * 1    3 subslices enabled (3x6)  -    0x00777000  (9+9)
+                 * 2    ss0 disabled (2x6)         -    0x00777000  (3+9)
+                 * 3    ss1 disabled (2x6)         -    0x00770000  (6+6)
+                 * 4    ss2 disabled (2x6)         -    0x00007000  (9+3)
+                 */
+                u32 eu_pool_config = 0x00777000;
+
+                OUT_BATCH(d, i, GEN9_MEDIA_POOL_STATE);
+                OUT_BATCH(d, i, GEN9_MEDIA_POOL_ENABLE);
+                OUT_BATCH(d, i, eu_pool_config);
+                OUT_BATCH(d, i, 0);
+                OUT_BATCH(d, i, 0);
+                OUT_BATCH(d, i, 0);
+        }
+
         OUT_BATCH(d, i, MI_BATCH_BUFFER_END);
         so->aux_batch_size = (i * sizeof(u32)) - so->aux_batch_offset;
 
@@ -177,7 +206,7 @@ int i915_gem_render_state_prepare(struct intel_engine_cs *engine,
         if (WARN_ON(engine->id != RCS))
                 return -ENOENT;
 
-        ret = render_state_init(so, engine->dev);
+        ret = render_state_init(so, engine->i915);
         if (ret)
                 return ret;
 
diff --git a/drivers/gpu/drm/i915/i915_gem_shrinker.c b/drivers/gpu/drm/i915/i915_gem_shrinker.c
index 66571466e9a8..6f10b421487b 100644
--- a/drivers/gpu/drm/i915/i915_gem_shrinker.c
+++ b/drivers/gpu/drm/i915/i915_gem_shrinker.c
@@ -131,7 +131,16 @@ i915_gem_shrink(struct drm_i915_private *dev_priv,
         unsigned long count = 0;
 
         trace_i915_gem_shrink(dev_priv, target, flags);
-        i915_gem_retire_requests(dev_priv->dev);
+        i915_gem_retire_requests(dev_priv);
+
+        /*
+         * Unbinding of objects will require HW access; Let us not wake the
+         * device just to recover a little memory. If absolutely necessary,
+         * we will force the wake during oom-notifier.
+         */
+        if ((flags & I915_SHRINK_BOUND) &&
+            !intel_runtime_pm_get_if_in_use(dev_priv))
+                flags &= ~I915_SHRINK_BOUND;
 
         /*
          * As we may completely rewrite the (un)bound list whilst unbinding
@@ -197,7 +206,10 @@ i915_gem_shrink(struct drm_i915_private *dev_priv,
                 list_splice(&still_in_list, phase->list);
         }
 
-        i915_gem_retire_requests(dev_priv->dev);
+        if (flags & I915_SHRINK_BOUND)
+                intel_runtime_pm_put(dev_priv);
+
+        i915_gem_retire_requests(dev_priv);
 
         return count;
 }
@@ -245,7 +257,7 @@ i915_gem_shrinker_count(struct shrinker *shrinker, struct shrink_control *sc)
 {
         struct drm_i915_private *dev_priv =
                 container_of(shrinker, struct drm_i915_private, mm.shrinker);
-        struct drm_device *dev = dev_priv->dev;
+        struct drm_device *dev = &dev_priv->drm;
         struct drm_i915_gem_object *obj;
         unsigned long count;
         bool unlock;
@@ -253,6 +265,8 @@ i915_gem_shrinker_count(struct shrinker *shrinker, struct shrink_control *sc)
         if (!i915_gem_shrinker_lock(dev, &unlock))
                 return 0;
 
+        i915_gem_retire_requests(dev_priv);
+
         count = 0;
         list_for_each_entry(obj, &dev_priv->mm.unbound_list, global_list)
                 if (can_release_pages(obj))
@@ -274,7 +288,7 @@ i915_gem_shrinker_scan(struct shrinker *shrinker, struct shrink_control *sc)
 {
         struct drm_i915_private *dev_priv =
                 container_of(shrinker, struct drm_i915_private, mm.shrinker);
-        struct drm_device *dev = dev_priv->dev;
+        struct drm_device *dev = &dev_priv->drm;
         unsigned long freed;
         bool unlock;
 
@@ -309,7 +323,7 @@ i915_gem_shrinker_lock_uninterruptible(struct drm_i915_private *dev_priv,
 {
         unsigned long timeout = msecs_to_jiffies(timeout_ms) + 1;
 
-        while (!i915_gem_shrinker_lock(dev_priv->dev, &slu->unlock)) {
+        while (!i915_gem_shrinker_lock(&dev_priv->drm, &slu->unlock)) {
                 schedule_timeout_killable(1);
                 if (fatal_signal_pending(current))
                         return false;
@@ -330,7 +344,7 @@ i915_gem_shrinker_unlock_uninterruptible(struct drm_i915_private *dev_priv,
 {
         dev_priv->mm.interruptible = slu->was_interruptible;
         if (slu->unlock)
-                mutex_unlock(&dev_priv->dev->struct_mutex);
+                mutex_unlock(&dev_priv->drm.struct_mutex);
 }
 
 static int
@@ -345,7 +359,9 @@ i915_gem_shrinker_oom(struct notifier_block *nb, unsigned long event, void *ptr)
         if (!i915_gem_shrinker_lock_uninterruptible(dev_priv, &slu, 5000))
                 return NOTIFY_DONE;
 
+        intel_runtime_pm_get(dev_priv);
         freed_pages = i915_gem_shrink_all(dev_priv);
+        intel_runtime_pm_put(dev_priv);
 
         /* Because we may be allocating inside our own driver, we cannot
          * assert that there are no objects with pinned pages that are not
@@ -386,17 +402,35 @@ i915_gem_shrinker_vmap(struct notifier_block *nb, unsigned long event, void *ptr
         struct drm_i915_private *dev_priv =
                 container_of(nb, struct drm_i915_private, mm.vmap_notifier);
         struct shrinker_lock_uninterruptible slu;
-        unsigned long freed_pages;
+        struct i915_vma *vma, *next;
+        unsigned long freed_pages = 0;
+        int ret;
 
         if (!i915_gem_shrinker_lock_uninterruptible(dev_priv, &slu, 5000))
                 return NOTIFY_DONE;
 
-        freed_pages = i915_gem_shrink(dev_priv, -1UL,
-                                      I915_SHRINK_BOUND |
-                                      I915_SHRINK_UNBOUND |
-                                      I915_SHRINK_ACTIVE |
-                                      I915_SHRINK_VMAPS);
+        /* Force everything onto the inactive lists */
+        ret = i915_gem_wait_for_idle(dev_priv);
+        if (ret)
+                goto out;
+
+        intel_runtime_pm_get(dev_priv);
+        freed_pages += i915_gem_shrink(dev_priv, -1UL,
+                                       I915_SHRINK_BOUND |
+                                       I915_SHRINK_UNBOUND |
+                                       I915_SHRINK_ACTIVE |
+                                       I915_SHRINK_VMAPS);
+        intel_runtime_pm_put(dev_priv);
+
+        /* We also want to clear any cached iomaps as they wrap vmap */
+        list_for_each_entry_safe(vma, next,
+                                 &dev_priv->ggtt.base.inactive_list, vm_link) {
+                unsigned long count = vma->node.size >> PAGE_SHIFT;
+                if (vma->iomap && i915_vma_unbind(vma) == 0)
+                        freed_pages += count;
+        }
 
+out:
         i915_gem_shrinker_unlock_uninterruptible(dev_priv, &slu);
 
         *(unsigned long *)ptr += freed_pages;
diff --git a/drivers/gpu/drm/i915/i915_gem_stolen.c b/drivers/gpu/drm/i915/i915_gem_stolen.c
index 44004e3f09e4..66be299a1486 100644
--- a/drivers/gpu/drm/i915/i915_gem_stolen.c
+++ b/drivers/gpu/drm/i915/i915_gem_stolen.c
@@ -111,9 +111,9 @@ static unsigned long i915_stolen_to_physical(struct drm_device *dev)
         if (INTEL_INFO(dev)->gen >= 3) {
                 u32 bsm;
 
-                pci_read_config_dword(dev->pdev, BSM, &bsm);
+                pci_read_config_dword(dev->pdev, INTEL_BSM, &bsm);
 
-                base = bsm & BSM_MASK;
+                base = bsm & INTEL_BSM_MASK;
         } else if (IS_I865G(dev)) {
                 u16 toud = 0;
 
@@ -270,7 +270,7 @@ static unsigned long i915_stolen_to_physical(struct drm_device *dev)
 
 void i915_gem_cleanup_stolen(struct drm_device *dev)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
 
         if (!drm_mm_initialized(&dev_priv->mm.stolen))
                 return;
@@ -550,7 +550,7 @@ static void i915_gem_object_put_pages_stolen(struct drm_i915_gem_object *obj)
 static void
 i915_gem_object_release_stolen(struct drm_i915_gem_object *obj)
 {
-        struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(obj->base.dev);
 
         if (obj->stolen) {
                 i915_gem_stolen_remove_node(dev_priv, obj->stolen);
@@ -601,7 +601,7 @@ cleanup:
 struct drm_i915_gem_object *
 i915_gem_object_create_stolen(struct drm_device *dev, u32 size)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         struct drm_i915_gem_object *obj;
         struct drm_mm_node *stolen;
         int ret;
diff --git a/drivers/gpu/drm/i915/i915_gem_tiling.c b/drivers/gpu/drm/i915/i915_gem_tiling.c
index b9bdb34032cd..8030199731db 100644
--- a/drivers/gpu/drm/i915/i915_gem_tiling.c
+++ b/drivers/gpu/drm/i915/i915_gem_tiling.c
@@ -125,7 +125,7 @@ i915_gem_object_fence_ok(struct drm_i915_gem_object *obj, int tiling_mode)
         if (INTEL_INFO(obj->base.dev)->gen >= 4)
                 return true;
 
-        if (INTEL_INFO(obj->base.dev)->gen == 3) {
+        if (IS_GEN3(obj->base.dev)) {
                 if (i915_gem_obj_ggtt_offset(obj) & ~I915_FENCE_START_MASK)
                         return false;
         } else {
@@ -162,7 +162,7 @@ i915_gem_set_tiling(struct drm_device *dev, void *data,
                    struct drm_file *file)
 {
         struct drm_i915_gem_set_tiling *args = data;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         struct drm_i915_gem_object *obj;
         int ret = 0;
 
@@ -229,7 +229,7 @@ i915_gem_set_tiling(struct drm_device *dev, void *data,
                  */
                 if (obj->map_and_fenceable &&
                     !i915_gem_object_fence_ok(obj, args->tiling_mode))
-                        ret = i915_gem_object_ggtt_unbind(obj);
+                        ret = i915_vma_unbind(i915_gem_obj_to_ggtt(obj));
 
                 if (ret == 0) {
                         if (obj->pages &&
@@ -294,7 +294,7 @@ i915_gem_get_tiling(struct drm_device *dev, void *data,
                    struct drm_file *file)
 {
         struct drm_i915_gem_get_tiling *args = data;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         struct drm_i915_gem_object *obj;
 
         obj = to_intel_bo(drm_gem_object_lookup(file, args->handle));
diff --git a/drivers/gpu/drm/i915/i915_gem_userptr.c b/drivers/gpu/drm/i915/i915_gem_userptr.c
index 32d9726e38b1..2314c88323e3 100644
--- a/drivers/gpu/drm/i915/i915_gem_userptr.c
+++ b/drivers/gpu/drm/i915/i915_gem_userptr.c
@@ -706,7 +706,8 @@ i915_gem_userptr_get_pages(struct drm_i915_gem_object *obj)
 static void
 i915_gem_userptr_put_pages(struct drm_i915_gem_object *obj)
 {
-        struct sg_page_iter sg_iter;
+        struct sgt_iter sgt_iter;
+        struct page *page;
 
         BUG_ON(obj->userptr.work != NULL);
         __i915_gem_userptr_set_active(obj, false);
@@ -716,9 +717,7 @@ i915_gem_userptr_put_pages(struct drm_i915_gem_object *obj)
 
         i915_gem_gtt_finish_object(obj);
 
-        for_each_sg_page(obj->pages->sgl, &sg_iter, obj->pages->nents, 0) {
-                struct page *page = sg_page_iter_page(&sg_iter);
-
+        for_each_sgt_page(page, sgt_iter, obj->pages) {
                 if (obj->dirty)
                         set_page_dirty(page);
 
@@ -855,11 +854,8 @@ i915_gem_userptr_ioctl(struct drm_device *dev, void *data, struct drm_file *file
         return 0;
 }
 
-int
-i915_gem_init_userptr(struct drm_device *dev)
+void i915_gem_init_userptr(struct drm_i915_private *dev_priv)
 {
-        struct drm_i915_private *dev_priv = to_i915(dev);
         mutex_init(&dev_priv->mm_lock);
         hash_init(dev_priv->mm_structs);
-        return 0;
 }
diff --git a/drivers/gpu/drm/i915/i915_gpu_error.c b/drivers/gpu/drm/i915/i915_gpu_error.c
index 89725c9efc25..9d73d2216adc 100644
--- a/drivers/gpu/drm/i915/i915_gpu_error.c
+++ b/drivers/gpu/drm/i915/i915_gpu_error.c
@@ -332,7 +332,7 @@ int i915_error_state_to_str(struct drm_i915_error_state_buf *m,
                             const struct i915_error_state_file_priv *error_priv)
 {
         struct drm_device *dev = error_priv->dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         struct drm_i915_error_state *error = error_priv->error;
         struct drm_i915_error_object *obj;
         int i, j, offset, elt;
@@ -411,7 +411,7 @@ int i915_error_state_to_str(struct drm_i915_error_state_buf *m,
                 err_printf(m, "DONE_REG: 0x%08x\n", error->done_reg);
         }
 
-        if (INTEL_INFO(dev)->gen == 7)
+        if (IS_GEN7(dev))
                 err_printf(m, "ERR_INT: 0x%08x\n", error->err_int);
 
         for (i = 0; i < ARRAY_SIZE(error->ring); i++)
@@ -463,6 +463,18 @@ int i915_error_state_to_str(struct drm_i915_error_state_buf *m,
                         }
                 }
 
+                if (error->ring[i].num_waiters) {
+                        err_printf(m, "%s --- %d waiters\n",
+                                   dev_priv->engine[i].name,
+                                   error->ring[i].num_waiters);
+                        for (j = 0; j < error->ring[i].num_waiters; j++) {
+                                err_printf(m, " seqno 0x%08x for %s [%d]\n",
+                                           error->ring[i].waiters[j].seqno,
+                                           error->ring[i].waiters[j].comm,
+                                           error->ring[i].waiters[j].pid);
+                        }
+                }
+
                 if ((obj = error->ring[i].ringbuffer)) {
                         err_printf(m, "%s --- ringbuffer = 0x%08x\n",
                                    dev_priv->engine[i].name,
@@ -488,7 +500,7 @@ int i915_error_state_to_str(struct drm_i915_error_state_buf *m,
                                            hws_page[elt+1],
                                            hws_page[elt+2],
                                            hws_page[elt+3]);
-                                        offset += 16;
+                                offset += 16;
                         }
                 }
 
@@ -605,8 +617,9 @@ static void i915_error_state_free(struct kref *error_ref)
                 i915_error_object_free(error->ring[i].ringbuffer);
                 i915_error_object_free(error->ring[i].hws_page);
                 i915_error_object_free(error->ring[i].ctx);
-                kfree(error->ring[i].requests);
                 i915_error_object_free(error->ring[i].wa_ctx);
+                kfree(error->ring[i].requests);
+                kfree(error->ring[i].waiters);
         }
 
         i915_error_object_free(error->semaphore_obj);
@@ -824,19 +837,18 @@ static uint32_t i915_error_generate_code(struct drm_i915_private *dev_priv,
         return error_code;
 }
 
-static void i915_gem_record_fences(struct drm_device *dev,
+static void i915_gem_record_fences(struct drm_i915_private *dev_priv,
                                    struct drm_i915_error_state *error)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
         int i;
 
-        if (IS_GEN3(dev) || IS_GEN2(dev)) {
+        if (IS_GEN3(dev_priv) || IS_GEN2(dev_priv)) {
                 for (i = 0; i < dev_priv->num_fence_regs; i++)
                         error->fence[i] = I915_READ(FENCE_REG(i));
-        } else if (IS_GEN5(dev) || IS_GEN4(dev)) {
+        } else if (IS_GEN5(dev_priv) || IS_GEN4(dev_priv)) {
                 for (i = 0; i < dev_priv->num_fence_regs; i++)
                         error->fence[i] = I915_READ64(FENCE_REG_965_LO(i));
-        } else if (INTEL_INFO(dev)->gen >= 6) {
+        } else if (INTEL_GEN(dev_priv) >= 6) {
                 for (i = 0; i < dev_priv->num_fence_regs; i++)
                         error->fence[i] = I915_READ64(FENCE_REG_GEN6_LO(i));
         }
@@ -851,7 +863,7 @@ static void gen8_record_semaphore_state(struct drm_i915_private *dev_priv,
         struct intel_engine_cs *to;
         enum intel_engine_id id;
 
-        if (!i915_semaphore_is_enabled(dev_priv->dev))
+        if (!i915_semaphore_is_enabled(dev_priv))
                 return;
 
         if (!error->semaphore_obj)
@@ -893,31 +905,71 @@ static void gen6_record_semaphore_state(struct drm_i915_private *dev_priv,
         }
 }
 
-static void i915_record_ring_state(struct drm_device *dev,
+static void engine_record_waiters(struct intel_engine_cs *engine,
+                                  struct drm_i915_error_ring *ering)
+{
+        struct intel_breadcrumbs *b = &engine->breadcrumbs;
+        struct drm_i915_error_waiter *waiter;
+        struct rb_node *rb;
+        int count;
+
+        ering->num_waiters = 0;
+        ering->waiters = NULL;
+
+        spin_lock(&b->lock);
+        count = 0;
+        for (rb = rb_first(&b->waiters); rb != NULL; rb = rb_next(rb))
+                count++;
+        spin_unlock(&b->lock);
+
+        waiter = NULL;
+        if (count)
+                waiter = kmalloc_array(count,
+                                       sizeof(struct drm_i915_error_waiter),
+                                       GFP_ATOMIC);
+        if (!waiter)
+                return;
+
+        ering->waiters = waiter;
+
+        spin_lock(&b->lock);
+        for (rb = rb_first(&b->waiters); rb; rb = rb_next(rb)) {
+                struct intel_wait *w = container_of(rb, typeof(*w), node);
+
+                strcpy(waiter->comm, w->tsk->comm);
+                waiter->pid = w->tsk->pid;
+                waiter->seqno = w->seqno;
+                waiter++;
+
+                if (++ering->num_waiters == count)
+                        break;
+        }
+        spin_unlock(&b->lock);
+}
+
+static void i915_record_ring_state(struct drm_i915_private *dev_priv,
                                    struct drm_i915_error_state *error,
                                    struct intel_engine_cs *engine,
                                    struct drm_i915_error_ring *ering)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
-
-        if (INTEL_INFO(dev)->gen >= 6) {
+        if (INTEL_GEN(dev_priv) >= 6) {
                 ering->rc_psmi = I915_READ(RING_PSMI_CTL(engine->mmio_base));
                 ering->fault_reg = I915_READ(RING_FAULT_REG(engine));
-                if (INTEL_INFO(dev)->gen >= 8)
+                if (INTEL_GEN(dev_priv) >= 8)
                         gen8_record_semaphore_state(dev_priv, error, engine,
                                                     ering);
                 else
                         gen6_record_semaphore_state(dev_priv, engine, ering);
         }
 
-        if (INTEL_INFO(dev)->gen >= 4) {
+        if (INTEL_GEN(dev_priv) >= 4) {
                 ering->faddr = I915_READ(RING_DMA_FADD(engine->mmio_base));
                 ering->ipeir = I915_READ(RING_IPEIR(engine->mmio_base));
                 ering->ipehr = I915_READ(RING_IPEHR(engine->mmio_base));
                 ering->instdone = I915_READ(RING_INSTDONE(engine->mmio_base));
                 ering->instps = I915_READ(RING_INSTPS(engine->mmio_base));
                 ering->bbaddr = I915_READ(RING_BBADDR(engine->mmio_base));
-                if (INTEL_INFO(dev)->gen >= 8) {
+                if (INTEL_GEN(dev_priv) >= 8) {
                         ering->faddr |= (u64) I915_READ(RING_DMA_FADD_UDW(engine->mmio_base)) << 32;
                         ering->bbaddr |= (u64) I915_READ(RING_BBADDR_UDW(engine->mmio_base)) << 32;
                 }
@@ -929,20 +981,20 @@ static void i915_record_ring_state(struct drm_device *dev,
                 ering->instdone = I915_READ(GEN2_INSTDONE);
         }
 
-        ering->waiting = waitqueue_active(&engine->irq_queue);
+        ering->waiting = intel_engine_has_waiter(engine);
         ering->instpm = I915_READ(RING_INSTPM(engine->mmio_base));
         ering->acthd = intel_ring_get_active_head(engine);
-        ering->seqno = engine->get_seqno(engine);
+        ering->seqno = intel_engine_get_seqno(engine);
         ering->last_seqno = engine->last_submitted_seqno;
         ering->start = I915_READ_START(engine);
         ering->head = I915_READ_HEAD(engine);
         ering->tail = I915_READ_TAIL(engine);
         ering->ctl = I915_READ_CTL(engine);
 
-        if (I915_NEED_GFX_HWS(dev)) {
+        if (I915_NEED_GFX_HWS(dev_priv)) {
                 i915_reg_t mmio;
 
-                if (IS_GEN7(dev)) {
+                if (IS_GEN7(dev_priv)) {
                         switch (engine->id) {
                         default:
                         case RCS:
@@ -958,7 +1010,7 @@ static void i915_record_ring_state(struct drm_device *dev,
                                 mmio = VEBOX_HWS_PGA_GEN7;
                                 break;
                         }
-                } else if (IS_GEN6(engine->dev)) {
+                } else if (IS_GEN6(engine->i915)) {
                         mmio = RING_HWS_PGA_GEN6(engine->mmio_base);
                 } else {
                         /* XXX: gen8 returns to sanity */
@@ -971,18 +1023,18 @@ static void i915_record_ring_state(struct drm_device *dev,
         ering->hangcheck_score = engine->hangcheck.score;
         ering->hangcheck_action = engine->hangcheck.action;
 
-        if (USES_PPGTT(dev)) {
+        if (USES_PPGTT(dev_priv)) {
                 int i;
 
                 ering->vm_info.gfx_mode = I915_READ(RING_MODE_GEN7(engine));
 
-                if (IS_GEN6(dev))
+                if (IS_GEN6(dev_priv))
                         ering->vm_info.pp_dir_base =
                                 I915_READ(RING_PP_DIR_BASE_READ(engine));
-                else if (IS_GEN7(dev))
+                else if (IS_GEN7(dev_priv))
                         ering->vm_info.pp_dir_base =
                                 I915_READ(RING_PP_DIR_BASE(engine));
-                else if (INTEL_INFO(dev)->gen >= 8)
+                else if (INTEL_GEN(dev_priv) >= 8)
                         for (i = 0; i < 4; i++) {
                                 ering->vm_info.pdp[i] =
                                         I915_READ(GEN8_RING_PDP_UDW(engine, i));
@@ -998,7 +1050,7 @@ static void i915_gem_record_active_context(struct intel_engine_cs *engine,
                                            struct drm_i915_error_state *error,
                                            struct drm_i915_error_ring *ering)
 {
-        struct drm_i915_private *dev_priv = engine->dev->dev_private;
+        struct drm_i915_private *dev_priv = engine->i915;
         struct drm_i915_gem_object *obj;
 
         /* Currently render ring is the only HW context user */
@@ -1016,34 +1068,33 @@ static void i915_gem_record_active_context(struct intel_engine_cs *engine,
         }
 }
 
-static void i915_gem_record_rings(struct drm_device *dev,
+static void i915_gem_record_rings(struct drm_i915_private *dev_priv,
                                   struct drm_i915_error_state *error)
 {
-        struct drm_i915_private *dev_priv = to_i915(dev);
         struct i915_ggtt *ggtt = &dev_priv->ggtt;
         struct drm_i915_gem_request *request;
         int i, count;
 
         for (i = 0; i < I915_NUM_ENGINES; i++) {
                 struct intel_engine_cs *engine = &dev_priv->engine[i];
-                struct intel_ringbuffer *rbuf;
 
                 error->ring[i].pid = -1;
 
-                if (engine->dev == NULL)
+                if (!intel_engine_initialized(engine))
                         continue;
 
                 error->ring[i].valid = true;
 
-                i915_record_ring_state(dev, error, engine, &error->ring[i]);
+                i915_record_ring_state(dev_priv, error, engine, &error->ring[i]);
+                engine_record_waiters(engine, &error->ring[i]);
 
                 request = i915_gem_find_active_request(engine);
                 if (request) {
                         struct i915_address_space *vm;
+                        struct intel_ringbuffer *rb;
 
-                        vm = request->ctx && request->ctx->ppgtt ?
-                                &request->ctx->ppgtt->base :
-                                &ggtt->base;
+                        vm = request->ctx->ppgtt ?
+                                &request->ctx->ppgtt->base : &ggtt->base;
 
                         /* We need to copy these to an anonymous buffer
                          * as the simplest method to avoid being overwritten
@@ -1070,26 +1121,17 @@ static void i915_gem_record_rings(struct drm_device *dev,
                                 }
                                 rcu_read_unlock();
                         }
-                }
 
-                if (i915.enable_execlists) {
-                        /* TODO: This is only a small fix to keep basic error
-                         * capture working, but we need to add more information
-                         * for it to be useful (e.g. dump the context being
-                         * executed).
-                         */
-                        if (request)
-                                rbuf = request->ctx->engine[engine->id].ringbuf;
-                        else
-                                rbuf = dev_priv->kernel_context->engine[engine->id].ringbuf;
-                } else
-                        rbuf = engine->buffer;
-
-                error->ring[i].cpu_ring_head = rbuf->head;
-                error->ring[i].cpu_ring_tail = rbuf->tail;
+                        error->simulated |=
+                                request->ctx->flags & CONTEXT_NO_ERROR_CAPTURE;
 
-                error->ring[i].ringbuffer =
-                        i915_error_ggtt_object_create(dev_priv, rbuf->obj);
+                        rb = request->ringbuf;
+                        error->ring[i].cpu_ring_head = rb->head;
+                        error->ring[i].cpu_ring_tail = rb->tail;
+                        error->ring[i].ringbuffer =
+                                i915_error_ggtt_object_create(dev_priv,
+                                                              rb->obj);
+                }
 
                 error->ring[i].hws_page =
                         i915_error_ggtt_object_create(dev_priv,
@@ -1234,7 +1276,7 @@ static void i915_gem_capture_buffers(struct drm_i915_private *dev_priv,
 static void i915_capture_reg_state(struct drm_i915_private *dev_priv,
                                    struct drm_i915_error_state *error)
 {
-        struct drm_device *dev = dev_priv->dev;
+        struct drm_device *dev = &dev_priv->drm;
         int i;
 
         /* General organization
@@ -1301,15 +1343,14 @@ static void i915_capture_reg_state(struct drm_i915_private *dev_priv,
         error->eir = I915_READ(EIR);
         error->pgtbl_er = I915_READ(PGTBL_ER);
 
-        i915_get_extra_instdone(dev, error->extra_instdone);
+        i915_get_extra_instdone(dev_priv, error->extra_instdone);
 }
 
-static void i915_error_capture_msg(struct drm_device *dev,
+static void i915_error_capture_msg(struct drm_i915_private *dev_priv,
                                    struct drm_i915_error_state *error,
                                    u32 engine_mask,
                                    const char *error_msg)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
         u32 ecode;
         int ring_id = -1, len;
 
@@ -1317,7 +1358,7 @@ static void i915_error_capture_msg(struct drm_device *dev,
 
         len = scnprintf(error->error_msg, sizeof(error->error_msg),
                         "GPU HANG: ecode %d:%d:0x%08x",
-                        INTEL_INFO(dev)->gen, ring_id, ecode);
+                        INTEL_GEN(dev_priv), ring_id, ecode);
 
         if (ring_id != -1 && error->ring[ring_id].pid != -1)
                 len += scnprintf(error->error_msg + len,
@@ -1352,14 +1393,17 @@ static void i915_capture_gen_state(struct drm_i915_private *dev_priv,
  * out a structure which becomes available in debugfs for user level tools
  * to pick up.
  */
-void i915_capture_error_state(struct drm_device *dev, u32 engine_mask,
+void i915_capture_error_state(struct drm_i915_private *dev_priv,
+                              u32 engine_mask,
                               const char *error_msg)
 {
         static bool warned;
-        struct drm_i915_private *dev_priv = dev->dev_private;
         struct drm_i915_error_state *error;
         unsigned long flags;
 
+        if (READ_ONCE(dev_priv->gpu_error.first_error))
+                return;
+
         /* Account for pipe specific data like PIPE*STAT */
         error = kzalloc(sizeof(*error), GFP_ATOMIC);
         if (!error) {
@@ -1372,23 +1416,25 @@ void i915_capture_error_state(struct drm_device *dev, u32 engine_mask,
         i915_capture_gen_state(dev_priv, error);
         i915_capture_reg_state(dev_priv, error);
         i915_gem_capture_buffers(dev_priv, error);
-        i915_gem_record_fences(dev, error);
-        i915_gem_record_rings(dev, error);
+        i915_gem_record_fences(dev_priv, error);
+        i915_gem_record_rings(dev_priv, error);
 
         do_gettimeofday(&error->time);
 
-        error->overlay = intel_overlay_capture_error_state(dev);
-        error->display = intel_display_capture_error_state(dev);
+        error->overlay = intel_overlay_capture_error_state(dev_priv);
+        error->display = intel_display_capture_error_state(dev_priv);
 
-        i915_error_capture_msg(dev, error, engine_mask, error_msg);
+        i915_error_capture_msg(dev_priv, error, engine_mask, error_msg);
         DRM_INFO("%s\n", error->error_msg);
 
-        spin_lock_irqsave(&dev_priv->gpu_error.lock, flags);
-        if (dev_priv->gpu_error.first_error == NULL) {
-                dev_priv->gpu_error.first_error = error;
-                error = NULL;
+        if (!error->simulated) {
+                spin_lock_irqsave(&dev_priv->gpu_error.lock, flags);
+                if (!dev_priv->gpu_error.first_error) {
+                        dev_priv->gpu_error.first_error = error;
+                        error = NULL;
+                }
+                spin_unlock_irqrestore(&dev_priv->gpu_error.lock, flags);
         }
-        spin_unlock_irqrestore(&dev_priv->gpu_error.lock, flags);
 
         if (error) {
                 i915_error_state_free(&error->ref);
@@ -1400,7 +1446,8 @@ void i915_capture_error_state(struct drm_device *dev, u32 engine_mask,
                 DRM_INFO("Please file a _new_ bug report on bugs.freedesktop.org against DRI -> DRM/Intel\n");
                 DRM_INFO("drm/i915 developers can then reassign to the right component if it's not a kernel issue.\n");
                 DRM_INFO("The gpu crash dump is required to analyze gpu hangs, so please always attach it.\n");
-                DRM_INFO("GPU crash dump saved to /sys/class/drm/card%d/error\n", dev->primary->index);
+                DRM_INFO("GPU crash dump saved to /sys/class/drm/card%d/error\n",
+                         dev_priv->drm.primary->index);
                 warned = true;
         }
 }
@@ -1408,7 +1455,7 @@ void i915_capture_error_state(struct drm_device *dev, u32 engine_mask,
 void i915_error_state_get(struct drm_device *dev,
                           struct i915_error_state_file_priv *error_priv)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
 
         spin_lock_irq(&dev_priv->gpu_error.lock);
         error_priv->error = dev_priv->gpu_error.first_error;
@@ -1426,7 +1473,7 @@ void i915_error_state_put(struct i915_error_state_file_priv *error_priv)
 
 void i915_destroy_error_state(struct drm_device *dev)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         struct drm_i915_error_state *error;
 
         spin_lock_irq(&dev_priv->gpu_error.lock);
@@ -1450,17 +1497,17 @@ const char *i915_cache_level_str(struct drm_i915_private *i915, int type)
 }
 
 /* NB: please notice the memset */
-void i915_get_extra_instdone(struct drm_device *dev, uint32_t *instdone)
+void i915_get_extra_instdone(struct drm_i915_private *dev_priv,
+                             uint32_t *instdone)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
         memset(instdone, 0, sizeof(*instdone) * I915_NUM_INSTDONE_REG);
 
-        if (IS_GEN2(dev) || IS_GEN3(dev))
+        if (IS_GEN2(dev_priv) || IS_GEN3(dev_priv))
                 instdone[0] = I915_READ(GEN2_INSTDONE);
-        else if (IS_GEN4(dev) || IS_GEN5(dev) || IS_GEN6(dev)) {
+        else if (IS_GEN4(dev_priv) || IS_GEN5(dev_priv) || IS_GEN6(dev_priv)) {
                 instdone[0] = I915_READ(RING_INSTDONE(RENDER_RING_BASE));
                 instdone[1] = I915_READ(GEN4_INSTDONE1);
-        } else if (INTEL_INFO(dev)->gen >= 7) {
+        } else if (INTEL_GEN(dev_priv) >= 7) {
                 instdone[0] = I915_READ(RING_INSTDONE(RENDER_RING_BASE));
                 instdone[1] = I915_READ(GEN7_SC_INSTDONE);
                 instdone[2] = I915_READ(GEN7_SAMPLER_INSTDONE);
diff --git a/drivers/gpu/drm/i915/i915_guc_reg.h b/drivers/gpu/drm/i915/i915_guc_reg.h
index 80786d9f9ad3..cf5a65be4fe0 100644
--- a/drivers/gpu/drm/i915/i915_guc_reg.h
+++ b/drivers/gpu/drm/i915/i915_guc_reg.h
@@ -67,11 +67,11 @@
 #define   GUC_WOPCM_OFFSET_VALUE          0x80000       /* 512KB */
 #define GUC_MAX_IDLE_COUNT              _MMIO(0xC3E4)
 
+/* Defines WOPCM space available to GuC firmware */
 #define GUC_WOPCM_SIZE                  _MMIO(0xc050)
-#define   GUC_WOPCM_SIZE_VALUE            (0x80 << 12)  /* 512KB */
-
 /* GuC addresses below GUC_WOPCM_TOP don't map through the GTT */
-#define GUC_WOPCM_TOP                   (GUC_WOPCM_SIZE_VALUE)
+#define   GUC_WOPCM_TOP                   (0x80 << 12)  /* 512KB */
+#define   BXT_GUC_WOPCM_RC6_RESERVED      (0x10 << 12)  /* 64KB  */
 
 #define GEN8_GT_PM_CONFIG               _MMIO(0x138140)
 #define GEN9LP_GT_PM_CONFIG             _MMIO(0x138140)
diff --git a/drivers/gpu/drm/i915/i915_guc_submission.c b/drivers/gpu/drm/i915/i915_guc_submission.c
index d40c13fb6643..2112e029db6a 100644
--- a/drivers/gpu/drm/i915/i915_guc_submission.c
+++ b/drivers/gpu/drm/i915/i915_guc_submission.c
@@ -97,8 +97,14 @@ static int host2guc_action(struct intel_guc *guc, u32 *data, u32 len)
 
         I915_WRITE(HOST2GUC_INTERRUPT, HOST2GUC_TRIGGER);
 
-        /* No HOST2GUC command should take longer than 10ms */
-        ret = wait_for_atomic(host2guc_action_response(dev_priv, &status), 10);
+        /*
+         * Fast commands should complete in less than 10us, so sample quickly
+         * up to that length of time, then switch to a slower sleep-wait loop.
+         * No HOST2GUC command should ever take longer than 10ms.
+         */
+        ret = wait_for_us(host2guc_action_response(dev_priv, &status), 10);
+        if (ret)
+                ret = wait_for(host2guc_action_response(dev_priv, &status), 10);
         if (status != GUC2HOST_STATUS_SUCCESS) {
                 /*
                  * Either the GuC explicitly returned an error (which
@@ -153,13 +159,11 @@ static int host2guc_sample_forcewake(struct intel_guc *guc,
                                      struct i915_guc_client *client)
 {
         struct drm_i915_private *dev_priv = guc_to_i915(guc);
-        struct drm_device *dev = dev_priv->dev;
         u32 data[2];
 
         data[0] = HOST2GUC_ACTION_SAMPLE_FORCEWAKE;
         /* WaRsDisableCoarsePowerGating:skl,bxt */
-        if (!intel_enable_rc6(dev) ||
-            NEEDS_WaRsDisableCoarsePowerGating(dev))
+        if (!intel_enable_rc6() || NEEDS_WaRsDisableCoarsePowerGating(dev_priv))
                 data[1] = 0;
         else
                 /* bit 0 and 1 are for Render and Media domain separately */
@@ -175,94 +179,88 @@ static int host2guc_sample_forcewake(struct intel_guc *guc,
  * client object which contains the page being used for the doorbell
  */
 
-static void guc_init_doorbell(struct intel_guc *guc,
-                              struct i915_guc_client *client)
+static int guc_update_doorbell_id(struct intel_guc *guc,
+                                  struct i915_guc_client *client,
+                                  u16 new_id)
 {
+        struct sg_table *sg = guc->ctx_pool_obj->pages;
+        void *doorbell_bitmap = guc->doorbell_bitmap;
         struct guc_doorbell_info *doorbell;
+        struct guc_context_desc desc;
+        size_t len;
 
         doorbell = client->client_base + client->doorbell_offset;
 
-        doorbell->db_status = GUC_DOORBELL_ENABLED;
-        doorbell->cookie = 0;
-}
-
-static int guc_ring_doorbell(struct i915_guc_client *gc)
-{
-        struct guc_process_desc *desc;
-        union guc_doorbell_qw db_cmp, db_exc, db_ret;
-        union guc_doorbell_qw *db;
-        int attempt = 2, ret = -EAGAIN;
-
-        desc = gc->client_base + gc->proc_desc_offset;
-
-        /* Update the tail so it is visible to GuC */
-        desc->tail = gc->wq_tail;
-
-        /* current cookie */
-        db_cmp.db_status = GUC_DOORBELL_ENABLED;
-        db_cmp.cookie = gc->cookie;
-
-        /* cookie to be updated */
-        db_exc.db_status = GUC_DOORBELL_ENABLED;
-        db_exc.cookie = gc->cookie + 1;
-        if (db_exc.cookie == 0)
-                db_exc.cookie = 1;
-
-        /* pointer of current doorbell cacheline */
-        db = gc->client_base + gc->doorbell_offset;
-
-        while (attempt--) {
-                /* lets ring the doorbell */
-                db_ret.value_qw = atomic64_cmpxchg((atomic64_t *)db,
-                        db_cmp.value_qw, db_exc.value_qw);
-
-                /* if the exchange was successfully executed */
-                if (db_ret.value_qw == db_cmp.value_qw) {
-                        /* db was successfully rung */
-                        gc->cookie = db_exc.cookie;
-                        ret = 0;
-                        break;
-                }
+        if (client->doorbell_id != GUC_INVALID_DOORBELL_ID &&
+            test_bit(client->doorbell_id, doorbell_bitmap)) {
+                /* Deactivate the old doorbell */
+                doorbell->db_status = GUC_DOORBELL_DISABLED;
+                (void)host2guc_release_doorbell(guc, client);
+                __clear_bit(client->doorbell_id, doorbell_bitmap);
+        }
 
-                /* XXX: doorbell was lost and need to acquire it again */
-                if (db_ret.db_status == GUC_DOORBELL_DISABLED)
-                        break;
+        /* Update the GuC's idea of the doorbell ID */
+        len = sg_pcopy_to_buffer(sg->sgl, sg->nents, &desc, sizeof(desc),
+                             sizeof(desc) * client->ctx_index);
+        if (len != sizeof(desc))
+                return -EFAULT;
+        desc.db_id = new_id;
+        len = sg_pcopy_from_buffer(sg->sgl, sg->nents, &desc, sizeof(desc),
+                             sizeof(desc) * client->ctx_index);
+        if (len != sizeof(desc))
+                return -EFAULT;
 
-                DRM_ERROR("Cookie mismatch. Expected %d, returned %d\n",
-                          db_cmp.cookie, db_ret.cookie);
+        client->doorbell_id = new_id;
+        if (new_id == GUC_INVALID_DOORBELL_ID)
+                return 0;
 
-                /* update the cookie to newly read cookie from GuC */
-                db_cmp.cookie = db_ret.cookie;
-                db_exc.cookie = db_ret.cookie + 1;
-                if (db_exc.cookie == 0)
-                        db_exc.cookie = 1;
-        }
+        /* Activate the new doorbell */
+        __set_bit(new_id, doorbell_bitmap);
+        doorbell->cookie = 0;
+        doorbell->db_status = GUC_DOORBELL_ENABLED;
+        return host2guc_allocate_doorbell(guc, client);
+}
 
-        return ret;
+static int guc_init_doorbell(struct intel_guc *guc,
+                              struct i915_guc_client *client,
+                              uint16_t db_id)
+{
+        return guc_update_doorbell_id(guc, client, db_id);
 }
 
 static void guc_disable_doorbell(struct intel_guc *guc,
                                  struct i915_guc_client *client)
 {
-        struct drm_i915_private *dev_priv = guc_to_i915(guc);
-        struct guc_doorbell_info *doorbell;
-        i915_reg_t drbreg = GEN8_DRBREGL(client->doorbell_id);
-        int value;
-
-        doorbell = client->client_base + client->doorbell_offset;
+        (void)guc_update_doorbell_id(guc, client, GUC_INVALID_DOORBELL_ID);
 
-        doorbell->db_status = GUC_DOORBELL_DISABLED;
+        /* XXX: wait for any interrupts */
+        /* XXX: wait for workqueue to drain */
+}
 
-        I915_WRITE(drbreg, I915_READ(drbreg) & ~GEN8_DRB_VALID);
+static uint16_t
+select_doorbell_register(struct intel_guc *guc, uint32_t priority)
+{
+        /*
+         * The bitmap tracks which doorbell registers are currently in use.
+         * It is split into two halves; the first half is used for normal
+         * priority contexts, the second half for high-priority ones.
+         * Note that logically higher priorities are numerically less than
+         * normal ones, so the test below means "is it high-priority?"
+         */
+        const bool hi_pri = (priority <= GUC_CTX_PRIORITY_HIGH);
+        const uint16_t half = GUC_MAX_DOORBELLS / 2;
+        const uint16_t start = hi_pri ? half : 0;
+        const uint16_t end = start + half;
+        uint16_t id;
 
-        value = I915_READ(drbreg);
-        WARN_ON((value & GEN8_DRB_VALID) != 0);
+        id = find_next_zero_bit(guc->doorbell_bitmap, end, start);
+        if (id == end)
+                id = GUC_INVALID_DOORBELL_ID;
 
-        I915_WRITE(GEN8_DRBREGU(client->doorbell_id), 0);
-        I915_WRITE(drbreg, 0);
+        DRM_DEBUG_DRIVER("assigned %s priority doorbell id 0x%x\n",
+                        hi_pri ? "high" : "normal", id);
 
-        /* XXX: wait for any interrupts */
-        /* XXX: wait for workqueue to drain */
+        return id;
 }
 
 /*
@@ -289,37 +287,6 @@ static uint32_t select_doorbell_cacheline(struct intel_guc *guc)
         return offset;
 }
 
-static uint16_t assign_doorbell(struct intel_guc *guc, uint32_t priority)
-{
-        /*
-         * The bitmap is split into two halves; the first half is used for
-         * normal priority contexts, the second half for high-priority ones.
-         * Note that logically higher priorities are numerically less than
-         * normal ones, so the test below means "is it high-priority?"
-         */
-        const bool hi_pri = (priority <= GUC_CTX_PRIORITY_HIGH);
-        const uint16_t half = GUC_MAX_DOORBELLS / 2;
-        const uint16_t start = hi_pri ? half : 0;
-        const uint16_t end = start + half;
-        uint16_t id;
-
-        id = find_next_zero_bit(guc->doorbell_bitmap, end, start);
-        if (id == end)
-                id = GUC_INVALID_DOORBELL_ID;
-        else
-                bitmap_set(guc->doorbell_bitmap, id, 1);
-
-        DRM_DEBUG_DRIVER("assigned %s priority doorbell id 0x%x\n",
-                        hi_pri ? "high" : "normal", id);
-
-        return id;
-}
-
-static void release_doorbell(struct intel_guc *guc, uint16_t id)
-{
-        bitmap_clear(guc->doorbell_bitmap, id, 1);
-}
-
 /*
  * Initialise the process descriptor shared with the GuC firmware.
  */
@@ -361,10 +328,9 @@ static void guc_init_ctx_desc(struct intel_guc *guc,
         struct drm_i915_gem_object *client_obj = client->client_obj;
         struct drm_i915_private *dev_priv = guc_to_i915(guc);
         struct intel_engine_cs *engine;
-        struct intel_context *ctx = client->owner;
+        struct i915_gem_context *ctx = client->owner;
         struct guc_context_desc desc;
         struct sg_table *sg;
-        enum intel_engine_id id;
         u32 gfx_addr;
 
         memset(&desc, 0, sizeof(desc));
@@ -374,10 +340,10 @@ static void guc_init_ctx_desc(struct intel_guc *guc,
         desc.priority = client->priority;
         desc.db_id = client->doorbell_id;
 
-        for_each_engine_id(engine, dev_priv, id) {
+        for_each_engine(engine, dev_priv) {
+                struct intel_context *ce = &ctx->engine[engine->id];
                 struct guc_execlist_context *lrc = &desc.lrc[engine->guc_id];
                 struct drm_i915_gem_object *obj;
-                uint64_t ctx_desc;
 
                 /* TODO: We have a design issue to be solved here. Only when we
                  * receive the first batch, we know which engine is used by the
@@ -386,20 +352,18 @@ static void guc_init_ctx_desc(struct intel_guc *guc,
                  * for now who owns a GuC client. But for future owner of GuC
                  * client, need to make sure lrc is pinned prior to enter here.
                  */
-                obj = ctx->engine[id].state;
-                if (!obj)
+                if (!ce->state)
                         break;  /* XXX: continue? */
 
-                ctx_desc = intel_lr_context_descriptor(ctx, engine);
-                lrc->context_desc = (u32)ctx_desc;
+                lrc->context_desc = lower_32_bits(ce->lrc_desc);
 
                 /* The state page is after PPHWSP */
-                gfx_addr = i915_gem_obj_ggtt_offset(obj);
+                gfx_addr = i915_gem_obj_ggtt_offset(ce->state);
                 lrc->ring_lcra = gfx_addr + LRC_STATE_PN * PAGE_SIZE;
                 lrc->context_id = (client->ctx_index << GUC_ELC_CTXID_OFFSET) |
                                 (engine->guc_id << GUC_ELC_ENGINE_OFFSET);
 
-                obj = ctx->engine[id].ringbuf->obj;
+                obj = ce->ringbuf->obj;
                 gfx_addr = i915_gem_obj_ggtt_offset(obj);
 
                 lrc->ring_begin = gfx_addr;
@@ -427,7 +391,7 @@ static void guc_init_ctx_desc(struct intel_guc *guc,
         desc.wq_size = client->wq_size;
 
         /*
-         * XXX: Take LRCs from an existing intel_context if this is not an
+         * XXX: Take LRCs from an existing context if this is not an
          * IsKMDCreatedContext client
          */
         desc.desc_private = (uintptr_t)client;
@@ -451,47 +415,64 @@ static void guc_fini_ctx_desc(struct intel_guc *guc,
                              sizeof(desc) * client->ctx_index);
 }
 
-int i915_guc_wq_check_space(struct i915_guc_client *gc)
+/**
+ * i915_guc_wq_check_space() - check that the GuC can accept a request
+ * @request:    request associated with the commands
+ *
+ * Return:      0 if space is available
+ *              -EAGAIN if space is not currently available
+ *
+ * This function must be called (and must return 0) before a request
+ * is submitted to the GuC via i915_guc_submit() below. Once a result
+ * of 0 has been returned, it remains valid until (but only until)
+ * the next call to submit().
+ *
+ * This precheck allows the caller to determine in advance that space
+ * will be available for the next submission before committing resources
+ * to it, and helps avoid late failures with complicated recovery paths.
+ */
+int i915_guc_wq_check_space(struct drm_i915_gem_request *request)
 {
+        const size_t wqi_size = sizeof(struct guc_wq_item);
+        struct i915_guc_client *gc = request->i915->guc.execbuf_client;
         struct guc_process_desc *desc;
-        u32 size = sizeof(struct guc_wq_item);
-        int ret = -ETIMEDOUT, timeout_counter = 200;
+        u32 freespace;
 
-        if (!gc)
-                return 0;
+        GEM_BUG_ON(gc == NULL);
 
         desc = gc->client_base + gc->proc_desc_offset;
 
-        while (timeout_counter-- > 0) {
-                if (CIRC_SPACE(gc->wq_tail, desc->head, gc->wq_size) >= size) {
-                        ret = 0;
-                        break;
-                }
+        freespace = CIRC_SPACE(gc->wq_tail, desc->head, gc->wq_size);
+        if (likely(freespace >= wqi_size))
+                return 0;
 
-                if (timeout_counter)
-                        usleep_range(1000, 2000);
-        };
+        gc->no_wq_space += 1;
 
-        return ret;
+        return -EAGAIN;
 }
 
-static int guc_add_workqueue_item(struct i915_guc_client *gc,
-                                  struct drm_i915_gem_request *rq)
+static void guc_add_workqueue_item(struct i915_guc_client *gc,
+                                   struct drm_i915_gem_request *rq)
 {
+        /* wqi_len is in DWords, and does not include the one-word header */
+        const size_t wqi_size = sizeof(struct guc_wq_item);
+        const u32 wqi_len = wqi_size/sizeof(u32) - 1;
         struct guc_process_desc *desc;
         struct guc_wq_item *wqi;
         void *base;
-        u32 tail, wq_len, wq_off, space;
+        u32 freespace, tail, wq_off, wq_page;
 
         desc = gc->client_base + gc->proc_desc_offset;
-        space = CIRC_SPACE(gc->wq_tail, desc->head, gc->wq_size);
-        if (WARN_ON(space < sizeof(struct guc_wq_item)))
-                return -ENOSPC; /* shouldn't happen */
 
-        /* postincrement WQ tail for next time */
-        wq_off = gc->wq_tail;
-        gc->wq_tail += sizeof(struct guc_wq_item);
-        gc->wq_tail &= gc->wq_size - 1;
+        /* Free space is guaranteed, see i915_guc_wq_check_space() above */
+        freespace = CIRC_SPACE(gc->wq_tail, desc->head, gc->wq_size);
+        GEM_BUG_ON(freespace < wqi_size);
+
+        /* The GuC firmware wants the tail index in QWords, not bytes */
+        tail = rq->tail;
+        GEM_BUG_ON(tail & 7);
+        tail >>= 3;
+        GEM_BUG_ON(tail > WQ_RING_TAIL_MAX);
 
         /* For now workqueue item is 4 DWs; workqueue buffer is 2 pages. So we
          * should not have the case where structure wqi is across page, neither
@@ -500,19 +481,23 @@ static int guc_add_workqueue_item(struct i915_guc_client *gc,
          * XXX: if not the case, we need save data to a temp wqi and copy it to
          * workqueue buffer dw by dw.
          */
-        WARN_ON(sizeof(struct guc_wq_item) != 16);
-        WARN_ON(wq_off & 3);
+        BUILD_BUG_ON(wqi_size != 16);
+
+        /* postincrement WQ tail for next time */
+        wq_off = gc->wq_tail;
+        gc->wq_tail += wqi_size;
+        gc->wq_tail &= gc->wq_size - 1;
+        GEM_BUG_ON(wq_off & (wqi_size - 1));
 
-        /* wq starts from the page after doorbell / process_desc */
-        base = kmap_atomic(i915_gem_object_get_page(gc->client_obj,
-                        (wq_off + GUC_DB_SIZE) >> PAGE_SHIFT));
+        /* WQ starts from the page after doorbell / process_desc */
+        wq_page = (wq_off + GUC_DB_SIZE) >> PAGE_SHIFT;
         wq_off &= PAGE_SIZE - 1;
+        base = kmap_atomic(i915_gem_object_get_page(gc->client_obj, wq_page));
         wqi = (struct guc_wq_item *)((char *)base + wq_off);
 
-        /* len does not include the header */
-        wq_len = sizeof(struct guc_wq_item) / sizeof(u32) - 1;
+        /* Now fill in the 4-word work queue item */
         wqi->header = WQ_TYPE_INORDER |
-                        (wq_len << WQ_LEN_SHIFT) |
+                        (wqi_len << WQ_LEN_SHIFT) |
                         (rq->engine->guc_id << WQ_TARGET_SHIFT) |
                         WQ_NO_WCFLUSH_WAIT;
 
@@ -520,48 +505,105 @@ static int guc_add_workqueue_item(struct i915_guc_client *gc,
         wqi->context_desc = (u32)intel_lr_context_descriptor(rq->ctx,
                                                              rq->engine);
 
-        /* The GuC firmware wants the tail index in QWords, not bytes */
-        tail = rq->ringbuf->tail >> 3;
         wqi->ring_tail = tail << WQ_RING_TAIL_SHIFT;
-        wqi->fence_id = 0; /*XXX: what fence to be here */
+        wqi->fence_id = rq->seqno;
 
         kunmap_atomic(base);
+}
 
-        return 0;
+static int guc_ring_doorbell(struct i915_guc_client *gc)
+{
+        struct guc_process_desc *desc;
+        union guc_doorbell_qw db_cmp, db_exc, db_ret;
+        union guc_doorbell_qw *db;
+        int attempt = 2, ret = -EAGAIN;
+
+        desc = gc->client_base + gc->proc_desc_offset;
+
+        /* Update the tail so it is visible to GuC */
+        desc->tail = gc->wq_tail;
+
+        /* current cookie */
+        db_cmp.db_status = GUC_DOORBELL_ENABLED;
+        db_cmp.cookie = gc->cookie;
+
+        /* cookie to be updated */
+        db_exc.db_status = GUC_DOORBELL_ENABLED;
+        db_exc.cookie = gc->cookie + 1;
+        if (db_exc.cookie == 0)
+                db_exc.cookie = 1;
+
+        /* pointer of current doorbell cacheline */
+        db = gc->client_base + gc->doorbell_offset;
+
+        while (attempt--) {
+                /* lets ring the doorbell */
+                db_ret.value_qw = atomic64_cmpxchg((atomic64_t *)db,
+                        db_cmp.value_qw, db_exc.value_qw);
+
+                /* if the exchange was successfully executed */
+                if (db_ret.value_qw == db_cmp.value_qw) {
+                        /* db was successfully rung */
+                        gc->cookie = db_exc.cookie;
+                        ret = 0;
+                        break;
+                }
+
+                /* XXX: doorbell was lost and need to acquire it again */
+                if (db_ret.db_status == GUC_DOORBELL_DISABLED)
+                        break;
+
+                DRM_ERROR("Cookie mismatch. Expected %d, returned %d\n",
+                          db_cmp.cookie, db_ret.cookie);
+
+                /* update the cookie to newly read cookie from GuC */
+                db_cmp.cookie = db_ret.cookie;
+                db_exc.cookie = db_ret.cookie + 1;
+                if (db_exc.cookie == 0)
+                        db_exc.cookie = 1;
+        }
+
+        return ret;
 }
 
 /**
  * i915_guc_submit() - Submit commands through GuC
- * @client:     the guc client where commands will go through
  * @rq:         request associated with the commands
  *
- * Return:      0 if succeed
+ * Return:      0 on success, otherwise an errno.
+ *              (Note: nonzero really shouldn't happen!)
+ *
+ * The caller must have already called i915_guc_wq_check_space() above
+ * with a result of 0 (success) since the last request submission. This
+ * guarantees that there is space in the work queue for the new request,
+ * so enqueuing the item cannot fail.
+ *
+ * Bad Things Will Happen if the caller violates this protocol e.g. calls
+ * submit() when check() says there's no space, or calls submit() multiple
+ * times with no intervening check().
+ *
+ * The only error here arises if the doorbell hardware isn't functioning
+ * as expected, which really shouln't happen.
  */
-int i915_guc_submit(struct i915_guc_client *client,
-                    struct drm_i915_gem_request *rq)
+int i915_guc_submit(struct drm_i915_gem_request *rq)
 {
-        struct intel_guc *guc = client->guc;
-        unsigned int engine_id = rq->engine->guc_id;
-        int q_ret, b_ret;
+        unsigned int engine_id = rq->engine->id;
+        struct intel_guc *guc = &rq->i915->guc;
+        struct i915_guc_client *client = guc->execbuf_client;
+        int b_ret;
 
-        q_ret = guc_add_workqueue_item(client, rq);
-        if (q_ret == 0)
-                b_ret = guc_ring_doorbell(client);
+        guc_add_workqueue_item(client, rq);
+        b_ret = guc_ring_doorbell(client);
 
         client->submissions[engine_id] += 1;
-        if (q_ret) {
-                client->q_fail += 1;
-                client->retcode = q_ret;
-        } else if (b_ret) {
+        client->retcode = b_ret;
+        if (b_ret)
                 client->b_fail += 1;
-                client->retcode = q_ret = b_ret;
-        } else {
-                client->retcode = 0;
-        }
+
         guc->submissions[engine_id] += 1;
         guc->last_seqno[engine_id] = rq->seqno;
 
-        return q_ret;
+        return b_ret;
 }
 
 /*
@@ -572,7 +614,7 @@ int i915_guc_submit(struct i915_guc_client *client,
 
 /**
  * gem_allocate_guc_obj() - Allocate gem object for GuC usage
- * @dev:        drm device
+ * @dev_priv:   driver private data structure
  * @size:       size of object
  *
  * This is a wrapper to create a gem obj. In order to use it inside GuC, the
@@ -581,14 +623,13 @@ int i915_guc_submit(struct i915_guc_client *client,
  *
  * Return:      A drm_i915_gem_object if successful, otherwise NULL.
  */
-static struct drm_i915_gem_object *gem_allocate_guc_obj(struct drm_device *dev,
-                                                        u32 size)
+static struct drm_i915_gem_object *
+gem_allocate_guc_obj(struct drm_i915_private *dev_priv, u32 size)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
         struct drm_i915_gem_object *obj;
 
-        obj = i915_gem_alloc_object(dev, size);
-        if (!obj)
+        obj = i915_gem_object_create(&dev_priv->drm, size);
+        if (IS_ERR(obj))
                 return NULL;
 
         if (i915_gem_object_get_pages(obj)) {
@@ -623,10 +664,10 @@ static void gem_release_guc_obj(struct drm_i915_gem_object *obj)
         drm_gem_object_unreference(&obj->base);
 }
 
-static void guc_client_free(struct drm_device *dev,
-                            struct i915_guc_client *client)
+static void
+guc_client_free(struct drm_i915_private *dev_priv,
+                struct i915_guc_client *client)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
         struct intel_guc *guc = &dev_priv->guc;
 
         if (!client)
@@ -639,17 +680,10 @@ static void guc_client_free(struct drm_device *dev,
 
         if (client->client_base) {
                 /*
-                 * If we got as far as setting up a doorbell, make sure
-                 * we shut it down before unmapping & deallocating the
-                 * memory. So first disable the doorbell, then tell the
-                 * GuC that we've finished with it, finally deallocate
-                 * it in our bitmap
+                 * If we got as far as setting up a doorbell, make sure we
+                 * shut it down before unmapping & deallocating the memory.
                  */
-                if (client->doorbell_id != GUC_INVALID_DOORBELL_ID) {
-                        guc_disable_doorbell(guc, client);
-                        host2guc_release_doorbell(guc, client);
-                        release_doorbell(guc, client->doorbell_id);
-                }
+                guc_disable_doorbell(guc, client);
 
                 kunmap(kmap_to_page(client->client_base));
         }
@@ -664,9 +698,51 @@ static void guc_client_free(struct drm_device *dev,
         kfree(client);
 }
 
+/*
+ * Borrow the first client to set up & tear down every doorbell
+ * in turn, to ensure that all doorbell h/w is (re)initialised.
+ */
+static void guc_init_doorbell_hw(struct intel_guc *guc)
+{
+        struct drm_i915_private *dev_priv = guc_to_i915(guc);
+        struct i915_guc_client *client = guc->execbuf_client;
+        uint16_t db_id, i;
+        int err;
+
+        db_id = client->doorbell_id;
+
+        for (i = 0; i < GUC_MAX_DOORBELLS; ++i) {
+                i915_reg_t drbreg = GEN8_DRBREGL(i);
+                u32 value = I915_READ(drbreg);
+
+                err = guc_update_doorbell_id(guc, client, i);
+
+                /* Report update failure or unexpectedly active doorbell */
+                if (err || (i != db_id && (value & GUC_DOORBELL_ENABLED)))
+                        DRM_DEBUG_DRIVER("Doorbell %d (reg 0x%x) was 0x%x, err %d\n",
+                                          i, drbreg.reg, value, err);
+        }
+
+        /* Restore to original value */
+        err = guc_update_doorbell_id(guc, client, db_id);
+        if (err)
+                DRM_ERROR("Failed to restore doorbell to %d, err %d\n",
+                        db_id, err);
+
+        for (i = 0; i < GUC_MAX_DOORBELLS; ++i) {
+                i915_reg_t drbreg = GEN8_DRBREGL(i);
+                u32 value = I915_READ(drbreg);
+
+                if (i != db_id && (value & GUC_DOORBELL_ENABLED))
+                        DRM_DEBUG_DRIVER("Doorbell %d (reg 0x%x) finally 0x%x\n",
+                                          i, drbreg.reg, value);
+
+        }
+}
+
 /**
  * guc_client_alloc() - Allocate an i915_guc_client
- * @dev:        drm device
+ * @dev_priv:   driver private data structure
  * @priority:   four levels priority _CRITICAL, _HIGH, _NORMAL and _LOW
  *              The kernel client to replace ExecList submission is created with
  *              NORMAL priority. Priority of a client for scheduler can be HIGH,
@@ -676,14 +752,15 @@ static void guc_client_free(struct drm_device *dev,
  *
  * Return:      An i915_guc_client object if success, else NULL.
  */
-static struct i915_guc_client *guc_client_alloc(struct drm_device *dev,
-                                                uint32_t priority,
-                                                struct intel_context *ctx)
+static struct i915_guc_client *
+guc_client_alloc(struct drm_i915_private *dev_priv,
+                 uint32_t priority,
+                 struct i915_gem_context *ctx)
 {
         struct i915_guc_client *client;
-        struct drm_i915_private *dev_priv = dev->dev_private;
         struct intel_guc *guc = &dev_priv->guc;
         struct drm_i915_gem_object *obj;
+        uint16_t db_id;
 
         client = kzalloc(sizeof(*client), GFP_KERNEL);
         if (!client)
@@ -702,7 +779,7 @@ static struct i915_guc_client *guc_client_alloc(struct drm_device *dev,
         }
 
         /* The first page is doorbell/proc_desc. Two followed pages are wq. */
-        obj = gem_allocate_guc_obj(dev, GUC_DB_SIZE + GUC_WQ_SIZE);
+        obj = gem_allocate_guc_obj(dev_priv, GUC_DB_SIZE + GUC_WQ_SIZE);
         if (!obj)
                 goto err;
 
@@ -712,6 +789,11 @@ static struct i915_guc_client *guc_client_alloc(struct drm_device *dev,
         client->wq_offset = GUC_DB_SIZE;
         client->wq_size = GUC_WQ_SIZE;
 
+        db_id = select_doorbell_register(guc, client->priority);
+        if (db_id == GUC_INVALID_DOORBELL_ID)
+                /* XXX: evict a doorbell instead? */
+                goto err;
+
         client->doorbell_offset = select_doorbell_cacheline(guc);
 
         /*
@@ -724,29 +806,22 @@ static struct i915_guc_client *guc_client_alloc(struct drm_device *dev,
         else
                 client->proc_desc_offset = (GUC_DB_SIZE / 2);
 
-        client->doorbell_id = assign_doorbell(guc, client->priority);
-        if (client->doorbell_id == GUC_INVALID_DOORBELL_ID)
-                /* XXX: evict a doorbell instead */
-                goto err;
-
         guc_init_proc_desc(guc, client);
         guc_init_ctx_desc(guc, client);
-        guc_init_doorbell(guc, client);
-
-        /* XXX: Any cache flushes needed? General domain mgmt calls? */
-
-        if (host2guc_allocate_doorbell(guc, client))
+        if (guc_init_doorbell(guc, client, db_id))
                 goto err;
 
-        DRM_DEBUG_DRIVER("new priority %u client %p: ctx_index %u db_id %u\n",
-                priority, client, client->ctx_index, client->doorbell_id);
+        DRM_DEBUG_DRIVER("new priority %u client %p: ctx_index %u\n",
+                priority, client, client->ctx_index);
+        DRM_DEBUG_DRIVER("doorbell id %u, cacheline offset 0x%x\n",
+                client->doorbell_id, client->doorbell_offset);
 
         return client;
 
 err:
         DRM_ERROR("FAILED to create priority %u GuC client!\n", priority);
 
-        guc_client_free(dev, client);
+        guc_client_free(dev_priv, client);
         return NULL;
 }
 
@@ -771,7 +846,7 @@ static void guc_create_log(struct intel_guc *guc)
 
         obj = guc->log_obj;
         if (!obj) {
-                obj = gem_allocate_guc_obj(dev_priv->dev, size);
+                obj = gem_allocate_guc_obj(dev_priv, size);
                 if (!obj) {
                         /* logging will be off */
                         i915.guc_log_level = -1;
@@ -831,7 +906,7 @@ static void guc_create_ads(struct intel_guc *guc)
 
         obj = guc->ads_obj;
         if (!obj) {
-                obj = gem_allocate_guc_obj(dev_priv->dev, PAGE_ALIGN(size));
+                obj = gem_allocate_guc_obj(dev_priv, PAGE_ALIGN(size));
                 if (!obj)
                         return;
 
@@ -885,66 +960,65 @@ static void guc_create_ads(struct intel_guc *guc)
  * Set up the memory resources to be shared with the GuC.  At this point,
  * we require just one object that can be mapped through the GGTT.
  */
-int i915_guc_submission_init(struct drm_device *dev)
+int i915_guc_submission_init(struct drm_i915_private *dev_priv)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
         const size_t ctxsize = sizeof(struct guc_context_desc);
         const size_t poolsize = GUC_MAX_GPU_CONTEXTS * ctxsize;
         const size_t gemsize = round_up(poolsize, PAGE_SIZE);
         struct intel_guc *guc = &dev_priv->guc;
 
+        /* Wipe bitmap & delete client in case of reinitialisation */
+        bitmap_clear(guc->doorbell_bitmap, 0, GUC_MAX_DOORBELLS);
+        i915_guc_submission_disable(dev_priv);
+
         if (!i915.enable_guc_submission)
                 return 0; /* not enabled  */
 
         if (guc->ctx_pool_obj)
                 return 0; /* already allocated */
 
-        guc->ctx_pool_obj = gem_allocate_guc_obj(dev_priv->dev, gemsize);
+        guc->ctx_pool_obj = gem_allocate_guc_obj(dev_priv, gemsize);
         if (!guc->ctx_pool_obj)
                 return -ENOMEM;
 
         ida_init(&guc->ctx_ids);
-
         guc_create_log(guc);
-
         guc_create_ads(guc);
 
         return 0;
 }
 
-int i915_guc_submission_enable(struct drm_device *dev)
+int i915_guc_submission_enable(struct drm_i915_private *dev_priv)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
         struct intel_guc *guc = &dev_priv->guc;
-        struct intel_context *ctx = dev_priv->kernel_context;
         struct i915_guc_client *client;
 
         /* client for execbuf submission */
-        client = guc_client_alloc(dev, GUC_CTX_PRIORITY_KMD_NORMAL, ctx);
+        client = guc_client_alloc(dev_priv,
+                                  GUC_CTX_PRIORITY_KMD_NORMAL,
+                                  dev_priv->kernel_context);
         if (!client) {
                 DRM_ERROR("Failed to create execbuf guc_client\n");
                 return -ENOMEM;
         }
 
         guc->execbuf_client = client;
-
         host2guc_sample_forcewake(guc, client);
+        guc_init_doorbell_hw(guc);
 
         return 0;
 }
 
-void i915_guc_submission_disable(struct drm_device *dev)
+void i915_guc_submission_disable(struct drm_i915_private *dev_priv)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
         struct intel_guc *guc = &dev_priv->guc;
 
-        guc_client_free(dev, guc->execbuf_client);
+        guc_client_free(dev_priv, guc->execbuf_client);
         guc->execbuf_client = NULL;
 }
 
-void i915_guc_submission_fini(struct drm_device *dev)
+void i915_guc_submission_fini(struct drm_i915_private *dev_priv)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
         struct intel_guc *guc = &dev_priv->guc;
 
         gem_release_guc_obj(dev_priv->guc.ads_obj);
@@ -965,12 +1039,12 @@ void i915_guc_submission_fini(struct drm_device *dev)
  */
 int intel_guc_suspend(struct drm_device *dev)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         struct intel_guc *guc = &dev_priv->guc;
-        struct intel_context *ctx;
+        struct i915_gem_context *ctx;
         u32 data[3];
 
-        if (!i915.enable_guc_submission)
+        if (guc->guc_fw.guc_fw_load_status != GUC_FIRMWARE_SUCCESS)
                 return 0;
 
         ctx = dev_priv->kernel_context;
@@ -991,12 +1065,12 @@ int intel_guc_suspend(struct drm_device *dev)
  */
 int intel_guc_resume(struct drm_device *dev)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         struct intel_guc *guc = &dev_priv->guc;
-        struct intel_context *ctx;
+        struct i915_gem_context *ctx;
         u32 data[3];
 
-        if (!i915.enable_guc_submission)
+        if (guc->guc_fw.guc_fw_load_status != GUC_FIRMWARE_SUCCESS)
                 return 0;
 
         ctx = dev_priv->kernel_context;
diff --git a/drivers/gpu/drm/i915/i915_irq.c b/drivers/gpu/drm/i915/i915_irq.c
index aab47f7bb61b..1c2aec392412 100644
--- a/drivers/gpu/drm/i915/i915_irq.c
+++ b/drivers/gpu/drm/i915/i915_irq.c
@@ -259,12 +259,12 @@ static void ilk_update_gt_irq(struct drm_i915_private *dev_priv,
         dev_priv->gt_irq_mask &= ~interrupt_mask;
         dev_priv->gt_irq_mask |= (~enabled_irq_mask & interrupt_mask);
         I915_WRITE(GTIMR, dev_priv->gt_irq_mask);
-        POSTING_READ(GTIMR);
 }
 
 void gen5_enable_gt_irq(struct drm_i915_private *dev_priv, uint32_t mask)
 {
         ilk_update_gt_irq(dev_priv, mask, mask);
+        POSTING_READ_FW(GTIMR);
 }
 
 void gen5_disable_gt_irq(struct drm_i915_private *dev_priv, uint32_t mask)
@@ -336,9 +336,8 @@ void gen6_disable_pm_irq(struct drm_i915_private *dev_priv, uint32_t mask)
         __gen6_disable_pm_irq(dev_priv, mask);
 }
 
-void gen6_reset_rps_interrupts(struct drm_device *dev)
+void gen6_reset_rps_interrupts(struct drm_i915_private *dev_priv)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
         i915_reg_t reg = gen6_pm_iir(dev_priv);
 
         spin_lock_irq(&dev_priv->irq_lock);
@@ -349,14 +348,11 @@ void gen6_reset_rps_interrupts(struct drm_device *dev)
         spin_unlock_irq(&dev_priv->irq_lock);
 }
 
-void gen6_enable_rps_interrupts(struct drm_device *dev)
+void gen6_enable_rps_interrupts(struct drm_i915_private *dev_priv)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
-
         spin_lock_irq(&dev_priv->irq_lock);
-
-        WARN_ON(dev_priv->rps.pm_iir);
-        WARN_ON(I915_READ(gen6_pm_iir(dev_priv)) & dev_priv->pm_rps_events);
+        WARN_ON_ONCE(dev_priv->rps.pm_iir);
+        WARN_ON_ONCE(I915_READ(gen6_pm_iir(dev_priv)) & dev_priv->pm_rps_events);
         dev_priv->rps.interrupts_enabled = true;
         I915_WRITE(gen6_pm_ier(dev_priv), I915_READ(gen6_pm_ier(dev_priv)) |
                                 dev_priv->pm_rps_events);
@@ -367,32 +363,13 @@ void gen6_enable_rps_interrupts(struct drm_device *dev)
 
 u32 gen6_sanitize_rps_pm_mask(struct drm_i915_private *dev_priv, u32 mask)
 {
-        /*
-         * SNB,IVB can while VLV,CHV may hard hang on looping batchbuffer
-         * if GEN6_PM_UP_EI_EXPIRED is masked.
-         *
-         * TODO: verify if this can be reproduced on VLV,CHV.
-         */
-        if (INTEL_INFO(dev_priv)->gen <= 7 && !IS_HASWELL(dev_priv))
-                mask &= ~GEN6_PM_RP_UP_EI_EXPIRED;
-
-        if (INTEL_INFO(dev_priv)->gen >= 8)
-                mask &= ~GEN8_PMINTR_REDIRECT_TO_NON_DISP;
-
-        return mask;
+        return (mask & ~dev_priv->rps.pm_intr_keep);
 }
 
-void gen6_disable_rps_interrupts(struct drm_device *dev)
+void gen6_disable_rps_interrupts(struct drm_i915_private *dev_priv)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
-
         spin_lock_irq(&dev_priv->irq_lock);
         dev_priv->rps.interrupts_enabled = false;
-        spin_unlock_irq(&dev_priv->irq_lock);
-
-        cancel_work_sync(&dev_priv->rps.work);
-
-        spin_lock_irq(&dev_priv->irq_lock);
 
         I915_WRITE(GEN6_PMINTRMSK, gen6_sanitize_rps_pm_mask(dev_priv, ~0));
 
@@ -401,8 +378,15 @@ void gen6_disable_rps_interrupts(struct drm_device *dev)
                                 ~dev_priv->pm_rps_events);
 
         spin_unlock_irq(&dev_priv->irq_lock);
+        synchronize_irq(dev_priv->drm.irq);
 
-        synchronize_irq(dev->irq);
+        /* Now that we will not be generating any more work, flush any
+         * outsanding tasks. As we are called on the RPS idle path,
+         * we will reset the GPU to minimum frequencies, so the current
+         * state of the worker can be discarded.
+         */
+        cancel_work_sync(&dev_priv->rps.work);
+        gen6_reset_rps_interrupts(dev_priv);
 }
 
 /**
@@ -582,7 +566,7 @@ i915_enable_pipestat(struct drm_i915_private *dev_priv, enum pipe pipe,
         u32 enable_mask;
 
         if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
-                enable_mask = vlv_get_pipestat_enable_mask(dev_priv->dev,
+                enable_mask = vlv_get_pipestat_enable_mask(&dev_priv->drm,
                                                            status_mask);
         else
                 enable_mask = status_mask << 16;
@@ -596,7 +580,7 @@ i915_disable_pipestat(struct drm_i915_private *dev_priv, enum pipe pipe,
         u32 enable_mask;
 
         if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
-                enable_mask = vlv_get_pipestat_enable_mask(dev_priv->dev,
+                enable_mask = vlv_get_pipestat_enable_mask(&dev_priv->drm,
                                                            status_mask);
         else
                 enable_mask = status_mask << 16;
@@ -605,19 +589,17 @@ i915_disable_pipestat(struct drm_i915_private *dev_priv, enum pipe pipe,
 
 /**
  * i915_enable_asle_pipestat - enable ASLE pipestat for OpRegion
- * @dev: drm device
+ * @dev_priv: i915 device private
  */
-static void i915_enable_asle_pipestat(struct drm_device *dev)
+static void i915_enable_asle_pipestat(struct drm_i915_private *dev_priv)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
-
-        if (!dev_priv->opregion.asle || !IS_MOBILE(dev))
+        if (!dev_priv->opregion.asle || !IS_MOBILE(dev_priv))
                 return;
 
         spin_lock_irq(&dev_priv->irq_lock);
 
         i915_enable_pipestat(dev_priv, PIPE_B, PIPE_LEGACY_BLC_EVENT_STATUS);
-        if (INTEL_INFO(dev)->gen >= 4)
+        if (INTEL_GEN(dev_priv) >= 4)
                 i915_enable_pipestat(dev_priv, PIPE_A,
                                      PIPE_LEGACY_BLC_EVENT_STATUS);
 
@@ -685,7 +667,7 @@ static u32 i8xx_get_vblank_counter(struct drm_device *dev, unsigned int pipe)
  */
 static u32 i915_get_vblank_counter(struct drm_device *dev, unsigned int pipe)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         i915_reg_t high_frame, low_frame;
         u32 high1, high2, low, pixel, vbl_start, hsync_start, htotal;
         struct intel_crtc *intel_crtc =
@@ -732,7 +714,7 @@ static u32 i915_get_vblank_counter(struct drm_device *dev, unsigned int pipe)
 
 static u32 g4x_get_vblank_counter(struct drm_device *dev, unsigned int pipe)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
 
         return I915_READ(PIPE_FRMCOUNT_G4X(pipe));
 }
@@ -741,7 +723,7 @@ static u32 g4x_get_vblank_counter(struct drm_device *dev, unsigned int pipe)
 static int __intel_get_crtc_scanline(struct intel_crtc *crtc)
 {
         struct drm_device *dev = crtc->base.dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         const struct drm_display_mode *mode = &crtc->base.hwmode;
         enum pipe pipe = crtc->pipe;
         int position, vtotal;
@@ -750,7 +732,7 @@ static int __intel_get_crtc_scanline(struct intel_crtc *crtc)
         if (mode->flags & DRM_MODE_FLAG_INTERLACE)
                 vtotal /= 2;
 
-        if (IS_GEN2(dev))
+        if (IS_GEN2(dev_priv))
                 position = I915_READ_FW(PIPEDSL(pipe)) & DSL_LINEMASK_GEN2;
         else
                 position = I915_READ_FW(PIPEDSL(pipe)) & DSL_LINEMASK_GEN3;
@@ -767,7 +749,7 @@ static int __intel_get_crtc_scanline(struct intel_crtc *crtc)
          * problem.  We may need to extend this to include other platforms,
          * but so far testing only shows the problem on HSW.
          */
-        if (HAS_DDI(dev) && !position) {
+        if (HAS_DDI(dev_priv) && !position) {
                 int i, temp;
 
                 for (i = 0; i < 100; i++) {
@@ -793,7 +775,7 @@ static int i915_get_crtc_scanoutpos(struct drm_device *dev, unsigned int pipe,
                                     ktime_t *stime, ktime_t *etime,
                                     const struct drm_display_mode *mode)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         struct drm_crtc *crtc = dev_priv->pipe_to_crtc_mapping[pipe];
         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
         int position;
@@ -835,7 +817,7 @@ static int i915_get_crtc_scanoutpos(struct drm_device *dev, unsigned int pipe,
         if (stime)
                 *stime = ktime_get();
 
-        if (IS_GEN2(dev) || IS_G4X(dev) || INTEL_INFO(dev)->gen >= 5) {
+        if (IS_GEN2(dev_priv) || IS_G4X(dev_priv) || INTEL_GEN(dev_priv) >= 5) {
                 /* No obvious pixelcount register. Only query vertical
                  * scanout position from Display scan line register.
                  */
@@ -897,7 +879,7 @@ static int i915_get_crtc_scanoutpos(struct drm_device *dev, unsigned int pipe,
         else
                 position += vtotal - vbl_end;
 
-        if (IS_GEN2(dev) || IS_G4X(dev) || INTEL_INFO(dev)->gen >= 5) {
+        if (IS_GEN2(dev_priv) || IS_G4X(dev_priv) || INTEL_GEN(dev_priv) >= 5) {
                 *vpos = position;
                 *hpos = 0;
         } else {
@@ -914,7 +896,7 @@ static int i915_get_crtc_scanoutpos(struct drm_device *dev, unsigned int pipe,
 
 int intel_get_crtc_scanline(struct intel_crtc *crtc)
 {
-        struct drm_i915_private *dev_priv = crtc->base.dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
         unsigned long irqflags;
         int position;
 
@@ -955,9 +937,8 @@ static int i915_get_vblank_timestamp(struct drm_device *dev, unsigned int pipe,
                                                      &crtc->hwmode);
 }
 
-static void ironlake_rps_change_irq_handler(struct drm_device *dev)
+static void ironlake_rps_change_irq_handler(struct drm_i915_private *dev_priv)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
         u32 busy_up, busy_down, max_avg, min_avg;
         u8 new_delay;
 
@@ -986,7 +967,7 @@ static void ironlake_rps_change_irq_handler(struct drm_device *dev)
                         new_delay = dev_priv->ips.min_delay;
         }
 
-        if (ironlake_set_drps(dev, new_delay))
+        if (ironlake_set_drps(dev_priv, new_delay))
                 dev_priv->ips.cur_delay = new_delay;
 
         spin_unlock(&mchdev_lock);
@@ -996,13 +977,11 @@ static void ironlake_rps_change_irq_handler(struct drm_device *dev)
 
 static void notify_ring(struct intel_engine_cs *engine)
 {
-        if (!intel_engine_initialized(engine))
-                return;
-
-        trace_i915_gem_request_notify(engine);
-        engine->user_interrupts++;
-
-        wake_up_all(&engine->irq_queue);
+        smp_store_mb(engine->breadcrumbs.irq_posted, true);
+        if (intel_engine_wakeup(engine)) {
+                trace_i915_gem_request_notify(engine);
+                engine->breadcrumbs.irq_wakeups++;
+        }
 }
 
 static void vlv_c0_read(struct drm_i915_private *dev_priv,
@@ -1083,7 +1062,7 @@ static bool any_waiters(struct drm_i915_private *dev_priv)
         struct intel_engine_cs *engine;
 
         for_each_engine(engine, dev_priv)
-                if (engine->irq_refcount)
+                if (intel_engine_has_waiter(engine))
                         return true;
 
         return false;
@@ -1104,13 +1083,6 @@ static void gen6_pm_rps_work(struct work_struct *work)
                 return;
         }
 
-        /*
-         * The RPS work is synced during runtime suspend, we don't require a
-         * wakeref. TODO: instead of disabling the asserts make sure that we
-         * always hold an RPM reference while the work is running.
-         */
-        DISABLE_RPM_WAKEREF_ASSERTS(dev_priv);
-
         pm_iir = dev_priv->rps.pm_iir;
         dev_priv->rps.pm_iir = 0;
         /* Make sure not to corrupt PMIMR state used by ringbuffer on GEN6 */
@@ -1123,7 +1095,7 @@ static void gen6_pm_rps_work(struct work_struct *work)
         WARN_ON(pm_iir & ~dev_priv->pm_rps_events);
 
         if ((pm_iir & dev_priv->pm_rps_events) == 0 && !client_boost)
-                goto out;
+                return;
 
         mutex_lock(&dev_priv->rps.hw_lock);
 
@@ -1175,11 +1147,9 @@ static void gen6_pm_rps_work(struct work_struct *work)
         new_delay += adj;
         new_delay = clamp_t(int, new_delay, min, max);
 
-        intel_set_rps(dev_priv->dev, new_delay);
+        intel_set_rps(dev_priv, new_delay);
 
         mutex_unlock(&dev_priv->rps.hw_lock);
-out:
-        ENABLE_RPM_WAKEREF_ASSERTS(dev_priv);
 }
 
 
@@ -1205,7 +1175,7 @@ static void ivybridge_parity_work(struct work_struct *work)
          * In order to prevent a get/put style interface, acquire struct mutex
          * any time we access those registers.
          */
-        mutex_lock(&dev_priv->dev->struct_mutex);
+        mutex_lock(&dev_priv->drm.struct_mutex);
 
         /* If we've screwed up tracking, just let the interrupt fire again */
         if (WARN_ON(!dev_priv->l3_parity.which_slice))
@@ -1241,7 +1211,7 @@ static void ivybridge_parity_work(struct work_struct *work)
                 parity_event[4] = kasprintf(GFP_KERNEL, "SLICE=%d", slice);
                 parity_event[5] = NULL;
 
-                kobject_uevent_env(&dev_priv->dev->primary->kdev->kobj,
+                kobject_uevent_env(&dev_priv->drm.primary->kdev->kobj,
                                    KOBJ_CHANGE, parity_event);
 
                 DRM_DEBUG("Parity error: Slice = %d, Row = %d, Bank = %d, Sub bank = %d.\n",
@@ -1261,7 +1231,7 @@ out:
         gen5_enable_gt_irq(dev_priv, GT_PARITY_ERROR(dev_priv));
         spin_unlock_irq(&dev_priv->irq_lock);
 
-        mutex_unlock(&dev_priv->dev->struct_mutex);
+        mutex_unlock(&dev_priv->drm.struct_mutex);
 }
 
 static void ivybridge_parity_error_irq_handler(struct drm_i915_private *dev_priv,
@@ -1287,8 +1257,7 @@ static void ivybridge_parity_error_irq_handler(struct drm_i915_private *dev_priv
 static void ilk_gt_irq_handler(struct drm_i915_private *dev_priv,
                                u32 gt_iir)
 {
-        if (gt_iir &
-            (GT_RENDER_USER_INTERRUPT | GT_RENDER_PIPECTL_NOTIFY_INTERRUPT))
+        if (gt_iir & GT_RENDER_USER_INTERRUPT)
                 notify_ring(&dev_priv->engine[RCS]);
         if (gt_iir & ILK_BSD_USER_INTERRUPT)
                 notify_ring(&dev_priv->engine[VCS]);
@@ -1297,9 +1266,7 @@ static void ilk_gt_irq_handler(struct drm_i915_private *dev_priv,
 static void snb_gt_irq_handler(struct drm_i915_private *dev_priv,
                                u32 gt_iir)
 {
-
-        if (gt_iir &
-            (GT_RENDER_USER_INTERRUPT | GT_RENDER_PIPECTL_NOTIFY_INTERRUPT))
+        if (gt_iir & GT_RENDER_USER_INTERRUPT)
                 notify_ring(&dev_priv->engine[RCS]);
         if (gt_iir & GT_BSD_USER_INTERRUPT)
                 notify_ring(&dev_priv->engine[VCS]);
@@ -1506,27 +1473,23 @@ static void intel_get_hpd_pins(u32 *pin_mask, u32 *long_mask,
 
 }
 
-static void gmbus_irq_handler(struct drm_device *dev)
+static void gmbus_irq_handler(struct drm_i915_private *dev_priv)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
-
         wake_up_all(&dev_priv->gmbus_wait_queue);
 }
 
-static void dp_aux_irq_handler(struct drm_device *dev)
+static void dp_aux_irq_handler(struct drm_i915_private *dev_priv)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
-
         wake_up_all(&dev_priv->gmbus_wait_queue);
 }
 
 #if defined(CONFIG_DEBUG_FS)
-static void display_pipe_crc_irq_handler(struct drm_device *dev, enum pipe pipe,
+static void display_pipe_crc_irq_handler(struct drm_i915_private *dev_priv,
+                                         enum pipe pipe,
                                          uint32_t crc0, uint32_t crc1,
                                          uint32_t crc2, uint32_t crc3,
                                          uint32_t crc4)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
         struct intel_pipe_crc *pipe_crc = &dev_priv->pipe_crc[pipe];
         struct intel_pipe_crc_entry *entry;
         int head, tail;
@@ -1550,7 +1513,8 @@ static void display_pipe_crc_irq_handler(struct drm_device *dev, enum pipe pipe,
 
         entry = &pipe_crc->entries[head];
 
-        entry->frame = dev->driver->get_vblank_counter(dev, pipe);
+        entry->frame = dev_priv->drm.driver->get_vblank_counter(&dev_priv->drm,
+                                                                 pipe);
         entry->crc[0] = crc0;
         entry->crc[1] = crc1;
         entry->crc[2] = crc2;
@@ -1566,27 +1530,26 @@ static void display_pipe_crc_irq_handler(struct drm_device *dev, enum pipe pipe,
 }
 #else
 static inline void
-display_pipe_crc_irq_handler(struct drm_device *dev, enum pipe pipe,
+display_pipe_crc_irq_handler(struct drm_i915_private *dev_priv,
+                             enum pipe pipe,
                              uint32_t crc0, uint32_t crc1,
                              uint32_t crc2, uint32_t crc3,
                              uint32_t crc4) {}
 #endif
 
 
-static void hsw_pipe_crc_irq_handler(struct drm_device *dev, enum pipe pipe)
+static void hsw_pipe_crc_irq_handler(struct drm_i915_private *dev_priv,
+                                     enum pipe pipe)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
-
-        display_pipe_crc_irq_handler(dev, pipe,
+        display_pipe_crc_irq_handler(dev_priv, pipe,
                                      I915_READ(PIPE_CRC_RES_1_IVB(pipe)),
                                      0, 0, 0, 0);
 }
 
-static void ivb_pipe_crc_irq_handler(struct drm_device *dev, enum pipe pipe)
+static void ivb_pipe_crc_irq_handler(struct drm_i915_private *dev_priv,
+                                     enum pipe pipe)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
-
-        display_pipe_crc_irq_handler(dev, pipe,
+        display_pipe_crc_irq_handler(dev_priv, pipe,
                                      I915_READ(PIPE_CRC_RES_1_IVB(pipe)),
                                      I915_READ(PIPE_CRC_RES_2_IVB(pipe)),
                                      I915_READ(PIPE_CRC_RES_3_IVB(pipe)),
@@ -1594,22 +1557,22 @@ static void ivb_pipe_crc_irq_handler(struct drm_device *dev, enum pipe pipe)
                                      I915_READ(PIPE_CRC_RES_5_IVB(pipe)));
 }
 
-static void i9xx_pipe_crc_irq_handler(struct drm_device *dev, enum pipe pipe)
+static void i9xx_pipe_crc_irq_handler(struct drm_i915_private *dev_priv,
+                                      enum pipe pipe)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
         uint32_t res1, res2;
 
-        if (INTEL_INFO(dev)->gen >= 3)
+        if (INTEL_GEN(dev_priv) >= 3)
                 res1 = I915_READ(PIPE_CRC_RES_RES1_I915(pipe));
         else
                 res1 = 0;
 
-        if (INTEL_INFO(dev)->gen >= 5 || IS_G4X(dev))
+        if (INTEL_GEN(dev_priv) >= 5 || IS_G4X(dev_priv))
                 res2 = I915_READ(PIPE_CRC_RES_RES2_G4X(pipe));
         else
                 res2 = 0;
 
-        display_pipe_crc_irq_handler(dev, pipe,
+        display_pipe_crc_irq_handler(dev_priv, pipe,
                                      I915_READ(PIPE_CRC_RES_RED(pipe)),
                                      I915_READ(PIPE_CRC_RES_GREEN(pipe)),
                                      I915_READ(PIPE_CRC_RES_BLUE(pipe)),
@@ -1626,7 +1589,7 @@ static void gen6_rps_irq_handler(struct drm_i915_private *dev_priv, u32 pm_iir)
                 gen6_disable_pm_irq(dev_priv, pm_iir & dev_priv->pm_rps_events);
                 if (dev_priv->rps.interrupts_enabled) {
                         dev_priv->rps.pm_iir |= pm_iir & dev_priv->pm_rps_events;
-                        queue_work(dev_priv->wq, &dev_priv->rps.work);
+                        schedule_work(&dev_priv->rps.work);
                 }
                 spin_unlock(&dev_priv->irq_lock);
         }
@@ -1643,18 +1606,21 @@ static void gen6_rps_irq_handler(struct drm_i915_private *dev_priv, u32 pm_iir)
         }
 }
 
-static bool intel_pipe_handle_vblank(struct drm_device *dev, enum pipe pipe)
+static bool intel_pipe_handle_vblank(struct drm_i915_private *dev_priv,
+                                     enum pipe pipe)
 {
-        if (!drm_handle_vblank(dev, pipe))
-                return false;
+        bool ret;
 
-        return true;
+        ret = drm_handle_vblank(&dev_priv->drm, pipe);
+        if (ret)
+                intel_finish_page_flip_mmio(dev_priv, pipe);
+
+        return ret;
 }
 
-static void valleyview_pipestat_irq_ack(struct drm_device *dev, u32 iir,
-                                        u32 pipe_stats[I915_MAX_PIPES])
+static void valleyview_pipestat_irq_ack(struct drm_i915_private *dev_priv,
+                                        u32 iir, u32 pipe_stats[I915_MAX_PIPES])
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
         int pipe;
 
         spin_lock(&dev_priv->irq_lock);
@@ -1710,31 +1676,28 @@ static void valleyview_pipestat_irq_ack(struct drm_device *dev, u32 iir,
         spin_unlock(&dev_priv->irq_lock);
 }
 
-static void valleyview_pipestat_irq_handler(struct drm_device *dev,
+static void valleyview_pipestat_irq_handler(struct drm_i915_private *dev_priv,
                                             u32 pipe_stats[I915_MAX_PIPES])
 {
-        struct drm_i915_private *dev_priv = to_i915(dev);
         enum pipe pipe;
 
         for_each_pipe(dev_priv, pipe) {
                 if (pipe_stats[pipe] & PIPE_START_VBLANK_INTERRUPT_STATUS &&
-                    intel_pipe_handle_vblank(dev, pipe))
-                        intel_check_page_flip(dev, pipe);
+                    intel_pipe_handle_vblank(dev_priv, pipe))
+                        intel_check_page_flip(dev_priv, pipe);
 
-                if (pipe_stats[pipe] & PLANE_FLIP_DONE_INT_STATUS_VLV) {
-                        intel_prepare_page_flip(dev, pipe);
-                        intel_finish_page_flip(dev, pipe);
-                }
+                if (pipe_stats[pipe] & PLANE_FLIP_DONE_INT_STATUS_VLV)
+                        intel_finish_page_flip_cs(dev_priv, pipe);
 
                 if (pipe_stats[pipe] & PIPE_CRC_DONE_INTERRUPT_STATUS)
-                        i9xx_pipe_crc_irq_handler(dev, pipe);
+                        i9xx_pipe_crc_irq_handler(dev_priv, pipe);
 
                 if (pipe_stats[pipe] & PIPE_FIFO_UNDERRUN_STATUS)
                         intel_cpu_fifo_underrun_irq_handler(dev_priv, pipe);
         }
 
         if (pipe_stats[0] & PIPE_GMBUS_INTERRUPT_STATUS)
-                gmbus_irq_handler(dev);
+                gmbus_irq_handler(dev_priv);
 }
 
 static u32 i9xx_hpd_irq_ack(struct drm_i915_private *dev_priv)
@@ -1747,12 +1710,13 @@ static u32 i9xx_hpd_irq_ack(struct drm_i915_private *dev_priv)
         return hotplug_status;
 }
 
-static void i9xx_hpd_irq_handler(struct drm_device *dev,
+static void i9xx_hpd_irq_handler(struct drm_i915_private *dev_priv,
                                  u32 hotplug_status)
 {
         u32 pin_mask = 0, long_mask = 0;
 
-        if (IS_G4X(dev) || IS_VALLEYVIEW(dev) || IS_CHERRYVIEW(dev)) {
+        if (IS_G4X(dev_priv) || IS_VALLEYVIEW(dev_priv) ||
+            IS_CHERRYVIEW(dev_priv)) {
                 u32 hotplug_trigger = hotplug_status & HOTPLUG_INT_STATUS_G4X;
 
                 if (hotplug_trigger) {
@@ -1760,11 +1724,11 @@ static void i9xx_hpd_irq_handler(struct drm_device *dev,
                                            hotplug_trigger, hpd_status_g4x,
                                            i9xx_port_hotplug_long_detect);
 
-                        intel_hpd_irq_handler(dev, pin_mask, long_mask);
+                        intel_hpd_irq_handler(dev_priv, pin_mask, long_mask);
                 }
 
                 if (hotplug_status & DP_AUX_CHANNEL_MASK_INT_STATUS_G4X)
-                        dp_aux_irq_handler(dev);
+                        dp_aux_irq_handler(dev_priv);
         } else {
                 u32 hotplug_trigger = hotplug_status & HOTPLUG_INT_STATUS_I915;
 
@@ -1772,7 +1736,7 @@ static void i9xx_hpd_irq_handler(struct drm_device *dev,
                         intel_get_hpd_pins(&pin_mask, &long_mask, hotplug_trigger,
                                            hotplug_trigger, hpd_status_i915,
                                            i9xx_port_hotplug_long_detect);
-                        intel_hpd_irq_handler(dev, pin_mask, long_mask);
+                        intel_hpd_irq_handler(dev_priv, pin_mask, long_mask);
                 }
         }
 }
@@ -1780,7 +1744,7 @@ static void i9xx_hpd_irq_handler(struct drm_device *dev,
 static irqreturn_t valleyview_irq_handler(int irq, void *arg)
 {
         struct drm_device *dev = arg;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         irqreturn_t ret = IRQ_NONE;
 
         if (!intel_irqs_enabled(dev_priv))
@@ -1831,7 +1795,7 @@ static irqreturn_t valleyview_irq_handler(int irq, void *arg)
 
                 /* Call regardless, as some status bits might not be
                  * signalled in iir */
-                valleyview_pipestat_irq_ack(dev, iir, pipe_stats);
+                valleyview_pipestat_irq_ack(dev_priv, iir, pipe_stats);
 
                 /*
                  * VLV_IIR is single buffered, and reflects the level
@@ -1850,9 +1814,9 @@ static irqreturn_t valleyview_irq_handler(int irq, void *arg)
                         gen6_rps_irq_handler(dev_priv, pm_iir);
 
                 if (hotplug_status)
-                        i9xx_hpd_irq_handler(dev, hotplug_status);
+                        i9xx_hpd_irq_handler(dev_priv, hotplug_status);
 
-                valleyview_pipestat_irq_handler(dev, pipe_stats);
+                valleyview_pipestat_irq_handler(dev_priv, pipe_stats);
         } while (0);
 
         enable_rpm_wakeref_asserts(dev_priv);
@@ -1863,7 +1827,7 @@ static irqreturn_t valleyview_irq_handler(int irq, void *arg)
 static irqreturn_t cherryview_irq_handler(int irq, void *arg)
 {
         struct drm_device *dev = arg;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         irqreturn_t ret = IRQ_NONE;
 
         if (!intel_irqs_enabled(dev_priv))
@@ -1911,7 +1875,7 @@ static irqreturn_t cherryview_irq_handler(int irq, void *arg)
 
                 /* Call regardless, as some status bits might not be
                  * signalled in iir */
-                valleyview_pipestat_irq_ack(dev, iir, pipe_stats);
+                valleyview_pipestat_irq_ack(dev_priv, iir, pipe_stats);
 
                 /*
                  * VLV_IIR is single buffered, and reflects the level
@@ -1927,9 +1891,9 @@ static irqreturn_t cherryview_irq_handler(int irq, void *arg)
                 gen8_gt_irq_handler(dev_priv, gt_iir);
 
                 if (hotplug_status)
-                        i9xx_hpd_irq_handler(dev, hotplug_status);
+                        i9xx_hpd_irq_handler(dev_priv, hotplug_status);
 
-                valleyview_pipestat_irq_handler(dev, pipe_stats);
+                valleyview_pipestat_irq_handler(dev_priv, pipe_stats);
         } while (0);
 
         enable_rpm_wakeref_asserts(dev_priv);
@@ -1937,10 +1901,10 @@ static irqreturn_t cherryview_irq_handler(int irq, void *arg)
         return ret;
 }
 
-static void ibx_hpd_irq_handler(struct drm_device *dev, u32 hotplug_trigger,
+static void ibx_hpd_irq_handler(struct drm_i915_private *dev_priv,
+                                u32 hotplug_trigger,
                                 const u32 hpd[HPD_NUM_PINS])
 {
-        struct drm_i915_private *dev_priv = to_i915(dev);
         u32 dig_hotplug_reg, pin_mask = 0, long_mask = 0;
 
         /*
@@ -1966,16 +1930,15 @@ static void ibx_hpd_irq_handler(struct drm_device *dev, u32 hotplug_trigger,
                            dig_hotplug_reg, hpd,
                            pch_port_hotplug_long_detect);
 
-        intel_hpd_irq_handler(dev, pin_mask, long_mask);
+        intel_hpd_irq_handler(dev_priv, pin_mask, long_mask);
 }
 
-static void ibx_irq_handler(struct drm_device *dev, u32 pch_iir)
+static void ibx_irq_handler(struct drm_i915_private *dev_priv, u32 pch_iir)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
         int pipe;
         u32 hotplug_trigger = pch_iir & SDE_HOTPLUG_MASK;
 
-        ibx_hpd_irq_handler(dev, hotplug_trigger, hpd_ibx);
+        ibx_hpd_irq_handler(dev_priv, hotplug_trigger, hpd_ibx);
 
         if (pch_iir & SDE_AUDIO_POWER_MASK) {
                 int port = ffs((pch_iir & SDE_AUDIO_POWER_MASK) >>
@@ -1985,10 +1948,10 @@ static void ibx_irq_handler(struct drm_device *dev, u32 pch_iir)
         }
 
         if (pch_iir & SDE_AUX_MASK)
-                dp_aux_irq_handler(dev);
+                dp_aux_irq_handler(dev_priv);
 
         if (pch_iir & SDE_GMBUS)
-                gmbus_irq_handler(dev);
+                gmbus_irq_handler(dev_priv);
 
         if (pch_iir & SDE_AUDIO_HDCP_MASK)
                 DRM_DEBUG_DRIVER("PCH HDCP audio interrupt\n");
@@ -2018,9 +1981,8 @@ static void ibx_irq_handler(struct drm_device *dev, u32 pch_iir)
                 intel_pch_fifo_underrun_irq_handler(dev_priv, TRANSCODER_B);
 }
 
-static void ivb_err_int_handler(struct drm_device *dev)
+static void ivb_err_int_handler(struct drm_i915_private *dev_priv)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
         u32 err_int = I915_READ(GEN7_ERR_INT);
         enum pipe pipe;
 
@@ -2032,19 +1994,18 @@ static void ivb_err_int_handler(struct drm_device *dev)
                         intel_cpu_fifo_underrun_irq_handler(dev_priv, pipe);
 
                 if (err_int & ERR_INT_PIPE_CRC_DONE(pipe)) {
-                        if (IS_IVYBRIDGE(dev))
-                                ivb_pipe_crc_irq_handler(dev, pipe);
+                        if (IS_IVYBRIDGE(dev_priv))
+                                ivb_pipe_crc_irq_handler(dev_priv, pipe);
                         else
-                                hsw_pipe_crc_irq_handler(dev, pipe);
+                                hsw_pipe_crc_irq_handler(dev_priv, pipe);
                 }
         }
 
         I915_WRITE(GEN7_ERR_INT, err_int);
 }
 
-static void cpt_serr_int_handler(struct drm_device *dev)
+static void cpt_serr_int_handler(struct drm_i915_private *dev_priv)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
         u32 serr_int = I915_READ(SERR_INT);
 
         if (serr_int & SERR_INT_POISON)
@@ -2062,13 +2023,12 @@ static void cpt_serr_int_handler(struct drm_device *dev)
         I915_WRITE(SERR_INT, serr_int);
 }
 
-static void cpt_irq_handler(struct drm_device *dev, u32 pch_iir)
+static void cpt_irq_handler(struct drm_i915_private *dev_priv, u32 pch_iir)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
         int pipe;
         u32 hotplug_trigger = pch_iir & SDE_HOTPLUG_MASK_CPT;
 
-        ibx_hpd_irq_handler(dev, hotplug_trigger, hpd_cpt);
+        ibx_hpd_irq_handler(dev_priv, hotplug_trigger, hpd_cpt);
 
         if (pch_iir & SDE_AUDIO_POWER_MASK_CPT) {
                 int port = ffs((pch_iir & SDE_AUDIO_POWER_MASK_CPT) >>
@@ -2078,10 +2038,10 @@ static void cpt_irq_handler(struct drm_device *dev, u32 pch_iir)
         }
 
         if (pch_iir & SDE_AUX_MASK_CPT)
-                dp_aux_irq_handler(dev);
+                dp_aux_irq_handler(dev_priv);
 
         if (pch_iir & SDE_GMBUS_CPT)
-                gmbus_irq_handler(dev);
+                gmbus_irq_handler(dev_priv);
 
         if (pch_iir & SDE_AUDIO_CP_REQ_CPT)
                 DRM_DEBUG_DRIVER("Audio CP request interrupt\n");
@@ -2096,12 +2056,11 @@ static void cpt_irq_handler(struct drm_device *dev, u32 pch_iir)
                                          I915_READ(FDI_RX_IIR(pipe)));
 
         if (pch_iir & SDE_ERROR_CPT)
-                cpt_serr_int_handler(dev);
+                cpt_serr_int_handler(dev_priv);
 }
 
-static void spt_irq_handler(struct drm_device *dev, u32 pch_iir)
+static void spt_irq_handler(struct drm_i915_private *dev_priv, u32 pch_iir)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
         u32 hotplug_trigger = pch_iir & SDE_HOTPLUG_MASK_SPT &
                 ~SDE_PORTE_HOTPLUG_SPT;
         u32 hotplug2_trigger = pch_iir & SDE_PORTE_HOTPLUG_SPT;
@@ -2130,16 +2089,16 @@ static void spt_irq_handler(struct drm_device *dev, u32 pch_iir)
         }
 
         if (pin_mask)
-                intel_hpd_irq_handler(dev, pin_mask, long_mask);
+                intel_hpd_irq_handler(dev_priv, pin_mask, long_mask);
 
         if (pch_iir & SDE_GMBUS_CPT)
-                gmbus_irq_handler(dev);
+                gmbus_irq_handler(dev_priv);
 }
 
-static void ilk_hpd_irq_handler(struct drm_device *dev, u32 hotplug_trigger,
+static void ilk_hpd_irq_handler(struct drm_i915_private *dev_priv,
+                                u32 hotplug_trigger,
                                 const u32 hpd[HPD_NUM_PINS])
 {
-        struct drm_i915_private *dev_priv = to_i915(dev);
         u32 dig_hotplug_reg, pin_mask = 0, long_mask = 0;
 
         dig_hotplug_reg = I915_READ(DIGITAL_PORT_HOTPLUG_CNTRL);
@@ -2149,97 +2108,93 @@ static void ilk_hpd_irq_handler(struct drm_device *dev, u32 hotplug_trigger,
                            dig_hotplug_reg, hpd,
                            ilk_port_hotplug_long_detect);
 
-        intel_hpd_irq_handler(dev, pin_mask, long_mask);
+        intel_hpd_irq_handler(dev_priv, pin_mask, long_mask);
 }
 
-static void ilk_display_irq_handler(struct drm_device *dev, u32 de_iir)
+static void ilk_display_irq_handler(struct drm_i915_private *dev_priv,
+                                    u32 de_iir)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
         enum pipe pipe;
         u32 hotplug_trigger = de_iir & DE_DP_A_HOTPLUG;
 
         if (hotplug_trigger)
-                ilk_hpd_irq_handler(dev, hotplug_trigger, hpd_ilk);
+                ilk_hpd_irq_handler(dev_priv, hotplug_trigger, hpd_ilk);
 
         if (de_iir & DE_AUX_CHANNEL_A)
-                dp_aux_irq_handler(dev);
+                dp_aux_irq_handler(dev_priv);
 
         if (de_iir & DE_GSE)
-                intel_opregion_asle_intr(dev);
+                intel_opregion_asle_intr(dev_priv);
 
         if (de_iir & DE_POISON)
                 DRM_ERROR("Poison interrupt\n");
 
         for_each_pipe(dev_priv, pipe) {
                 if (de_iir & DE_PIPE_VBLANK(pipe) &&
-                    intel_pipe_handle_vblank(dev, pipe))
-                        intel_check_page_flip(dev, pipe);
+                    intel_pipe_handle_vblank(dev_priv, pipe))
+                        intel_check_page_flip(dev_priv, pipe);
 
                 if (de_iir & DE_PIPE_FIFO_UNDERRUN(pipe))
                         intel_cpu_fifo_underrun_irq_handler(dev_priv, pipe);
 
                 if (de_iir & DE_PIPE_CRC_DONE(pipe))
-                        i9xx_pipe_crc_irq_handler(dev, pipe);
+                        i9xx_pipe_crc_irq_handler(dev_priv, pipe);
 
                 /* plane/pipes map 1:1 on ilk+ */
-                if (de_iir & DE_PLANE_FLIP_DONE(pipe)) {
-                        intel_prepare_page_flip(dev, pipe);
-                        intel_finish_page_flip_plane(dev, pipe);
-                }
+                if (de_iir & DE_PLANE_FLIP_DONE(pipe))
+                        intel_finish_page_flip_cs(dev_priv, pipe);
         }
 
         /* check event from PCH */
         if (de_iir & DE_PCH_EVENT) {
                 u32 pch_iir = I915_READ(SDEIIR);
 
-                if (HAS_PCH_CPT(dev))
-                        cpt_irq_handler(dev, pch_iir);
+                if (HAS_PCH_CPT(dev_priv))
+                        cpt_irq_handler(dev_priv, pch_iir);
                 else
-                        ibx_irq_handler(dev, pch_iir);
+                        ibx_irq_handler(dev_priv, pch_iir);
 
                 /* should clear PCH hotplug event before clear CPU irq */
                 I915_WRITE(SDEIIR, pch_iir);
         }
 
-        if (IS_GEN5(dev) && de_iir & DE_PCU_EVENT)
-                ironlake_rps_change_irq_handler(dev);
+        if (IS_GEN5(dev_priv) && de_iir & DE_PCU_EVENT)
+                ironlake_rps_change_irq_handler(dev_priv);
 }
 
-static void ivb_display_irq_handler(struct drm_device *dev, u32 de_iir)
+static void ivb_display_irq_handler(struct drm_i915_private *dev_priv,
+                                    u32 de_iir)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
         enum pipe pipe;
         u32 hotplug_trigger = de_iir & DE_DP_A_HOTPLUG_IVB;
 
         if (hotplug_trigger)
-                ilk_hpd_irq_handler(dev, hotplug_trigger, hpd_ivb);
+                ilk_hpd_irq_handler(dev_priv, hotplug_trigger, hpd_ivb);
 
         if (de_iir & DE_ERR_INT_IVB)
-                ivb_err_int_handler(dev);
+                ivb_err_int_handler(dev_priv);
 
         if (de_iir & DE_AUX_CHANNEL_A_IVB)
-                dp_aux_irq_handler(dev);
+                dp_aux_irq_handler(dev_priv);
 
         if (de_iir & DE_GSE_IVB)
-                intel_opregion_asle_intr(dev);
+                intel_opregion_asle_intr(dev_priv);
 
         for_each_pipe(dev_priv, pipe) {
                 if (de_iir & (DE_PIPE_VBLANK_IVB(pipe)) &&
-                    intel_pipe_handle_vblank(dev, pipe))
-                        intel_check_page_flip(dev, pipe);
+                    intel_pipe_handle_vblank(dev_priv, pipe))
+                        intel_check_page_flip(dev_priv, pipe);
 
                 /* plane/pipes map 1:1 on ilk+ */
-                if (de_iir & DE_PLANE_FLIP_DONE_IVB(pipe)) {
-                        intel_prepare_page_flip(dev, pipe);
-                        intel_finish_page_flip_plane(dev, pipe);
-                }
+                if (de_iir & DE_PLANE_FLIP_DONE_IVB(pipe))
+                        intel_finish_page_flip_cs(dev_priv, pipe);
         }
 
         /* check event from PCH */
-        if (!HAS_PCH_NOP(dev) && (de_iir & DE_PCH_EVENT_IVB)) {
+        if (!HAS_PCH_NOP(dev_priv) && (de_iir & DE_PCH_EVENT_IVB)) {
                 u32 pch_iir = I915_READ(SDEIIR);
 
-                cpt_irq_handler(dev, pch_iir);
+                cpt_irq_handler(dev_priv, pch_iir);
 
                 /* clear PCH hotplug event before clear CPU irq */
                 I915_WRITE(SDEIIR, pch_iir);
@@ -2257,7 +2212,7 @@ static void ivb_display_irq_handler(struct drm_device *dev, u32 de_iir)
 static irqreturn_t ironlake_irq_handler(int irq, void *arg)
 {
         struct drm_device *dev = arg;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         u32 de_iir, gt_iir, de_ier, sde_ier = 0;
         irqreturn_t ret = IRQ_NONE;
 
@@ -2277,7 +2232,7 @@ static irqreturn_t ironlake_irq_handler(int irq, void *arg)
          * able to process them after we restore SDEIER (as soon as we restore
          * it, we'll get an interrupt if SDEIIR still has something to process
          * due to its back queue). */
-        if (!HAS_PCH_NOP(dev)) {
+        if (!HAS_PCH_NOP(dev_priv)) {
                 sde_ier = I915_READ(SDEIER);
                 I915_WRITE(SDEIER, 0);
                 POSTING_READ(SDEIER);
@@ -2289,7 +2244,7 @@ static irqreturn_t ironlake_irq_handler(int irq, void *arg)
         if (gt_iir) {
                 I915_WRITE(GTIIR, gt_iir);
                 ret = IRQ_HANDLED;
-                if (INTEL_INFO(dev)->gen >= 6)
+                if (INTEL_GEN(dev_priv) >= 6)
                         snb_gt_irq_handler(dev_priv, gt_iir);
                 else
                         ilk_gt_irq_handler(dev_priv, gt_iir);
@@ -2299,13 +2254,13 @@ static irqreturn_t ironlake_irq_handler(int irq, void *arg)
         if (de_iir) {
                 I915_WRITE(DEIIR, de_iir);
                 ret = IRQ_HANDLED;
-                if (INTEL_INFO(dev)->gen >= 7)
-                        ivb_display_irq_handler(dev, de_iir);
+                if (INTEL_GEN(dev_priv) >= 7)
+                        ivb_display_irq_handler(dev_priv, de_iir);
                 else
-                        ilk_display_irq_handler(dev, de_iir);
+                        ilk_display_irq_handler(dev_priv, de_iir);
         }
 
-        if (INTEL_INFO(dev)->gen >= 6) {
+        if (INTEL_GEN(dev_priv) >= 6) {
                 u32 pm_iir = I915_READ(GEN6_PMIIR);
                 if (pm_iir) {
                         I915_WRITE(GEN6_PMIIR, pm_iir);
@@ -2316,7 +2271,7 @@ static irqreturn_t ironlake_irq_handler(int irq, void *arg)
 
         I915_WRITE(DEIER, de_ier);
         POSTING_READ(DEIER);
-        if (!HAS_PCH_NOP(dev)) {
+        if (!HAS_PCH_NOP(dev_priv)) {
                 I915_WRITE(SDEIER, sde_ier);
                 POSTING_READ(SDEIER);
         }
@@ -2327,10 +2282,10 @@ static irqreturn_t ironlake_irq_handler(int irq, void *arg)
         return ret;
 }
 
-static void bxt_hpd_irq_handler(struct drm_device *dev, u32 hotplug_trigger,
+static void bxt_hpd_irq_handler(struct drm_i915_private *dev_priv,
+                                u32 hotplug_trigger,
                                 const u32 hpd[HPD_NUM_PINS])
 {
-        struct drm_i915_private *dev_priv = to_i915(dev);
         u32 dig_hotplug_reg, pin_mask = 0, long_mask = 0;
 
         dig_hotplug_reg = I915_READ(PCH_PORT_HOTPLUG);
@@ -2340,13 +2295,12 @@ static void bxt_hpd_irq_handler(struct drm_device *dev, u32 hotplug_trigger,
                            dig_hotplug_reg, hpd,
                            bxt_port_hotplug_long_detect);
 
-        intel_hpd_irq_handler(dev, pin_mask, long_mask);
+        intel_hpd_irq_handler(dev_priv, pin_mask, long_mask);
 }
 
 static irqreturn_t
 gen8_de_irq_handler(struct drm_i915_private *dev_priv, u32 master_ctl)
 {
-        struct drm_device *dev = dev_priv->dev;
         irqreturn_t ret = IRQ_NONE;
         u32 iir;
         enum pipe pipe;
@@ -2357,7 +2311,7 @@ gen8_de_irq_handler(struct drm_i915_private *dev_priv, u32 master_ctl)
                         I915_WRITE(GEN8_DE_MISC_IIR, iir);
                         ret = IRQ_HANDLED;
                         if (iir & GEN8_DE_MISC_GSE)
-                                intel_opregion_asle_intr(dev);
+                                intel_opregion_asle_intr(dev_priv);
                         else
                                 DRM_ERROR("Unexpected DE Misc interrupt\n");
                 }
@@ -2381,26 +2335,28 @@ gen8_de_irq_handler(struct drm_i915_private *dev_priv, u32 master_ctl)
                                             GEN9_AUX_CHANNEL_D;
 
                         if (iir & tmp_mask) {
-                                dp_aux_irq_handler(dev);
+                                dp_aux_irq_handler(dev_priv);
                                 found = true;
                         }
 
                         if (IS_BROXTON(dev_priv)) {
                                 tmp_mask = iir & BXT_DE_PORT_HOTPLUG_MASK;
                                 if (tmp_mask) {
-                                        bxt_hpd_irq_handler(dev, tmp_mask, hpd_bxt);
+                                        bxt_hpd_irq_handler(dev_priv, tmp_mask,
+                                                            hpd_bxt);
                                         found = true;
                                 }
                         } else if (IS_BROADWELL(dev_priv)) {
                                 tmp_mask = iir & GEN8_PORT_DP_A_HOTPLUG;
                                 if (tmp_mask) {
-                                        ilk_hpd_irq_handler(dev, tmp_mask, hpd_bdw);
+                                        ilk_hpd_irq_handler(dev_priv,
+                                                            tmp_mask, hpd_bdw);
                                         found = true;
                                 }
                         }
 
-                        if (IS_BROXTON(dev) && (iir & BXT_DE_PORT_GMBUS)) {
-                                gmbus_irq_handler(dev);
+                        if (IS_BROXTON(dev_priv) && (iir & BXT_DE_PORT_GMBUS)) {
+                                gmbus_irq_handler(dev_priv);
                                 found = true;
                         }
 
@@ -2427,8 +2383,8 @@ gen8_de_irq_handler(struct drm_i915_private *dev_priv, u32 master_ctl)
                 I915_WRITE(GEN8_DE_PIPE_IIR(pipe), iir);
 
                 if (iir & GEN8_PIPE_VBLANK &&
-                    intel_pipe_handle_vblank(dev, pipe))
-                        intel_check_page_flip(dev, pipe);
+                    intel_pipe_handle_vblank(dev_priv, pipe))
+                        intel_check_page_flip(dev_priv, pipe);
 
                 flip_done = iir;
                 if (INTEL_INFO(dev_priv)->gen >= 9)
@@ -2436,13 +2392,11 @@ gen8_de_irq_handler(struct drm_i915_private *dev_priv, u32 master_ctl)
                 else
                         flip_done &= GEN8_PIPE_PRIMARY_FLIP_DONE;
 
-                if (flip_done) {
-                        intel_prepare_page_flip(dev, pipe);
-                        intel_finish_page_flip_plane(dev, pipe);
-                }
+                if (flip_done)
+                        intel_finish_page_flip_cs(dev_priv, pipe);
 
                 if (iir & GEN8_PIPE_CDCLK_CRC_DONE)
-                        hsw_pipe_crc_irq_handler(dev, pipe);
+                        hsw_pipe_crc_irq_handler(dev_priv, pipe);
 
                 if (iir & GEN8_PIPE_FIFO_UNDERRUN)
                         intel_cpu_fifo_underrun_irq_handler(dev_priv, pipe);
@@ -2459,7 +2413,7 @@ gen8_de_irq_handler(struct drm_i915_private *dev_priv, u32 master_ctl)
                                   fault_errors);
         }
 
-        if (HAS_PCH_SPLIT(dev) && !HAS_PCH_NOP(dev) &&
+        if (HAS_PCH_SPLIT(dev_priv) && !HAS_PCH_NOP(dev_priv) &&
             master_ctl & GEN8_DE_PCH_IRQ) {
                 /*
                  * FIXME(BDW): Assume for now that the new interrupt handling
@@ -2472,9 +2426,9 @@ gen8_de_irq_handler(struct drm_i915_private *dev_priv, u32 master_ctl)
                         ret = IRQ_HANDLED;
 
                         if (HAS_PCH_SPT(dev_priv) || HAS_PCH_KBP(dev_priv))
-                                spt_irq_handler(dev, iir);
+                                spt_irq_handler(dev_priv, iir);
                         else
-                                cpt_irq_handler(dev, iir);
+                                cpt_irq_handler(dev_priv, iir);
                 } else {
                         /*
                          * Like on previous PCH there seems to be something
@@ -2490,7 +2444,7 @@ gen8_de_irq_handler(struct drm_i915_private *dev_priv, u32 master_ctl)
 static irqreturn_t gen8_irq_handler(int irq, void *arg)
 {
         struct drm_device *dev = arg;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         u32 master_ctl;
         u32 gt_iir[4] = {};
         irqreturn_t ret;
@@ -2521,11 +2475,8 @@ static irqreturn_t gen8_irq_handler(int irq, void *arg)
         return ret;
 }
 
-static void i915_error_wake_up(struct drm_i915_private *dev_priv,
-                               bool reset_completed)
+static void i915_error_wake_up(struct drm_i915_private *dev_priv)
 {
-        struct intel_engine_cs *engine;
-
         /*
          * Notify all waiters for GPU completion events that reset state has
          * been changed, and that they need to restart their wait after
@@ -2534,36 +2485,28 @@ static void i915_error_wake_up(struct drm_i915_private *dev_priv,
          */
 
         /* Wake up __wait_seqno, potentially holding dev->struct_mutex. */
-        for_each_engine(engine, dev_priv)
-                wake_up_all(&engine->irq_queue);
+        wake_up_all(&dev_priv->gpu_error.wait_queue);
 
         /* Wake up intel_crtc_wait_for_pending_flips, holding crtc->mutex. */
         wake_up_all(&dev_priv->pending_flip_queue);
-
-        /*
-         * Signal tasks blocked in i915_gem_wait_for_error that the pending
-         * reset state is cleared.
-         */
-        if (reset_completed)
-                wake_up_all(&dev_priv->gpu_error.reset_queue);
 }
 
 /**
  * i915_reset_and_wakeup - do process context error handling work
- * @dev: drm device
+ * @dev_priv: i915 device private
  *
  * Fire an error uevent so userspace can see that a hang or error
  * was detected.
  */
-static void i915_reset_and_wakeup(struct drm_device *dev)
+static void i915_reset_and_wakeup(struct drm_i915_private *dev_priv)
 {
-        struct drm_i915_private *dev_priv = to_i915(dev);
+        struct kobject *kobj = &dev_priv->drm.primary->kdev->kobj;
         char *error_event[] = { I915_ERROR_UEVENT "=1", NULL };
         char *reset_event[] = { I915_RESET_UEVENT "=1", NULL };
         char *reset_done_event[] = { I915_ERROR_UEVENT "=0", NULL };
         int ret;
 
-        kobject_uevent_env(&dev->primary->kdev->kobj, KOBJ_CHANGE, error_event);
+        kobject_uevent_env(kobj, KOBJ_CHANGE, error_event);
 
         /*
          * Note that there's only one work item which does gpu resets, so we
@@ -2577,8 +2520,7 @@ static void i915_reset_and_wakeup(struct drm_device *dev)
          */
         if (i915_reset_in_progress(&dev_priv->gpu_error)) {
                 DRM_DEBUG_DRIVER("resetting chip\n");
-                kobject_uevent_env(&dev->primary->kdev->kobj, KOBJ_CHANGE,
-                                   reset_event);
+                kobject_uevent_env(kobj, KOBJ_CHANGE, reset_event);
 
                 /*
                  * In most cases it's guaranteed that we get here with an RPM
@@ -2589,7 +2531,7 @@ static void i915_reset_and_wakeup(struct drm_device *dev)
                  */
                 intel_runtime_pm_get(dev_priv);
 
-                intel_prepare_reset(dev);
+                intel_prepare_reset(dev_priv);
 
                 /*
                  * All state reset _must_ be completed before we update the
@@ -2597,27 +2539,26 @@ static void i915_reset_and_wakeup(struct drm_device *dev)
                  * pending state and not properly drop locks, resulting in
                  * deadlocks with the reset work.
                  */
-                ret = i915_reset(dev);
+                ret = i915_reset(dev_priv);
 
-                intel_finish_reset(dev);
+                intel_finish_reset(dev_priv);
 
                 intel_runtime_pm_put(dev_priv);
 
                 if (ret == 0)
-                        kobject_uevent_env(&dev->primary->kdev->kobj,
+                        kobject_uevent_env(kobj,
                                            KOBJ_CHANGE, reset_done_event);
 
                 /*
                  * Note: The wake_up also serves as a memory barrier so that
                  * waiters see the update value of the reset counter atomic_t.
                  */
-                i915_error_wake_up(dev_priv, true);
+                wake_up_all(&dev_priv->gpu_error.reset_queue);
         }
 }
 
-static void i915_report_and_clear_eir(struct drm_device *dev)
+static void i915_report_and_clear_eir(struct drm_i915_private *dev_priv)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
         uint32_t instdone[I915_NUM_INSTDONE_REG];
         u32 eir = I915_READ(EIR);
         int pipe, i;
@@ -2627,9 +2568,9 @@ static void i915_report_and_clear_eir(struct drm_device *dev)
 
         pr_err("render error detected, EIR: 0x%08x\n", eir);
 
-        i915_get_extra_instdone(dev, instdone);
+        i915_get_extra_instdone(dev_priv, instdone);
 
-        if (IS_G4X(dev)) {
+        if (IS_G4X(dev_priv)) {
                 if (eir & (GM45_ERROR_MEM_PRIV | GM45_ERROR_CP_PRIV)) {
                         u32 ipeir = I915_READ(IPEIR_I965);
 
@@ -2651,7 +2592,7 @@ static void i915_report_and_clear_eir(struct drm_device *dev)
                 }
         }
 
-        if (!IS_GEN2(dev)) {
+        if (!IS_GEN2(dev_priv)) {
                 if (eir & I915_ERROR_PAGE_TABLE) {
                         u32 pgtbl_err = I915_READ(PGTBL_ER);
                         pr_err("page table error\n");
@@ -2673,7 +2614,7 @@ static void i915_report_and_clear_eir(struct drm_device *dev)
                 pr_err("  INSTPM: 0x%08x\n", I915_READ(INSTPM));
                 for (i = 0; i < ARRAY_SIZE(instdone); i++)
                         pr_err("  INSTDONE_%d: 0x%08x\n", i, instdone[i]);
-                if (INTEL_INFO(dev)->gen < 4) {
+                if (INTEL_GEN(dev_priv) < 4) {
                         u32 ipeir = I915_READ(IPEIR);
 
                         pr_err("  IPEIR: 0x%08x\n", I915_READ(IPEIR));
@@ -2709,18 +2650,19 @@ static void i915_report_and_clear_eir(struct drm_device *dev)
 
 /**
  * i915_handle_error - handle a gpu error
- * @dev: drm device
+ * @dev_priv: i915 device private
  * @engine_mask: mask representing engines that are hung
  * Do some basic checking of register state at error time and
  * dump it to the syslog.  Also call i915_capture_error_state() to make
  * sure we get a record and make it available in debugfs.  Fire a uevent
  * so userspace knows something bad happened (should trigger collection
  * of a ring dump etc.).
+ * @fmt: Error message format string
  */
-void i915_handle_error(struct drm_device *dev, u32 engine_mask,
+void i915_handle_error(struct drm_i915_private *dev_priv,
+                       u32 engine_mask,
                        const char *fmt, ...)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
         va_list args;
         char error_msg[80];
 
@@ -2728,8 +2670,8 @@ void i915_handle_error(struct drm_device *dev, u32 engine_mask,
         vscnprintf(error_msg, sizeof(error_msg), fmt, args);
         va_end(args);
 
-        i915_capture_error_state(dev, engine_mask, error_msg);
-        i915_report_and_clear_eir(dev);
+        i915_capture_error_state(dev_priv, engine_mask, error_msg);
+        i915_report_and_clear_eir(dev_priv);
 
         if (engine_mask) {
                 atomic_or(I915_RESET_IN_PROGRESS_FLAG,
@@ -2748,10 +2690,10 @@ void i915_handle_error(struct drm_device *dev, u32 engine_mask,
                  * ensure that the waiters see the updated value of the reset
                  * counter atomic_t.
                  */
-                i915_error_wake_up(dev_priv, false);
+                i915_error_wake_up(dev_priv);
         }
 
-        i915_reset_and_wakeup(dev);
+        i915_reset_and_wakeup(dev_priv);
 }
 
 /* Called from drm generic code, passed 'crtc' which
@@ -2759,7 +2701,7 @@ void i915_handle_error(struct drm_device *dev, u32 engine_mask,
  */
 static int i915_enable_vblank(struct drm_device *dev, unsigned int pipe)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         unsigned long irqflags;
 
         spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
@@ -2776,7 +2718,7 @@ static int i915_enable_vblank(struct drm_device *dev, unsigned int pipe)
 
 static int ironlake_enable_vblank(struct drm_device *dev, unsigned int pipe)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         unsigned long irqflags;
         uint32_t bit = (INTEL_INFO(dev)->gen >= 7) ? DE_PIPE_VBLANK_IVB(pipe) :
                                                      DE_PIPE_VBLANK(pipe);
@@ -2790,7 +2732,7 @@ static int ironlake_enable_vblank(struct drm_device *dev, unsigned int pipe)
 
 static int valleyview_enable_vblank(struct drm_device *dev, unsigned int pipe)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         unsigned long irqflags;
 
         spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
@@ -2803,7 +2745,7 @@ static int valleyview_enable_vblank(struct drm_device *dev, unsigned int pipe)
 
 static int gen8_enable_vblank(struct drm_device *dev, unsigned int pipe)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         unsigned long irqflags;
 
         spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
@@ -2818,7 +2760,7 @@ static int gen8_enable_vblank(struct drm_device *dev, unsigned int pipe)
  */
 static void i915_disable_vblank(struct drm_device *dev, unsigned int pipe)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         unsigned long irqflags;
 
         spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
@@ -2830,7 +2772,7 @@ static void i915_disable_vblank(struct drm_device *dev, unsigned int pipe)
 
 static void ironlake_disable_vblank(struct drm_device *dev, unsigned int pipe)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         unsigned long irqflags;
         uint32_t bit = (INTEL_INFO(dev)->gen >= 7) ? DE_PIPE_VBLANK_IVB(pipe) :
                                                      DE_PIPE_VBLANK(pipe);
@@ -2842,7 +2784,7 @@ static void ironlake_disable_vblank(struct drm_device *dev, unsigned int pipe)
 
 static void valleyview_disable_vblank(struct drm_device *dev, unsigned int pipe)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         unsigned long irqflags;
 
         spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
@@ -2853,7 +2795,7 @@ static void valleyview_disable_vblank(struct drm_device *dev, unsigned int pipe)
 
 static void gen8_disable_vblank(struct drm_device *dev, unsigned int pipe)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         unsigned long irqflags;
 
         spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
@@ -2869,9 +2811,9 @@ ring_idle(struct intel_engine_cs *engine, u32 seqno)
 }
 
 static bool
-ipehr_is_semaphore_wait(struct drm_device *dev, u32 ipehr)
+ipehr_is_semaphore_wait(struct intel_engine_cs *engine, u32 ipehr)
 {
-        if (INTEL_INFO(dev)->gen >= 8) {
+        if (INTEL_GEN(engine->i915) >= 8) {
                 return (ipehr >> 23) == 0x1c;
         } else {
                 ipehr &= ~MI_SEMAPHORE_SYNC_MASK;
@@ -2884,10 +2826,10 @@ static struct intel_engine_cs *
 semaphore_wait_to_signaller_ring(struct intel_engine_cs *engine, u32 ipehr,
                                  u64 offset)
 {
-        struct drm_i915_private *dev_priv = engine->dev->dev_private;
+        struct drm_i915_private *dev_priv = engine->i915;
         struct intel_engine_cs *signaller;
 
-        if (INTEL_INFO(dev_priv)->gen >= 8) {
+        if (INTEL_GEN(dev_priv) >= 8) {
                 for_each_engine(signaller, dev_priv) {
                         if (engine == signaller)
                                 continue;
@@ -2916,7 +2858,7 @@ semaphore_wait_to_signaller_ring(struct intel_engine_cs *engine, u32 ipehr,
 static struct intel_engine_cs *
 semaphore_waits_for(struct intel_engine_cs *engine, u32 *seqno)
 {
-        struct drm_i915_private *dev_priv = engine->dev->dev_private;
+        struct drm_i915_private *dev_priv = engine->i915;
         u32 cmd, ipehr, head;
         u64 offset = 0;
         int i, backwards;
@@ -2942,7 +2884,7 @@ semaphore_waits_for(struct intel_engine_cs *engine, u32 *seqno)
                 return NULL;
 
         ipehr = I915_READ(RING_IPEHR(engine->mmio_base));
-        if (!ipehr_is_semaphore_wait(engine->dev, ipehr))
+        if (!ipehr_is_semaphore_wait(engine, ipehr))
                 return NULL;
 
         /*
@@ -2954,7 +2896,7 @@ semaphore_waits_for(struct intel_engine_cs *engine, u32 *seqno)
          * ringbuffer itself.
          */
         head = I915_READ_HEAD(engine) & HEAD_ADDR;
-        backwards = (INTEL_INFO(engine->dev)->gen >= 8) ? 5 : 4;
+        backwards = (INTEL_GEN(dev_priv) >= 8) ? 5 : 4;
 
         for (i = backwards; i; --i) {
                 /*
@@ -2976,7 +2918,7 @@ semaphore_waits_for(struct intel_engine_cs *engine, u32 *seqno)
                 return NULL;
 
         *seqno = ioread32(engine->buffer->virtual_start + head + 4) + 1;
-        if (INTEL_INFO(engine->dev)->gen >= 8) {
+        if (INTEL_GEN(dev_priv) >= 8) {
                 offset = ioread32(engine->buffer->virtual_start + head + 12);
                 offset <<= 32;
                 offset = ioread32(engine->buffer->virtual_start + head + 8);
@@ -2986,7 +2928,7 @@ semaphore_waits_for(struct intel_engine_cs *engine, u32 *seqno)
 
 static int semaphore_passed(struct intel_engine_cs *engine)
 {
-        struct drm_i915_private *dev_priv = engine->dev->dev_private;
+        struct drm_i915_private *dev_priv = engine->i915;
         struct intel_engine_cs *signaller;
         u32 seqno;
 
@@ -3000,7 +2942,7 @@ static int semaphore_passed(struct intel_engine_cs *engine)
         if (signaller->hangcheck.deadlock >= I915_NUM_ENGINES)
                 return -1;
 
-        if (i915_seqno_passed(signaller->get_seqno(signaller), seqno))
+        if (i915_seqno_passed(intel_engine_get_seqno(signaller), seqno))
                 return 1;
 
         /* cursory check for an unkickable deadlock */
@@ -3028,7 +2970,7 @@ static bool subunits_stuck(struct intel_engine_cs *engine)
         if (engine->id != RCS)
                 return true;
 
-        i915_get_extra_instdone(engine->dev, instdone);
+        i915_get_extra_instdone(engine->i915, instdone);
 
         /* There might be unstable subunit states even when
          * actual head is not moving. Filter out the unstable ones by
@@ -3069,8 +3011,7 @@ head_stuck(struct intel_engine_cs *engine, u64 acthd)
 static enum intel_ring_hangcheck_action
 ring_stuck(struct intel_engine_cs *engine, u64 acthd)
 {
-        struct drm_device *dev = engine->dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = engine->i915;
         enum intel_ring_hangcheck_action ha;
         u32 tmp;
 
@@ -3078,7 +3019,7 @@ ring_stuck(struct intel_engine_cs *engine, u64 acthd)
         if (ha != HANGCHECK_HUNG)
                 return ha;
 
-        if (IS_GEN2(dev))
+        if (IS_GEN2(dev_priv))
                 return HANGCHECK_HUNG;
 
         /* Is the chip hanging on a WAIT_FOR_EVENT?
@@ -3088,19 +3029,19 @@ ring_stuck(struct intel_engine_cs *engine, u64 acthd)
          */
         tmp = I915_READ_CTL(engine);
         if (tmp & RING_WAIT) {
-                i915_handle_error(dev, 0,
+                i915_handle_error(dev_priv, 0,
                                   "Kicking stuck wait on %s",
                                   engine->name);
                 I915_WRITE_CTL(engine, tmp);
                 return HANGCHECK_KICK;
         }
 
-        if (INTEL_INFO(dev)->gen >= 6 && tmp & RING_WAIT_SEMAPHORE) {
+        if (INTEL_GEN(dev_priv) >= 6 && tmp & RING_WAIT_SEMAPHORE) {
                 switch (semaphore_passed(engine)) {
                 default:
                         return HANGCHECK_HUNG;
                 case 1:
-                        i915_handle_error(dev, 0,
+                        i915_handle_error(dev_priv, 0,
                                           "Kicking stuck semaphore on %s",
                                           engine->name);
                         I915_WRITE_CTL(engine, tmp);
@@ -3113,23 +3054,21 @@ ring_stuck(struct intel_engine_cs *engine, u64 acthd)
         return HANGCHECK_HUNG;
 }
 
-static unsigned kick_waiters(struct intel_engine_cs *engine)
+static unsigned long kick_waiters(struct intel_engine_cs *engine)
 {
-        struct drm_i915_private *i915 = to_i915(engine->dev);
-        unsigned user_interrupts = READ_ONCE(engine->user_interrupts);
+        struct drm_i915_private *i915 = engine->i915;
+        unsigned long irq_count = READ_ONCE(engine->breadcrumbs.irq_wakeups);
 
-        if (engine->hangcheck.user_interrupts == user_interrupts &&
+        if (engine->hangcheck.user_interrupts == irq_count &&
             !test_and_set_bit(engine->id, &i915->gpu_error.missed_irq_rings)) {
-                if (!(i915->gpu_error.test_irq_rings & intel_engine_flag(engine)))
+                if (!test_bit(engine->id, &i915->gpu_error.test_irq_rings))
                         DRM_ERROR("Hangcheck timer elapsed... %s idle\n",
                                   engine->name);
-                else
-                        DRM_INFO("Fake missed irq on %s\n",
-                                 engine->name);
-                wake_up_all(&engine->irq_queue);
+
+                intel_engine_enable_fake_irq(engine);
         }
 
-        return user_interrupts;
+        return irq_count;
 }
 /*
  * This is called when the chip hasn't reported back with completed
@@ -3144,11 +3083,9 @@ static void i915_hangcheck_elapsed(struct work_struct *work)
         struct drm_i915_private *dev_priv =
                 container_of(work, typeof(*dev_priv),
                              gpu_error.hangcheck_work.work);
-        struct drm_device *dev = dev_priv->dev;
         struct intel_engine_cs *engine;
-        enum intel_engine_id id;
-        int busy_count = 0, rings_hung = 0;
-        bool stuck[I915_NUM_ENGINES] = { 0 };
+        unsigned int hung = 0, stuck = 0;
+        int busy_count = 0;
 #define BUSY 1
 #define KICK 5
 #define HUNG 20
@@ -3157,12 +3094,8 @@ static void i915_hangcheck_elapsed(struct work_struct *work)
         if (!i915.enable_hangcheck)
                 return;
 
-        /*
-         * The hangcheck work is synced during runtime suspend, we don't
-         * require a wakeref. TODO: instead of disabling the asserts make
-         * sure that we hold a reference when this work is running.
-         */
-        DISABLE_RPM_WAKEREF_ASSERTS(dev_priv);
+        if (!READ_ONCE(dev_priv->gt.awake))
+                return;
 
         /* As enabling the GPU requires fairly extensive mmio access,
          * periodically arm the mmio checker to see if we are triggering
@@ -3170,11 +3103,11 @@ static void i915_hangcheck_elapsed(struct work_struct *work)
          */
         intel_uncore_arm_unclaimed_mmio_detection(dev_priv);
 
-        for_each_engine_id(engine, dev_priv, id) {
+        for_each_engine(engine, dev_priv) {
+                bool busy = intel_engine_has_waiter(engine);
                 u64 acthd;
                 u32 seqno;
                 unsigned user_interrupts;
-                bool busy = true;
 
                 semaphore_clear_deadlocks(dev_priv);
 
@@ -3189,7 +3122,7 @@ static void i915_hangcheck_elapsed(struct work_struct *work)
                         engine->irq_seqno_barrier(engine);
 
                 acthd = intel_ring_get_active_head(engine);
-                seqno = engine->get_seqno(engine);
+                seqno = intel_engine_get_seqno(engine);
 
                 /* Reset stuck interrupts between batch advances */
                 user_interrupts = 0;
@@ -3197,12 +3130,11 @@ static void i915_hangcheck_elapsed(struct work_struct *work)
                 if (engine->hangcheck.seqno == seqno) {
                         if (ring_idle(engine, seqno)) {
                                 engine->hangcheck.action = HANGCHECK_IDLE;
-                                if (waitqueue_active(&engine->irq_queue)) {
+                                if (busy) {
                                         /* Safeguard against driver failure */
                                         user_interrupts = kick_waiters(engine);
                                         engine->hangcheck.score += BUSY;
-                                } else
-                                        busy = false;
+                                }
                         } else {
                                 /* We always increment the hangcheck score
                                  * if the ring is busy and still processing
@@ -3234,10 +3166,15 @@ static void i915_hangcheck_elapsed(struct work_struct *work)
                                         break;
                                 case HANGCHECK_HUNG:
                                         engine->hangcheck.score += HUNG;
-                                        stuck[id] = true;
                                         break;
                                 }
                         }
+
+                        if (engine->hangcheck.score >= HANGCHECK_SCORE_RING_HUNG) {
+                                hung |= intel_engine_flag(engine);
+                                if (engine->hangcheck.action != HANGCHECK_HUNG)
+                                        stuck |= intel_engine_flag(engine);
+                        }
                 } else {
                         engine->hangcheck.action = HANGCHECK_ACTIVE;
 
@@ -3262,48 +3199,33 @@ static void i915_hangcheck_elapsed(struct work_struct *work)
                 busy_count += busy;
         }
 
-        for_each_engine_id(engine, dev_priv, id) {
-                if (engine->hangcheck.score >= HANGCHECK_SCORE_RING_HUNG) {
-                        DRM_INFO("%s on %s\n",
-                                 stuck[id] ? "stuck" : "no progress",
-                                 engine->name);
-                        rings_hung |= intel_engine_flag(engine);
-                }
-        }
+        if (hung) {
+                char msg[80];
+                int len;
 
-        if (rings_hung) {
-                i915_handle_error(dev, rings_hung, "Engine(s) hung");
-                goto out;
+                /* If some rings hung but others were still busy, only
+                 * blame the hanging rings in the synopsis.
+                 */
+                if (stuck != hung)
+                        hung &= ~stuck;
+                len = scnprintf(msg, sizeof(msg),
+                                "%s on ", stuck == hung ? "No progress" : "Hang");
+                for_each_engine_masked(engine, dev_priv, hung)
+                        len += scnprintf(msg + len, sizeof(msg) - len,
+                                         "%s, ", engine->name);
+                msg[len-2] = '\0';
+
+                return i915_handle_error(dev_priv, hung, msg);
         }
 
+        /* Reset timer in case GPU hangs without another request being added */
         if (busy_count)
-                /* Reset timer case chip hangs without another request
-                 * being added */
-                i915_queue_hangcheck(dev);
-
-out:
-        ENABLE_RPM_WAKEREF_ASSERTS(dev_priv);
-}
-
-void i915_queue_hangcheck(struct drm_device *dev)
-{
-        struct i915_gpu_error *e = &to_i915(dev)->gpu_error;
-
-        if (!i915.enable_hangcheck)
-                return;
-
-        /* Don't continually defer the hangcheck so that it is always run at
-         * least once after work has been scheduled on any ring. Otherwise,
-         * we will ignore a hung ring if a second ring is kept busy.
-         */
-
-        queue_delayed_work(e->hangcheck_wq, &e->hangcheck_work,
-                           round_jiffies_up_relative(DRM_I915_HANGCHECK_JIFFIES));
+                i915_queue_hangcheck(dev_priv);
 }
 
 static void ibx_irq_reset(struct drm_device *dev)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
 
         if (HAS_PCH_NOP(dev))
                 return;
@@ -3324,7 +3246,7 @@ static void ibx_irq_reset(struct drm_device *dev)
  */
 static void ibx_irq_pre_postinstall(struct drm_device *dev)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
 
         if (HAS_PCH_NOP(dev))
                 return;
@@ -3336,7 +3258,7 @@ static void ibx_irq_pre_postinstall(struct drm_device *dev)
 
 static void gen5_gt_irq_reset(struct drm_device *dev)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
 
         GEN5_IRQ_RESET(GT);
         if (INTEL_INFO(dev)->gen >= 6)
@@ -3396,7 +3318,7 @@ static void vlv_display_irq_postinstall(struct drm_i915_private *dev_priv)
 */
 static void ironlake_irq_reset(struct drm_device *dev)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
 
         I915_WRITE(HWSTAM, 0xffffffff);
 
@@ -3411,7 +3333,7 @@ static void ironlake_irq_reset(struct drm_device *dev)
 
 static void valleyview_irq_preinstall(struct drm_device *dev)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
 
         I915_WRITE(VLV_MASTER_IER, 0);
         POSTING_READ(VLV_MASTER_IER);
@@ -3434,7 +3356,7 @@ static void gen8_gt_irq_reset(struct drm_i915_private *dev_priv)
 
 static void gen8_irq_reset(struct drm_device *dev)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         int pipe;
 
         I915_WRITE(GEN8_MASTER_IRQ, 0);
@@ -3480,12 +3402,12 @@ void gen8_irq_power_well_pre_disable(struct drm_i915_private *dev_priv,
         spin_unlock_irq(&dev_priv->irq_lock);
 
         /* make sure we're done processing display irqs */
-        synchronize_irq(dev_priv->dev->irq);
+        synchronize_irq(dev_priv->drm.irq);
 }
 
 static void cherryview_irq_preinstall(struct drm_device *dev)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
 
         I915_WRITE(GEN8_MASTER_IRQ, 0);
         POSTING_READ(GEN8_MASTER_IRQ);
@@ -3500,31 +3422,29 @@ static void cherryview_irq_preinstall(struct drm_device *dev)
         spin_unlock_irq(&dev_priv->irq_lock);
 }
 
-static u32 intel_hpd_enabled_irqs(struct drm_device *dev,
+static u32 intel_hpd_enabled_irqs(struct drm_i915_private *dev_priv,
                                   const u32 hpd[HPD_NUM_PINS])
 {
-        struct drm_i915_private *dev_priv = to_i915(dev);
         struct intel_encoder *encoder;
         u32 enabled_irqs = 0;
 
-        for_each_intel_encoder(dev, encoder)
+        for_each_intel_encoder(&dev_priv->drm, encoder)
                 if (dev_priv->hotplug.stats[encoder->hpd_pin].state == HPD_ENABLED)
                         enabled_irqs |= hpd[encoder->hpd_pin];
 
         return enabled_irqs;
 }
 
-static void ibx_hpd_irq_setup(struct drm_device *dev)
+static void ibx_hpd_irq_setup(struct drm_i915_private *dev_priv)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
         u32 hotplug_irqs, hotplug, enabled_irqs;
 
-        if (HAS_PCH_IBX(dev)) {
+        if (HAS_PCH_IBX(dev_priv)) {
                 hotplug_irqs = SDE_HOTPLUG_MASK;
-                enabled_irqs = intel_hpd_enabled_irqs(dev, hpd_ibx);
+                enabled_irqs = intel_hpd_enabled_irqs(dev_priv, hpd_ibx);
         } else {
                 hotplug_irqs = SDE_HOTPLUG_MASK_CPT;
-                enabled_irqs = intel_hpd_enabled_irqs(dev, hpd_cpt);
+                enabled_irqs = intel_hpd_enabled_irqs(dev_priv, hpd_cpt);
         }
 
         ibx_display_interrupt_update(dev_priv, hotplug_irqs, enabled_irqs);
@@ -3543,18 +3463,17 @@ static void ibx_hpd_irq_setup(struct drm_device *dev)
          * When CPU and PCH are on the same package, port A
          * HPD must be enabled in both north and south.
          */
-        if (HAS_PCH_LPT_LP(dev))
+        if (HAS_PCH_LPT_LP(dev_priv))
                 hotplug |= PORTA_HOTPLUG_ENABLE;
         I915_WRITE(PCH_PORT_HOTPLUG, hotplug);
 }
 
-static void spt_hpd_irq_setup(struct drm_device *dev)
+static void spt_hpd_irq_setup(struct drm_i915_private *dev_priv)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
         u32 hotplug_irqs, hotplug, enabled_irqs;
 
         hotplug_irqs = SDE_HOTPLUG_MASK_SPT;
-        enabled_irqs = intel_hpd_enabled_irqs(dev, hpd_spt);
+        enabled_irqs = intel_hpd_enabled_irqs(dev_priv, hpd_spt);
 
         ibx_display_interrupt_update(dev_priv, hotplug_irqs, enabled_irqs);
 
@@ -3569,24 +3488,23 @@ static void spt_hpd_irq_setup(struct drm_device *dev)
         I915_WRITE(PCH_PORT_HOTPLUG2, hotplug);
 }
 
-static void ilk_hpd_irq_setup(struct drm_device *dev)
+static void ilk_hpd_irq_setup(struct drm_i915_private *dev_priv)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
         u32 hotplug_irqs, hotplug, enabled_irqs;
 
-        if (INTEL_INFO(dev)->gen >= 8) {
+        if (INTEL_GEN(dev_priv) >= 8) {
                 hotplug_irqs = GEN8_PORT_DP_A_HOTPLUG;
-                enabled_irqs = intel_hpd_enabled_irqs(dev, hpd_bdw);
+                enabled_irqs = intel_hpd_enabled_irqs(dev_priv, hpd_bdw);
 
                 bdw_update_port_irq(dev_priv, hotplug_irqs, enabled_irqs);
-        } else if (INTEL_INFO(dev)->gen >= 7) {
+        } else if (INTEL_GEN(dev_priv) >= 7) {
                 hotplug_irqs = DE_DP_A_HOTPLUG_IVB;
-                enabled_irqs = intel_hpd_enabled_irqs(dev, hpd_ivb);
+                enabled_irqs = intel_hpd_enabled_irqs(dev_priv, hpd_ivb);
 
                 ilk_update_display_irq(dev_priv, hotplug_irqs, enabled_irqs);
         } else {
                 hotplug_irqs = DE_DP_A_HOTPLUG;
-                enabled_irqs = intel_hpd_enabled_irqs(dev, hpd_ilk);
+                enabled_irqs = intel_hpd_enabled_irqs(dev_priv, hpd_ilk);
 
                 ilk_update_display_irq(dev_priv, hotplug_irqs, enabled_irqs);
         }
@@ -3601,15 +3519,14 @@ static void ilk_hpd_irq_setup(struct drm_device *dev)
         hotplug |= DIGITAL_PORTA_HOTPLUG_ENABLE | DIGITAL_PORTA_PULSE_DURATION_2ms;
         I915_WRITE(DIGITAL_PORT_HOTPLUG_CNTRL, hotplug);
 
-        ibx_hpd_irq_setup(dev);
+        ibx_hpd_irq_setup(dev_priv);
 }
 
-static void bxt_hpd_irq_setup(struct drm_device *dev)
+static void bxt_hpd_irq_setup(struct drm_i915_private *dev_priv)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
         u32 hotplug_irqs, hotplug, enabled_irqs;
 
-        enabled_irqs = intel_hpd_enabled_irqs(dev, hpd_bxt);
+        enabled_irqs = intel_hpd_enabled_irqs(dev_priv, hpd_bxt);
         hotplug_irqs = BXT_DE_PORT_HOTPLUG_MASK;
 
         bdw_update_port_irq(dev_priv, hotplug_irqs, enabled_irqs);
@@ -3642,7 +3559,7 @@ static void bxt_hpd_irq_setup(struct drm_device *dev)
 
 static void ibx_irq_postinstall(struct drm_device *dev)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         u32 mask;
 
         if (HAS_PCH_NOP(dev))
@@ -3659,7 +3576,7 @@ static void ibx_irq_postinstall(struct drm_device *dev)
 
 static void gen5_gt_irq_postinstall(struct drm_device *dev)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         u32 pm_irqs, gt_irqs;
 
         pm_irqs = gt_irqs = 0;
@@ -3673,8 +3590,7 @@ static void gen5_gt_irq_postinstall(struct drm_device *dev)
 
         gt_irqs |= GT_RENDER_USER_INTERRUPT;
         if (IS_GEN5(dev)) {
-                gt_irqs |= GT_RENDER_PIPECTL_NOTIFY_INTERRUPT |
-                           ILK_BSD_USER_INTERRUPT;
+                gt_irqs |= ILK_BSD_USER_INTERRUPT;
         } else {
                 gt_irqs |= GT_BLT_USER_INTERRUPT | GT_BSD_USER_INTERRUPT;
         }
@@ -3696,7 +3612,7 @@ static void gen5_gt_irq_postinstall(struct drm_device *dev)
 
 static int ironlake_irq_postinstall(struct drm_device *dev)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         u32 display_mask, extra_mask;
 
         if (INTEL_INFO(dev)->gen >= 7) {
@@ -3775,7 +3691,7 @@ void valleyview_disable_display_irqs(struct drm_i915_private *dev_priv)
 
 static int valleyview_irq_postinstall(struct drm_device *dev)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
 
         gen5_gt_irq_postinstall(dev);
 
@@ -3827,6 +3743,7 @@ static void gen8_de_irq_postinstall(struct drm_i915_private *dev_priv)
         uint32_t de_pipe_enables;
         u32 de_port_masked = GEN8_AUX_CHANNEL_A;
         u32 de_port_enables;
+        u32 de_misc_masked = GEN8_DE_MISC_GSE;
         enum pipe pipe;
 
         if (INTEL_INFO(dev_priv)->gen >= 9) {
@@ -3862,11 +3779,12 @@ static void gen8_de_irq_postinstall(struct drm_i915_private *dev_priv)
                                           de_pipe_enables);
 
         GEN5_IRQ_INIT(GEN8_DE_PORT_, ~de_port_masked, de_port_enables);
+        GEN5_IRQ_INIT(GEN8_DE_MISC_, ~de_misc_masked, de_misc_masked);
 }
 
 static int gen8_irq_postinstall(struct drm_device *dev)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
 
         if (HAS_PCH_SPLIT(dev))
                 ibx_irq_pre_postinstall(dev);
@@ -3885,7 +3803,7 @@ static int gen8_irq_postinstall(struct drm_device *dev)
 
 static int cherryview_irq_postinstall(struct drm_device *dev)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
 
         gen8_gt_irq_postinstall(dev_priv);
 
@@ -3902,7 +3820,7 @@ static int cherryview_irq_postinstall(struct drm_device *dev)
 
 static void gen8_irq_uninstall(struct drm_device *dev)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
 
         if (!dev_priv)
                 return;
@@ -3912,7 +3830,7 @@ static void gen8_irq_uninstall(struct drm_device *dev)
 
 static void valleyview_irq_uninstall(struct drm_device *dev)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
 
         if (!dev_priv)
                 return;
@@ -3932,7 +3850,7 @@ static void valleyview_irq_uninstall(struct drm_device *dev)
 
 static void cherryview_irq_uninstall(struct drm_device *dev)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
 
         if (!dev_priv)
                 return;
@@ -3952,7 +3870,7 @@ static void cherryview_irq_uninstall(struct drm_device *dev)
 
 static void ironlake_irq_uninstall(struct drm_device *dev)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
 
         if (!dev_priv)
                 return;
@@ -3962,7 +3880,7 @@ static void ironlake_irq_uninstall(struct drm_device *dev)
 
 static void i8xx_irq_preinstall(struct drm_device * dev)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         int pipe;
 
         for_each_pipe(dev_priv, pipe)
@@ -3974,7 +3892,7 @@ static void i8xx_irq_preinstall(struct drm_device * dev)
 
 static int i8xx_irq_postinstall(struct drm_device *dev)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
 
         I915_WRITE16(EMR,
                      ~(I915_ERROR_PAGE_TABLE | I915_ERROR_MEMORY_REFRESH));
@@ -4006,13 +3924,12 @@ static int i8xx_irq_postinstall(struct drm_device *dev)
 /*
  * Returns true when a page flip has completed.
  */
-static bool i8xx_handle_vblank(struct drm_device *dev,
+static bool i8xx_handle_vblank(struct drm_i915_private *dev_priv,
                                int plane, int pipe, u32 iir)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
         u16 flip_pending = DISPLAY_PLANE_FLIP_PENDING(plane);
 
-        if (!intel_pipe_handle_vblank(dev, pipe))
+        if (!intel_pipe_handle_vblank(dev_priv, pipe))
                 return false;
 
         if ((iir & flip_pending) == 0)
@@ -4027,19 +3944,18 @@ static bool i8xx_handle_vblank(struct drm_device *dev,
         if (I915_READ16(ISR) & flip_pending)
                 goto check_page_flip;
 
-        intel_prepare_page_flip(dev, plane);
-        intel_finish_page_flip(dev, pipe);
+        intel_finish_page_flip_cs(dev_priv, pipe);
         return true;
 
 check_page_flip:
-        intel_check_page_flip(dev, pipe);
+        intel_check_page_flip(dev_priv, pipe);
         return false;
 }
 
 static irqreturn_t i8xx_irq_handler(int irq, void *arg)
 {
         struct drm_device *dev = arg;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         u16 iir, new_iir;
         u32 pipe_stats[2];
         int pipe;
@@ -4089,15 +4005,15 @@ static irqreturn_t i8xx_irq_handler(int irq, void *arg)
 
                 for_each_pipe(dev_priv, pipe) {
                         int plane = pipe;
-                        if (HAS_FBC(dev))
+                        if (HAS_FBC(dev_priv))
                                 plane = !plane;
 
                         if (pipe_stats[pipe] & PIPE_VBLANK_INTERRUPT_STATUS &&
-                            i8xx_handle_vblank(dev, plane, pipe, iir))
+                            i8xx_handle_vblank(dev_priv, plane, pipe, iir))
                                 flip_mask &= ~DISPLAY_PLANE_FLIP_PENDING(plane);
 
                         if (pipe_stats[pipe] & PIPE_CRC_DONE_INTERRUPT_STATUS)
-                                i9xx_pipe_crc_irq_handler(dev, pipe);
+                                i9xx_pipe_crc_irq_handler(dev_priv, pipe);
 
                         if (pipe_stats[pipe] & PIPE_FIFO_UNDERRUN_STATUS)
                                 intel_cpu_fifo_underrun_irq_handler(dev_priv,
@@ -4116,7 +4032,7 @@ out:
 
 static void i8xx_irq_uninstall(struct drm_device * dev)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         int pipe;
 
         for_each_pipe(dev_priv, pipe) {
@@ -4131,7 +4047,7 @@ static void i8xx_irq_uninstall(struct drm_device * dev)
 
 static void i915_irq_preinstall(struct drm_device * dev)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         int pipe;
 
         if (I915_HAS_HOTPLUG(dev)) {
@@ -4149,7 +4065,7 @@ static void i915_irq_preinstall(struct drm_device * dev)
 
 static int i915_irq_postinstall(struct drm_device *dev)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         u32 enable_mask;
 
         I915_WRITE(EMR, ~(I915_ERROR_PAGE_TABLE | I915_ERROR_MEMORY_REFRESH));
@@ -4182,7 +4098,7 @@ static int i915_irq_postinstall(struct drm_device *dev)
         I915_WRITE(IER, enable_mask);
         POSTING_READ(IER);
 
-        i915_enable_asle_pipestat(dev);
+        i915_enable_asle_pipestat(dev_priv);
 
         /* Interrupt setup is already guaranteed to be single-threaded, this is
          * just to make the assert_spin_locked check happy. */
@@ -4197,13 +4113,12 @@ static int i915_irq_postinstall(struct drm_device *dev)
 /*
  * Returns true when a page flip has completed.
  */
-static bool i915_handle_vblank(struct drm_device *dev,
+static bool i915_handle_vblank(struct drm_i915_private *dev_priv,
                                int plane, int pipe, u32 iir)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
         u32 flip_pending = DISPLAY_PLANE_FLIP_PENDING(plane);
 
-        if (!intel_pipe_handle_vblank(dev, pipe))
+        if (!intel_pipe_handle_vblank(dev_priv, pipe))
                 return false;
 
         if ((iir & flip_pending) == 0)
@@ -4218,19 +4133,18 @@ static bool i915_handle_vblank(struct drm_device *dev,
         if (I915_READ(ISR) & flip_pending)
                 goto check_page_flip;
 
-        intel_prepare_page_flip(dev, plane);
-        intel_finish_page_flip(dev, pipe);
+        intel_finish_page_flip_cs(dev_priv, pipe);
         return true;
 
 check_page_flip:
-        intel_check_page_flip(dev, pipe);
+        intel_check_page_flip(dev_priv, pipe);
         return false;
 }
 
 static irqreturn_t i915_irq_handler(int irq, void *arg)
 {
         struct drm_device *dev = arg;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         u32 iir, new_iir, pipe_stats[I915_MAX_PIPES];
         u32 flip_mask =
                 I915_DISPLAY_PLANE_A_FLIP_PENDING_INTERRUPT |
@@ -4273,11 +4187,11 @@ static irqreturn_t i915_irq_handler(int irq, void *arg)
                         break;
 
                 /* Consume port.  Then clear IIR or we'll miss events */
-                if (I915_HAS_HOTPLUG(dev) &&
+                if (I915_HAS_HOTPLUG(dev_priv) &&
                     iir & I915_DISPLAY_PORT_INTERRUPT) {
                         u32 hotplug_status = i9xx_hpd_irq_ack(dev_priv);
                         if (hotplug_status)
-                                i9xx_hpd_irq_handler(dev, hotplug_status);
+                                i9xx_hpd_irq_handler(dev_priv, hotplug_status);
                 }
 
                 I915_WRITE(IIR, iir & ~flip_mask);
@@ -4288,18 +4202,18 @@ static irqreturn_t i915_irq_handler(int irq, void *arg)
 
                 for_each_pipe(dev_priv, pipe) {
                         int plane = pipe;
-                        if (HAS_FBC(dev))
+                        if (HAS_FBC(dev_priv))
                                 plane = !plane;
 
                         if (pipe_stats[pipe] & PIPE_VBLANK_INTERRUPT_STATUS &&
-                            i915_handle_vblank(dev, plane, pipe, iir))
+                            i915_handle_vblank(dev_priv, plane, pipe, iir))
                                 flip_mask &= ~DISPLAY_PLANE_FLIP_PENDING(plane);
 
                         if (pipe_stats[pipe] & PIPE_LEGACY_BLC_EVENT_STATUS)
                                 blc_event = true;
 
                         if (pipe_stats[pipe] & PIPE_CRC_DONE_INTERRUPT_STATUS)
-                                i9xx_pipe_crc_irq_handler(dev, pipe);
+                                i9xx_pipe_crc_irq_handler(dev_priv, pipe);
 
                         if (pipe_stats[pipe] & PIPE_FIFO_UNDERRUN_STATUS)
                                 intel_cpu_fifo_underrun_irq_handler(dev_priv,
@@ -4307,7 +4221,7 @@ static irqreturn_t i915_irq_handler(int irq, void *arg)
                 }
 
                 if (blc_event || (iir & I915_ASLE_INTERRUPT))
-                        intel_opregion_asle_intr(dev);
+                        intel_opregion_asle_intr(dev_priv);
 
                 /* With MSI, interrupts are only generated when iir
                  * transitions from zero to nonzero.  If another bit got
@@ -4335,7 +4249,7 @@ static irqreturn_t i915_irq_handler(int irq, void *arg)
 
 static void i915_irq_uninstall(struct drm_device * dev)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         int pipe;
 
         if (I915_HAS_HOTPLUG(dev)) {
@@ -4357,7 +4271,7 @@ static void i915_irq_uninstall(struct drm_device * dev)
 
 static void i965_irq_preinstall(struct drm_device * dev)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         int pipe;
 
         i915_hotplug_interrupt_update(dev_priv, 0xffffffff, 0);
@@ -4373,7 +4287,7 @@ static void i965_irq_preinstall(struct drm_device * dev)
 
 static int i965_irq_postinstall(struct drm_device *dev)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         u32 enable_mask;
         u32 error_mask;
 
@@ -4391,7 +4305,7 @@ static int i965_irq_postinstall(struct drm_device *dev)
                          I915_DISPLAY_PLANE_B_FLIP_PENDING_INTERRUPT);
         enable_mask |= I915_USER_INTERRUPT;
 
-        if (IS_G4X(dev))
+        if (IS_G4X(dev_priv))
                 enable_mask |= I915_BSD_USER_INTERRUPT;
 
         /* Interrupt setup is already guaranteed to be single-threaded, this is
@@ -4406,7 +4320,7 @@ static int i965_irq_postinstall(struct drm_device *dev)
          * Enable some error detection, note the instruction error mask
          * bit is reserved, so we leave it masked.
          */
-        if (IS_G4X(dev)) {
+        if (IS_G4X(dev_priv)) {
                 error_mask = ~(GM45_ERROR_PAGE_TABLE |
                                GM45_ERROR_MEM_PRIV |
                                GM45_ERROR_CP_PRIV |
@@ -4424,26 +4338,25 @@ static int i965_irq_postinstall(struct drm_device *dev)
         i915_hotplug_interrupt_update(dev_priv, 0xffffffff, 0);
         POSTING_READ(PORT_HOTPLUG_EN);
 
-        i915_enable_asle_pipestat(dev);
+        i915_enable_asle_pipestat(dev_priv);
 
         return 0;
 }
 
-static void i915_hpd_irq_setup(struct drm_device *dev)
+static void i915_hpd_irq_setup(struct drm_i915_private *dev_priv)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
         u32 hotplug_en;
 
         assert_spin_locked(&dev_priv->irq_lock);
 
         /* Note HDMI and DP share hotplug bits */
         /* enable bits are the same for all generations */
-        hotplug_en = intel_hpd_enabled_irqs(dev, hpd_mask_i915);
+        hotplug_en = intel_hpd_enabled_irqs(dev_priv, hpd_mask_i915);
         /* Programming the CRT detection parameters tends
            to generate a spurious hotplug event about three
            seconds later.  So just do it once.
         */
-        if (IS_G4X(dev))
+        if (IS_G4X(dev_priv))
                 hotplug_en |= CRT_HOTPLUG_ACTIVATION_PERIOD_64;
         hotplug_en |= CRT_HOTPLUG_VOLTAGE_COMPARE_50;
 
@@ -4458,7 +4371,7 @@ static void i915_hpd_irq_setup(struct drm_device *dev)
 static irqreturn_t i965_irq_handler(int irq, void *arg)
 {
         struct drm_device *dev = arg;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         u32 iir, new_iir;
         u32 pipe_stats[I915_MAX_PIPES];
         int ret = IRQ_NONE, pipe;
@@ -4510,7 +4423,7 @@ static irqreturn_t i965_irq_handler(int irq, void *arg)
                 if (iir & I915_DISPLAY_PORT_INTERRUPT) {
                         u32 hotplug_status = i9xx_hpd_irq_ack(dev_priv);
                         if (hotplug_status)
-                                i9xx_hpd_irq_handler(dev, hotplug_status);
+                                i9xx_hpd_irq_handler(dev_priv, hotplug_status);
                 }
 
                 I915_WRITE(IIR, iir & ~flip_mask);
@@ -4523,24 +4436,24 @@ static irqreturn_t i965_irq_handler(int irq, void *arg)
 
                 for_each_pipe(dev_priv, pipe) {
                         if (pipe_stats[pipe] & PIPE_START_VBLANK_INTERRUPT_STATUS &&
-                            i915_handle_vblank(dev, pipe, pipe, iir))
+                            i915_handle_vblank(dev_priv, pipe, pipe, iir))
                                 flip_mask &= ~DISPLAY_PLANE_FLIP_PENDING(pipe);
 
                         if (pipe_stats[pipe] & PIPE_LEGACY_BLC_EVENT_STATUS)
                                 blc_event = true;
 
                         if (pipe_stats[pipe] & PIPE_CRC_DONE_INTERRUPT_STATUS)
-                                i9xx_pipe_crc_irq_handler(dev, pipe);
+                                i9xx_pipe_crc_irq_handler(dev_priv, pipe);
 
                         if (pipe_stats[pipe] & PIPE_FIFO_UNDERRUN_STATUS)
                                 intel_cpu_fifo_underrun_irq_handler(dev_priv, pipe);
                 }
 
                 if (blc_event || (iir & I915_ASLE_INTERRUPT))
-                        intel_opregion_asle_intr(dev);
+                        intel_opregion_asle_intr(dev_priv);
 
                 if (pipe_stats[0] & PIPE_GMBUS_INTERRUPT_STATUS)
-                        gmbus_irq_handler(dev);
+                        gmbus_irq_handler(dev_priv);
 
                 /* With MSI, interrupts are only generated when iir
                  * transitions from zero to nonzero.  If another bit got
@@ -4567,7 +4480,7 @@ static irqreturn_t i965_irq_handler(int irq, void *arg)
 
 static void i965_irq_uninstall(struct drm_device * dev)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         int pipe;
 
         if (!dev_priv)
@@ -4597,7 +4510,7 @@ static void i965_irq_uninstall(struct drm_device * dev)
  */
 void intel_irq_init(struct drm_i915_private *dev_priv)
 {
-        struct drm_device *dev = dev_priv->dev;
+        struct drm_device *dev = &dev_priv->drm;
 
         intel_hpd_init_work(dev_priv);
 
@@ -4611,6 +4524,20 @@ void intel_irq_init(struct drm_i915_private *dev_priv)
         else
                 dev_priv->pm_rps_events = GEN6_PM_RPS_EVENTS;
 
+        dev_priv->rps.pm_intr_keep = 0;
+
+        /*
+         * SNB,IVB can while VLV,CHV may hard hang on looping batchbuffer
+         * if GEN6_PM_UP_EI_EXPIRED is masked.
+         *
+         * TODO: verify if this can be reproduced on VLV,CHV.
+         */
+        if (INTEL_INFO(dev_priv)->gen <= 7 && !IS_HASWELL(dev_priv))
+                dev_priv->rps.pm_intr_keep |= GEN6_PM_RP_UP_EI_EXPIRED;
+
+        if (INTEL_INFO(dev_priv)->gen >= 8)
+                dev_priv->rps.pm_intr_keep |= GEN8_PMINTR_REDIRECT_TO_NON_DISP;
+
         INIT_DELAYED_WORK(&dev_priv->gpu_error.hangcheck_work,
                           i915_hangcheck_elapsed);
 
@@ -4674,12 +4601,12 @@ void intel_irq_init(struct drm_i915_private *dev_priv)
                 dev->driver->disable_vblank = ironlake_disable_vblank;
                 dev_priv->display.hpd_irq_setup = ilk_hpd_irq_setup;
         } else {
-                if (INTEL_INFO(dev_priv)->gen == 2) {
+                if (IS_GEN2(dev_priv)) {
                         dev->driver->irq_preinstall = i8xx_irq_preinstall;
                         dev->driver->irq_postinstall = i8xx_irq_postinstall;
                         dev->driver->irq_handler = i8xx_irq_handler;
                         dev->driver->irq_uninstall = i8xx_irq_uninstall;
-                } else if (INTEL_INFO(dev_priv)->gen == 3) {
+                } else if (IS_GEN3(dev_priv)) {
                         dev->driver->irq_preinstall = i915_irq_preinstall;
                         dev->driver->irq_postinstall = i915_irq_postinstall;
                         dev->driver->irq_uninstall = i915_irq_uninstall;
@@ -4717,7 +4644,7 @@ int intel_irq_install(struct drm_i915_private *dev_priv)
          */
         dev_priv->pm.irqs_enabled = true;
 
-        return drm_irq_install(dev_priv->dev, dev_priv->dev->pdev->irq);
+        return drm_irq_install(&dev_priv->drm, dev_priv->drm.pdev->irq);
 }
 
 /**
@@ -4729,7 +4656,7 @@ int intel_irq_install(struct drm_i915_private *dev_priv)
  */
 void intel_irq_uninstall(struct drm_i915_private *dev_priv)
 {
-        drm_irq_uninstall(dev_priv->dev);
+        drm_irq_uninstall(&dev_priv->drm);
         intel_hpd_cancel_work(dev_priv);
         dev_priv->pm.irqs_enabled = false;
 }
@@ -4743,9 +4670,9 @@ void intel_irq_uninstall(struct drm_i915_private *dev_priv)
  */
 void intel_runtime_pm_disable_interrupts(struct drm_i915_private *dev_priv)
 {
-        dev_priv->dev->driver->irq_uninstall(dev_priv->dev);
+        dev_priv->drm.driver->irq_uninstall(&dev_priv->drm);
         dev_priv->pm.irqs_enabled = false;
-        synchronize_irq(dev_priv->dev->irq);
+        synchronize_irq(dev_priv->drm.irq);
 }
 
 /**
@@ -4758,6 +4685,6 @@ void intel_runtime_pm_disable_interrupts(struct drm_i915_private *dev_priv)
 void intel_runtime_pm_enable_interrupts(struct drm_i915_private *dev_priv)
 {
         dev_priv->pm.irqs_enabled = true;
-        dev_priv->dev->driver->irq_preinstall(dev_priv->dev);
-        dev_priv->dev->driver->irq_postinstall(dev_priv->dev);
+        dev_priv->drm.driver->irq_preinstall(&dev_priv->drm);
+        dev_priv->drm.driver->irq_postinstall(&dev_priv->drm);
 }
diff --git a/drivers/gpu/drm/i915/i915_params.c b/drivers/gpu/drm/i915/i915_params.c
index 1779f02e6df8..b6e404c91eed 100644
--- a/drivers/gpu/drm/i915/i915_params.c
+++ b/drivers/gpu/drm/i915/i915_params.c
@@ -54,10 +54,13 @@ struct i915_params i915 __read_mostly = {
         .verbose_state_checks = 1,
         .nuclear_pageflip = 0,
         .edp_vswing = 0,
-        .enable_guc_submission = false,
+        .enable_guc_loading = 0,
+        .enable_guc_submission = 0,
         .guc_log_level = -1,
         .enable_dp_mst = true,
         .inject_load_failure = 0,
+        .enable_dpcd_backlight = false,
+        .enable_gvt = false,
 };
 
 module_param_named(modeset, i915.modeset, int, 0400);
@@ -197,8 +200,15 @@ MODULE_PARM_DESC(edp_vswing,
                  "(0=use value from vbt [default], 1=low power swing(200mV),"
                  "2=default swing(400mV))");
 
-module_param_named_unsafe(enable_guc_submission, i915.enable_guc_submission, bool, 0400);
-MODULE_PARM_DESC(enable_guc_submission, "Enable GuC submission (default:false)");
+module_param_named_unsafe(enable_guc_loading, i915.enable_guc_loading, int, 0400);
+MODULE_PARM_DESC(enable_guc_loading,
+                "Enable GuC firmware loading "
+                "(-1=auto, 0=never [default], 1=if available, 2=required)");
+
+module_param_named_unsafe(enable_guc_submission, i915.enable_guc_submission, int, 0400);
+MODULE_PARM_DESC(enable_guc_submission,
+                "Enable GuC submission "
+                "(-1=auto, 0=never [default], 1=if available, 2=required)");
 
 module_param_named(guc_log_level, i915.guc_log_level, int, 0400);
 MODULE_PARM_DESC(guc_log_level,
@@ -210,3 +220,10 @@ MODULE_PARM_DESC(enable_dp_mst,
 module_param_named_unsafe(inject_load_failure, i915.inject_load_failure, uint, 0400);
 MODULE_PARM_DESC(inject_load_failure,
         "Force an error after a number of failure check points (0:disabled (default), N:force failure at the Nth failure check point)");
+module_param_named(enable_dpcd_backlight, i915.enable_dpcd_backlight, bool, 0600);
+MODULE_PARM_DESC(enable_dpcd_backlight,
+        "Enable support for DPCD backlight control (default:false)");
+
+module_param_named(enable_gvt, i915.enable_gvt, bool, 0400);
+MODULE_PARM_DESC(enable_gvt,
+        "Enable support for Intel GVT-g graphics virtualization host support(default:false)");
diff --git a/drivers/gpu/drm/i915/i915_params.h b/drivers/gpu/drm/i915/i915_params.h
index 02bc27804291..0ad020b4a925 100644
--- a/drivers/gpu/drm/i915/i915_params.h
+++ b/drivers/gpu/drm/i915/i915_params.h
@@ -45,6 +45,8 @@ struct i915_params {
         int enable_ips;
         int invert_brightness;
         int enable_cmd_parser;
+        int enable_guc_loading;
+        int enable_guc_submission;
         int guc_log_level;
         int use_mmio_flip;
         int mmio_debug;
@@ -57,10 +59,11 @@ struct i915_params {
         bool load_detect_test;
         bool reset;
         bool disable_display;
-        bool enable_guc_submission;
         bool verbose_state_checks;
         bool nuclear_pageflip;
         bool enable_dp_mst;
+        bool enable_dpcd_backlight;
+        bool enable_gvt;
 };
 
 extern struct i915_params i915 __read_mostly;
diff --git a/drivers/gpu/drm/i915/i915_pci.c b/drivers/gpu/drm/i915/i915_pci.c
new file mode 100644
index 000000000000..949c01686a66
--- /dev/null
+++ b/drivers/gpu/drm/i915/i915_pci.c
@@ -0,0 +1,503 @@
+/*
+ * Copyright © 2016 Intel Corporation
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ *
+ */
+
+#include <linux/console.h>
+#include <linux/vgaarb.h>
+#include <linux/vga_switcheroo.h>
+
+#include "i915_drv.h"
+
+#define GEN_DEFAULT_PIPEOFFSETS \
+        .pipe_offsets = { PIPE_A_OFFSET, PIPE_B_OFFSET, \
+                          PIPE_C_OFFSET, PIPE_EDP_OFFSET }, \
+        .trans_offsets = { TRANSCODER_A_OFFSET, TRANSCODER_B_OFFSET, \
+                           TRANSCODER_C_OFFSET, TRANSCODER_EDP_OFFSET }, \
+        .palette_offsets = { PALETTE_A_OFFSET, PALETTE_B_OFFSET }
+
+#define GEN_CHV_PIPEOFFSETS \
+        .pipe_offsets = { PIPE_A_OFFSET, PIPE_B_OFFSET, \
+                          CHV_PIPE_C_OFFSET }, \
+        .trans_offsets = { TRANSCODER_A_OFFSET, TRANSCODER_B_OFFSET, \
+                           CHV_TRANSCODER_C_OFFSET, }, \
+        .palette_offsets = { PALETTE_A_OFFSET, PALETTE_B_OFFSET, \
+                             CHV_PALETTE_C_OFFSET }
+
+#define CURSOR_OFFSETS \
+        .cursor_offsets = { CURSOR_A_OFFSET, CURSOR_B_OFFSET, CHV_CURSOR_C_OFFSET }
+
+#define IVB_CURSOR_OFFSETS \
+        .cursor_offsets = { CURSOR_A_OFFSET, IVB_CURSOR_B_OFFSET, IVB_CURSOR_C_OFFSET }
+
+#define BDW_COLORS \
+        .color = { .degamma_lut_size = 512, .gamma_lut_size = 512 }
+#define CHV_COLORS \
+        .color = { .degamma_lut_size = 65, .gamma_lut_size = 257 }
+
+static const struct intel_device_info intel_i830_info = {
+        .gen = 2, .is_mobile = 1, .cursor_needs_physical = 1, .num_pipes = 2,
+        .has_overlay = 1, .overlay_needs_physical = 1,
+        .ring_mask = RENDER_RING,
+        GEN_DEFAULT_PIPEOFFSETS,
+        CURSOR_OFFSETS,
+};
+
+static const struct intel_device_info intel_845g_info = {
+        .gen = 2, .num_pipes = 1,
+        .has_overlay = 1, .overlay_needs_physical = 1,
+        .ring_mask = RENDER_RING,
+        GEN_DEFAULT_PIPEOFFSETS,
+        CURSOR_OFFSETS,
+};
+
+static const struct intel_device_info intel_i85x_info = {
+        .gen = 2, .is_i85x = 1, .is_mobile = 1, .num_pipes = 2,
+        .cursor_needs_physical = 1,
+        .has_overlay = 1, .overlay_needs_physical = 1,
+        .has_fbc = 1,
+        .ring_mask = RENDER_RING,
+        GEN_DEFAULT_PIPEOFFSETS,
+        CURSOR_OFFSETS,
+};
+
+static const struct intel_device_info intel_i865g_info = {
+        .gen = 2, .num_pipes = 1,
+        .has_overlay = 1, .overlay_needs_physical = 1,
+        .ring_mask = RENDER_RING,
+        GEN_DEFAULT_PIPEOFFSETS,
+        CURSOR_OFFSETS,
+};
+
+static const struct intel_device_info intel_i915g_info = {
+        .gen = 3, .is_i915g = 1, .cursor_needs_physical = 1, .num_pipes = 2,
+        .has_overlay = 1, .overlay_needs_physical = 1,
+        .ring_mask = RENDER_RING,
+        GEN_DEFAULT_PIPEOFFSETS,
+        CURSOR_OFFSETS,
+};
+static const struct intel_device_info intel_i915gm_info = {
+        .gen = 3, .is_mobile = 1, .num_pipes = 2,
+        .cursor_needs_physical = 1,
+        .has_overlay = 1, .overlay_needs_physical = 1,
+        .supports_tv = 1,
+        .has_fbc = 1,
+        .ring_mask = RENDER_RING,
+        GEN_DEFAULT_PIPEOFFSETS,
+        CURSOR_OFFSETS,
+};
+static const struct intel_device_info intel_i945g_info = {
+        .gen = 3, .has_hotplug = 1, .cursor_needs_physical = 1, .num_pipes = 2,
+        .has_overlay = 1, .overlay_needs_physical = 1,
+        .ring_mask = RENDER_RING,
+        GEN_DEFAULT_PIPEOFFSETS,
+        CURSOR_OFFSETS,
+};
+static const struct intel_device_info intel_i945gm_info = {
+        .gen = 3, .is_i945gm = 1, .is_mobile = 1, .num_pipes = 2,
+        .has_hotplug = 1, .cursor_needs_physical = 1,
+        .has_overlay = 1, .overlay_needs_physical = 1,
+        .supports_tv = 1,
+        .has_fbc = 1,
+        .ring_mask = RENDER_RING,
+        GEN_DEFAULT_PIPEOFFSETS,
+        CURSOR_OFFSETS,
+};
+
+static const struct intel_device_info intel_i965g_info = {
+        .gen = 4, .is_broadwater = 1, .num_pipes = 2,
+        .has_hotplug = 1,
+        .has_overlay = 1,
+        .ring_mask = RENDER_RING,
+        GEN_DEFAULT_PIPEOFFSETS,
+        CURSOR_OFFSETS,
+};
+
+static const struct intel_device_info intel_i965gm_info = {
+        .gen = 4, .is_crestline = 1, .num_pipes = 2,
+        .is_mobile = 1, .has_fbc = 1, .has_hotplug = 1,
+        .has_overlay = 1,
+        .supports_tv = 1,
+        .ring_mask = RENDER_RING,
+        GEN_DEFAULT_PIPEOFFSETS,
+        CURSOR_OFFSETS,
+};
+
+static const struct intel_device_info intel_g33_info = {
+        .gen = 3, .is_g33 = 1, .num_pipes = 2,
+        .need_gfx_hws = 1, .has_hotplug = 1,
+        .has_overlay = 1,
+        .ring_mask = RENDER_RING,
+        GEN_DEFAULT_PIPEOFFSETS,
+        CURSOR_OFFSETS,
+};
+
+static const struct intel_device_info intel_g45_info = {
+        .gen = 4, .is_g4x = 1, .need_gfx_hws = 1, .num_pipes = 2,
+        .has_pipe_cxsr = 1, .has_hotplug = 1,
+        .ring_mask = RENDER_RING | BSD_RING,
+        GEN_DEFAULT_PIPEOFFSETS,
+        CURSOR_OFFSETS,
+};
+
+static const struct intel_device_info intel_gm45_info = {
+        .gen = 4, .is_g4x = 1, .num_pipes = 2,
+        .is_mobile = 1, .need_gfx_hws = 1, .has_fbc = 1,
+        .has_pipe_cxsr = 1, .has_hotplug = 1,
+        .supports_tv = 1,
+        .ring_mask = RENDER_RING | BSD_RING,
+        GEN_DEFAULT_PIPEOFFSETS,
+        CURSOR_OFFSETS,
+};
+
+static const struct intel_device_info intel_pineview_info = {
+        .gen = 3, .is_g33 = 1, .is_pineview = 1, .is_mobile = 1, .num_pipes = 2,
+        .need_gfx_hws = 1, .has_hotplug = 1,
+        .has_overlay = 1,
+        GEN_DEFAULT_PIPEOFFSETS,
+        CURSOR_OFFSETS,
+};
+
+static const struct intel_device_info intel_ironlake_d_info = {
+        .gen = 5, .num_pipes = 2,
+        .need_gfx_hws = 1, .has_hotplug = 1,
+        .ring_mask = RENDER_RING | BSD_RING,
+        GEN_DEFAULT_PIPEOFFSETS,
+        CURSOR_OFFSETS,
+};
+
+static const struct intel_device_info intel_ironlake_m_info = {
+        .gen = 5, .is_mobile = 1, .num_pipes = 2,
+        .need_gfx_hws = 1, .has_hotplug = 1,
+        .has_fbc = 1,
+        .ring_mask = RENDER_RING | BSD_RING,
+        GEN_DEFAULT_PIPEOFFSETS,
+        CURSOR_OFFSETS,
+};
+
+static const struct intel_device_info intel_sandybridge_d_info = {
+        .gen = 6, .num_pipes = 2,
+        .need_gfx_hws = 1, .has_hotplug = 1,
+        .has_fbc = 1,
+        .ring_mask = RENDER_RING | BSD_RING | BLT_RING,
+        .has_llc = 1,
+        GEN_DEFAULT_PIPEOFFSETS,
+        CURSOR_OFFSETS,
+};
+
+static const struct intel_device_info intel_sandybridge_m_info = {
+        .gen = 6, .is_mobile = 1, .num_pipes = 2,
+        .need_gfx_hws = 1, .has_hotplug = 1,
+        .has_fbc = 1,
+        .ring_mask = RENDER_RING | BSD_RING | BLT_RING,
+        .has_llc = 1,
+        GEN_DEFAULT_PIPEOFFSETS,
+        CURSOR_OFFSETS,
+};
+
+#define GEN7_FEATURES  \
+        .gen = 7, .num_pipes = 3, \
+        .need_gfx_hws = 1, .has_hotplug = 1, \
+        .has_fbc = 1, \
+        .ring_mask = RENDER_RING | BSD_RING | BLT_RING, \
+        .has_llc = 1, \
+        GEN_DEFAULT_PIPEOFFSETS, \
+        IVB_CURSOR_OFFSETS
+
+static const struct intel_device_info intel_ivybridge_d_info = {
+        GEN7_FEATURES,
+        .is_ivybridge = 1,
+};
+
+static const struct intel_device_info intel_ivybridge_m_info = {
+        GEN7_FEATURES,
+        .is_ivybridge = 1,
+        .is_mobile = 1,
+};
+
+static const struct intel_device_info intel_ivybridge_q_info = {
+        GEN7_FEATURES,
+        .is_ivybridge = 1,
+        .num_pipes = 0, /* legal, last one wins */
+};
+
+#define VLV_FEATURES  \
+        .gen = 7, .num_pipes = 2, \
+        .need_gfx_hws = 1, .has_hotplug = 1, \
+        .ring_mask = RENDER_RING | BSD_RING | BLT_RING, \
+        .display_mmio_offset = VLV_DISPLAY_BASE, \
+        GEN_DEFAULT_PIPEOFFSETS, \
+        CURSOR_OFFSETS
+
+static const struct intel_device_info intel_valleyview_m_info = {
+        VLV_FEATURES,
+        .is_valleyview = 1,
+        .is_mobile = 1,
+};
+
+static const struct intel_device_info intel_valleyview_d_info = {
+        VLV_FEATURES,
+        .is_valleyview = 1,
+};
+
+#define HSW_FEATURES  \
+        GEN7_FEATURES, \
+        .ring_mask = RENDER_RING | BSD_RING | BLT_RING | VEBOX_RING, \
+        .has_ddi = 1, \
+        .has_fpga_dbg = 1
+
+static const struct intel_device_info intel_haswell_d_info = {
+        HSW_FEATURES,
+        .is_haswell = 1,
+};
+
+static const struct intel_device_info intel_haswell_m_info = {
+        HSW_FEATURES,
+        .is_haswell = 1,
+        .is_mobile = 1,
+};
+
+#define BDW_FEATURES \
+        HSW_FEATURES, \
+        BDW_COLORS
+
+static const struct intel_device_info intel_broadwell_d_info = {
+        BDW_FEATURES,
+        .gen = 8,
+        .is_broadwell = 1,
+};
+
+static const struct intel_device_info intel_broadwell_m_info = {
+        BDW_FEATURES,
+        .gen = 8, .is_mobile = 1,
+        .is_broadwell = 1,
+};
+
+static const struct intel_device_info intel_broadwell_gt3d_info = {
+        BDW_FEATURES,
+        .gen = 8,
+        .is_broadwell = 1,
+        .ring_mask = RENDER_RING | BSD_RING | BLT_RING | VEBOX_RING | BSD2_RING,
+};
+
+static const struct intel_device_info intel_broadwell_gt3m_info = {
+        BDW_FEATURES,
+        .gen = 8, .is_mobile = 1,
+        .is_broadwell = 1,
+        .ring_mask = RENDER_RING | BSD_RING | BLT_RING | VEBOX_RING | BSD2_RING,
+};
+
+static const struct intel_device_info intel_cherryview_info = {
+        .gen = 8, .num_pipes = 3,
+        .need_gfx_hws = 1, .has_hotplug = 1,
+        .ring_mask = RENDER_RING | BSD_RING | BLT_RING | VEBOX_RING,
+        .is_cherryview = 1,
+        .display_mmio_offset = VLV_DISPLAY_BASE,
+        GEN_CHV_PIPEOFFSETS,
+        CURSOR_OFFSETS,
+        CHV_COLORS,
+};
+
+static const struct intel_device_info intel_skylake_info = {
+        BDW_FEATURES,
+        .is_skylake = 1,
+        .gen = 9,
+};
+
+static const struct intel_device_info intel_skylake_gt3_info = {
+        BDW_FEATURES,
+        .is_skylake = 1,
+        .gen = 9,
+        .ring_mask = RENDER_RING | BSD_RING | BLT_RING | VEBOX_RING | BSD2_RING,
+};
+
+static const struct intel_device_info intel_broxton_info = {
+        .is_broxton = 1,
+        .gen = 9,
+        .need_gfx_hws = 1, .has_hotplug = 1,
+        .ring_mask = RENDER_RING | BSD_RING | BLT_RING | VEBOX_RING,
+        .num_pipes = 3,
+        .has_ddi = 1,
+        .has_fpga_dbg = 1,
+        .has_fbc = 1,
+        .has_pooled_eu = 0,
+        GEN_DEFAULT_PIPEOFFSETS,
+        IVB_CURSOR_OFFSETS,
+        BDW_COLORS,
+};
+
+static const struct intel_device_info intel_kabylake_info = {
+        BDW_FEATURES,
+        .is_kabylake = 1,
+        .gen = 9,
+};
+
+static const struct intel_device_info intel_kabylake_gt3_info = {
+        BDW_FEATURES,
+        .is_kabylake = 1,
+        .gen = 9,
+        .ring_mask = RENDER_RING | BSD_RING | BLT_RING | VEBOX_RING | BSD2_RING,
+};
+
+/*
+ * Make sure any device matches here are from most specific to most
+ * general.  For example, since the Quanta match is based on the subsystem
+ * and subvendor IDs, we need it to come before the more general IVB
+ * PCI ID matches, otherwise we'll use the wrong info struct above.
+ */
+static const struct pci_device_id pciidlist[] = {
+        INTEL_I830_IDS(&intel_i830_info),
+        INTEL_I845G_IDS(&intel_845g_info),
+        INTEL_I85X_IDS(&intel_i85x_info),
+        INTEL_I865G_IDS(&intel_i865g_info),
+        INTEL_I915G_IDS(&intel_i915g_info),
+        INTEL_I915GM_IDS(&intel_i915gm_info),
+        INTEL_I945G_IDS(&intel_i945g_info),
+        INTEL_I945GM_IDS(&intel_i945gm_info),
+        INTEL_I965G_IDS(&intel_i965g_info),
+        INTEL_G33_IDS(&intel_g33_info),
+        INTEL_I965GM_IDS(&intel_i965gm_info),
+        INTEL_GM45_IDS(&intel_gm45_info),
+        INTEL_G45_IDS(&intel_g45_info),
+        INTEL_PINEVIEW_IDS(&intel_pineview_info),
+        INTEL_IRONLAKE_D_IDS(&intel_ironlake_d_info),
+        INTEL_IRONLAKE_M_IDS(&intel_ironlake_m_info),
+        INTEL_SNB_D_IDS(&intel_sandybridge_d_info),
+        INTEL_SNB_M_IDS(&intel_sandybridge_m_info),
+        INTEL_IVB_Q_IDS(&intel_ivybridge_q_info), /* must be first IVB */
+        INTEL_IVB_M_IDS(&intel_ivybridge_m_info),
+        INTEL_IVB_D_IDS(&intel_ivybridge_d_info),
+        INTEL_HSW_D_IDS(&intel_haswell_d_info),
+        INTEL_HSW_M_IDS(&intel_haswell_m_info),
+        INTEL_VLV_M_IDS(&intel_valleyview_m_info),
+        INTEL_VLV_D_IDS(&intel_valleyview_d_info),
+        INTEL_BDW_GT12M_IDS(&intel_broadwell_m_info),
+        INTEL_BDW_GT12D_IDS(&intel_broadwell_d_info),
+        INTEL_BDW_GT3M_IDS(&intel_broadwell_gt3m_info),
+        INTEL_BDW_GT3D_IDS(&intel_broadwell_gt3d_info),
+        INTEL_CHV_IDS(&intel_cherryview_info),
+        INTEL_SKL_GT1_IDS(&intel_skylake_info),
+        INTEL_SKL_GT2_IDS(&intel_skylake_info),
+        INTEL_SKL_GT3_IDS(&intel_skylake_gt3_info),
+        INTEL_SKL_GT4_IDS(&intel_skylake_gt3_info),
+        INTEL_BXT_IDS(&intel_broxton_info),
+        INTEL_KBL_GT1_IDS(&intel_kabylake_info),
+        INTEL_KBL_GT2_IDS(&intel_kabylake_info),
+        INTEL_KBL_GT3_IDS(&intel_kabylake_gt3_info),
+        INTEL_KBL_GT4_IDS(&intel_kabylake_gt3_info),
+        {0, 0, 0}
+};
+MODULE_DEVICE_TABLE(pci, pciidlist);
+
+extern int i915_driver_load(struct pci_dev *pdev,
+                            const struct pci_device_id *ent);
+
+static int i915_pci_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
+{
+        struct intel_device_info *intel_info =
+                (struct intel_device_info *) ent->driver_data;
+
+        if (IS_PRELIMINARY_HW(intel_info) && !i915.preliminary_hw_support) {
+                DRM_INFO("This hardware requires preliminary hardware support.\n"
+                         "See CONFIG_DRM_I915_PRELIMINARY_HW_SUPPORT, and/or modparam preliminary_hw_support\n");
+                return -ENODEV;
+        }
+
+        /* Only bind to function 0 of the device. Early generations
+         * used function 1 as a placeholder for multi-head. This causes
+         * us confusion instead, especially on the systems where both
+         * functions have the same PCI-ID!
+         */
+        if (PCI_FUNC(pdev->devfn))
+                return -ENODEV;
+
+        /*
+         * apple-gmux is needed on dual GPU MacBook Pro
+         * to probe the panel if we're the inactive GPU.
+         */
+        if (vga_switcheroo_client_probe_defer(pdev))
+                return -EPROBE_DEFER;
+
+        return i915_driver_load(pdev, ent);
+}
+
+extern void i915_driver_unload(struct drm_device *dev);
+
+static void i915_pci_remove(struct pci_dev *pdev)
+{
+        struct drm_device *dev = pci_get_drvdata(pdev);
+
+        i915_driver_unload(dev);
+        drm_dev_unref(dev);
+}
+
+extern const struct dev_pm_ops i915_pm_ops;
+
+static struct pci_driver i915_pci_driver = {
+        .name = DRIVER_NAME,
+        .id_table = pciidlist,
+        .probe = i915_pci_probe,
+        .remove = i915_pci_remove,
+        .driver.pm = &i915_pm_ops,
+};
+
+static int __init i915_init(void)
+{
+        bool use_kms = true;
+
+        /*
+         * Enable KMS by default, unless explicitly overriden by
+         * either the i915.modeset prarameter or by the
+         * vga_text_mode_force boot option.
+         */
+
+        if (i915.modeset == 0)
+                use_kms = false;
+
+        if (vgacon_text_force() && i915.modeset == -1)
+                use_kms = false;
+
+        if (!use_kms) {
+                /* Silently fail loading to not upset userspace. */
+                DRM_DEBUG_DRIVER("KMS disabled.\n");
+                return 0;
+        }
+
+        return pci_register_driver(&i915_pci_driver);
+}
+
+static void __exit i915_exit(void)
+{
+        if (!i915_pci_driver.driver.owner)
+                return;
+
+        pci_unregister_driver(&i915_pci_driver);
+}
+
+module_init(i915_init);
+module_exit(i915_exit);
+
+MODULE_AUTHOR("Tungsten Graphics, Inc.");
+MODULE_AUTHOR("Intel Corporation");
+
+MODULE_DESCRIPTION(DRIVER_DESC);
+MODULE_LICENSE("GPL and additional rights");
diff --git a/drivers/gpu/drm/i915/i915_pvinfo.h b/drivers/gpu/drm/i915/i915_pvinfo.h
new file mode 100644
index 000000000000..c0cb2974caac
--- /dev/null
+++ b/drivers/gpu/drm/i915/i915_pvinfo.h
@@ -0,0 +1,113 @@
+/*
+ * Copyright(c) 2011-2016 Intel Corporation. All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#ifndef _I915_PVINFO_H_
+#define _I915_PVINFO_H_
+
+/* The MMIO offset of the shared info between guest and host emulator */
+#define VGT_PVINFO_PAGE 0x78000
+#define VGT_PVINFO_SIZE 0x1000
+
+/*
+ * The following structure pages are defined in GEN MMIO space
+ * for virtualization. (One page for now)
+ */
+#define VGT_MAGIC         0x4776544776544776ULL /* 'vGTvGTvG' */
+#define VGT_VERSION_MAJOR 1
+#define VGT_VERSION_MINOR 0
+
+#define INTEL_VGT_IF_VERSION_ENCODE(major, minor) ((major) << 16 | (minor))
+#define INTEL_VGT_IF_VERSION \
+        INTEL_VGT_IF_VERSION_ENCODE(VGT_VERSION_MAJOR, VGT_VERSION_MINOR)
+
+/*
+ * notifications from guest to vgpu device model
+ */
+enum vgt_g2v_type {
+        VGT_G2V_PPGTT_L3_PAGE_TABLE_CREATE = 2,
+        VGT_G2V_PPGTT_L3_PAGE_TABLE_DESTROY,
+        VGT_G2V_PPGTT_L4_PAGE_TABLE_CREATE,
+        VGT_G2V_PPGTT_L4_PAGE_TABLE_DESTROY,
+        VGT_G2V_EXECLIST_CONTEXT_CREATE,
+        VGT_G2V_EXECLIST_CONTEXT_DESTROY,
+        VGT_G2V_MAX,
+};
+
+struct vgt_if {
+        u64 magic;              /* VGT_MAGIC */
+        uint16_t version_major;
+        uint16_t version_minor;
+        u32 vgt_id;             /* ID of vGT instance */
+        u32 rsv1[12];           /* pad to offset 0x40 */
+        /*
+         *  Data structure to describe the balooning info of resources.
+         *  Each VM can only have one portion of continuous area for now.
+         *  (May support scattered resource in future)
+         *  (starting from offset 0x40)
+         */
+        struct {
+                /* Aperture register balooning */
+                struct {
+                        u32 base;
+                        u32 size;
+                } mappable_gmadr;       /* aperture */
+                /* GMADR register balooning */
+                struct {
+                        u32 base;
+                        u32 size;
+                } nonmappable_gmadr;    /* non aperture */
+                /* allowed fence registers */
+                u32 fence_num;
+                u32 rsv2[3];
+        } avail_rs;             /* available/assigned resource */
+        u32 rsv3[0x200 - 24];   /* pad to half page */
+        /*
+         * The bottom half page is for response from Gfx driver to hypervisor.
+         */
+        u32 rsv4;
+        u32 display_ready;      /* ready for display owner switch */
+
+        u32 rsv5[4];
+
+        u32 g2v_notify;
+        u32 rsv6[7];
+
+        struct {
+                u32 lo;
+                u32 hi;
+        } pdp[4];
+
+        u32 execlist_context_descriptor_lo;
+        u32 execlist_context_descriptor_hi;
+
+        u32  rsv7[0x200 - 24];    /* pad to one page */
+} __packed;
+
+#define vgtif_reg(x) \
+        _MMIO((VGT_PVINFO_PAGE + offsetof(struct vgt_if, x)))
+
+/* vGPU display status to be used by the host side */
+#define VGT_DRV_DISPLAY_NOT_READY 0
+#define VGT_DRV_DISPLAY_READY     1  /* ready for display switch */
+
+#endif /* _I915_PVINFO_H_ */
diff --git a/drivers/gpu/drm/i915/i915_reg.h b/drivers/gpu/drm/i915/i915_reg.h
index 3fcf7dd5b6ca..ce14fe09d962 100644
--- a/drivers/gpu/drm/i915/i915_reg.h
+++ b/drivers/gpu/drm/i915/i915_reg.h
@@ -445,6 +445,8 @@ static inline bool i915_mmio_reg_valid(i915_reg_t reg)
  */
 #define GFX_INSTR(opcode, flags) ((0x3 << 29) | ((opcode) << 24) | (flags))
 
+#define GEN9_MEDIA_POOL_STATE     ((0x3 << 29) | (0x2 << 27) | (0x5 << 16) | 4)
+#define   GEN9_MEDIA_POOL_ENABLE  (1 << 31)
 #define GFX_OP_RASTER_RULES    ((0x3<<29)|(0x7<<24))
 #define GFX_OP_SCISSOR         ((0x3<<29)|(0x1c<<24)|(0x10<<19))
 #define   SC_UPDATE_SCISSOR       (0x1<<1)
@@ -716,6 +718,9 @@ enum skl_disp_power_wells {
         /* Not actual bit groups. Used as IDs for lookup_power_well() */
         SKL_DISP_PW_ALWAYS_ON,
         SKL_DISP_PW_DC_OFF,
+
+        BXT_DPIO_CMN_A,
+        BXT_DPIO_CMN_BC,
 };
 
 #define SKL_POWER_WELL_STATE(pw) (1 << ((pw) * 2))
@@ -889,7 +894,7 @@ enum skl_disp_power_wells {
  * PLLs can be routed to any transcoder A/B/C.
  *
  * Note: DDI0 is digital port B, DD1 is digital port C, and DDI2 is
- * digital port D (CHV) or port A (BXT).
+ * digital port D (CHV) or port A (BXT). ::
  *
  *
  *     Dual channel PHY (VLV/CHV/BXT)
@@ -1276,6 +1281,15 @@ enum skl_disp_power_wells {
 #define BXT_P_CR_GT_DISP_PWRON          _MMIO(0x138090)
 #define   GT_DISPLAY_POWER_ON(phy)      (1 << (phy))
 
+#define _BXT_PHY_CTL_DDI_A              0x64C00
+#define _BXT_PHY_CTL_DDI_B              0x64C10
+#define _BXT_PHY_CTL_DDI_C              0x64C20
+#define   BXT_PHY_CMNLANE_POWERDOWN_ACK (1 << 10)
+#define   BXT_PHY_LANE_POWERDOWN_ACK    (1 << 9)
+#define   BXT_PHY_LANE_ENABLED          (1 << 8)
+#define BXT_PHY_CTL(port)               _MMIO_PORT(port, _BXT_PHY_CTL_DDI_A, \
+                                                         _BXT_PHY_CTL_DDI_B)
+
 #define _PHY_CTL_FAMILY_EDP             0x64C80
 #define _PHY_CTL_FAMILY_DDI             0x64C90
 #define   COMMON_RESET_DIS              (1 << 31)
@@ -1672,6 +1686,9 @@ enum skl_disp_power_wells {
 
 #define GEN7_TLB_RD_ADDR        _MMIO(0x4700)
 
+#define GEN9_GAMT_ECO_REG_RW_IA _MMIO(0x4ab0)
+#define   GAMT_ECO_ENABLE_IN_PLACE_DECOMPRESS   (1<<18)
+
 #define GAMT_CHKN_BIT_REG       _MMIO(0x4ab8)
 #define   GAMT_CHKN_DISABLE_DYNAMIC_CREDIT_SHARING      (1<<28)
 
@@ -2171,6 +2188,9 @@ enum skl_disp_power_wells {
 
 #define FBC_LL_SIZE             (1536)
 
+#define FBC_LLC_READ_CTRL       _MMIO(0x9044)
+#define   FBC_LLC_FULLY_OPEN    (1<<30)
+
 /* Framebuffer compression for GM45+ */
 #define DPFC_CB_BASE            _MMIO(0x3200)
 #define DPFC_CONTROL            _MMIO(0x3208)
@@ -2461,6 +2481,8 @@ enum skl_disp_power_wells {
 #define   DPLL_MD_VGA_UDI_MULTIPLIER_MASK       0x0000003f
 #define   DPLL_MD_VGA_UDI_MULTIPLIER_SHIFT      0
 
+#define RAWCLK_FREQ_VLV         _MMIO(VLV_DISPLAY_BASE + 0x6024)
+
 #define _FPA0   0x6040
 #define _FPA1   0x6044
 #define _FPB0   0x6048
@@ -3032,6 +3054,18 @@ enum skl_disp_power_wells {
 /* Same as Haswell, but 72064 bytes now. */
 #define GEN8_CXT_TOTAL_SIZE             (18 * PAGE_SIZE)
 
+enum {
+        INTEL_ADVANCED_CONTEXT = 0,
+        INTEL_LEGACY_32B_CONTEXT,
+        INTEL_ADVANCED_AD_CONTEXT,
+        INTEL_LEGACY_64B_CONTEXT
+};
+
+#define GEN8_CTX_ADDRESSING_MODE_SHIFT 3
+#define GEN8_CTX_ADDRESSING_MODE(dev_priv) (USES_FULL_48BIT_PPGTT(dev_priv) ?\
+                                INTEL_LEGACY_64B_CONTEXT : \
+                                INTEL_LEGACY_32B_CONTEXT)
+
 #define CHV_CLK_CTL1                    _MMIO(0x101100)
 #define VLV_CLK_CTL2                    _MMIO(0x101104)
 #define   CLK_CTL2_CZCOUNT_30NS_SHIFT   28
@@ -6045,6 +6079,9 @@ enum skl_disp_power_wells {
 #define  FORCE_ARB_IDLE_PLANES  (1 << 14)
 #define  SKL_EDP_PSR_FIX_RDWRAP (1 << 3)
 
+#define CHICKEN_PAR2_1          _MMIO(0x42090)
+#define  KVM_CONFIG_CHANGE_NOTIFICATION_SELECT  (1 << 14)
+
 #define _CHICKEN_PIPESL_1_A     0x420b0
 #define _CHICKEN_PIPESL_1_B     0x420b4
 #define  HSW_FBCQ_DIS                   (1 << 22)
@@ -6084,6 +6121,7 @@ enum skl_disp_power_wells {
 
 #define FF_SLICE_CS_CHICKEN2                    _MMIO(0x20e4)
 #define  GEN9_TSG_BARRIER_ACK_DISABLE           (1<<8)
+#define  GEN9_POOLED_EU_LOAD_BALANCING_FIX_DISABLE  (1<<10)
 
 #define GEN9_CS_DEBUG_MODE1             _MMIO(0x20ec)
 #define GEN9_CTX_PREEMPT_REG            _MMIO(0x2248)
@@ -6108,7 +6146,14 @@ enum skl_disp_power_wells {
 #define  VLV_B0_WA_L3SQCREG1_VALUE              0x00D30000
 
 #define GEN8_L3SQCREG1                          _MMIO(0xB100)
-#define  BDW_WA_L3SQCREG1_DEFAULT               0x784000
+/*
+ * Note that on CHV the following has an off-by-one error wrt. to BSpec.
+ * Using the formula in BSpec leads to a hang, while the formula here works
+ * fine and matches the formulas for all other platforms. A BSpec change
+ * request has been filed to clarify this.
+ */
+#define  L3_GENERAL_PRIO_CREDITS(x)             (((x) >> 1) << 19)
+#define  L3_HIGH_PRIO_CREDITS(x)                (((x) >> 1) << 14)
 
 #define GEN7_L3CNTLREG1                         _MMIO(0xB01C)
 #define  GEN7_WA_FOR_GEN7_L3_CONTROL                    0x3C47FF8C
@@ -7028,7 +7073,8 @@ enum skl_disp_power_wells {
 #define GEN6_RPDEUC                             _MMIO(0xA084)
 #define GEN6_RPDEUCSW                           _MMIO(0xA088)
 #define GEN6_RC_STATE                           _MMIO(0xA094)
-#define   RC6_STATE                             (1 << 18)
+#define   RC_SW_TARGET_STATE_SHIFT              16
+#define   RC_SW_TARGET_STATE_MASK               (7 << RC_SW_TARGET_STATE_SHIFT)
 #define GEN6_RC1_WAKE_RATE_LIMIT                _MMIO(0xA098)
 #define GEN6_RC6_WAKE_RATE_LIMIT                _MMIO(0xA09C)
 #define GEN6_RC6pp_WAKE_RATE_LIMIT              _MMIO(0xA0A0)
@@ -7042,13 +7088,17 @@ enum skl_disp_power_wells {
 #define VLV_RCEDATA                             _MMIO(0xA0BC)
 #define GEN6_RC6pp_THRESHOLD                    _MMIO(0xA0C0)
 #define GEN6_PMINTRMSK                          _MMIO(0xA168)
-#define GEN8_PMINTR_REDIRECT_TO_NON_DISP        (1<<31)
+#define   GEN8_PMINTR_REDIRECT_TO_NON_DISP      (1<<31)
+#define GEN8_MISC_CTRL0                         _MMIO(0xA180)
 #define VLV_PWRDWNUPCTL                         _MMIO(0xA294)
 #define GEN9_MEDIA_PG_IDLE_HYSTERESIS           _MMIO(0xA0C4)
 #define GEN9_RENDER_PG_IDLE_HYSTERESIS          _MMIO(0xA0C8)
 #define GEN9_PG_ENABLE                          _MMIO(0xA210)
 #define GEN9_RENDER_PG_ENABLE                   (1<<0)
 #define GEN9_MEDIA_PG_ENABLE                    (1<<1)
+#define GEN8_PUSHBUS_CONTROL                    _MMIO(0xA248)
+#define GEN8_PUSHBUS_ENABLE                     _MMIO(0xA250)
+#define GEN8_PUSHBUS_SHIFT                      _MMIO(0xA25C)
 
 #define VLV_CHICKEN_3                           _MMIO(VLV_DISPLAY_BASE + 0x7040C)
 #define  PIXEL_OVERLAP_CNT_MASK                 (3 << 30)
@@ -7578,14 +7628,15 @@ enum skl_disp_power_wells {
 #define  CDCLK_FREQ_540                 (1<<26)
 #define  CDCLK_FREQ_337_308             (2<<26)
 #define  CDCLK_FREQ_675_617             (3<<26)
-#define  CDCLK_FREQ_DECIMAL_MASK        (0x7ff)
-
 #define  BXT_CDCLK_CD2X_DIV_SEL_MASK    (3<<22)
 #define  BXT_CDCLK_CD2X_DIV_SEL_1       (0<<22)
 #define  BXT_CDCLK_CD2X_DIV_SEL_1_5     (1<<22)
 #define  BXT_CDCLK_CD2X_DIV_SEL_2       (2<<22)
 #define  BXT_CDCLK_CD2X_DIV_SEL_4       (3<<22)
+#define  BXT_CDCLK_CD2X_PIPE(pipe)      ((pipe)<<20)
+#define  BXT_CDCLK_CD2X_PIPE_NONE       BXT_CDCLK_CD2X_PIPE(3)
 #define  BXT_CDCLK_SSA_PRECHARGE_ENABLE (1<<16)
+#define  CDCLK_FREQ_DECIMAL_MASK        (0x7ff)
 
 /* LCPLL_CTL */
 #define LCPLL1_CTL              _MMIO(0x46010)
@@ -8161,6 +8212,8 @@ enum skl_disp_power_wells {
 #define _MIPIA_EOT_DISABLE              (dev_priv->mipi_mmio_base + 0xb05c)
 #define _MIPIC_EOT_DISABLE              (dev_priv->mipi_mmio_base + 0xb85c)
 #define MIPI_EOT_DISABLE(port)          _MMIO_MIPI(port, _MIPIA_EOT_DISABLE, _MIPIC_EOT_DISABLE)
+#define  BXT_DEFEATURE_DPI_FIFO_CTR                     (1 << 9)
+#define  BXT_DPHY_DEFEATURE_EN                          (1 << 8)
 #define  LP_RX_TIMEOUT_ERROR_RECOVERY_DISABLE           (1 << 7)
 #define  HS_RX_TIMEOUT_ERROR_RECOVERY_DISABLE           (1 << 6)
 #define  LOW_CONTENTION_RECOVERY_DISABLE                (1 << 5)
diff --git a/drivers/gpu/drm/i915/i915_suspend.c b/drivers/gpu/drm/i915/i915_suspend.c
index 34e061a9ef06..5cfe4c7716b4 100644
--- a/drivers/gpu/drm/i915/i915_suspend.c
+++ b/drivers/gpu/drm/i915/i915_suspend.c
@@ -31,7 +31,7 @@
 
 static void i915_save_display(struct drm_device *dev)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
 
         /* Display arbitration control */
         if (INTEL_INFO(dev)->gen <= 4)
@@ -63,7 +63,7 @@ static void i915_save_display(struct drm_device *dev)
 
 static void i915_restore_display(struct drm_device *dev)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         u32 mask = 0xffffffff;
 
         /* Display arbitration */
@@ -103,7 +103,7 @@ static void i915_restore_display(struct drm_device *dev)
 
 int i915_save_state(struct drm_device *dev)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         int i;
 
         mutex_lock(&dev->struct_mutex);
@@ -148,7 +148,7 @@ int i915_save_state(struct drm_device *dev)
 
 int i915_restore_state(struct drm_device *dev)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         int i;
 
         mutex_lock(&dev->struct_mutex);
diff --git a/drivers/gpu/drm/i915/i915_sysfs.c b/drivers/gpu/drm/i915/i915_sysfs.c
index 2d576b7ff299..d61829e54f93 100644
--- a/drivers/gpu/drm/i915/i915_sysfs.c
+++ b/drivers/gpu/drm/i915/i915_sysfs.c
@@ -38,12 +38,12 @@
 static u32 calc_residency(struct drm_device *dev,
                           i915_reg_t reg)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         u64 raw_time; /* 32b value may overflow during fixed point math */
         u64 units = 128ULL, div = 100000ULL;
         u32 ret;
 
-        if (!intel_enable_rc6(dev))
+        if (!intel_enable_rc6())
                 return 0;
 
         intel_runtime_pm_get(dev_priv);
@@ -70,8 +70,7 @@ static u32 calc_residency(struct drm_device *dev,
 static ssize_t
 show_rc6_mask(struct device *kdev, struct device_attribute *attr, char *buf)
 {
-        struct drm_minor *dminor = dev_to_drm_minor(kdev);
-        return snprintf(buf, PAGE_SIZE, "%x\n", intel_enable_rc6(dminor->dev));
+        return snprintf(buf, PAGE_SIZE, "%x\n", intel_enable_rc6());
 }
 
 static ssize_t
@@ -167,7 +166,7 @@ i915_l3_read(struct file *filp, struct kobject *kobj,
         struct device *dev = kobj_to_dev(kobj);
         struct drm_minor *dminor = dev_to_drm_minor(dev);
         struct drm_device *drm_dev = dminor->dev;
-        struct drm_i915_private *dev_priv = drm_dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(drm_dev);
         int slice = (int)(uintptr_t)attr->private;
         int ret;
 
@@ -203,8 +202,8 @@ i915_l3_write(struct file *filp, struct kobject *kobj,
         struct device *dev = kobj_to_dev(kobj);
         struct drm_minor *dminor = dev_to_drm_minor(dev);
         struct drm_device *drm_dev = dminor->dev;
-        struct drm_i915_private *dev_priv = drm_dev->dev_private;
-        struct intel_context *ctx;
+        struct drm_i915_private *dev_priv = to_i915(drm_dev);
+        struct i915_gem_context *ctx;
         u32 *temp = NULL; /* Just here to make handling failures easy */
         int slice = (int)(uintptr_t)attr->private;
         int ret;
@@ -228,13 +227,6 @@ i915_l3_write(struct file *filp, struct kobject *kobj,
                 }
         }
 
-        ret = i915_gpu_idle(drm_dev);
-        if (ret) {
-                kfree(temp);
-                mutex_unlock(&drm_dev->struct_mutex);
-                return ret;
-        }
-
         /* TODO: Ideally we really want a GPU reset here to make sure errors
          * aren't propagated. Since I cannot find a stable way to reset the GPU
          * at this point it is left as a TODO.
@@ -276,7 +268,7 @@ static ssize_t gt_act_freq_mhz_show(struct device *kdev,
 {
         struct drm_minor *minor = dev_to_drm_minor(kdev);
         struct drm_device *dev = minor->dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         int ret;
 
         flush_delayed_work(&dev_priv->rps.delayed_resume_work);
@@ -310,7 +302,7 @@ static ssize_t gt_cur_freq_mhz_show(struct device *kdev,
 {
         struct drm_minor *minor = dev_to_drm_minor(kdev);
         struct drm_device *dev = minor->dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         int ret;
 
         flush_delayed_work(&dev_priv->rps.delayed_resume_work);
@@ -331,7 +323,7 @@ static ssize_t vlv_rpe_freq_mhz_show(struct device *kdev,
 {
         struct drm_minor *minor = dev_to_drm_minor(kdev);
         struct drm_device *dev = minor->dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
 
         return snprintf(buf, PAGE_SIZE,
                         "%d\n",
@@ -342,7 +334,7 @@ static ssize_t gt_max_freq_mhz_show(struct device *kdev, struct device_attribute
 {
         struct drm_minor *minor = dev_to_drm_minor(kdev);
         struct drm_device *dev = minor->dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         int ret;
 
         flush_delayed_work(&dev_priv->rps.delayed_resume_work);
@@ -360,7 +352,7 @@ static ssize_t gt_max_freq_mhz_store(struct device *kdev,
 {
         struct drm_minor *minor = dev_to_drm_minor(kdev);
         struct drm_device *dev = minor->dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         u32 val;
         ssize_t ret;
 
@@ -397,7 +389,7 @@ static ssize_t gt_max_freq_mhz_store(struct device *kdev,
         /* We still need *_set_rps to process the new max_delay and
          * update the interrupt limits and PMINTRMSK even though
          * frequency request may be unchanged. */
-        intel_set_rps(dev, val);
+        intel_set_rps(dev_priv, val);
 
         mutex_unlock(&dev_priv->rps.hw_lock);
 
@@ -410,7 +402,7 @@ static ssize_t gt_min_freq_mhz_show(struct device *kdev, struct device_attribute
 {
         struct drm_minor *minor = dev_to_drm_minor(kdev);
         struct drm_device *dev = minor->dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         int ret;
 
         flush_delayed_work(&dev_priv->rps.delayed_resume_work);
@@ -428,7 +420,7 @@ static ssize_t gt_min_freq_mhz_store(struct device *kdev,
 {
         struct drm_minor *minor = dev_to_drm_minor(kdev);
         struct drm_device *dev = minor->dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         u32 val;
         ssize_t ret;
 
@@ -461,7 +453,7 @@ static ssize_t gt_min_freq_mhz_store(struct device *kdev,
         /* We still need *_set_rps to process the new min_delay and
          * update the interrupt limits and PMINTRMSK even though
          * frequency request may be unchanged. */
-        intel_set_rps(dev, val);
+        intel_set_rps(dev_priv, val);
 
         mutex_unlock(&dev_priv->rps.hw_lock);
 
@@ -488,7 +480,7 @@ static ssize_t gt_rp_mhz_show(struct device *kdev, struct device_attribute *attr
 {
         struct drm_minor *minor = dev_to_drm_minor(kdev);
         struct drm_device *dev = minor->dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         u32 val;
 
         if (attr == &dev_attr_gt_RP0_freq_mhz)
diff --git a/drivers/gpu/drm/i915/i915_trace.h b/drivers/gpu/drm/i915/i915_trace.h
index dc0def210097..534154e05fbe 100644
--- a/drivers/gpu/drm/i915/i915_trace.h
+++ b/drivers/gpu/drm/i915/i915_trace.h
@@ -118,7 +118,7 @@ TRACE_EVENT(i915_gem_shrink,
                              ),
 
             TP_fast_assign(
-                           __entry->dev = i915->dev->primary->index;
+                           __entry->dev = i915->drm.primary->index;
                            __entry->target = target;
                            __entry->flags = flags;
                            ),
@@ -462,7 +462,7 @@ TRACE_EVENT(i915_gem_ring_sync_to,
                              ),
 
             TP_fast_assign(
-                           __entry->dev = from->dev->primary->index;
+                           __entry->dev = from->i915->drm.primary->index;
                            __entry->sync_from = from->id;
                            __entry->sync_to = to_req->engine->id;
                            __entry->seqno = i915_gem_request_get_seqno(req);
@@ -486,13 +486,11 @@ TRACE_EVENT(i915_gem_ring_dispatch,
                              ),
 
             TP_fast_assign(
-                           struct intel_engine_cs *engine =
-                                                i915_gem_request_get_engine(req);
-                           __entry->dev = engine->dev->primary->index;
-                           __entry->ring = engine->id;
-                           __entry->seqno = i915_gem_request_get_seqno(req);
+                           __entry->dev = req->i915->drm.primary->index;
+                           __entry->ring = req->engine->id;
+                           __entry->seqno = req->seqno;
                            __entry->flags = flags;
-                           i915_trace_irq_get(engine, req);
+                           intel_engine_enable_signaling(req);
                            ),
 
             TP_printk("dev=%u, ring=%u, seqno=%u, flags=%x",
@@ -511,7 +509,7 @@ TRACE_EVENT(i915_gem_ring_flush,
                              ),
 
             TP_fast_assign(
-                           __entry->dev = req->engine->dev->primary->index;
+                           __entry->dev = req->i915->drm.primary->index;
                            __entry->ring = req->engine->id;
                            __entry->invalidate = invalidate;
                            __entry->flush = flush;
@@ -533,11 +531,9 @@ DECLARE_EVENT_CLASS(i915_gem_request,
                              ),
 
             TP_fast_assign(
-                           struct intel_engine_cs *engine =
-                                                i915_gem_request_get_engine(req);
-                           __entry->dev = engine->dev->primary->index;
-                           __entry->ring = engine->id;
-                           __entry->seqno = i915_gem_request_get_seqno(req);
+                           __entry->dev = req->i915->drm.primary->index;
+                           __entry->ring = req->engine->id;
+                           __entry->seqno = req->seqno;
                            ),
 
             TP_printk("dev=%u, ring=%u, seqno=%u",
@@ -560,9 +556,9 @@ TRACE_EVENT(i915_gem_request_notify,
                              ),
 
             TP_fast_assign(
-                           __entry->dev = engine->dev->primary->index;
+                           __entry->dev = engine->i915->drm.primary->index;
                            __entry->ring = engine->id;
-                           __entry->seqno = engine->get_seqno(engine);
+                           __entry->seqno = intel_engine_get_seqno(engine);
                            ),
 
             TP_printk("dev=%u, ring=%u, seqno=%u",
@@ -597,13 +593,11 @@ TRACE_EVENT(i915_gem_request_wait_begin,
              * less desirable.
              */
             TP_fast_assign(
-                           struct intel_engine_cs *engine =
-                                                i915_gem_request_get_engine(req);
-                           __entry->dev = engine->dev->primary->index;
-                           __entry->ring = engine->id;
-                           __entry->seqno = i915_gem_request_get_seqno(req);
+                           __entry->dev = req->i915->drm.primary->index;
+                           __entry->ring = req->engine->id;
+                           __entry->seqno = req->seqno;
                            __entry->blocking =
-                                     mutex_is_locked(&engine->dev->struct_mutex);
+                                     mutex_is_locked(&req->i915->drm.struct_mutex);
                            ),
 
             TP_printk("dev=%u, ring=%u, seqno=%u, blocking=%s",
@@ -740,19 +734,19 @@ DEFINE_EVENT(i915_ppgtt, i915_ppgtt_release,
  * the context.
  */
 DECLARE_EVENT_CLASS(i915_context,
-        TP_PROTO(struct intel_context *ctx),
+        TP_PROTO(struct i915_gem_context *ctx),
         TP_ARGS(ctx),
 
         TP_STRUCT__entry(
                         __field(u32, dev)
-                        __field(struct intel_context *, ctx)
+                        __field(struct i915_gem_context *, ctx)
                         __field(struct i915_address_space *, vm)
         ),
 
         TP_fast_assign(
                         __entry->ctx = ctx;
                         __entry->vm = ctx->ppgtt ? &ctx->ppgtt->base : NULL;
-                        __entry->dev = ctx->i915->dev->primary->index;
+                        __entry->dev = ctx->i915->drm.primary->index;
         ),
 
         TP_printk("dev=%u, ctx=%p, ctx_vm=%p",
@@ -760,12 +754,12 @@ DECLARE_EVENT_CLASS(i915_context,
 )
 
 DEFINE_EVENT(i915_context, i915_context_create,
-        TP_PROTO(struct intel_context *ctx),
+        TP_PROTO(struct i915_gem_context *ctx),
         TP_ARGS(ctx)
 );
 
 DEFINE_EVENT(i915_context, i915_context_free,
-        TP_PROTO(struct intel_context *ctx),
+        TP_PROTO(struct i915_gem_context *ctx),
         TP_ARGS(ctx)
 );
 
@@ -777,13 +771,13 @@ DEFINE_EVENT(i915_context, i915_context_free,
  * called only if full ppgtt is enabled.
  */
 TRACE_EVENT(switch_mm,
-        TP_PROTO(struct intel_engine_cs *engine, struct intel_context *to),
+        TP_PROTO(struct intel_engine_cs *engine, struct i915_gem_context *to),
 
         TP_ARGS(engine, to),
 
         TP_STRUCT__entry(
                         __field(u32, ring)
-                        __field(struct intel_context *, to)
+                        __field(struct i915_gem_context *, to)
                         __field(struct i915_address_space *, vm)
                         __field(u32, dev)
         ),
@@ -792,7 +786,7 @@ TRACE_EVENT(switch_mm,
                         __entry->ring = engine->id;
                         __entry->to = to;
                         __entry->vm = to->ppgtt? &to->ppgtt->base : NULL;
-                        __entry->dev = engine->dev->primary->index;
+                        __entry->dev = engine->i915->drm.primary->index;
         ),
 
         TP_printk("dev=%u, ring=%u, ctx=%p, ctx_vm=%p",
diff --git a/drivers/gpu/drm/i915/i915_vgpu.c b/drivers/gpu/drm/i915/i915_vgpu.c
index d02efb8cad4d..f6acb5a0e701 100644
--- a/drivers/gpu/drm/i915/i915_vgpu.c
+++ b/drivers/gpu/drm/i915/i915_vgpu.c
@@ -53,20 +53,19 @@
 
 /**
  * i915_check_vgpu - detect virtual GPU
- * @dev: drm device *
+ * @dev_priv: i915 device private
  *
  * This function is called at the initialization stage, to detect whether
  * running on a vGPU.
  */
-void i915_check_vgpu(struct drm_device *dev)
+void i915_check_vgpu(struct drm_i915_private *dev_priv)
 {
-        struct drm_i915_private *dev_priv = to_i915(dev);
         uint64_t magic;
         uint32_t version;
 
         BUILD_BUG_ON(sizeof(struct vgt_if) != VGT_PVINFO_SIZE);
 
-        if (!IS_HASWELL(dev))
+        if (!IS_HASWELL(dev_priv))
                 return;
 
         magic = __raw_i915_read64(dev_priv, vgtif_reg(magic));
@@ -102,10 +101,13 @@ static struct _balloon_info_ bl_info;
  * This function is called to deallocate the ballooned-out graphic memory, when
  * driver is unloaded or when ballooning fails.
  */
-void intel_vgt_deballoon(void)
+void intel_vgt_deballoon(struct drm_i915_private *dev_priv)
 {
         int i;
 
+        if (!intel_vgpu_active(dev_priv))
+                return;
+
         DRM_DEBUG("VGT deballoon.\n");
 
         for (i = 0; i < 4; i++) {
@@ -151,36 +153,35 @@ static int vgt_balloon_space(struct drm_mm *mm,
  * of its graphic space being zero. Yet there are some portions ballooned out(
  * the shadow part, which are marked as reserved by drm allocator). From the
  * host point of view, the graphic address space is partitioned by multiple
- * vGPUs in different VMs.
+ * vGPUs in different VMs. ::
  *
  *                        vGPU1 view         Host view
  *             0 ------> +-----------+     +-----------+
- *               ^       |///////////|     |   vGPU3   |
- *               |       |///////////|     +-----------+
- *               |       |///////////|     |   vGPU2   |
+ *               ^       |###########|     |   vGPU3   |
+ *               |       |###########|     +-----------+
+ *               |       |###########|     |   vGPU2   |
  *               |       +-----------+     +-----------+
  *        mappable GM    | available | ==> |   vGPU1   |
  *               |       +-----------+     +-----------+
- *               |       |///////////|     |           |
- *               v       |///////////|     |   Host    |
+ *               |       |###########|     |           |
+ *               v       |###########|     |   Host    |
  *               +=======+===========+     +===========+
- *               ^       |///////////|     |   vGPU3   |
- *               |       |///////////|     +-----------+
- *               |       |///////////|     |   vGPU2   |
+ *               ^       |###########|     |   vGPU3   |
+ *               |       |###########|     +-----------+
+ *               |       |###########|     |   vGPU2   |
  *               |       +-----------+     +-----------+
  *      unmappable GM    | available | ==> |   vGPU1   |
  *               |       +-----------+     +-----------+
- *               |       |///////////|     |           |
- *               |       |///////////|     |   Host    |
- *               v       |///////////|     |           |
+ *               |       |###########|     |           |
+ *               |       |###########|     |   Host    |
+ *               v       |###########|     |           |
  * total GM size ------> +-----------+     +-----------+
  *
  * Returns:
  * zero on success, non-zero if configuration invalid or ballooning failed
  */
-int intel_vgt_balloon(struct drm_device *dev)
+int intel_vgt_balloon(struct drm_i915_private *dev_priv)
 {
-        struct drm_i915_private *dev_priv = to_i915(dev);
         struct i915_ggtt *ggtt = &dev_priv->ggtt;
         unsigned long ggtt_end = ggtt->base.start + ggtt->base.total;
 
@@ -188,6 +189,9 @@ int intel_vgt_balloon(struct drm_device *dev)
         unsigned long unmappable_base, unmappable_size, unmappable_end;
         int ret;
 
+        if (!intel_vgpu_active(dev_priv))
+                return 0;
+
         mappable_base = I915_READ(vgtif_reg(avail_rs.mappable_gmadr.base));
         mappable_size = I915_READ(vgtif_reg(avail_rs.mappable_gmadr.size));
         unmappable_base = I915_READ(vgtif_reg(avail_rs.nonmappable_gmadr.base));
@@ -259,6 +263,6 @@ int intel_vgt_balloon(struct drm_device *dev)
 
 err:
         DRM_ERROR("VGT balloon fail\n");
-        intel_vgt_deballoon();
+        intel_vgt_deballoon(dev_priv);
         return ret;
 }
diff --git a/drivers/gpu/drm/i915/i915_vgpu.h b/drivers/gpu/drm/i915/i915_vgpu.h
index 3c83b47b5f69..3c3b2d24e830 100644
--- a/drivers/gpu/drm/i915/i915_vgpu.h
+++ b/drivers/gpu/drm/i915/i915_vgpu.h
@@ -24,94 +24,10 @@
 #ifndef _I915_VGPU_H_
 #define _I915_VGPU_H_
 
-/* The MMIO offset of the shared info between guest and host emulator */
-#define VGT_PVINFO_PAGE 0x78000
-#define VGT_PVINFO_SIZE 0x1000
+#include "i915_pvinfo.h"
 
-/*
- * The following structure pages are defined in GEN MMIO space
- * for virtualization. (One page for now)
- */
-#define VGT_MAGIC         0x4776544776544776ULL /* 'vGTvGTvG' */
-#define VGT_VERSION_MAJOR 1
-#define VGT_VERSION_MINOR 0
-
-#define INTEL_VGT_IF_VERSION_ENCODE(major, minor) ((major) << 16 | (minor))
-#define INTEL_VGT_IF_VERSION \
-        INTEL_VGT_IF_VERSION_ENCODE(VGT_VERSION_MAJOR, VGT_VERSION_MINOR)
-
-/*
- * notifications from guest to vgpu device model
- */
-enum vgt_g2v_type {
-        VGT_G2V_PPGTT_L3_PAGE_TABLE_CREATE = 2,
-        VGT_G2V_PPGTT_L3_PAGE_TABLE_DESTROY,
-        VGT_G2V_PPGTT_L4_PAGE_TABLE_CREATE,
-        VGT_G2V_PPGTT_L4_PAGE_TABLE_DESTROY,
-        VGT_G2V_EXECLIST_CONTEXT_CREATE,
-        VGT_G2V_EXECLIST_CONTEXT_DESTROY,
-        VGT_G2V_MAX,
-};
-
-struct vgt_if {
-        uint64_t magic;         /* VGT_MAGIC */
-        uint16_t version_major;
-        uint16_t version_minor;
-        uint32_t vgt_id;        /* ID of vGT instance */
-        uint32_t rsv1[12];      /* pad to offset 0x40 */
-        /*
-         *  Data structure to describe the balooning info of resources.
-         *  Each VM can only have one portion of continuous area for now.
-         *  (May support scattered resource in future)
-         *  (starting from offset 0x40)
-         */
-        struct {
-                /* Aperture register balooning */
-                struct {
-                        uint32_t base;
-                        uint32_t size;
-                } mappable_gmadr;       /* aperture */
-                /* GMADR register balooning */
-                struct {
-                        uint32_t base;
-                        uint32_t size;
-                } nonmappable_gmadr;    /* non aperture */
-                /* allowed fence registers */
-                uint32_t fence_num;
-                uint32_t rsv2[3];
-        } avail_rs;             /* available/assigned resource */
-        uint32_t rsv3[0x200 - 24];      /* pad to half page */
-        /*
-         * The bottom half page is for response from Gfx driver to hypervisor.
-         */
-        uint32_t rsv4;
-        uint32_t display_ready; /* ready for display owner switch */
-
-        uint32_t rsv5[4];
-
-        uint32_t g2v_notify;
-        uint32_t rsv6[7];
-
-        struct {
-                uint32_t lo;
-                uint32_t hi;
-        } pdp[4];
-
-        uint32_t execlist_context_descriptor_lo;
-        uint32_t execlist_context_descriptor_hi;
-
-        uint32_t  rsv7[0x200 - 24];    /* pad to one page */
-} __packed;
-
-#define vgtif_reg(x) \
-        _MMIO((VGT_PVINFO_PAGE + (long)&((struct vgt_if *)NULL)->x))
-
-/* vGPU display status to be used by the host side */
-#define VGT_DRV_DISPLAY_NOT_READY 0
-#define VGT_DRV_DISPLAY_READY     1  /* ready for display switch */
-
-extern void i915_check_vgpu(struct drm_device *dev);
-extern int intel_vgt_balloon(struct drm_device *dev);
-extern void intel_vgt_deballoon(void);
+void i915_check_vgpu(struct drm_i915_private *dev_priv);
+int intel_vgt_balloon(struct drm_i915_private *dev_priv);
+void intel_vgt_deballoon(struct drm_i915_private *dev_priv);
 
 #endif /* _I915_VGPU_H_ */
diff --git a/drivers/gpu/drm/i915/intel_atomic.c b/drivers/gpu/drm/i915/intel_atomic.c
index 50ff90aea721..c5a166752eda 100644
--- a/drivers/gpu/drm/i915/intel_atomic.c
+++ b/drivers/gpu/drm/i915/intel_atomic.c
@@ -191,7 +191,7 @@ int intel_atomic_setup_scalers(struct drm_device *dev,
 
                         /* plane scaler case: assign as a plane scaler */
                         /* find the plane that set the bit as scaler_user */
-                        plane = drm_state->planes[i];
+                        plane = drm_state->planes[i].ptr;
 
                         /*
                          * to enable/disable hq mode, add planes that are using scaler
@@ -223,7 +223,8 @@ int intel_atomic_setup_scalers(struct drm_device *dev,
                                 continue;
                         }
 
-                        plane_state = to_intel_plane_state(drm_state->plane_states[i]);
+                        plane_state = intel_atomic_get_existing_plane_state(drm_state,
+                                                                            intel_plane);
                         scaler_id = &plane_state->scaler_id;
                 }
 
diff --git a/drivers/gpu/drm/i915/intel_audio.c b/drivers/gpu/drm/i915/intel_audio.c
index 02a7527ce7bb..6700a7be7f78 100644
--- a/drivers/gpu/drm/i915/intel_audio.c
+++ b/drivers/gpu/drm/i915/intel_audio.c
@@ -154,7 +154,7 @@ static bool audio_rate_need_prog(struct intel_crtc *crtc,
 {
         if (((mode->clock == TMDS_297M) ||
                  (mode->clock == TMDS_296M)) &&
-                intel_pipe_has_type(crtc, INTEL_OUTPUT_HDMI))
+                intel_crtc_has_type(crtc->config, INTEL_OUTPUT_HDMI))
                 return true;
         else
                 return false;
@@ -165,7 +165,7 @@ static bool intel_eld_uptodate(struct drm_connector *connector,
                                i915_reg_t reg_elda, uint32_t bits_elda,
                                i915_reg_t reg_edid)
 {
-        struct drm_i915_private *dev_priv = connector->dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(connector->dev);
         uint8_t *eld = connector->eld;
         uint32_t tmp;
         int i;
@@ -189,7 +189,7 @@ static bool intel_eld_uptodate(struct drm_connector *connector,
 
 static void g4x_audio_codec_disable(struct intel_encoder *encoder)
 {
-        struct drm_i915_private *dev_priv = encoder->base.dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
         uint32_t eldv, tmp;
 
         DRM_DEBUG_KMS("Disable audio codec\n");
@@ -210,7 +210,7 @@ static void g4x_audio_codec_enable(struct drm_connector *connector,
                                    struct intel_encoder *encoder,
                                    const struct drm_display_mode *adjusted_mode)
 {
-        struct drm_i915_private *dev_priv = connector->dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(connector->dev);
         uint8_t *eld = connector->eld;
         uint32_t eldv;
         uint32_t tmp;
@@ -247,7 +247,7 @@ static void g4x_audio_codec_enable(struct drm_connector *connector,
 
 static void hsw_audio_codec_disable(struct intel_encoder *encoder)
 {
-        struct drm_i915_private *dev_priv = encoder->base.dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
         struct intel_crtc *intel_crtc = to_intel_crtc(encoder->base.crtc);
         enum pipe pipe = intel_crtc->pipe;
         uint32_t tmp;
@@ -262,7 +262,7 @@ static void hsw_audio_codec_disable(struct intel_encoder *encoder)
         tmp |= AUD_CONFIG_N_PROG_ENABLE;
         tmp &= ~AUD_CONFIG_UPPER_N_MASK;
         tmp &= ~AUD_CONFIG_LOWER_N_MASK;
-        if (intel_pipe_has_type(intel_crtc, INTEL_OUTPUT_DISPLAYPORT))
+        if (intel_crtc_has_dp_encoder(intel_crtc->config))
                 tmp |= AUD_CONFIG_N_VALUE_INDEX;
         I915_WRITE(HSW_AUD_CFG(pipe), tmp);
 
@@ -279,7 +279,7 @@ static void hsw_audio_codec_enable(struct drm_connector *connector,
                                    struct intel_encoder *encoder,
                                    const struct drm_display_mode *adjusted_mode)
 {
-        struct drm_i915_private *dev_priv = connector->dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(connector->dev);
         struct intel_crtc *intel_crtc = to_intel_crtc(encoder->base.crtc);
         enum pipe pipe = intel_crtc->pipe;
         struct i915_audio_component *acomp = dev_priv->audio_component;
@@ -328,7 +328,7 @@ static void hsw_audio_codec_enable(struct drm_connector *connector,
         tmp = I915_READ(HSW_AUD_CFG(pipe));
         tmp &= ~AUD_CONFIG_N_VALUE_INDEX;
         tmp &= ~AUD_CONFIG_PIXEL_CLOCK_HDMI_MASK;
-        if (intel_pipe_has_type(intel_crtc, INTEL_OUTPUT_DISPLAYPORT))
+        if (intel_crtc_has_dp_encoder(intel_crtc->config))
                 tmp |= AUD_CONFIG_N_VALUE_INDEX;
         else
                 tmp |= audio_config_hdmi_pixel_clock(adjusted_mode);
@@ -357,7 +357,7 @@ static void hsw_audio_codec_enable(struct drm_connector *connector,
 
 static void ilk_audio_codec_disable(struct intel_encoder *encoder)
 {
-        struct drm_i915_private *dev_priv = encoder->base.dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
         struct intel_crtc *intel_crtc = to_intel_crtc(encoder->base.crtc);
         struct intel_digital_port *intel_dig_port =
                 enc_to_dig_port(&encoder->base);
@@ -389,7 +389,7 @@ static void ilk_audio_codec_disable(struct intel_encoder *encoder)
         tmp |= AUD_CONFIG_N_PROG_ENABLE;
         tmp &= ~AUD_CONFIG_UPPER_N_MASK;
         tmp &= ~AUD_CONFIG_LOWER_N_MASK;
-        if (intel_pipe_has_type(intel_crtc, INTEL_OUTPUT_DISPLAYPORT))
+        if (intel_crtc_has_dp_encoder(intel_crtc->config))
                 tmp |= AUD_CONFIG_N_VALUE_INDEX;
         I915_WRITE(aud_config, tmp);
 
@@ -405,7 +405,7 @@ static void ilk_audio_codec_enable(struct drm_connector *connector,
                                    struct intel_encoder *encoder,
                                    const struct drm_display_mode *adjusted_mode)
 {
-        struct drm_i915_private *dev_priv = connector->dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(connector->dev);
         struct intel_crtc *intel_crtc = to_intel_crtc(encoder->base.crtc);
         struct intel_digital_port *intel_dig_port =
                 enc_to_dig_port(&encoder->base);
@@ -475,7 +475,7 @@ static void ilk_audio_codec_enable(struct drm_connector *connector,
         tmp &= ~AUD_CONFIG_N_VALUE_INDEX;
         tmp &= ~AUD_CONFIG_N_PROG_ENABLE;
         tmp &= ~AUD_CONFIG_PIXEL_CLOCK_HDMI_MASK;
-        if (intel_pipe_has_type(intel_crtc, INTEL_OUTPUT_DISPLAYPORT))
+        if (intel_crtc_has_dp_encoder(intel_crtc->config))
                 tmp |= AUD_CONFIG_N_VALUE_INDEX;
         else
                 tmp |= audio_config_hdmi_pixel_clock(adjusted_mode);
@@ -496,7 +496,7 @@ void intel_audio_codec_enable(struct intel_encoder *intel_encoder)
         const struct drm_display_mode *adjusted_mode = &crtc->config->base.adjusted_mode;
         struct drm_connector *connector;
         struct drm_device *dev = encoder->dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         struct i915_audio_component *acomp = dev_priv->audio_component;
         struct intel_digital_port *intel_dig_port = enc_to_dig_port(encoder);
         enum port port = intel_dig_port->port;
@@ -513,7 +513,7 @@ void intel_audio_codec_enable(struct intel_encoder *intel_encoder)
 
         /* ELD Conn_Type */
         connector->eld[5] &= ~(3 << 2);
-        if (intel_pipe_has_type(crtc, INTEL_OUTPUT_DISPLAYPORT))
+        if (intel_crtc_has_dp_encoder(crtc->config))
                 connector->eld[5] |= (1 << 2);
 
         connector->eld[6] = drm_av_sync_delay(connector, adjusted_mode) / 2;
@@ -543,7 +543,7 @@ void intel_audio_codec_disable(struct intel_encoder *intel_encoder)
 {
         struct drm_encoder *encoder = &intel_encoder->base;
         struct drm_device *dev = encoder->dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         struct i915_audio_component *acomp = dev_priv->audio_component;
         struct intel_digital_port *intel_dig_port = enc_to_dig_port(encoder);
         enum port port = intel_dig_port->port;
@@ -621,17 +621,11 @@ static void i915_audio_component_codec_wake_override(struct device *dev,
 static int i915_audio_component_get_cdclk_freq(struct device *dev)
 {
         struct drm_i915_private *dev_priv = dev_to_i915(dev);
-        int ret;
 
         if (WARN_ON_ONCE(!HAS_DDI(dev_priv)))
                 return -ENODEV;
 
-        intel_display_power_get(dev_priv, POWER_DOMAIN_AUDIO);
-        ret = dev_priv->display.get_display_clock_speed(dev_priv->dev);
-
-        intel_display_power_put(dev_priv, POWER_DOMAIN_AUDIO);
-
-        return ret;
+        return dev_priv->cdclk_freq;
 }
 
 static int i915_audio_component_sync_audio_rate(struct device *dev,
@@ -755,14 +749,14 @@ static int i915_audio_component_bind(struct device *i915_dev,
         if (WARN_ON(acomp->ops || acomp->dev))
                 return -EEXIST;
 
-        drm_modeset_lock_all(dev_priv->dev);
+        drm_modeset_lock_all(&dev_priv->drm);
         acomp->ops = &i915_audio_component_ops;
         acomp->dev = i915_dev;
         BUILD_BUG_ON(MAX_PORTS != I915_MAX_PORTS);
         for (i = 0; i < ARRAY_SIZE(acomp->aud_sample_rate); i++)
                 acomp->aud_sample_rate[i] = 0;
         dev_priv->audio_component = acomp;
-        drm_modeset_unlock_all(dev_priv->dev);
+        drm_modeset_unlock_all(&dev_priv->drm);
 
         return 0;
 }
@@ -773,11 +767,11 @@ static void i915_audio_component_unbind(struct device *i915_dev,
         struct i915_audio_component *acomp = data;
         struct drm_i915_private *dev_priv = dev_to_i915(i915_dev);
 
-        drm_modeset_lock_all(dev_priv->dev);
+        drm_modeset_lock_all(&dev_priv->drm);
         acomp->ops = NULL;
         acomp->dev = NULL;
         dev_priv->audio_component = NULL;
-        drm_modeset_unlock_all(dev_priv->dev);
+        drm_modeset_unlock_all(&dev_priv->drm);
 }
 
 static const struct component_ops i915_audio_component_bind_ops = {
@@ -805,7 +799,7 @@ void i915_audio_component_init(struct drm_i915_private *dev_priv)
 {
         int ret;
 
-        ret = component_add(dev_priv->dev->dev, &i915_audio_component_bind_ops);
+        ret = component_add(dev_priv->drm.dev, &i915_audio_component_bind_ops);
         if (ret < 0) {
                 DRM_ERROR("failed to add audio component (%d)\n", ret);
                 /* continue with reduced functionality */
@@ -827,6 +821,6 @@ void i915_audio_component_cleanup(struct drm_i915_private *dev_priv)
         if (!dev_priv->audio_component_registered)
                 return;
 
-        component_del(dev_priv->dev->dev, &i915_audio_component_bind_ops);
+        component_del(dev_priv->drm.dev, &i915_audio_component_bind_ops);
         dev_priv->audio_component_registered = false;
 }
diff --git a/drivers/gpu/drm/i915/intel_bios.c b/drivers/gpu/drm/i915/intel_bios.c
index b9022fa053d6..c6e69e4cfa83 100644
--- a/drivers/gpu/drm/i915/intel_bios.c
+++ b/drivers/gpu/drm/i915/intel_bios.c
@@ -218,7 +218,7 @@ parse_lfp_panel_data(struct drm_i915_private *dev_priv,
 
         dev_priv->vbt.lvds_dither = lvds_options->pixel_dither;
 
-        ret = intel_opregion_get_panel_type(dev_priv->dev);
+        ret = intel_opregion_get_panel_type(dev_priv);
         if (ret >= 0) {
                 WARN_ON(ret > 0xf);
                 panel_type = ret;
@@ -323,6 +323,15 @@ parse_lfp_backlight(struct drm_i915_private *dev_priv,
                 return;
         }
 
+        dev_priv->vbt.backlight.type = INTEL_BACKLIGHT_DISPLAY_DDI;
+        if (bdb->version >= 191 &&
+            get_blocksize(backlight_data) >= sizeof(*backlight_data)) {
+                const struct bdb_lfp_backlight_control_method *method;
+
+                method = &backlight_data->backlight_control[panel_type];
+                dev_priv->vbt.backlight.type = method->type;
+        }
+
         dev_priv->vbt.backlight.pwm_freq_hz = entry->pwm_freq_hz;
         dev_priv->vbt.backlight.active_low_pwm = entry->active_low_pwm;
         dev_priv->vbt.backlight.min_brightness = entry->min_brightness;
@@ -768,6 +777,16 @@ parse_mipi_config(struct drm_i915_private *dev_priv,
                 return;
         }
 
+        /*
+         * These fields are introduced from the VBT version 197 onwards,
+         * so making sure that these bits are set zero in the previous
+         * versions.
+         */
+        if (dev_priv->vbt.dsi.config->dual_link && bdb->version < 197) {
+                dev_priv->vbt.dsi.config->dl_dcs_cabc_ports = 0;
+                dev_priv->vbt.dsi.config->dl_dcs_backlight_ports = 0;
+        }
+
         /* We have mandatory mipi config blocks. Initialize as generic panel */
         dev_priv->vbt.dsi.panel_id = MIPI_DSI_GENERIC_PANEL_ID;
 }
@@ -1407,7 +1426,7 @@ static const struct vbt_header *find_vbt(void __iomem *bios, size_t size)
 int
 intel_bios_init(struct drm_i915_private *dev_priv)
 {
-        struct pci_dev *pdev = dev_priv->dev->pdev;
+        struct pci_dev *pdev = dev_priv->drm.pdev;
         const struct vbt_header *vbt = dev_priv->opregion.vbt;
         const struct bdb_header *bdb;
         u8 __iomem *bios = NULL;
diff --git a/drivers/gpu/drm/i915/intel_bios.h b/drivers/gpu/drm/i915/intel_bios.h
index ab0ea315eddb..8405b5a367d7 100644
--- a/drivers/gpu/drm/i915/intel_bios.h
+++ b/drivers/gpu/drm/i915/intel_bios.h
@@ -30,6 +30,14 @@
 #ifndef _INTEL_BIOS_H_
 #define _INTEL_BIOS_H_
 
+enum intel_backlight_type {
+        INTEL_BACKLIGHT_PMIC,
+        INTEL_BACKLIGHT_LPSS,
+        INTEL_BACKLIGHT_DISPLAY_DDI,
+        INTEL_BACKLIGHT_DSI_DCS,
+        INTEL_BACKLIGHT_PANEL_DRIVER_INTERFACE,
+};
+
 struct edp_power_seq {
         u16 t1_t3;
         u16 t8;
@@ -113,7 +121,13 @@ struct mipi_config {
         u16 dual_link:2;
         u16 lane_cnt:2;
         u16 pixel_overlap:3;
-        u16 rsvd3:9;
+        u16 rgb_flip:1;
+#define DL_DCS_PORT_A                   0x00
+#define DL_DCS_PORT_C                   0x01
+#define DL_DCS_PORT_A_AND_C             0x02
+        u16 dl_dcs_cabc_ports:2;
+        u16 dl_dcs_backlight_ports:2;
+        u16 rsvd3:4;
 
         u16 rsvd4;
 
diff --git a/drivers/gpu/drm/i915/intel_breadcrumbs.c b/drivers/gpu/drm/i915/intel_breadcrumbs.c
new file mode 100644
index 000000000000..b074f3d6d127
--- /dev/null
+++ b/drivers/gpu/drm/i915/intel_breadcrumbs.c
@@ -0,0 +1,595 @@
+/*
+ * Copyright © 2015 Intel Corporation
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ *
+ */
+
+#include <linux/kthread.h>
+
+#include "i915_drv.h"
+
+static void intel_breadcrumbs_fake_irq(unsigned long data)
+{
+        struct intel_engine_cs *engine = (struct intel_engine_cs *)data;
+
+        /*
+         * The timer persists in case we cannot enable interrupts,
+         * or if we have previously seen seqno/interrupt incoherency
+         * ("missed interrupt" syndrome). Here the worker will wake up
+         * every jiffie in order to kick the oldest waiter to do the
+         * coherent seqno check.
+         */
+        rcu_read_lock();
+        if (intel_engine_wakeup(engine))
+                mod_timer(&engine->breadcrumbs.fake_irq, jiffies + 1);
+        rcu_read_unlock();
+}
+
+static void irq_enable(struct intel_engine_cs *engine)
+{
+        /* Enabling the IRQ may miss the generation of the interrupt, but
+         * we still need to force the barrier before reading the seqno,
+         * just in case.
+         */
+        engine->breadcrumbs.irq_posted = true;
+
+        spin_lock_irq(&engine->i915->irq_lock);
+        engine->irq_enable(engine);
+        spin_unlock_irq(&engine->i915->irq_lock);
+}
+
+static void irq_disable(struct intel_engine_cs *engine)
+{
+        spin_lock_irq(&engine->i915->irq_lock);
+        engine->irq_disable(engine);
+        spin_unlock_irq(&engine->i915->irq_lock);
+
+        engine->breadcrumbs.irq_posted = false;
+}
+
+static void __intel_breadcrumbs_enable_irq(struct intel_breadcrumbs *b)
+{
+        struct intel_engine_cs *engine =
+                container_of(b, struct intel_engine_cs, breadcrumbs);
+        struct drm_i915_private *i915 = engine->i915;
+
+        assert_spin_locked(&b->lock);
+        if (b->rpm_wakelock)
+                return;
+
+        /* Since we are waiting on a request, the GPU should be busy
+         * and should have its own rpm reference. For completeness,
+         * record an rpm reference for ourselves to cover the
+         * interrupt we unmask.
+         */
+        intel_runtime_pm_get_noresume(i915);
+        b->rpm_wakelock = true;
+
+        /* No interrupts? Kick the waiter every jiffie! */
+        if (intel_irqs_enabled(i915)) {
+                if (!test_bit(engine->id, &i915->gpu_error.test_irq_rings))
+                        irq_enable(engine);
+                b->irq_enabled = true;
+        }
+
+        if (!b->irq_enabled ||
+            test_bit(engine->id, &i915->gpu_error.missed_irq_rings))
+                mod_timer(&b->fake_irq, jiffies + 1);
+
+        /* Ensure that even if the GPU hangs, we get woken up.
+         *
+         * However, note that if no one is waiting, we never notice
+         * a gpu hang. Eventually, we will have to wait for a resource
+         * held by the GPU and so trigger a hangcheck. In the most
+         * pathological case, this will be upon memory starvation!
+         */
+        i915_queue_hangcheck(i915);
+}
+
+static void __intel_breadcrumbs_disable_irq(struct intel_breadcrumbs *b)
+{
+        struct intel_engine_cs *engine =
+                container_of(b, struct intel_engine_cs, breadcrumbs);
+
+        assert_spin_locked(&b->lock);
+        if (!b->rpm_wakelock)
+                return;
+
+        if (b->irq_enabled) {
+                irq_disable(engine);
+                b->irq_enabled = false;
+        }
+
+        intel_runtime_pm_put(engine->i915);
+        b->rpm_wakelock = false;
+}
+
+static inline struct intel_wait *to_wait(struct rb_node *node)
+{
+        return container_of(node, struct intel_wait, node);
+}
+
+static inline void __intel_breadcrumbs_finish(struct intel_breadcrumbs *b,
+                                              struct intel_wait *wait)
+{
+        assert_spin_locked(&b->lock);
+
+        /* This request is completed, so remove it from the tree, mark it as
+         * complete, and *then* wake up the associated task.
+         */
+        rb_erase(&wait->node, &b->waiters);
+        RB_CLEAR_NODE(&wait->node);
+
+        wake_up_process(wait->tsk); /* implicit smp_wmb() */
+}
+
+static bool __intel_engine_add_wait(struct intel_engine_cs *engine,
+                                    struct intel_wait *wait)
+{
+        struct intel_breadcrumbs *b = &engine->breadcrumbs;
+        struct rb_node **p, *parent, *completed;
+        bool first;
+        u32 seqno;
+
+        /* Insert the request into the retirement ordered list
+         * of waiters by walking the rbtree. If we are the oldest
+         * seqno in the tree (the first to be retired), then
+         * set ourselves as the bottom-half.
+         *
+         * As we descend the tree, prune completed branches since we hold the
+         * spinlock we know that the first_waiter must be delayed and can
+         * reduce some of the sequential wake up latency if we take action
+         * ourselves and wake up the completed tasks in parallel. Also, by
+         * removing stale elements in the tree, we may be able to reduce the
+         * ping-pong between the old bottom-half and ourselves as first-waiter.
+         */
+        first = true;
+        parent = NULL;
+        completed = NULL;
+        seqno = intel_engine_get_seqno(engine);
+
+         /* If the request completed before we managed to grab the spinlock,
+          * return now before adding ourselves to the rbtree. We let the
+          * current bottom-half handle any pending wakeups and instead
+          * try and get out of the way quickly.
+          */
+        if (i915_seqno_passed(seqno, wait->seqno)) {
+                RB_CLEAR_NODE(&wait->node);
+                return first;
+        }
+
+        p = &b->waiters.rb_node;
+        while (*p) {
+                parent = *p;
+                if (wait->seqno == to_wait(parent)->seqno) {
+                        /* We have multiple waiters on the same seqno, select
+                         * the highest priority task (that with the smallest
+                         * task->prio) to serve as the bottom-half for this
+                         * group.
+                         */
+                        if (wait->tsk->prio > to_wait(parent)->tsk->prio) {
+                                p = &parent->rb_right;
+                                first = false;
+                        } else {
+                                p = &parent->rb_left;
+                        }
+                } else if (i915_seqno_passed(wait->seqno,
+                                             to_wait(parent)->seqno)) {
+                        p = &parent->rb_right;
+                        if (i915_seqno_passed(seqno, to_wait(parent)->seqno))
+                                completed = parent;
+                        else
+                                first = false;
+                } else {
+                        p = &parent->rb_left;
+                }
+        }
+        rb_link_node(&wait->node, parent, p);
+        rb_insert_color(&wait->node, &b->waiters);
+        GEM_BUG_ON(!first && !b->irq_seqno_bh);
+
+        if (completed) {
+                struct rb_node *next = rb_next(completed);
+
+                GEM_BUG_ON(!next && !first);
+                if (next && next != &wait->node) {
+                        GEM_BUG_ON(first);
+                        b->first_wait = to_wait(next);
+                        smp_store_mb(b->irq_seqno_bh, b->first_wait->tsk);
+                        /* As there is a delay between reading the current
+                         * seqno, processing the completed tasks and selecting
+                         * the next waiter, we may have missed the interrupt
+                         * and so need for the next bottom-half to wakeup.
+                         *
+                         * Also as we enable the IRQ, we may miss the
+                         * interrupt for that seqno, so we have to wake up
+                         * the next bottom-half in order to do a coherent check
+                         * in case the seqno passed.
+                         */
+                        __intel_breadcrumbs_enable_irq(b);
+                        if (READ_ONCE(b->irq_posted))
+                                wake_up_process(to_wait(next)->tsk);
+                }
+
+                do {
+                        struct intel_wait *crumb = to_wait(completed);
+                        completed = rb_prev(completed);
+                        __intel_breadcrumbs_finish(b, crumb);
+                } while (completed);
+        }
+
+        if (first) {
+                GEM_BUG_ON(rb_first(&b->waiters) != &wait->node);
+                b->first_wait = wait;
+                smp_store_mb(b->irq_seqno_bh, wait->tsk);
+                /* After assigning ourselves as the new bottom-half, we must
+                 * perform a cursory check to prevent a missed interrupt.
+                 * Either we miss the interrupt whilst programming the hardware,
+                 * or if there was a previous waiter (for a later seqno) they
+                 * may be woken instead of us (due to the inherent race
+                 * in the unlocked read of b->irq_seqno_bh in the irq handler)
+                 * and so we miss the wake up.
+                 */
+                __intel_breadcrumbs_enable_irq(b);
+        }
+        GEM_BUG_ON(!b->irq_seqno_bh);
+        GEM_BUG_ON(!b->first_wait);
+        GEM_BUG_ON(rb_first(&b->waiters) != &b->first_wait->node);
+
+        return first;
+}
+
+bool intel_engine_add_wait(struct intel_engine_cs *engine,
+                           struct intel_wait *wait)
+{
+        struct intel_breadcrumbs *b = &engine->breadcrumbs;
+        bool first;
+
+        spin_lock(&b->lock);
+        first = __intel_engine_add_wait(engine, wait);
+        spin_unlock(&b->lock);
+
+        return first;
+}
+
+void intel_engine_enable_fake_irq(struct intel_engine_cs *engine)
+{
+        mod_timer(&engine->breadcrumbs.fake_irq, jiffies + 1);
+}
+
+static inline bool chain_wakeup(struct rb_node *rb, int priority)
+{
+        return rb && to_wait(rb)->tsk->prio <= priority;
+}
+
+static inline int wakeup_priority(struct intel_breadcrumbs *b,
+                                  struct task_struct *tsk)
+{
+        if (tsk == b->signaler)
+                return INT_MIN;
+        else
+                return tsk->prio;
+}
+
+void intel_engine_remove_wait(struct intel_engine_cs *engine,
+                              struct intel_wait *wait)
+{
+        struct intel_breadcrumbs *b = &engine->breadcrumbs;
+
+        /* Quick check to see if this waiter was already decoupled from
+         * the tree by the bottom-half to avoid contention on the spinlock
+         * by the herd.
+         */
+        if (RB_EMPTY_NODE(&wait->node))
+                return;
+
+        spin_lock(&b->lock);
+
+        if (RB_EMPTY_NODE(&wait->node))
+                goto out_unlock;
+
+        if (b->first_wait == wait) {
+                const int priority = wakeup_priority(b, wait->tsk);
+                struct rb_node *next;
+
+                GEM_BUG_ON(b->irq_seqno_bh != wait->tsk);
+
+                /* We are the current bottom-half. Find the next candidate,
+                 * the first waiter in the queue on the remaining oldest
+                 * request. As multiple seqnos may complete in the time it
+                 * takes us to wake up and find the next waiter, we have to
+                 * wake up that waiter for it to perform its own coherent
+                 * completion check.
+                 */
+                next = rb_next(&wait->node);
+                if (chain_wakeup(next, priority)) {
+                        /* If the next waiter is already complete,
+                         * wake it up and continue onto the next waiter. So
+                         * if have a small herd, they will wake up in parallel
+                         * rather than sequentially, which should reduce
+                         * the overall latency in waking all the completed
+                         * clients.
+                         *
+                         * However, waking up a chain adds extra latency to
+                         * the first_waiter. This is undesirable if that
+                         * waiter is a high priority task.
+                         */
+                        u32 seqno = intel_engine_get_seqno(engine);
+
+                        while (i915_seqno_passed(seqno, to_wait(next)->seqno)) {
+                                struct rb_node *n = rb_next(next);
+
+                                __intel_breadcrumbs_finish(b, to_wait(next));
+                                next = n;
+                                if (!chain_wakeup(next, priority))
+                                        break;
+                        }
+                }
+
+                if (next) {
+                        /* In our haste, we may have completed the first waiter
+                         * before we enabled the interrupt. Do so now as we
+                         * have a second waiter for a future seqno. Afterwards,
+                         * we have to wake up that waiter in case we missed
+                         * the interrupt, or if we have to handle an
+                         * exception rather than a seqno completion.
+                         */
+                        b->first_wait = to_wait(next);
+                        smp_store_mb(b->irq_seqno_bh, b->first_wait->tsk);
+                        if (b->first_wait->seqno != wait->seqno)
+                                __intel_breadcrumbs_enable_irq(b);
+                        wake_up_process(b->irq_seqno_bh);
+                } else {
+                        b->first_wait = NULL;
+                        WRITE_ONCE(b->irq_seqno_bh, NULL);
+                        __intel_breadcrumbs_disable_irq(b);
+                }
+        } else {
+                GEM_BUG_ON(rb_first(&b->waiters) == &wait->node);
+        }
+
+        GEM_BUG_ON(RB_EMPTY_NODE(&wait->node));
+        rb_erase(&wait->node, &b->waiters);
+
+out_unlock:
+        GEM_BUG_ON(b->first_wait == wait);
+        GEM_BUG_ON(rb_first(&b->waiters) !=
+                   (b->first_wait ? &b->first_wait->node : NULL));
+        GEM_BUG_ON(!b->irq_seqno_bh ^ RB_EMPTY_ROOT(&b->waiters));
+        spin_unlock(&b->lock);
+}
+
+static bool signal_complete(struct drm_i915_gem_request *request)
+{
+        if (!request)
+                return false;
+
+        /* If another process served as the bottom-half it may have already
+         * signalled that this wait is already completed.
+         */
+        if (intel_wait_complete(&request->signaling.wait))
+                return true;
+
+        /* Carefully check if the request is complete, giving time for the
+         * seqno to be visible or if the GPU hung.
+         */
+        if (__i915_request_irq_complete(request))
+                return true;
+
+        return false;
+}
+
+static struct drm_i915_gem_request *to_signaler(struct rb_node *rb)
+{
+        return container_of(rb, struct drm_i915_gem_request, signaling.node);
+}
+
+static void signaler_set_rtpriority(void)
+{
+         struct sched_param param = { .sched_priority = 1 };
+
+         sched_setscheduler_nocheck(current, SCHED_FIFO, &param);
+}
+
+static int intel_breadcrumbs_signaler(void *arg)
+{
+        struct intel_engine_cs *engine = arg;
+        struct intel_breadcrumbs *b = &engine->breadcrumbs;
+        struct drm_i915_gem_request *request;
+
+        /* Install ourselves with high priority to reduce signalling latency */
+        signaler_set_rtpriority();
+
+        do {
+                set_current_state(TASK_INTERRUPTIBLE);
+
+                /* We are either woken up by the interrupt bottom-half,
+                 * or by a client adding a new signaller. In both cases,
+                 * the GPU seqno may have advanced beyond our oldest signal.
+                 * If it has, propagate the signal, remove the waiter and
+                 * check again with the next oldest signal. Otherwise we
+                 * need to wait for a new interrupt from the GPU or for
+                 * a new client.
+                 */
+                request = READ_ONCE(b->first_signal);
+                if (signal_complete(request)) {
+                        /* Wake up all other completed waiters and select the
+                         * next bottom-half for the next user interrupt.
+                         */
+                        intel_engine_remove_wait(engine,
+                                                 &request->signaling.wait);
+
+                        /* Find the next oldest signal. Note that as we have
+                         * not been holding the lock, another client may
+                         * have installed an even older signal than the one
+                         * we just completed - so double check we are still
+                         * the oldest before picking the next one.
+                         */
+                        spin_lock(&b->lock);
+                        if (request == b->first_signal) {
+                                struct rb_node *rb =
+                                        rb_next(&request->signaling.node);
+                                b->first_signal = rb ? to_signaler(rb) : NULL;
+                        }
+                        rb_erase(&request->signaling.node, &b->signals);
+                        spin_unlock(&b->lock);
+
+                        i915_gem_request_unreference(request);
+                } else {
+                        if (kthread_should_stop())
+                                break;
+
+                        schedule();
+                }
+        } while (1);
+        __set_current_state(TASK_RUNNING);
+
+        return 0;
+}
+
+void intel_engine_enable_signaling(struct drm_i915_gem_request *request)
+{
+        struct intel_engine_cs *engine = request->engine;
+        struct intel_breadcrumbs *b = &engine->breadcrumbs;
+        struct rb_node *parent, **p;
+        bool first, wakeup;
+
+        if (unlikely(READ_ONCE(request->signaling.wait.tsk)))
+                return;
+
+        spin_lock(&b->lock);
+        if (unlikely(request->signaling.wait.tsk)) {
+                wakeup = false;
+                goto unlock;
+        }
+
+        request->signaling.wait.tsk = b->signaler;
+        request->signaling.wait.seqno = request->seqno;
+        i915_gem_request_reference(request);
+
+        /* First add ourselves into the list of waiters, but register our
+         * bottom-half as the signaller thread. As per usual, only the oldest
+         * waiter (not just signaller) is tasked as the bottom-half waking
+         * up all completed waiters after the user interrupt.
+         *
+         * If we are the oldest waiter, enable the irq (after which we
+         * must double check that the seqno did not complete).
+         */
+        wakeup = __intel_engine_add_wait(engine, &request->signaling.wait);
+
+        /* Now insert ourselves into the retirement ordered list of signals
+         * on this engine. We track the oldest seqno as that will be the
+         * first signal to complete.
+         */
+        parent = NULL;
+        first = true;
+        p = &b->signals.rb_node;
+        while (*p) {
+                parent = *p;
+                if (i915_seqno_passed(request->seqno,
+                                      to_signaler(parent)->seqno)) {
+                        p = &parent->rb_right;
+                        first = false;
+                } else {
+                        p = &parent->rb_left;
+                }
+        }
+        rb_link_node(&request->signaling.node, parent, p);
+        rb_insert_color(&request->signaling.node, &b->signals);
+        if (first)
+                smp_store_mb(b->first_signal, request);
+
+unlock:
+        spin_unlock(&b->lock);
+
+        if (wakeup)
+                wake_up_process(b->signaler);
+}
+
+int intel_engine_init_breadcrumbs(struct intel_engine_cs *engine)
+{
+        struct intel_breadcrumbs *b = &engine->breadcrumbs;
+        struct task_struct *tsk;
+
+        spin_lock_init(&b->lock);
+        setup_timer(&b->fake_irq,
+                    intel_breadcrumbs_fake_irq,
+                    (unsigned long)engine);
+
+        /* Spawn a thread to provide a common bottom-half for all signals.
+         * As this is an asynchronous interface we cannot steal the current
+         * task for handling the bottom-half to the user interrupt, therefore
+         * we create a thread to do the coherent seqno dance after the
+         * interrupt and then signal the waitqueue (via the dma-buf/fence).
+         */
+        tsk = kthread_run(intel_breadcrumbs_signaler, engine,
+                          "i915/signal:%d", engine->id);
+        if (IS_ERR(tsk))
+                return PTR_ERR(tsk);
+
+        b->signaler = tsk;
+
+        return 0;
+}
+
+void intel_engine_fini_breadcrumbs(struct intel_engine_cs *engine)
+{
+        struct intel_breadcrumbs *b = &engine->breadcrumbs;
+
+        if (!IS_ERR_OR_NULL(b->signaler))
+                kthread_stop(b->signaler);
+
+        del_timer_sync(&b->fake_irq);
+}
+
+unsigned int intel_kick_waiters(struct drm_i915_private *i915)
+{
+        struct intel_engine_cs *engine;
+        unsigned int mask = 0;
+
+        /* To avoid the task_struct disappearing beneath us as we wake up
+         * the process, we must first inspect the task_struct->state under the
+         * RCU lock, i.e. as we call wake_up_process() we must be holding the
+         * rcu_read_lock().
+         */
+        rcu_read_lock();
+        for_each_engine(engine, i915)
+                if (unlikely(intel_engine_wakeup(engine)))
+                        mask |= intel_engine_flag(engine);
+        rcu_read_unlock();
+
+        return mask;
+}
+
+unsigned int intel_kick_signalers(struct drm_i915_private *i915)
+{
+        struct intel_engine_cs *engine;
+        unsigned int mask = 0;
+
+        for_each_engine(engine, i915) {
+                if (unlikely(READ_ONCE(engine->breadcrumbs.first_signal))) {
+                        wake_up_process(engine->breadcrumbs.signaler);
+                        mask |= intel_engine_flag(engine);
+                }
+        }
+
+        return mask;
+}
diff --git a/drivers/gpu/drm/i915/intel_color.c b/drivers/gpu/drm/i915/intel_color.c
index 1b3f97449395..bc0fef3d3335 100644
--- a/drivers/gpu/drm/i915/intel_color.c
+++ b/drivers/gpu/drm/i915/intel_color.c
@@ -96,7 +96,7 @@ static void i9xx_load_csc_matrix(struct drm_crtc_state *crtc_state)
 {
         struct drm_crtc *crtc = crtc_state->crtc;
         struct drm_device *dev = crtc->dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
         int i, pipe = intel_crtc->pipe;
         uint16_t coeffs[9] = { 0, };
@@ -207,7 +207,7 @@ static void cherryview_load_csc_matrix(struct drm_crtc_state *state)
 {
         struct drm_crtc *crtc = state->crtc;
         struct drm_device *dev = crtc->dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         int pipe = to_intel_crtc(crtc)->pipe;
         uint32_t mode;
 
@@ -255,7 +255,7 @@ static void cherryview_load_csc_matrix(struct drm_crtc_state *state)
 void intel_color_set_csc(struct drm_crtc_state *crtc_state)
 {
         struct drm_device *dev = crtc_state->crtc->dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
 
         if (dev_priv->display.load_csc_matrix)
                 dev_priv->display.load_csc_matrix(crtc_state);
@@ -266,13 +266,13 @@ static void i9xx_load_luts_internal(struct drm_crtc *crtc,
                                     struct drm_property_blob *blob)
 {
         struct drm_device *dev = crtc->dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
         enum pipe pipe = intel_crtc->pipe;
         int i;
 
         if (HAS_GMCH_DISPLAY(dev)) {
-                if (intel_crtc->config->has_dsi_encoder)
+                if (intel_crtc_has_type(intel_crtc->config, INTEL_OUTPUT_DSI))
                         assert_dsi_pll_enabled(dev_priv);
                 else
                         assert_pll_enabled(dev_priv, pipe);
@@ -313,7 +313,7 @@ static void haswell_load_luts(struct drm_crtc_state *crtc_state)
 {
         struct drm_crtc *crtc = crtc_state->crtc;
         struct drm_device *dev = crtc->dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
         struct intel_crtc_state *intel_crtc_state =
                 to_intel_crtc_state(crtc_state);
@@ -343,7 +343,7 @@ static void broadwell_load_luts(struct drm_crtc_state *state)
 {
         struct drm_crtc *crtc = state->crtc;
         struct drm_device *dev = crtc->dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         struct intel_crtc_state *intel_state = to_intel_crtc_state(state);
         enum pipe pipe = to_intel_crtc(crtc)->pipe;
         uint32_t i, lut_size = INTEL_INFO(dev)->color.degamma_lut_size;
@@ -426,7 +426,7 @@ static void cherryview_load_luts(struct drm_crtc_state *state)
 {
         struct drm_crtc *crtc = state->crtc;
         struct drm_device *dev = crtc->dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         enum pipe pipe = to_intel_crtc(crtc)->pipe;
         struct drm_color_lut *lut;
         uint32_t i, lut_size;
@@ -485,7 +485,7 @@ static void cherryview_load_luts(struct drm_crtc_state *state)
 void intel_color_load_luts(struct drm_crtc_state *crtc_state)
 {
         struct drm_device *dev = crtc_state->crtc->dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
 
         dev_priv->display.load_luts(crtc_state);
 }
@@ -526,7 +526,7 @@ int intel_color_check(struct drm_crtc *crtc,
 void intel_color_init(struct drm_crtc *crtc)
 {
         struct drm_device *dev = crtc->dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
 
         drm_mode_crtc_set_gamma_size(crtc, 256);
 
@@ -547,7 +547,8 @@ void intel_color_init(struct drm_crtc *crtc)
         /* Enable color management support when we have degamma & gamma LUTs. */
         if (INTEL_INFO(dev)->color.degamma_lut_size != 0 &&
             INTEL_INFO(dev)->color.gamma_lut_size != 0)
-                drm_helper_crtc_enable_color_mgmt(crtc,
+                drm_crtc_enable_color_mgmt(crtc,
                                         INTEL_INFO(dev)->color.degamma_lut_size,
+                                        true,
                                         INTEL_INFO(dev)->color.gamma_lut_size);
 }
diff --git a/drivers/gpu/drm/i915/intel_crt.c b/drivers/gpu/drm/i915/intel_crt.c
index 3fbb6fc66451..827b6ef4e9ae 100644
--- a/drivers/gpu/drm/i915/intel_crt.c
+++ b/drivers/gpu/drm/i915/intel_crt.c
@@ -67,7 +67,7 @@ static bool intel_crt_get_hw_state(struct intel_encoder *encoder,
                                    enum pipe *pipe)
 {
         struct drm_device *dev = encoder->base.dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         struct intel_crt *crt = intel_encoder_to_crt(encoder);
         enum intel_display_power_domain power_domain;
         u32 tmp;
@@ -98,7 +98,7 @@ out:
 
 static unsigned int intel_crt_get_flags(struct intel_encoder *encoder)
 {
-        struct drm_i915_private *dev_priv = encoder->base.dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
         struct intel_crt *crt = intel_encoder_to_crt(encoder);
         u32 tmp, flags = 0;
 
@@ -146,7 +146,7 @@ static void hsw_crt_get_config(struct intel_encoder *encoder,
 static void intel_crt_set_dpms(struct intel_encoder *encoder, int mode)
 {
         struct drm_device *dev = encoder->base.dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         struct intel_crt *crt = intel_encoder_to_crt(encoder);
         struct intel_crtc *crtc = to_intel_crtc(encoder->base.crtc);
         const struct drm_display_mode *adjusted_mode = &crtc->config->base.adjusted_mode;
@@ -281,7 +281,7 @@ static bool intel_ironlake_crt_detect_hotplug(struct drm_connector *connector)
 {
         struct drm_device *dev = connector->dev;
         struct intel_crt *crt = intel_attached_crt(connector);
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         u32 adpa;
         bool ret;
 
@@ -301,8 +301,10 @@ static bool intel_ironlake_crt_detect_hotplug(struct drm_connector *connector)
 
                 I915_WRITE(crt->adpa_reg, adpa);
 
-                if (wait_for((I915_READ(crt->adpa_reg) & ADPA_CRT_HOTPLUG_FORCE_TRIGGER) == 0,
-                             1000))
+                if (intel_wait_for_register(dev_priv,
+                                            crt->adpa_reg,
+                                            ADPA_CRT_HOTPLUG_FORCE_TRIGGER, 0,
+                                            1000))
                         DRM_DEBUG_KMS("timed out waiting for FORCE_TRIGGER");
 
                 if (turn_off_dac) {
@@ -326,11 +328,26 @@ static bool valleyview_crt_detect_hotplug(struct drm_connector *connector)
 {
         struct drm_device *dev = connector->dev;
         struct intel_crt *crt = intel_attached_crt(connector);
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
+        bool reenable_hpd;
         u32 adpa;
         bool ret;
         u32 save_adpa;
 
+        /*
+         * Doing a force trigger causes a hpd interrupt to get sent, which can
+         * get us stuck in a loop if we're polling:
+         *  - We enable power wells and reset the ADPA
+         *  - output_poll_exec does force probe on VGA, triggering a hpd
+         *  - HPD handler waits for poll to unlock dev->mode_config.mutex
+         *  - output_poll_exec shuts off the ADPA, unlocks
+         *    dev->mode_config.mutex
+         *  - HPD handler runs, resets ADPA and brings us back to the start
+         *
+         * Just disable HPD interrupts here to prevent this
+         */
+        reenable_hpd = intel_hpd_disable(dev_priv, crt->base.hpd_pin);
+
         save_adpa = adpa = I915_READ(crt->adpa_reg);
         DRM_DEBUG_KMS("trigger hotplug detect cycle: adpa=0x%x\n", adpa);
 
@@ -338,8 +355,10 @@ static bool valleyview_crt_detect_hotplug(struct drm_connector *connector)
 
         I915_WRITE(crt->adpa_reg, adpa);
 
-        if (wait_for((I915_READ(crt->adpa_reg) & ADPA_CRT_HOTPLUG_FORCE_TRIGGER) == 0,
-                     1000)) {
+        if (intel_wait_for_register(dev_priv,
+                                    crt->adpa_reg,
+                                    ADPA_CRT_HOTPLUG_FORCE_TRIGGER, 0,
+                                    1000)) {
                 DRM_DEBUG_KMS("timed out waiting for FORCE_TRIGGER");
                 I915_WRITE(crt->adpa_reg, save_adpa);
         }
@@ -353,6 +372,9 @@ static bool valleyview_crt_detect_hotplug(struct drm_connector *connector)
 
         DRM_DEBUG_KMS("valleyview hotplug adpa=0x%x, result %d\n", adpa, ret);
 
+        if (reenable_hpd)
+                intel_hpd_enable(dev_priv, crt->base.hpd_pin);
+
         return ret;
 }
 
@@ -367,7 +389,7 @@ static bool valleyview_crt_detect_hotplug(struct drm_connector *connector)
 static bool intel_crt_detect_hotplug(struct drm_connector *connector)
 {
         struct drm_device *dev = connector->dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         u32 stat;
         bool ret = false;
         int i, tries = 0;
@@ -394,9 +416,9 @@ static bool intel_crt_detect_hotplug(struct drm_connector *connector)
                                               CRT_HOTPLUG_FORCE_DETECT,
                                               CRT_HOTPLUG_FORCE_DETECT);
                 /* wait for FORCE_DETECT to go off */
-                if (wait_for((I915_READ(PORT_HOTPLUG_EN) &
-                              CRT_HOTPLUG_FORCE_DETECT) == 0,
-                             1000))
+                if (intel_wait_for_register(dev_priv, PORT_HOTPLUG_EN,
+                                            CRT_HOTPLUG_FORCE_DETECT, 0,
+                                            1000))
                         DRM_DEBUG_KMS("timed out waiting for FORCE_DETECT to go off");
         }
 
@@ -449,7 +471,7 @@ static int intel_crt_ddc_get_modes(struct drm_connector *connector,
 static bool intel_crt_detect_ddc(struct drm_connector *connector)
 {
         struct intel_crt *crt = intel_attached_crt(connector);
-        struct drm_i915_private *dev_priv = crt->base.base.dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(crt->base.base.dev);
         struct edid *edid;
         struct i2c_adapter *i2c;
 
@@ -485,7 +507,7 @@ static enum drm_connector_status
 intel_crt_load_detect(struct intel_crt *crt, uint32_t pipe)
 {
         struct drm_device *dev = crt->base.base.dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         uint32_t save_bclrpat;
         uint32_t save_vtotal;
         uint32_t vtotal, vactive;
@@ -600,7 +622,7 @@ static enum drm_connector_status
 intel_crt_detect(struct drm_connector *connector, bool force)
 {
         struct drm_device *dev = connector->dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         struct intel_crt *crt = intel_attached_crt(connector);
         struct intel_encoder *intel_encoder = &crt->base;
         enum intel_display_power_domain power_domain;
@@ -681,7 +703,7 @@ static void intel_crt_destroy(struct drm_connector *connector)
 static int intel_crt_get_modes(struct drm_connector *connector)
 {
         struct drm_device *dev = connector->dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         struct intel_crt *crt = intel_attached_crt(connector);
         struct intel_encoder *intel_encoder = &crt->base;
         enum intel_display_power_domain power_domain;
@@ -713,11 +735,11 @@ static int intel_crt_set_property(struct drm_connector *connector,
         return 0;
 }
 
-static void intel_crt_reset(struct drm_connector *connector)
+void intel_crt_reset(struct drm_encoder *encoder)
 {
-        struct drm_device *dev = connector->dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
-        struct intel_crt *crt = intel_attached_crt(connector);
+        struct drm_device *dev = encoder->dev;
+        struct drm_i915_private *dev_priv = to_i915(dev);
+        struct intel_crt *crt = intel_encoder_to_crt(to_intel_encoder(encoder));
 
         if (INTEL_INFO(dev)->gen >= 5) {
                 u32 adpa;
@@ -739,10 +761,11 @@ static void intel_crt_reset(struct drm_connector *connector)
  */
 
 static const struct drm_connector_funcs intel_crt_connector_funcs = {
-        .reset = intel_crt_reset,
         .dpms = drm_atomic_helper_connector_dpms,
         .detect = intel_crt_detect,
         .fill_modes = drm_helper_probe_single_connector_modes,
+        .late_register = intel_connector_register,
+        .early_unregister = intel_connector_unregister,
         .destroy = intel_crt_destroy,
         .set_property = intel_crt_set_property,
         .atomic_destroy_state = drm_atomic_helper_connector_destroy_state,
@@ -753,10 +776,10 @@ static const struct drm_connector_funcs intel_crt_connector_funcs = {
 static const struct drm_connector_helper_funcs intel_crt_connector_helper_funcs = {
         .mode_valid = intel_crt_mode_valid,
         .get_modes = intel_crt_get_modes,
-        .best_encoder = intel_best_encoder,
 };
 
 static const struct drm_encoder_funcs intel_crt_enc_funcs = {
+        .reset = intel_crt_reset,
         .destroy = intel_encoder_destroy,
 };
 
@@ -791,7 +814,7 @@ void intel_crt_init(struct drm_device *dev)
         struct drm_connector *connector;
         struct intel_crt *crt;
         struct intel_connector *intel_connector;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         i915_reg_t adpa_reg;
         u32 adpa;
 
@@ -839,7 +862,7 @@ void intel_crt_init(struct drm_device *dev)
                            &intel_crt_connector_funcs, DRM_MODE_CONNECTOR_VGA);
 
         drm_encoder_init(dev, &crt->base.base, &intel_crt_enc_funcs,
-                         DRM_MODE_ENCODER_DAC, NULL);
+                         DRM_MODE_ENCODER_DAC, "CRT");
 
         intel_connector_attach_encoder(intel_connector, &crt->base);
 
@@ -876,12 +899,9 @@ void intel_crt_init(struct drm_device *dev)
                 crt->base.get_hw_state = intel_crt_get_hw_state;
         }
         intel_connector->get_hw_state = intel_connector_get_hw_state;
-        intel_connector->unregister = intel_connector_unregister;
 
         drm_connector_helper_add(connector, &intel_crt_connector_helper_funcs);
 
-        drm_connector_register(connector);
-
         if (!I915_HAS_HOTPLUG(dev))
                 intel_connector->polled = DRM_CONNECTOR_POLL_CONNECT;
 
@@ -902,5 +922,5 @@ void intel_crt_init(struct drm_device *dev)
                 dev_priv->fdi_rx_config = I915_READ(FDI_RX_CTL(PIPE_A)) & fdi_config;
         }
 
-        intel_crt_reset(connector);
+        intel_crt_reset(&crt->base.base);
 }
diff --git a/drivers/gpu/drm/i915/intel_csr.c b/drivers/gpu/drm/i915/intel_csr.c
index 2b3b428d9cd2..3edb9580928e 100644
--- a/drivers/gpu/drm/i915/intel_csr.c
+++ b/drivers/gpu/drm/i915/intel_csr.c
@@ -286,7 +286,7 @@ static uint32_t *parse_csr_fw(struct drm_i915_private *dev_priv,
         uint32_t dmc_offset = CSR_DEFAULT_FW_OFFSET, readcount = 0, nbytes;
         uint32_t i;
         uint32_t *dmc_payload;
-        uint32_t required_min_version;
+        uint32_t required_version;
 
         if (!fw)
                 return NULL;
@@ -303,24 +303,23 @@ static uint32_t *parse_csr_fw(struct drm_i915_private *dev_priv,
         csr->version = css_header->version;
 
         if (IS_KABYLAKE(dev_priv)) {
-                required_min_version = KBL_CSR_VERSION_REQUIRED;
+                required_version = KBL_CSR_VERSION_REQUIRED;
         } else if (IS_SKYLAKE(dev_priv)) {
-                required_min_version = SKL_CSR_VERSION_REQUIRED;
+                required_version = SKL_CSR_VERSION_REQUIRED;
         } else if (IS_BROXTON(dev_priv)) {
-                required_min_version = BXT_CSR_VERSION_REQUIRED;
+                required_version = BXT_CSR_VERSION_REQUIRED;
         } else {
                 MISSING_CASE(INTEL_REVID(dev_priv));
-                required_min_version = 0;
+                required_version = 0;
         }
 
-        if (csr->version < required_min_version) {
-                DRM_INFO("Refusing to load old DMC firmware v%u.%u,"
-                         " please upgrade to v%u.%u or later"
-                           " [" FIRMWARE_URL "].\n",
+        if (csr->version != required_version) {
+                DRM_INFO("Refusing to load DMC firmware v%u.%u,"
+                         " please use v%u.%u [" FIRMWARE_URL "].\n",
                          CSR_VERSION_MAJOR(csr->version),
                          CSR_VERSION_MINOR(csr->version),
-                         CSR_VERSION_MAJOR(required_min_version),
-                         CSR_VERSION_MINOR(required_min_version));
+                         CSR_VERSION_MAJOR(required_version),
+                         CSR_VERSION_MINOR(required_version));
                 return NULL;
         }
 
@@ -413,7 +412,7 @@ static void csr_load_work_fn(struct work_struct *work)
         csr = &dev_priv->csr;
 
         ret = request_firmware(&fw, dev_priv->csr.fw_path,
-                               &dev_priv->dev->pdev->dev);
+                               &dev_priv->drm.pdev->dev);
         if (fw)
                 dev_priv->csr.dmc_payload = parse_csr_fw(dev_priv, fw);
 
@@ -427,7 +426,7 @@ static void csr_load_work_fn(struct work_struct *work)
                          CSR_VERSION_MAJOR(csr->version),
                          CSR_VERSION_MINOR(csr->version));
         } else {
-                dev_notice(dev_priv->dev->dev,
+                dev_notice(dev_priv->drm.dev,
                            "Failed to load DMC firmware"
                            " [" FIRMWARE_URL "],"
                            " disabling runtime power management.\n");
diff --git a/drivers/gpu/drm/i915/intel_ddi.c b/drivers/gpu/drm/i915/intel_ddi.c
index 01e523df363b..dd1d6fe12297 100644
--- a/drivers/gpu/drm/i915/intel_ddi.c
+++ b/drivers/gpu/drm/i915/intel_ddi.c
@@ -318,7 +318,7 @@ static void ddi_get_encoder_port(struct intel_encoder *intel_encoder,
         default:
                 WARN(1, "Invalid DDI encoder type %d\n", intel_encoder->type);
                 /* fallthrough and treat as unknown */
-        case INTEL_OUTPUT_DISPLAYPORT:
+        case INTEL_OUTPUT_DP:
         case INTEL_OUTPUT_EDP:
         case INTEL_OUTPUT_HDMI:
         case INTEL_OUTPUT_UNKNOWN:
@@ -482,7 +482,7 @@ void intel_prepare_ddi_buffer(struct intel_encoder *encoder)
                 ddi_translations = ddi_translations_edp;
                 size = n_edp_entries;
                 break;
-        case INTEL_OUTPUT_DISPLAYPORT:
+        case INTEL_OUTPUT_DP:
         case INTEL_OUTPUT_HDMI:
                 ddi_translations = ddi_translations_dp;
                 size = n_dp_entries;
@@ -543,7 +543,7 @@ static void intel_wait_ddi_buf_idle(struct drm_i915_private *dev_priv,
 void hsw_fdi_link_train(struct drm_crtc *crtc)
 {
         struct drm_device *dev = crtc->dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
         struct intel_encoder *encoder;
         u32 temp, i, rx_ctl_val;
@@ -834,7 +834,7 @@ static void ddi_dotclock_get(struct intel_crtc_state *pipe_config)
         if (pipe_config->has_pch_encoder)
                 dotclock = intel_dotclock_calculate(pipe_config->port_clock,
                                                     &pipe_config->fdi_m_n);
-        else if (pipe_config->has_dp_encoder)
+        else if (intel_crtc_has_dp_encoder(pipe_config))
                 dotclock = intel_dotclock_calculate(pipe_config->port_clock,
                                                     &pipe_config->dp_m_n);
         else if (pipe_config->has_hdmi_sink && pipe_config->pipe_bpp == 36)
@@ -851,7 +851,7 @@ static void ddi_dotclock_get(struct intel_crtc_state *pipe_config)
 static void skl_ddi_clock_get(struct intel_encoder *encoder,
                                 struct intel_crtc_state *pipe_config)
 {
-        struct drm_i915_private *dev_priv = encoder->base.dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
         int link_clock = 0;
         uint32_t dpll_ctl1, dpll;
 
@@ -899,7 +899,7 @@ static void skl_ddi_clock_get(struct intel_encoder *encoder,
 static void hsw_ddi_clock_get(struct intel_encoder *encoder,
                               struct intel_crtc_state *pipe_config)
 {
-        struct drm_i915_private *dev_priv = encoder->base.dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
         int link_clock = 0;
         u32 val, pll;
 
@@ -948,7 +948,7 @@ static int bxt_calc_pll_link(struct drm_i915_private *dev_priv,
 {
         struct intel_shared_dpll *pll;
         struct intel_dpll_hw_state *state;
-        intel_clock_t clock;
+        struct dpll clock;
 
         /* For DDI ports we always use a shared PLL. */
         if (WARN_ON(dpll == DPLL_ID_PRIVATE))
@@ -971,7 +971,7 @@ static int bxt_calc_pll_link(struct drm_i915_private *dev_priv,
 static void bxt_ddi_clock_get(struct intel_encoder *encoder,
                                 struct intel_crtc_state *pipe_config)
 {
-        struct drm_i915_private *dev_priv = encoder->base.dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
         enum port port = intel_ddi_get_encoder_port(encoder);
         uint32_t dpll = port;
 
@@ -1061,14 +1061,14 @@ bool intel_ddi_pll_select(struct intel_crtc *intel_crtc,
 
 void intel_ddi_set_pipe_settings(struct drm_crtc *crtc)
 {
-        struct drm_i915_private *dev_priv = crtc->dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(crtc->dev);
         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
         struct intel_encoder *intel_encoder = intel_ddi_get_crtc_encoder(crtc);
         enum transcoder cpu_transcoder = intel_crtc->config->cpu_transcoder;
         int type = intel_encoder->type;
         uint32_t temp;
 
-        if (type == INTEL_OUTPUT_DISPLAYPORT || type == INTEL_OUTPUT_EDP || type == INTEL_OUTPUT_DP_MST) {
+        if (type == INTEL_OUTPUT_DP || type == INTEL_OUTPUT_EDP || type == INTEL_OUTPUT_DP_MST) {
                 WARN_ON(transcoder_is_dsi(cpu_transcoder));
 
                 temp = TRANS_MSA_SYNC_CLK;
@@ -1096,7 +1096,7 @@ void intel_ddi_set_vc_payload_alloc(struct drm_crtc *crtc, bool state)
 {
         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
         struct drm_device *dev = crtc->dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         enum transcoder cpu_transcoder = intel_crtc->config->cpu_transcoder;
         uint32_t temp;
         temp = I915_READ(TRANS_DDI_FUNC_CTL(cpu_transcoder));
@@ -1113,7 +1113,7 @@ void intel_ddi_enable_transcoder_func(struct drm_crtc *crtc)
         struct intel_encoder *intel_encoder = intel_ddi_get_crtc_encoder(crtc);
         struct drm_encoder *encoder = &intel_encoder->base;
         struct drm_device *dev = crtc->dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         enum pipe pipe = intel_crtc->pipe;
         enum transcoder cpu_transcoder = intel_crtc->config->cpu_transcoder;
         enum port port = intel_ddi_get_encoder_port(intel_encoder);
@@ -1182,7 +1182,7 @@ void intel_ddi_enable_transcoder_func(struct drm_crtc *crtc)
                 temp |= TRANS_DDI_MODE_SELECT_FDI;
                 temp |= (intel_crtc->config->fdi_lanes - 1) << 1;
 
-        } else if (type == INTEL_OUTPUT_DISPLAYPORT ||
+        } else if (type == INTEL_OUTPUT_DP ||
                    type == INTEL_OUTPUT_EDP) {
                 struct intel_dp *intel_dp = enc_to_intel_dp(encoder);
 
@@ -1223,7 +1223,7 @@ void intel_ddi_disable_transcoder_func(struct drm_i915_private *dev_priv,
 bool intel_ddi_connector_get_hw_state(struct intel_connector *intel_connector)
 {
         struct drm_device *dev = intel_connector->base.dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         struct intel_encoder *intel_encoder = intel_connector->encoder;
         int type = intel_connector->base.connector_type;
         enum port port = intel_ddi_get_encoder_port(intel_encoder);
@@ -1285,7 +1285,7 @@ bool intel_ddi_get_hw_state(struct intel_encoder *encoder,
                             enum pipe *pipe)
 {
         struct drm_device *dev = encoder->base.dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         enum port port = intel_ddi_get_encoder_port(encoder);
         enum intel_display_power_domain power_domain;
         u32 tmp;
@@ -1342,6 +1342,14 @@ bool intel_ddi_get_hw_state(struct intel_encoder *encoder,
         DRM_DEBUG_KMS("No pipe for ddi port %c found\n", port_name(port));
 
 out:
+        if (ret && IS_BROXTON(dev_priv)) {
+                tmp = I915_READ(BXT_PHY_CTL(port));
+                if ((tmp & (BXT_PHY_LANE_POWERDOWN_ACK |
+                            BXT_PHY_LANE_ENABLED)) != BXT_PHY_LANE_ENABLED)
+                        DRM_ERROR("Port %c enabled but PHY powered down? "
+                                  "(PHY_CTL %08x)\n", port_name(port), tmp);
+        }
+
         intel_display_power_put(dev_priv, power_domain);
 
         return ret;
@@ -1351,7 +1359,7 @@ void intel_ddi_enable_pipe_clock(struct intel_crtc *intel_crtc)
 {
         struct drm_crtc *crtc = &intel_crtc->base;
         struct drm_device *dev = crtc->dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         struct intel_encoder *intel_encoder = intel_ddi_get_crtc_encoder(crtc);
         enum port port = intel_ddi_get_encoder_port(intel_encoder);
         enum transcoder cpu_transcoder = intel_crtc->config->cpu_transcoder;
@@ -1363,7 +1371,7 @@ void intel_ddi_enable_pipe_clock(struct intel_crtc *intel_crtc)
 
 void intel_ddi_disable_pipe_clock(struct intel_crtc *intel_crtc)
 {
-        struct drm_i915_private *dev_priv = intel_crtc->base.dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(intel_crtc->base.dev);
         enum transcoder cpu_transcoder = intel_crtc->config->cpu_transcoder;
 
         if (cpu_transcoder != TRANSCODER_EDP)
@@ -1384,7 +1392,7 @@ static void skl_ddi_set_iboost(struct drm_i915_private *dev_priv,
         dp_iboost = dev_priv->vbt.ddi_port_info[port].dp_boost_level;
         hdmi_iboost = dev_priv->vbt.ddi_port_info[port].hdmi_boost_level;
 
-        if (type == INTEL_OUTPUT_DISPLAYPORT) {
+        if (type == INTEL_OUTPUT_DP) {
                 if (dp_iboost) {
                         iboost = dp_iboost;
                 } else {
@@ -1442,7 +1450,7 @@ static void bxt_ddi_vswing_sequence(struct drm_i915_private *dev_priv,
         if (type == INTEL_OUTPUT_EDP && dev_priv->vbt.edp.low_vswing) {
                 n_entries = ARRAY_SIZE(bxt_ddi_translations_edp);
                 ddi_translations = bxt_ddi_translations_edp;
-        } else if (type == INTEL_OUTPUT_DISPLAYPORT
+        } else if (type == INTEL_OUTPUT_DP
                         || type == INTEL_OUTPUT_EDP) {
                 n_entries = ARRAY_SIZE(bxt_ddi_translations_dp);
                 ddi_translations = bxt_ddi_translations_dp;
@@ -1616,7 +1624,7 @@ static void intel_ddi_pre_enable(struct intel_encoder *intel_encoder)
 
         intel_ddi_clk_select(intel_encoder, crtc->config);
 
-        if (type == INTEL_OUTPUT_DISPLAYPORT || type == INTEL_OUTPUT_EDP) {
+        if (type == INTEL_OUTPUT_DP || type == INTEL_OUTPUT_EDP) {
                 struct intel_dp *intel_dp = enc_to_intel_dp(encoder);
 
                 intel_dp_set_link_params(intel_dp, crtc->config);
@@ -1640,7 +1648,7 @@ static void intel_ddi_post_disable(struct intel_encoder *intel_encoder)
 {
         struct drm_encoder *encoder = &intel_encoder->base;
         struct drm_device *dev = encoder->dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         enum port port = intel_ddi_get_encoder_port(intel_encoder);
         int type = intel_encoder->type;
         uint32_t val;
@@ -1661,7 +1669,7 @@ static void intel_ddi_post_disable(struct intel_encoder *intel_encoder)
         if (wait)
                 intel_wait_ddi_buf_idle(dev_priv, port);
 
-        if (type == INTEL_OUTPUT_DISPLAYPORT || type == INTEL_OUTPUT_EDP) {
+        if (type == INTEL_OUTPUT_DP || type == INTEL_OUTPUT_EDP) {
                 struct intel_dp *intel_dp = enc_to_intel_dp(encoder);
                 intel_dp_sink_dpms(intel_dp, DRM_MODE_DPMS_OFF);
                 intel_edp_panel_vdd_on(intel_dp);
@@ -1687,7 +1695,7 @@ static void intel_enable_ddi(struct intel_encoder *intel_encoder)
         struct drm_crtc *crtc = encoder->crtc;
         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
         struct drm_device *dev = encoder->dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         enum port port = intel_ddi_get_encoder_port(intel_encoder);
         int type = intel_encoder->type;
 
@@ -1726,7 +1734,7 @@ static void intel_disable_ddi(struct intel_encoder *intel_encoder)
         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
         int type = intel_encoder->type;
         struct drm_device *dev = encoder->dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
 
         if (intel_crtc->config->has_audio) {
                 intel_audio_codec_disable(intel_encoder);
@@ -1742,9 +1750,11 @@ static void intel_disable_ddi(struct intel_encoder *intel_encoder)
         }
 }
 
-static bool broxton_phy_is_enabled(struct drm_i915_private *dev_priv,
-                                   enum dpio_phy phy)
+bool bxt_ddi_phy_is_enabled(struct drm_i915_private *dev_priv,
+                            enum dpio_phy phy)
 {
+        enum port port;
+
         if (!(I915_READ(BXT_P_CR_GT_DISP_PWRON) & GT_DISPLAY_POWER_ON(phy)))
                 return false;
 
@@ -1770,38 +1780,51 @@ static bool broxton_phy_is_enabled(struct drm_i915_private *dev_priv,
                 return false;
         }
 
+        for_each_port_masked(port,
+                             phy == DPIO_PHY0 ? BIT(PORT_B) | BIT(PORT_C) :
+                                                BIT(PORT_A)) {
+                u32 tmp = I915_READ(BXT_PHY_CTL(port));
+
+                if (tmp & BXT_PHY_CMNLANE_POWERDOWN_ACK) {
+                        DRM_DEBUG_DRIVER("DDI PHY %d powered, but common lane "
+                                         "for port %c powered down "
+                                         "(PHY_CTL %08x)\n",
+                                         phy, port_name(port), tmp);
+
+                        return false;
+                }
+        }
+
         return true;
 }
 
-static u32 broxton_get_grc(struct drm_i915_private *dev_priv, enum dpio_phy phy)
+static u32 bxt_get_grc(struct drm_i915_private *dev_priv, enum dpio_phy phy)
 {
         u32 val = I915_READ(BXT_PORT_REF_DW6(phy));
 
         return (val & GRC_CODE_MASK) >> GRC_CODE_SHIFT;
 }
 
-static void broxton_phy_wait_grc_done(struct drm_i915_private *dev_priv,
-                                      enum dpio_phy phy)
+static void bxt_phy_wait_grc_done(struct drm_i915_private *dev_priv,
+                                  enum dpio_phy phy)
 {
-        if (wait_for(I915_READ(BXT_PORT_REF_DW3(phy)) & GRC_DONE, 10))
+        if (intel_wait_for_register(dev_priv,
+                                    BXT_PORT_REF_DW3(phy),
+                                    GRC_DONE, GRC_DONE,
+                                    10))
                 DRM_ERROR("timeout waiting for PHY%d GRC\n", phy);
 }
 
-static bool broxton_phy_verify_state(struct drm_i915_private *dev_priv,
-                                     enum dpio_phy phy);
-
-static void broxton_phy_init(struct drm_i915_private *dev_priv,
-                             enum dpio_phy phy)
+void bxt_ddi_phy_init(struct drm_i915_private *dev_priv, enum dpio_phy phy)
 {
-        enum port port;
-        u32 ports, val;
+        u32 val;
 
-        if (broxton_phy_is_enabled(dev_priv, phy)) {
+        if (bxt_ddi_phy_is_enabled(dev_priv, phy)) {
                 /* Still read out the GRC value for state verification */
                 if (phy == DPIO_PHY0)
-                        dev_priv->bxt_phy_grc = broxton_get_grc(dev_priv, phy);
+                        dev_priv->bxt_phy_grc = bxt_get_grc(dev_priv, phy);
 
-                if (broxton_phy_verify_state(dev_priv, phy)) {
+                if (bxt_ddi_phy_verify_state(dev_priv, phy)) {
                         DRM_DEBUG_DRIVER("DDI PHY %d already enabled, "
                                          "won't reprogram it\n", phy);
 
@@ -1810,8 +1833,6 @@ static void broxton_phy_init(struct drm_i915_private *dev_priv,
 
                 DRM_DEBUG_DRIVER("DDI PHY %d enabled with invalid state, "
                                  "force reprogramming it\n", phy);
-        } else {
-                DRM_DEBUG_DRIVER("DDI PHY %d not enabled, enabling it\n", phy);
         }
 
         val = I915_READ(BXT_P_CR_GT_DISP_PWRON);
@@ -1831,28 +1852,6 @@ static void broxton_phy_init(struct drm_i915_private *dev_priv,
                 DRM_ERROR("timeout during PHY%d power on\n", phy);
         }
 
-        if (phy == DPIO_PHY0)
-                ports = BIT(PORT_B) | BIT(PORT_C);
-        else
-                ports = BIT(PORT_A);
-
-        for_each_port_masked(port, ports) {
-                int lane;
-
-                for (lane = 0; lane < 4; lane++) {
-                        val = I915_READ(BXT_PORT_TX_DW14_LN(port, lane));
-                        /*
-                         * Note that on CHV this flag is called UPAR, but has
-                         * the same function.
-                         */
-                        val &= ~LATENCY_OPTIM;
-                        if (lane != 1)
-                                val |= LATENCY_OPTIM;
-
-                        I915_WRITE(BXT_PORT_TX_DW14_LN(port, lane), val);
-                }
-        }
-
         /* Program PLL Rcomp code offset */
         val = I915_READ(BXT_PORT_CL1CM_DW9(phy));
         val &= ~IREF0RC_OFFSET_MASK;
@@ -1899,10 +1898,7 @@ static void broxton_phy_init(struct drm_i915_private *dev_priv,
                  * the corresponding calibrated value from PHY1, and disable
                  * the automatic calibration on PHY0.
                  */
-                broxton_phy_wait_grc_done(dev_priv, DPIO_PHY1);
-
-                val = dev_priv->bxt_phy_grc = broxton_get_grc(dev_priv,
-                                                              DPIO_PHY1);
+                val = dev_priv->bxt_phy_grc = bxt_get_grc(dev_priv, DPIO_PHY1);
                 grc_code = val << GRC_CODE_FAST_SHIFT |
                            val << GRC_CODE_SLOW_SHIFT |
                            val;
@@ -1912,31 +1908,16 @@ static void broxton_phy_init(struct drm_i915_private *dev_priv,
                 val |= GRC_DIS | GRC_RDY_OVRD;
                 I915_WRITE(BXT_PORT_REF_DW8(DPIO_PHY0), val);
         }
-        /*
-         * During PHY1 init delay waiting for GRC calibration to finish, since
-         * it can happen in parallel with the subsequent PHY0 init.
-         */
 
         val = I915_READ(BXT_PHY_CTL_FAMILY(phy));
         val |= COMMON_RESET_DIS;
         I915_WRITE(BXT_PHY_CTL_FAMILY(phy), val);
-}
-
-void broxton_ddi_phy_init(struct drm_i915_private *dev_priv)
-{
-        /* Enable PHY1 first since it provides Rcomp for PHY0 */
-        broxton_phy_init(dev_priv, DPIO_PHY1);
-        broxton_phy_init(dev_priv, DPIO_PHY0);
 
-        /*
-         * If BIOS enabled only PHY0 and not PHY1, we skipped waiting for the
-         * PHY1 GRC calibration to finish, so wait for it here.
-         */
-        broxton_phy_wait_grc_done(dev_priv, DPIO_PHY1);
+        if (phy == DPIO_PHY1)
+                bxt_phy_wait_grc_done(dev_priv, DPIO_PHY1);
 }
 
-static void broxton_phy_uninit(struct drm_i915_private *dev_priv,
-                               enum dpio_phy phy)
+void bxt_ddi_phy_uninit(struct drm_i915_private *dev_priv, enum dpio_phy phy)
 {
         uint32_t val;
 
@@ -1949,12 +1930,6 @@ static void broxton_phy_uninit(struct drm_i915_private *dev_priv,
         I915_WRITE(BXT_P_CR_GT_DISP_PWRON, val);
 }
 
-void broxton_ddi_phy_uninit(struct drm_i915_private *dev_priv)
-{
-        broxton_phy_uninit(dev_priv, DPIO_PHY1);
-        broxton_phy_uninit(dev_priv, DPIO_PHY0);
-}
-
 static bool __printf(6, 7)
 __phy_reg_verify_state(struct drm_i915_private *dev_priv, enum dpio_phy phy,
                        i915_reg_t reg, u32 mask, u32 expected,
@@ -1982,11 +1957,9 @@ __phy_reg_verify_state(struct drm_i915_private *dev_priv, enum dpio_phy phy,
         return false;
 }
 
-static bool broxton_phy_verify_state(struct drm_i915_private *dev_priv,
-                                     enum dpio_phy phy)
+bool bxt_ddi_phy_verify_state(struct drm_i915_private *dev_priv,
+                              enum dpio_phy phy)
 {
-        enum port port;
-        u32 ports;
         uint32_t mask;
         bool ok;
 
@@ -1994,27 +1967,11 @@ static bool broxton_phy_verify_state(struct drm_i915_private *dev_priv,
         __phy_reg_verify_state(dev_priv, phy, reg, mask, exp, fmt,      \
                                ## __VA_ARGS__)
 
-        /* We expect the PHY to be always enabled */
-        if (!broxton_phy_is_enabled(dev_priv, phy))
+        if (!bxt_ddi_phy_is_enabled(dev_priv, phy))
                 return false;
 
         ok = true;
 
-        if (phy == DPIO_PHY0)
-                ports = BIT(PORT_B) | BIT(PORT_C);
-        else
-                ports = BIT(PORT_A);
-
-        for_each_port_masked(port, ports) {
-                int lane;
-
-                for (lane = 0; lane < 4; lane++)
-                        ok &= _CHK(BXT_PORT_TX_DW14_LN(port, lane),
-                                    LATENCY_OPTIM,
-                                    lane != 1 ? LATENCY_OPTIM : 0,
-                                    "BXT_PORT_TX_DW14_LN(%d, %d)", port, lane);
-        }
-
         /* PLL Rcomp code offset */
         ok &= _CHK(BXT_PORT_CL1CM_DW9(phy),
                     IREF0RC_OFFSET_MASK, 0xe4 << IREF0RC_OFFSET_SHIFT,
@@ -2058,11 +2015,65 @@ static bool broxton_phy_verify_state(struct drm_i915_private *dev_priv,
 #undef _CHK
 }
 
-void broxton_ddi_phy_verify_state(struct drm_i915_private *dev_priv)
+static uint8_t
+bxt_ddi_phy_calc_lane_lat_optim_mask(struct intel_encoder *encoder,
+                                     struct intel_crtc_state *pipe_config)
 {
-        if (!broxton_phy_verify_state(dev_priv, DPIO_PHY0) ||
-            !broxton_phy_verify_state(dev_priv, DPIO_PHY1))
-                i915_report_error(dev_priv, "DDI PHY state mismatch\n");
+        switch (pipe_config->lane_count) {
+        case 1:
+                return 0;
+        case 2:
+                return BIT(2) | BIT(0);
+        case 4:
+                return BIT(3) | BIT(2) | BIT(0);
+        default:
+                MISSING_CASE(pipe_config->lane_count);
+
+                return 0;
+        }
+}
+
+static void bxt_ddi_pre_pll_enable(struct intel_encoder *encoder)
+{
+        struct intel_digital_port *dport = enc_to_dig_port(&encoder->base);
+        struct drm_i915_private *dev_priv = to_i915(dport->base.base.dev);
+        enum port port = dport->port;
+        struct intel_crtc *intel_crtc = to_intel_crtc(encoder->base.crtc);
+        int lane;
+
+        for (lane = 0; lane < 4; lane++) {
+                u32 val = I915_READ(BXT_PORT_TX_DW14_LN(port, lane));
+
+                /*
+                 * Note that on CHV this flag is called UPAR, but has
+                 * the same function.
+                 */
+                val &= ~LATENCY_OPTIM;
+                if (intel_crtc->config->lane_lat_optim_mask & BIT(lane))
+                        val |= LATENCY_OPTIM;
+
+                I915_WRITE(BXT_PORT_TX_DW14_LN(port, lane), val);
+        }
+}
+
+static uint8_t
+bxt_ddi_phy_get_lane_lat_optim_mask(struct intel_encoder *encoder)
+{
+        struct intel_digital_port *dport = enc_to_dig_port(&encoder->base);
+        struct drm_i915_private *dev_priv = to_i915(dport->base.base.dev);
+        enum port port = dport->port;
+        int lane;
+        uint8_t mask;
+
+        mask = 0;
+        for (lane = 0; lane < 4; lane++) {
+                u32 val = I915_READ(BXT_PORT_TX_DW14_LN(port, lane));
+
+                if (val & LATENCY_OPTIM)
+                        mask |= BIT(lane);
+        }
+
+        return mask;
 }
 
 void intel_ddi_prepare_link_retrain(struct intel_dp *intel_dp)
@@ -2113,7 +2124,7 @@ void intel_ddi_prepare_link_retrain(struct intel_dp *intel_dp)
 
 void intel_ddi_fdi_disable(struct drm_crtc *crtc)
 {
-        struct drm_i915_private *dev_priv = crtc->dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(crtc->dev);
         struct intel_encoder *intel_encoder = intel_ddi_get_crtc_encoder(crtc);
         uint32_t val;
 
@@ -2146,7 +2157,7 @@ void intel_ddi_fdi_disable(struct drm_crtc *crtc)
 void intel_ddi_get_config(struct intel_encoder *encoder,
                           struct intel_crtc_state *pipe_config)
 {
-        struct drm_i915_private *dev_priv = encoder->base.dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
         struct intel_crtc *intel_crtc = to_intel_crtc(encoder->base.crtc);
         enum transcoder cpu_transcoder = pipe_config->cpu_transcoder;
         struct intel_hdmi *intel_hdmi;
@@ -2200,7 +2211,6 @@ void intel_ddi_get_config(struct intel_encoder *encoder,
                 break;
         case TRANS_DDI_MODE_SELECT_DP_SST:
         case TRANS_DDI_MODE_SELECT_DP_MST:
-                pipe_config->has_dp_encoder = true;
                 pipe_config->lane_count =
                         ((temp & DDI_PORT_WIDTH_MASK) >> DDI_PORT_WIDTH_SHIFT) + 1;
                 intel_dp_get_m_n(intel_crtc, pipe_config);
@@ -2236,13 +2246,19 @@ void intel_ddi_get_config(struct intel_encoder *encoder,
         }
 
         intel_ddi_clock_get(encoder, pipe_config);
+
+        if (IS_BROXTON(dev_priv))
+                pipe_config->lane_lat_optim_mask =
+                        bxt_ddi_phy_get_lane_lat_optim_mask(encoder);
 }
 
 static bool intel_ddi_compute_config(struct intel_encoder *encoder,
                                      struct intel_crtc_state *pipe_config)
 {
+        struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
         int type = encoder->type;
         int port = intel_ddi_get_encoder_port(encoder);
+        int ret;
 
         WARN(type == INTEL_OUTPUT_UNKNOWN, "compute_config() on unknown output!\n");
 
@@ -2250,9 +2266,17 @@ static bool intel_ddi_compute_config(struct intel_encoder *encoder,
                 pipe_config->cpu_transcoder = TRANSCODER_EDP;
 
         if (type == INTEL_OUTPUT_HDMI)
-                return intel_hdmi_compute_config(encoder, pipe_config);
+                ret = intel_hdmi_compute_config(encoder, pipe_config);
         else
-                return intel_dp_compute_config(encoder, pipe_config);
+                ret = intel_dp_compute_config(encoder, pipe_config);
+
+        if (IS_BROXTON(dev_priv) && ret)
+                pipe_config->lane_lat_optim_mask =
+                        bxt_ddi_phy_calc_lane_lat_optim_mask(encoder,
+                                                             pipe_config);
+
+        return ret;
+
 }
 
 static const struct drm_encoder_funcs intel_ddi_funcs = {
@@ -2297,7 +2321,7 @@ intel_ddi_init_hdmi_connector(struct intel_digital_port *intel_dig_port)
 
 void intel_ddi_init(struct drm_device *dev, enum port port)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         struct intel_digital_port *intel_dig_port;
         struct intel_encoder *intel_encoder;
         struct drm_encoder *encoder;
@@ -2347,10 +2371,12 @@ void intel_ddi_init(struct drm_device *dev, enum port port)
         encoder = &intel_encoder->base;
 
         drm_encoder_init(dev, encoder, &intel_ddi_funcs,
-                         DRM_MODE_ENCODER_TMDS, NULL);
+                         DRM_MODE_ENCODER_TMDS, "DDI %c", port_name(port));
 
         intel_encoder->compute_config = intel_ddi_compute_config;
         intel_encoder->enable = intel_enable_ddi;
+        if (IS_BROXTON(dev_priv))
+                intel_encoder->pre_pll_enable = bxt_ddi_pre_pll_enable;
         intel_encoder->pre_enable = intel_ddi_pre_enable;
         intel_encoder->disable = intel_disable_ddi;
         intel_encoder->post_disable = intel_ddi_post_disable;
diff --git a/drivers/gpu/drm/i915/intel_device_info.c b/drivers/gpu/drm/i915/intel_device_info.c
new file mode 100644
index 000000000000..cba137f9ad3e
--- /dev/null
+++ b/drivers/gpu/drm/i915/intel_device_info.c
@@ -0,0 +1,388 @@
+/*
+ * Copyright © 2016 Intel Corporation
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ *
+ */
+
+#include "i915_drv.h"
+
+void intel_device_info_dump(struct drm_i915_private *dev_priv)
+{
+        const struct intel_device_info *info = &dev_priv->info;
+
+#define PRINT_S(name) "%s"
+#define SEP_EMPTY
+#define PRINT_FLAG(name) info->name ? #name "," : ""
+#define SEP_COMMA ,
+        DRM_DEBUG_DRIVER("i915 device info: gen=%i, pciid=0x%04x rev=0x%02x flags="
+                         DEV_INFO_FOR_EACH_FLAG(PRINT_S, SEP_EMPTY),
+                         info->gen,
+                         dev_priv->drm.pdev->device,
+                         dev_priv->drm.pdev->revision,
+                         DEV_INFO_FOR_EACH_FLAG(PRINT_FLAG, SEP_COMMA));
+#undef PRINT_S
+#undef SEP_EMPTY
+#undef PRINT_FLAG
+#undef SEP_COMMA
+}
+
+static void cherryview_sseu_info_init(struct drm_i915_private *dev_priv)
+{
+        struct intel_device_info *info = mkwrite_device_info(dev_priv);
+        u32 fuse, eu_dis;
+
+        fuse = I915_READ(CHV_FUSE_GT);
+
+        info->slice_total = 1;
+
+        if (!(fuse & CHV_FGT_DISABLE_SS0)) {
+                info->subslice_per_slice++;
+                eu_dis = fuse & (CHV_FGT_EU_DIS_SS0_R0_MASK |
+                                 CHV_FGT_EU_DIS_SS0_R1_MASK);
+                info->eu_total += 8 - hweight32(eu_dis);
+        }
+
+        if (!(fuse & CHV_FGT_DISABLE_SS1)) {
+                info->subslice_per_slice++;
+                eu_dis = fuse & (CHV_FGT_EU_DIS_SS1_R0_MASK |
+                                 CHV_FGT_EU_DIS_SS1_R1_MASK);
+                info->eu_total += 8 - hweight32(eu_dis);
+        }
+
+        info->subslice_total = info->subslice_per_slice;
+        /*
+         * CHV expected to always have a uniform distribution of EU
+         * across subslices.
+        */
+        info->eu_per_subslice = info->subslice_total ?
+                                info->eu_total / info->subslice_total :
+                                0;
+        /*
+         * CHV supports subslice power gating on devices with more than
+         * one subslice, and supports EU power gating on devices with
+         * more than one EU pair per subslice.
+        */
+        info->has_slice_pg = 0;
+        info->has_subslice_pg = (info->subslice_total > 1);
+        info->has_eu_pg = (info->eu_per_subslice > 2);
+}
+
+static void gen9_sseu_info_init(struct drm_i915_private *dev_priv)
+{
+        struct intel_device_info *info = mkwrite_device_info(dev_priv);
+        int s_max = 3, ss_max = 4, eu_max = 8;
+        int s, ss;
+        u32 fuse2, s_enable, ss_disable, eu_disable;
+        u8 eu_mask = 0xff;
+
+        fuse2 = I915_READ(GEN8_FUSE2);
+        s_enable = (fuse2 & GEN8_F2_S_ENA_MASK) >> GEN8_F2_S_ENA_SHIFT;
+        ss_disable = (fuse2 & GEN9_F2_SS_DIS_MASK) >> GEN9_F2_SS_DIS_SHIFT;
+
+        info->slice_total = hweight32(s_enable);
+        /*
+         * The subslice disable field is global, i.e. it applies
+         * to each of the enabled slices.
+        */
+        info->subslice_per_slice = ss_max - hweight32(ss_disable);
+        info->subslice_total = info->slice_total * info->subslice_per_slice;
+
+        /*
+         * Iterate through enabled slices and subslices to
+         * count the total enabled EU.
+        */
+        for (s = 0; s < s_max; s++) {
+                if (!(s_enable & BIT(s)))
+                        /* skip disabled slice */
+                        continue;
+
+                eu_disable = I915_READ(GEN9_EU_DISABLE(s));
+                for (ss = 0; ss < ss_max; ss++) {
+                        int eu_per_ss;
+
+                        if (ss_disable & BIT(ss))
+                                /* skip disabled subslice */
+                                continue;
+
+                        eu_per_ss = eu_max - hweight8((eu_disable >> (ss*8)) &
+                                                      eu_mask);
+
+                        /*
+                         * Record which subslice(s) has(have) 7 EUs. we
+                         * can tune the hash used to spread work among
+                         * subslices if they are unbalanced.
+                         */
+                        if (eu_per_ss == 7)
+                                info->subslice_7eu[s] |= BIT(ss);
+
+                        info->eu_total += eu_per_ss;
+                }
+        }
+
+        /*
+         * SKL is expected to always have a uniform distribution
+         * of EU across subslices with the exception that any one
+         * EU in any one subslice may be fused off for die
+         * recovery. BXT is expected to be perfectly uniform in EU
+         * distribution.
+        */
+        info->eu_per_subslice = info->subslice_total ?
+                                DIV_ROUND_UP(info->eu_total,
+                                             info->subslice_total) : 0;
+        /*
+         * SKL supports slice power gating on devices with more than
+         * one slice, and supports EU power gating on devices with
+         * more than one EU pair per subslice. BXT supports subslice
+         * power gating on devices with more than one subslice, and
+         * supports EU power gating on devices with more than one EU
+         * pair per subslice.
+        */
+        info->has_slice_pg =
+                (IS_SKYLAKE(dev_priv) || IS_KABYLAKE(dev_priv)) &&
+                info->slice_total > 1;
+        info->has_subslice_pg =
+                IS_BROXTON(dev_priv) && info->subslice_total > 1;
+        info->has_eu_pg = info->eu_per_subslice > 2;
+
+        if (IS_BROXTON(dev_priv)) {
+#define IS_SS_DISABLED(_ss_disable, ss)    (_ss_disable & BIT(ss))
+                /*
+                 * There is a HW issue in 2x6 fused down parts that requires
+                 * Pooled EU to be enabled as a WA. The pool configuration
+                 * changes depending upon which subslice is fused down. This
+                 * doesn't affect if the device has all 3 subslices enabled.
+                 */
+                /* WaEnablePooledEuFor2x6:bxt */
+                info->has_pooled_eu = ((info->subslice_per_slice == 3) ||
+                                       (info->subslice_per_slice == 2 &&
+                                        INTEL_REVID(dev_priv) < BXT_REVID_C0));
+
+                info->min_eu_in_pool = 0;
+                if (info->has_pooled_eu) {
+                        if (IS_SS_DISABLED(ss_disable, 0) ||
+                            IS_SS_DISABLED(ss_disable, 2))
+                                info->min_eu_in_pool = 3;
+                        else if (IS_SS_DISABLED(ss_disable, 1))
+                                info->min_eu_in_pool = 6;
+                        else
+                                info->min_eu_in_pool = 9;
+                }
+#undef IS_SS_DISABLED
+        }
+}
+
+static void broadwell_sseu_info_init(struct drm_i915_private *dev_priv)
+{
+        struct intel_device_info *info = mkwrite_device_info(dev_priv);
+        const int s_max = 3, ss_max = 3, eu_max = 8;
+        int s, ss;
+        u32 fuse2, eu_disable[s_max], s_enable, ss_disable;
+
+        fuse2 = I915_READ(GEN8_FUSE2);
+        s_enable = (fuse2 & GEN8_F2_S_ENA_MASK) >> GEN8_F2_S_ENA_SHIFT;
+        ss_disable = (fuse2 & GEN8_F2_SS_DIS_MASK) >> GEN8_F2_SS_DIS_SHIFT;
+
+        eu_disable[0] = I915_READ(GEN8_EU_DISABLE0) & GEN8_EU_DIS0_S0_MASK;
+        eu_disable[1] = (I915_READ(GEN8_EU_DISABLE0) >> GEN8_EU_DIS0_S1_SHIFT) |
+                        ((I915_READ(GEN8_EU_DISABLE1) & GEN8_EU_DIS1_S1_MASK) <<
+                         (32 - GEN8_EU_DIS0_S1_SHIFT));
+        eu_disable[2] = (I915_READ(GEN8_EU_DISABLE1) >> GEN8_EU_DIS1_S2_SHIFT) |
+                        ((I915_READ(GEN8_EU_DISABLE2) & GEN8_EU_DIS2_S2_MASK) <<
+                         (32 - GEN8_EU_DIS1_S2_SHIFT));
+
+        info->slice_total = hweight32(s_enable);
+
+        /*
+         * The subslice disable field is global, i.e. it applies
+         * to each of the enabled slices.
+         */
+        info->subslice_per_slice = ss_max - hweight32(ss_disable);
+        info->subslice_total = info->slice_total * info->subslice_per_slice;
+
+        /*
+         * Iterate through enabled slices and subslices to
+         * count the total enabled EU.
+         */
+        for (s = 0; s < s_max; s++) {
+                if (!(s_enable & (0x1 << s)))
+                        /* skip disabled slice */
+                        continue;
+
+                for (ss = 0; ss < ss_max; ss++) {
+                        u32 n_disabled;
+
+                        if (ss_disable & (0x1 << ss))
+                                /* skip disabled subslice */
+                                continue;
+
+                        n_disabled = hweight8(eu_disable[s] >> (ss * eu_max));
+
+                        /*
+                         * Record which subslices have 7 EUs.
+                         */
+                        if (eu_max - n_disabled == 7)
+                                info->subslice_7eu[s] |= 1 << ss;
+
+                        info->eu_total += eu_max - n_disabled;
+                }
+        }
+
+        /*
+         * BDW is expected to always have a uniform distribution of EU across
+         * subslices with the exception that any one EU in any one subslice may
+         * be fused off for die recovery.
+         */
+        info->eu_per_subslice = info->subslice_total ?
+                DIV_ROUND_UP(info->eu_total, info->subslice_total) : 0;
+
+        /*
+         * BDW supports slice power gating on devices with more than
+         * one slice.
+         */
+        info->has_slice_pg = (info->slice_total > 1);
+        info->has_subslice_pg = 0;
+        info->has_eu_pg = 0;
+}
+
+/*
+ * Determine various intel_device_info fields at runtime.
+ *
+ * Use it when either:
+ *   - it's judged too laborious to fill n static structures with the limit
+ *     when a simple if statement does the job,
+ *   - run-time checks (eg read fuse/strap registers) are needed.
+ *
+ * This function needs to be called:
+ *   - after the MMIO has been setup as we are reading registers,
+ *   - after the PCH has been detected,
+ *   - before the first usage of the fields it can tweak.
+ */
+void intel_device_info_runtime_init(struct drm_i915_private *dev_priv)
+{
+        struct intel_device_info *info = mkwrite_device_info(dev_priv);
+        enum pipe pipe;
+
+        /*
+         * Skylake and Broxton currently don't expose the topmost plane as its
+         * use is exclusive with the legacy cursor and we only want to expose
+         * one of those, not both. Until we can safely expose the topmost plane
+         * as a DRM_PLANE_TYPE_CURSOR with all the features exposed/supported,
+         * we don't expose the topmost plane at all to prevent ABI breakage
+         * down the line.
+         */
+        if (IS_BROXTON(dev_priv)) {
+                info->num_sprites[PIPE_A] = 2;
+                info->num_sprites[PIPE_B] = 2;
+                info->num_sprites[PIPE_C] = 1;
+        } else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
+                for_each_pipe(dev_priv, pipe)
+                        info->num_sprites[pipe] = 2;
+        else
+                for_each_pipe(dev_priv, pipe)
+                        info->num_sprites[pipe] = 1;
+
+        if (i915.disable_display) {
+                DRM_INFO("Display disabled (module parameter)\n");
+                info->num_pipes = 0;
+        } else if (info->num_pipes > 0 &&
+                   (IS_GEN7(dev_priv) || IS_GEN8(dev_priv)) &&
+                   HAS_PCH_SPLIT(dev_priv)) {
+                u32 fuse_strap = I915_READ(FUSE_STRAP);
+                u32 sfuse_strap = I915_READ(SFUSE_STRAP);
+
+                /*
+                 * SFUSE_STRAP is supposed to have a bit signalling the display
+                 * is fused off. Unfortunately it seems that, at least in
+                 * certain cases, fused off display means that PCH display
+                 * reads don't land anywhere. In that case, we read 0s.
+                 *
+                 * On CPT/PPT, we can detect this case as SFUSE_STRAP_FUSE_LOCK
+                 * should be set when taking over after the firmware.
+                 */
+                if (fuse_strap & ILK_INTERNAL_DISPLAY_DISABLE ||
+                    sfuse_strap & SFUSE_STRAP_DISPLAY_DISABLED ||
+                    (dev_priv->pch_type == PCH_CPT &&
+                     !(sfuse_strap & SFUSE_STRAP_FUSE_LOCK))) {
+                        DRM_INFO("Display fused off, disabling\n");
+                        info->num_pipes = 0;
+                } else if (fuse_strap & IVB_PIPE_C_DISABLE) {
+                        DRM_INFO("PipeC fused off\n");
+                        info->num_pipes -= 1;
+                }
+        } else if (info->num_pipes > 0 && IS_GEN9(dev_priv)) {
+                u32 dfsm = I915_READ(SKL_DFSM);
+                u8 disabled_mask = 0;
+                bool invalid;
+                int num_bits;
+
+                if (dfsm & SKL_DFSM_PIPE_A_DISABLE)
+                        disabled_mask |= BIT(PIPE_A);
+                if (dfsm & SKL_DFSM_PIPE_B_DISABLE)
+                        disabled_mask |= BIT(PIPE_B);
+                if (dfsm & SKL_DFSM_PIPE_C_DISABLE)
+                        disabled_mask |= BIT(PIPE_C);
+
+                num_bits = hweight8(disabled_mask);
+
+                switch (disabled_mask) {
+                case BIT(PIPE_A):
+                case BIT(PIPE_B):
+                case BIT(PIPE_A) | BIT(PIPE_B):
+                case BIT(PIPE_A) | BIT(PIPE_C):
+                        invalid = true;
+                        break;
+                default:
+                        invalid = false;
+                }
+
+                if (num_bits > info->num_pipes || invalid)
+                        DRM_ERROR("invalid pipe fuse configuration: 0x%x\n",
+                                  disabled_mask);
+                else
+                        info->num_pipes -= num_bits;
+        }
+
+        /* Initialize slice/subslice/EU info */
+        if (IS_CHERRYVIEW(dev_priv))
+                cherryview_sseu_info_init(dev_priv);
+        else if (IS_BROADWELL(dev_priv))
+                broadwell_sseu_info_init(dev_priv);
+        else if (INTEL_INFO(dev_priv)->gen >= 9)
+                gen9_sseu_info_init(dev_priv);
+
+        info->has_snoop = !info->has_llc;
+
+        /* Snooping is broken on BXT A stepping. */
+        if (IS_BXT_REVID(dev_priv, 0, BXT_REVID_A1))
+                info->has_snoop = false;
+
+        DRM_DEBUG_DRIVER("slice total: %u\n", info->slice_total);
+        DRM_DEBUG_DRIVER("subslice total: %u\n", info->subslice_total);
+        DRM_DEBUG_DRIVER("subslice per slice: %u\n", info->subslice_per_slice);
+        DRM_DEBUG_DRIVER("EU total: %u\n", info->eu_total);
+        DRM_DEBUG_DRIVER("EU per subslice: %u\n", info->eu_per_subslice);
+        DRM_DEBUG_DRIVER("has slice power gating: %s\n",
+                         info->has_slice_pg ? "y" : "n");
+        DRM_DEBUG_DRIVER("has subslice power gating: %s\n",
+                         info->has_subslice_pg ? "y" : "n");
+        DRM_DEBUG_DRIVER("has EU power gating: %s\n",
+                         info->has_eu_pg ? "y" : "n");
+}
diff --git a/drivers/gpu/drm/i915/intel_display.c b/drivers/gpu/drm/i915/intel_display.c
index 3074c56a643d..c457eed76f1f 100644
--- a/drivers/gpu/drm/i915/intel_display.c
+++ b/drivers/gpu/drm/i915/intel_display.c
@@ -36,6 +36,7 @@
 #include "intel_drv.h"
 #include <drm/i915_drm.h>
 #include "i915_drv.h"
+#include "i915_gem_dmabuf.h"
 #include "intel_dsi.h"
 #include "i915_trace.h"
 #include <drm/drm_atomic.h>
@@ -46,7 +47,11 @@
 #include <drm/drm_rect.h>
 #include <linux/dma_remapping.h>
 #include <linux/reservation.h>
-#include <linux/dma-buf.h>
+
+static bool is_mmio_work(struct intel_flip_work *work)
+{
+        return work->mmio_work.func;
+}
 
 /* Primary plane formats for gen <= 3 */
 static const uint32_t i8xx_primary_formats[] = {
@@ -117,20 +122,18 @@ static void ironlake_pfit_disable(struct intel_crtc *crtc, bool force);
 static void ironlake_pfit_enable(struct intel_crtc *crtc);
 static void intel_modeset_setup_hw_state(struct drm_device *dev);
 static void intel_pre_disable_primary_noatomic(struct drm_crtc *crtc);
+static int ilk_max_pixel_rate(struct drm_atomic_state *state);
+static int bxt_calc_cdclk(int max_pixclk);
 
-typedef struct {
-        int     min, max;
-} intel_range_t;
-
-typedef struct {
-        int     dot_limit;
-        int     p2_slow, p2_fast;
-} intel_p2_t;
-
-typedef struct intel_limit intel_limit_t;
 struct intel_limit {
-        intel_range_t   dot, vco, n, m, m1, m2, p, p1;
-        intel_p2_t          p2;
+        struct {
+                int min, max;
+        } dot, vco, n, m, m1, m2, p, p1;
+
+        struct {
+                int dot_limit;
+                int p2_slow, p2_fast;
+        } p2;
 };
 
 /* returns HPLL frequency in kHz */
@@ -185,6 +188,7 @@ intel_pch_rawclk(struct drm_i915_private *dev_priv)
 static int
 intel_vlv_hrawclk(struct drm_i915_private *dev_priv)
 {
+        /* RAWCLK_FREQ_VLV register updated from power well code */
         return vlv_get_cck_clock_hpll(dev_priv, "hrawclk",
                                       CCK_DISPLAY_REF_CLOCK_CONTROL);
 }
@@ -218,7 +222,7 @@ intel_g4x_hrawclk(struct drm_i915_private *dev_priv)
         }
 }
 
-static void intel_update_rawclk(struct drm_i915_private *dev_priv)
+void intel_update_rawclk(struct drm_i915_private *dev_priv)
 {
         if (HAS_PCH_SPLIT(dev_priv))
                 dev_priv->rawclk_freq = intel_pch_rawclk(dev_priv);
@@ -255,7 +259,7 @@ intel_fdi_link_freq(struct drm_i915_private *dev_priv,
                 return 270000;
 }
 
-static const intel_limit_t intel_limits_i8xx_dac = {
+static const struct intel_limit intel_limits_i8xx_dac = {
         .dot = { .min = 25000, .max = 350000 },
         .vco = { .min = 908000, .max = 1512000 },
         .n = { .min = 2, .max = 16 },
@@ -268,7 +272,7 @@ static const intel_limit_t intel_limits_i8xx_dac = {
                 .p2_slow = 4, .p2_fast = 2 },
 };
 
-static const intel_limit_t intel_limits_i8xx_dvo = {
+static const struct intel_limit intel_limits_i8xx_dvo = {
         .dot = { .min = 25000, .max = 350000 },
         .vco = { .min = 908000, .max = 1512000 },
         .n = { .min = 2, .max = 16 },
@@ -281,7 +285,7 @@ static const intel_limit_t intel_limits_i8xx_dvo = {
                 .p2_slow = 4, .p2_fast = 4 },
 };
 
-static const intel_limit_t intel_limits_i8xx_lvds = {
+static const struct intel_limit intel_limits_i8xx_lvds = {
         .dot = { .min = 25000, .max = 350000 },
         .vco = { .min = 908000, .max = 1512000 },
         .n = { .min = 2, .max = 16 },
@@ -294,7 +298,7 @@ static const intel_limit_t intel_limits_i8xx_lvds = {
                 .p2_slow = 14, .p2_fast = 7 },
 };
 
-static const intel_limit_t intel_limits_i9xx_sdvo = {
+static const struct intel_limit intel_limits_i9xx_sdvo = {
         .dot = { .min = 20000, .max = 400000 },
         .vco = { .min = 1400000, .max = 2800000 },
         .n = { .min = 1, .max = 6 },
@@ -307,7 +311,7 @@ static const intel_limit_t intel_limits_i9xx_sdvo = {
                 .p2_slow = 10, .p2_fast = 5 },
 };
 
-static const intel_limit_t intel_limits_i9xx_lvds = {
+static const struct intel_limit intel_limits_i9xx_lvds = {
         .dot = { .min = 20000, .max = 400000 },
         .vco = { .min = 1400000, .max = 2800000 },
         .n = { .min = 1, .max = 6 },
@@ -321,7 +325,7 @@ static const intel_limit_t intel_limits_i9xx_lvds = {
 };
 
 
-static const intel_limit_t intel_limits_g4x_sdvo = {
+static const struct intel_limit intel_limits_g4x_sdvo = {
         .dot = { .min = 25000, .max = 270000 },
         .vco = { .min = 1750000, .max = 3500000},
         .n = { .min = 1, .max = 4 },
@@ -336,7 +340,7 @@ static const intel_limit_t intel_limits_g4x_sdvo = {
         },
 };
 
-static const intel_limit_t intel_limits_g4x_hdmi = {
+static const struct intel_limit intel_limits_g4x_hdmi = {
         .dot = { .min = 22000, .max = 400000 },
         .vco = { .min = 1750000, .max = 3500000},
         .n = { .min = 1, .max = 4 },
@@ -349,7 +353,7 @@ static const intel_limit_t intel_limits_g4x_hdmi = {
                 .p2_slow = 10, .p2_fast = 5 },
 };
 
-static const intel_limit_t intel_limits_g4x_single_channel_lvds = {
+static const struct intel_limit intel_limits_g4x_single_channel_lvds = {
         .dot = { .min = 20000, .max = 115000 },
         .vco = { .min = 1750000, .max = 3500000 },
         .n = { .min = 1, .max = 3 },
@@ -363,7 +367,7 @@ static const intel_limit_t intel_limits_g4x_single_channel_lvds = {
         },
 };
 
-static const intel_limit_t intel_limits_g4x_dual_channel_lvds = {
+static const struct intel_limit intel_limits_g4x_dual_channel_lvds = {
         .dot = { .min = 80000, .max = 224000 },
         .vco = { .min = 1750000, .max = 3500000 },
         .n = { .min = 1, .max = 3 },
@@ -377,7 +381,7 @@ static const intel_limit_t intel_limits_g4x_dual_channel_lvds = {
         },
 };
 
-static const intel_limit_t intel_limits_pineview_sdvo = {
+static const struct intel_limit intel_limits_pineview_sdvo = {
         .dot = { .min = 20000, .max = 400000},
         .vco = { .min = 1700000, .max = 3500000 },
         /* Pineview's Ncounter is a ring counter */
@@ -392,7 +396,7 @@ static const intel_limit_t intel_limits_pineview_sdvo = {
                 .p2_slow = 10, .p2_fast = 5 },
 };
 
-static const intel_limit_t intel_limits_pineview_lvds = {
+static const struct intel_limit intel_limits_pineview_lvds = {
         .dot = { .min = 20000, .max = 400000 },
         .vco = { .min = 1700000, .max = 3500000 },
         .n = { .min = 3, .max = 6 },
@@ -410,7 +414,7 @@ static const intel_limit_t intel_limits_pineview_lvds = {
  * We calculate clock using (register_value + 2) for N/M1/M2, so here
  * the range value for them is (actual_value - 2).
  */
-static const intel_limit_t intel_limits_ironlake_dac = {
+static const struct intel_limit intel_limits_ironlake_dac = {
         .dot = { .min = 25000, .max = 350000 },
         .vco = { .min = 1760000, .max = 3510000 },
         .n = { .min = 1, .max = 5 },
@@ -423,7 +427,7 @@ static const intel_limit_t intel_limits_ironlake_dac = {
                 .p2_slow = 10, .p2_fast = 5 },
 };
 
-static const intel_limit_t intel_limits_ironlake_single_lvds = {
+static const struct intel_limit intel_limits_ironlake_single_lvds = {
         .dot = { .min = 25000, .max = 350000 },
         .vco = { .min = 1760000, .max = 3510000 },
         .n = { .min = 1, .max = 3 },
@@ -436,7 +440,7 @@ static const intel_limit_t intel_limits_ironlake_single_lvds = {
                 .p2_slow = 14, .p2_fast = 14 },
 };
 
-static const intel_limit_t intel_limits_ironlake_dual_lvds = {
+static const struct intel_limit intel_limits_ironlake_dual_lvds = {
         .dot = { .min = 25000, .max = 350000 },
         .vco = { .min = 1760000, .max = 3510000 },
         .n = { .min = 1, .max = 3 },
@@ -450,7 +454,7 @@ static const intel_limit_t intel_limits_ironlake_dual_lvds = {
 };
 
 /* LVDS 100mhz refclk limits. */
-static const intel_limit_t intel_limits_ironlake_single_lvds_100m = {
+static const struct intel_limit intel_limits_ironlake_single_lvds_100m = {
         .dot = { .min = 25000, .max = 350000 },
         .vco = { .min = 1760000, .max = 3510000 },
         .n = { .min = 1, .max = 2 },
@@ -463,7 +467,7 @@ static const intel_limit_t intel_limits_ironlake_single_lvds_100m = {
                 .p2_slow = 14, .p2_fast = 14 },
 };
 
-static const intel_limit_t intel_limits_ironlake_dual_lvds_100m = {
+static const struct intel_limit intel_limits_ironlake_dual_lvds_100m = {
         .dot = { .min = 25000, .max = 350000 },
         .vco = { .min = 1760000, .max = 3510000 },
         .n = { .min = 1, .max = 3 },
@@ -476,7 +480,7 @@ static const intel_limit_t intel_limits_ironlake_dual_lvds_100m = {
                 .p2_slow = 7, .p2_fast = 7 },
 };
 
-static const intel_limit_t intel_limits_vlv = {
+static const struct intel_limit intel_limits_vlv = {
          /*
           * These are the data rate limits (measured in fast clocks)
           * since those are the strictest limits we have. The fast
@@ -492,7 +496,7 @@ static const intel_limit_t intel_limits_vlv = {
         .p2 = { .p2_slow = 2, .p2_fast = 20 }, /* slow=min, fast=max */
 };
 
-static const intel_limit_t intel_limits_chv = {
+static const struct intel_limit intel_limits_chv = {
         /*
          * These are the data rate limits (measured in fast clocks)
          * since those are the strictest limits we have.  The fast
@@ -508,7 +512,7 @@ static const intel_limit_t intel_limits_chv = {
         .p2 = { .p2_slow = 1, .p2_fast = 14 },
 };
 
-static const intel_limit_t intel_limits_bxt = {
+static const struct intel_limit intel_limits_bxt = {
         /* FIXME: find real dot limits */
         .dot = { .min = 0, .max = INT_MAX },
         .vco = { .min = 4800000, .max = 6700000 },
@@ -526,52 +530,6 @@ needs_modeset(struct drm_crtc_state *state)
         return drm_atomic_crtc_needs_modeset(state);
 }
 
-/**
- * Returns whether any output on the specified pipe is of the specified type
- */
-bool intel_pipe_has_type(struct intel_crtc *crtc, enum intel_output_type type)
-{
-        struct drm_device *dev = crtc->base.dev;
-        struct intel_encoder *encoder;
-
-        for_each_encoder_on_crtc(dev, &crtc->base, encoder)
-                if (encoder->type == type)
-                        return true;
-
-        return false;
-}
-
-/**
- * Returns whether any output on the specified pipe will have the specified
- * type after a staged modeset is complete, i.e., the same as
- * intel_pipe_has_type() but looking at encoder->new_crtc instead of
- * encoder->crtc.
- */
-static bool intel_pipe_will_have_type(const struct intel_crtc_state *crtc_state,
-                                      int type)
-{
-        struct drm_atomic_state *state = crtc_state->base.state;
-        struct drm_connector *connector;
-        struct drm_connector_state *connector_state;
-        struct intel_encoder *encoder;
-        int i, num_connectors = 0;
-
-        for_each_connector_in_state(state, connector, connector_state, i) {
-                if (connector_state->crtc != crtc_state->base.crtc)
-                        continue;
-
-                num_connectors++;
-
-                encoder = to_intel_encoder(connector_state->best_encoder);
-                if (encoder->type == type)
-                        return true;
-        }
-
-        WARN_ON(num_connectors == 0);
-
-        return false;
-}
-
 /*
  * Platform specific helpers to calculate the port PLL loopback- (clock.m),
  * and post-divider (clock.p) values, pre- (clock.vco) and post-divided fast
@@ -581,7 +539,7 @@ static bool intel_pipe_will_have_type(const struct intel_crtc_state *crtc_state,
  * divided-down version of it.
  */
 /* m1 is reserved as 0 in Pineview, n is a ring counter */
-static int pnv_calc_dpll_params(int refclk, intel_clock_t *clock)
+static int pnv_calc_dpll_params(int refclk, struct dpll *clock)
 {
         clock->m = clock->m2 + 2;
         clock->p = clock->p1 * clock->p2;
@@ -598,7 +556,7 @@ static uint32_t i9xx_dpll_compute_m(struct dpll *dpll)
         return 5 * (dpll->m1 + 2) + (dpll->m2 + 2);
 }
 
-static int i9xx_calc_dpll_params(int refclk, intel_clock_t *clock)
+static int i9xx_calc_dpll_params(int refclk, struct dpll *clock)
 {
         clock->m = i9xx_dpll_compute_m(clock);
         clock->p = clock->p1 * clock->p2;
@@ -610,7 +568,7 @@ static int i9xx_calc_dpll_params(int refclk, intel_clock_t *clock)
         return clock->dot;
 }
 
-static int vlv_calc_dpll_params(int refclk, intel_clock_t *clock)
+static int vlv_calc_dpll_params(int refclk, struct dpll *clock)
 {
         clock->m = clock->m1 * clock->m2;
         clock->p = clock->p1 * clock->p2;
@@ -622,7 +580,7 @@ static int vlv_calc_dpll_params(int refclk, intel_clock_t *clock)
         return clock->dot / 5;
 }
 
-int chv_calc_dpll_params(int refclk, intel_clock_t *clock)
+int chv_calc_dpll_params(int refclk, struct dpll *clock)
 {
         clock->m = clock->m1 * clock->m2;
         clock->p = clock->p1 * clock->p2;
@@ -642,8 +600,8 @@ int chv_calc_dpll_params(int refclk, intel_clock_t *clock)
  */
 
 static bool intel_PLL_is_valid(struct drm_device *dev,
-                               const intel_limit_t *limit,
-                               const intel_clock_t *clock)
+                               const struct intel_limit *limit,
+                               const struct dpll *clock)
 {
         if (clock->n   < limit->n.min   || limit->n.max   < clock->n)
                 INTELPllInvalid("n out of range\n");
@@ -678,13 +636,13 @@ static bool intel_PLL_is_valid(struct drm_device *dev,
 }
 
 static int
-i9xx_select_p2_div(const intel_limit_t *limit,
+i9xx_select_p2_div(const struct intel_limit *limit,
                    const struct intel_crtc_state *crtc_state,
                    int target)
 {
         struct drm_device *dev = crtc_state->base.crtc->dev;
 
-        if (intel_pipe_will_have_type(crtc_state, INTEL_OUTPUT_LVDS)) {
+        if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_LVDS)) {
                 /*
                  * For LVDS just rely on its current settings for dual-channel.
                  * We haven't figured out how to reliably set up different
@@ -713,13 +671,13 @@ i9xx_select_p2_div(const intel_limit_t *limit,
  * divider from @match_clock used for LVDS downclocking.
  */
 static bool
-i9xx_find_best_dpll(const intel_limit_t *limit,
+i9xx_find_best_dpll(const struct intel_limit *limit,
                     struct intel_crtc_state *crtc_state,
-                    int target, int refclk, intel_clock_t *match_clock,
-                    intel_clock_t *best_clock)
+                    int target, int refclk, struct dpll *match_clock,
+                    struct dpll *best_clock)
 {
         struct drm_device *dev = crtc_state->base.crtc->dev;
-        intel_clock_t clock;
+        struct dpll clock;
         int err = target;
 
         memset(best_clock, 0, sizeof(*best_clock));
@@ -770,13 +728,13 @@ i9xx_find_best_dpll(const intel_limit_t *limit,
  * divider from @match_clock used for LVDS downclocking.
  */
 static bool
-pnv_find_best_dpll(const intel_limit_t *limit,
+pnv_find_best_dpll(const struct intel_limit *limit,
                    struct intel_crtc_state *crtc_state,
-                   int target, int refclk, intel_clock_t *match_clock,
-                   intel_clock_t *best_clock)
+                   int target, int refclk, struct dpll *match_clock,
+                   struct dpll *best_clock)
 {
         struct drm_device *dev = crtc_state->base.crtc->dev;
-        intel_clock_t clock;
+        struct dpll clock;
         int err = target;
 
         memset(best_clock, 0, sizeof(*best_clock));
@@ -825,13 +783,13 @@ pnv_find_best_dpll(const intel_limit_t *limit,
  * divider from @match_clock used for LVDS downclocking.
  */
 static bool
-g4x_find_best_dpll(const intel_limit_t *limit,
+g4x_find_best_dpll(const struct intel_limit *limit,
                    struct intel_crtc_state *crtc_state,
-                   int target, int refclk, intel_clock_t *match_clock,
-                   intel_clock_t *best_clock)
+                   int target, int refclk, struct dpll *match_clock,
+                   struct dpll *best_clock)
 {
         struct drm_device *dev = crtc_state->base.crtc->dev;
-        intel_clock_t clock;
+        struct dpll clock;
         int max_n;
         bool found = false;
         /* approximately equals target * 0.00585 */
@@ -877,8 +835,8 @@ g4x_find_best_dpll(const intel_limit_t *limit,
  * best configuration and error found so far. Return the calculated error.
  */
 static bool vlv_PLL_is_optimal(struct drm_device *dev, int target_freq,
-                               const intel_clock_t *calculated_clock,
-                               const intel_clock_t *best_clock,
+                               const struct dpll *calculated_clock,
+                               const struct dpll *best_clock,
                                unsigned int best_error_ppm,
                                unsigned int *error_ppm)
 {
@@ -918,14 +876,14 @@ static bool vlv_PLL_is_optimal(struct drm_device *dev, int target_freq,
  * reflck * (5 * (m1 + 2) + (m2 + 2)) / (n + 2) / p1 / p2.
  */
 static bool
-vlv_find_best_dpll(const intel_limit_t *limit,
+vlv_find_best_dpll(const struct intel_limit *limit,
                    struct intel_crtc_state *crtc_state,
-                   int target, int refclk, intel_clock_t *match_clock,
-                   intel_clock_t *best_clock)
+                   int target, int refclk, struct dpll *match_clock,
+                   struct dpll *best_clock)
 {
         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
         struct drm_device *dev = crtc->base.dev;
-        intel_clock_t clock;
+        struct dpll clock;
         unsigned int bestppm = 1000000;
         /* min update 19.2 MHz */
         int max_n = min(limit->n.max, refclk / 19200);
@@ -977,15 +935,15 @@ vlv_find_best_dpll(const intel_limit_t *limit,
  * reflck * (5 * (m1 + 2) + (m2 + 2)) / (n + 2) / p1 / p2.
  */
 static bool
-chv_find_best_dpll(const intel_limit_t *limit,
+chv_find_best_dpll(const struct intel_limit *limit,
                    struct intel_crtc_state *crtc_state,
-                   int target, int refclk, intel_clock_t *match_clock,
-                   intel_clock_t *best_clock)
+                   int target, int refclk, struct dpll *match_clock,
+                   struct dpll *best_clock)
 {
         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
         struct drm_device *dev = crtc->base.dev;
         unsigned int best_error_ppm;
-        intel_clock_t clock;
+        struct dpll clock;
         uint64_t m2;
         int found = false;
 
@@ -1035,10 +993,10 @@ chv_find_best_dpll(const intel_limit_t *limit,
 }
 
 bool bxt_find_best_dpll(struct intel_crtc_state *crtc_state, int target_clock,
-                        intel_clock_t *best_clock)
+                        struct dpll *best_clock)
 {
         int refclk = 100000;
-        const intel_limit_t *limit = &intel_limits_bxt;
+        const struct intel_limit *limit = &intel_limits_bxt;
 
         return chv_find_best_dpll(limit, crtc_state,
                                   target_clock, refclk, NULL, best_clock);
@@ -1076,7 +1034,7 @@ enum transcoder intel_pipe_to_cpu_transcoder(struct drm_i915_private *dev_priv,
 
 static bool pipe_dsl_stopped(struct drm_device *dev, enum pipe pipe)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         i915_reg_t reg = PIPEDSL(pipe);
         u32 line1, line2;
         u32 line_mask;
@@ -1112,7 +1070,7 @@ static bool pipe_dsl_stopped(struct drm_device *dev, enum pipe pipe)
 static void intel_wait_for_pipe_off(struct intel_crtc *crtc)
 {
         struct drm_device *dev = crtc->base.dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         enum transcoder cpu_transcoder = crtc->config->cpu_transcoder;
         enum pipe pipe = crtc->pipe;
 
@@ -1120,8 +1078,9 @@ static void intel_wait_for_pipe_off(struct intel_crtc *crtc)
                 i915_reg_t reg = PIPECONF(cpu_transcoder);
 
                 /* Wait for the Pipe State to go off */
-                if (wait_for((I915_READ(reg) & I965_PIPECONF_ACTIVE) == 0,
-                             100))
+                if (intel_wait_for_register(dev_priv,
+                                            reg, I965_PIPECONF_ACTIVE, 0,
+                                            100))
                         WARN(1, "pipe_off wait timed out\n");
         } else {
                 /* Wait for the display line to settle */
@@ -1203,7 +1162,7 @@ static void assert_fdi_tx_pll_enabled(struct drm_i915_private *dev_priv,
         u32 val;
 
         /* ILK FDI PLL is always enabled */
-        if (INTEL_INFO(dev_priv)->gen == 5)
+        if (IS_GEN5(dev_priv))
                 return;
 
         /* On Haswell, DDI ports are responsible for the FDI PLL setup */
@@ -1230,7 +1189,7 @@ void assert_fdi_rx_pll(struct drm_i915_private *dev_priv,
 void assert_panel_unlocked(struct drm_i915_private *dev_priv,
                            enum pipe pipe)
 {
-        struct drm_device *dev = dev_priv->dev;
+        struct drm_device *dev = &dev_priv->drm;
         i915_reg_t pp_reg;
         u32 val;
         enum pipe panel_pipe = PIPE_A;
@@ -1272,7 +1231,7 @@ void assert_panel_unlocked(struct drm_i915_private *dev_priv,
 static void assert_cursor(struct drm_i915_private *dev_priv,
                           enum pipe pipe, bool state)
 {
-        struct drm_device *dev = dev_priv->dev;
+        struct drm_device *dev = &dev_priv->drm;
         bool cur_state;
 
         if (IS_845G(dev) || IS_I865G(dev))
@@ -1334,7 +1293,7 @@ static void assert_plane(struct drm_i915_private *dev_priv,
 static void assert_planes_disabled(struct drm_i915_private *dev_priv,
                                    enum pipe pipe)
 {
-        struct drm_device *dev = dev_priv->dev;
+        struct drm_device *dev = &dev_priv->drm;
         int i;
 
         /* Primary planes are fixed to pipes on gen4+ */
@@ -1360,7 +1319,7 @@ static void assert_planes_disabled(struct drm_i915_private *dev_priv,
 static void assert_sprites_disabled(struct drm_i915_private *dev_priv,
                                     enum pipe pipe)
 {
-        struct drm_device *dev = dev_priv->dev;
+        struct drm_device *dev = &dev_priv->drm;
         int sprite;
 
         if (INTEL_INFO(dev)->gen >= 9) {
@@ -1540,7 +1499,11 @@ static void _vlv_enable_pll(struct intel_crtc *crtc,
         POSTING_READ(DPLL(pipe));
         udelay(150);
 
-        if (wait_for(((I915_READ(DPLL(pipe)) & DPLL_LOCK_VLV) == DPLL_LOCK_VLV), 1))
+        if (intel_wait_for_register(dev_priv,
+                                    DPLL(pipe),
+                                    DPLL_LOCK_VLV,
+                                    DPLL_LOCK_VLV,
+                                    1))
                 DRM_ERROR("DPLL %d failed to lock\n", pipe);
 }
 
@@ -1589,7 +1552,9 @@ static void _chv_enable_pll(struct intel_crtc *crtc,
         I915_WRITE(DPLL(pipe), pipe_config->dpll_hw_state.dpll);
 
         /* Check PLL is locked */
-        if (wait_for(((I915_READ(DPLL(pipe)) & DPLL_LOCK_VLV) == DPLL_LOCK_VLV), 1))
+        if (intel_wait_for_register(dev_priv,
+                                    DPLL(pipe), DPLL_LOCK_VLV, DPLL_LOCK_VLV,
+                                    1))
                 DRM_ERROR("PLL %d failed to lock\n", pipe);
 }
 
@@ -1635,9 +1600,10 @@ static int intel_num_dvo_pipes(struct drm_device *dev)
         struct intel_crtc *crtc;
         int count = 0;
 
-        for_each_intel_crtc(dev, crtc)
+        for_each_intel_crtc(dev, crtc) {
                 count += crtc->base.state->active &&
-                        intel_pipe_has_type(crtc, INTEL_OUTPUT_DVO);
+                        intel_crtc_has_type(crtc->config, INTEL_OUTPUT_DVO);
+        }
 
         return count;
 }
@@ -1645,7 +1611,7 @@ static int intel_num_dvo_pipes(struct drm_device *dev)
 static void i9xx_enable_pll(struct intel_crtc *crtc)
 {
         struct drm_device *dev = crtc->base.dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         i915_reg_t reg = DPLL(crtc->pipe);
         u32 dpll = crtc->config->dpll_hw_state.dpll;
 
@@ -1717,12 +1683,12 @@ static void i9xx_enable_pll(struct intel_crtc *crtc)
 static void i9xx_disable_pll(struct intel_crtc *crtc)
 {
         struct drm_device *dev = crtc->base.dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         enum pipe pipe = crtc->pipe;
 
         /* Disable DVO 2x clock on both PLLs if necessary */
         if (IS_I830(dev) &&
-            intel_pipe_has_type(crtc, INTEL_OUTPUT_DVO) &&
+            intel_crtc_has_type(crtc->config, INTEL_OUTPUT_DVO) &&
             !intel_num_dvo_pipes(dev)) {
                 I915_WRITE(DPLL(PIPE_B),
                            I915_READ(DPLL(PIPE_B)) & ~DPLL_DVO_2X_MODE);
@@ -1809,7 +1775,9 @@ void vlv_wait_port_ready(struct drm_i915_private *dev_priv,
                 BUG();
         }
 
-        if (wait_for((I915_READ(dpll_reg) & port_mask) == expected_mask, 1000))
+        if (intel_wait_for_register(dev_priv,
+                                    dpll_reg, port_mask, expected_mask,
+                                    1000))
                 WARN(1, "timed out waiting for port %c ready: got 0x%x, expected 0x%x\n",
                      port_name(dport->port), I915_READ(dpll_reg) & port_mask, expected_mask);
 }
@@ -1817,7 +1785,7 @@ void vlv_wait_port_ready(struct drm_i915_private *dev_priv,
 static void ironlake_enable_pch_transcoder(struct drm_i915_private *dev_priv,
                                            enum pipe pipe)
 {
-        struct drm_device *dev = dev_priv->dev;
+        struct drm_device *dev = &dev_priv->drm;
         struct drm_crtc *crtc = dev_priv->pipe_to_crtc_mapping[pipe];
         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
         i915_reg_t reg;
@@ -1850,7 +1818,7 @@ static void ironlake_enable_pch_transcoder(struct drm_i915_private *dev_priv,
                  * here for both 8bpc and 12bpc.
                  */
                 val &= ~PIPECONF_BPC_MASK;
-                if (intel_pipe_has_type(intel_crtc, INTEL_OUTPUT_HDMI))
+                if (intel_crtc_has_type(intel_crtc->config, INTEL_OUTPUT_HDMI))
                         val |= PIPECONF_8BPC;
                 else
                         val |= pipeconf_val & PIPECONF_BPC_MASK;
@@ -1859,7 +1827,7 @@ static void ironlake_enable_pch_transcoder(struct drm_i915_private *dev_priv,
         val &= ~TRANS_INTERLACE_MASK;
         if ((pipeconf_val & PIPECONF_INTERLACE_MASK) == PIPECONF_INTERLACED_ILK)
                 if (HAS_PCH_IBX(dev_priv) &&
-                    intel_pipe_has_type(intel_crtc, INTEL_OUTPUT_SDVO))
+                    intel_crtc_has_type(intel_crtc->config, INTEL_OUTPUT_SDVO))
                         val |= TRANS_LEGACY_INTERLACED_ILK;
                 else
                         val |= TRANS_INTERLACED;
@@ -1867,7 +1835,9 @@ static void ironlake_enable_pch_transcoder(struct drm_i915_private *dev_priv,
                 val |= TRANS_PROGRESSIVE;
 
         I915_WRITE(reg, val | TRANS_ENABLE);
-        if (wait_for(I915_READ(reg) & TRANS_STATE_ENABLE, 100))
+        if (intel_wait_for_register(dev_priv,
+                                    reg, TRANS_STATE_ENABLE, TRANS_STATE_ENABLE,
+                                    100))
                 DRM_ERROR("failed to enable transcoder %c\n", pipe_name(pipe));
 }
 
@@ -1895,14 +1865,18 @@ static void lpt_enable_pch_transcoder(struct drm_i915_private *dev_priv,
                 val |= TRANS_PROGRESSIVE;
 
         I915_WRITE(LPT_TRANSCONF, val);
-        if (wait_for(I915_READ(LPT_TRANSCONF) & TRANS_STATE_ENABLE, 100))
+        if (intel_wait_for_register(dev_priv,
+                                    LPT_TRANSCONF,
+                                    TRANS_STATE_ENABLE,
+                                    TRANS_STATE_ENABLE,
+                                    100))
                 DRM_ERROR("Failed to enable PCH transcoder\n");
 }
 
 static void ironlake_disable_pch_transcoder(struct drm_i915_private *dev_priv,
                                             enum pipe pipe)
 {
-        struct drm_device *dev = dev_priv->dev;
+        struct drm_device *dev = &dev_priv->drm;
         i915_reg_t reg;
         uint32_t val;
 
@@ -1918,7 +1892,9 @@ static void ironlake_disable_pch_transcoder(struct drm_i915_private *dev_priv,
         val &= ~TRANS_ENABLE;
         I915_WRITE(reg, val);
         /* wait for PCH transcoder off, transcoder state */
-        if (wait_for((I915_READ(reg) & TRANS_STATE_ENABLE) == 0, 50))
+        if (intel_wait_for_register(dev_priv,
+                                    reg, TRANS_STATE_ENABLE, 0,
+                                    50))
                 DRM_ERROR("failed to disable transcoder %c\n", pipe_name(pipe));
 
         if (HAS_PCH_CPT(dev)) {
@@ -1938,7 +1914,9 @@ static void lpt_disable_pch_transcoder(struct drm_i915_private *dev_priv)
         val &= ~TRANS_ENABLE;
         I915_WRITE(LPT_TRANSCONF, val);
         /* wait for PCH transcoder off, transcoder state */
-        if (wait_for((I915_READ(LPT_TRANSCONF) & TRANS_STATE_ENABLE) == 0, 50))
+        if (intel_wait_for_register(dev_priv,
+                                    LPT_TRANSCONF, TRANS_STATE_ENABLE, 0,
+                                    50))
                 DRM_ERROR("Failed to disable PCH transcoder\n");
 
         /* Workaround: clear timing override bit. */
@@ -1957,7 +1935,7 @@ static void lpt_disable_pch_transcoder(struct drm_i915_private *dev_priv)
 static void intel_enable_pipe(struct intel_crtc *crtc)
 {
         struct drm_device *dev = crtc->base.dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         enum pipe pipe = crtc->pipe;
         enum transcoder cpu_transcoder = crtc->config->cpu_transcoder;
         enum pipe pch_transcoder;
@@ -1981,7 +1959,7 @@ static void intel_enable_pipe(struct intel_crtc *crtc)
          * need the check.
          */
         if (HAS_GMCH_DISPLAY(dev_priv))
-                if (crtc->config->has_dsi_encoder)
+                if (intel_crtc_has_type(crtc->config, INTEL_OUTPUT_DSI))
                         assert_dsi_pll_enabled(dev_priv);
                 else
                         assert_pll_enabled(dev_priv, pipe);
@@ -2030,7 +2008,7 @@ static void intel_enable_pipe(struct intel_crtc *crtc)
  */
 static void intel_disable_pipe(struct intel_crtc *crtc)
 {
-        struct drm_i915_private *dev_priv = crtc->base.dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
         enum transcoder cpu_transcoder = crtc->config->cpu_transcoder;
         enum pipe pipe = crtc->pipe;
         i915_reg_t reg;
@@ -2068,15 +2046,6 @@ static void intel_disable_pipe(struct intel_crtc *crtc)
                 intel_wait_for_pipe_off(crtc);
 }
 
-static bool need_vtd_wa(struct drm_device *dev)
-{
-#ifdef CONFIG_INTEL_IOMMU
-        if (INTEL_INFO(dev)->gen >= 6 && intel_iommu_gfx_mapped)
-                return true;
-#endif
-        return false;
-}
-
 static unsigned int intel_tile_size(const struct drm_i915_private *dev_priv)
 {
         return IS_GEN2(dev_priv) ? 2048 : 4096;
@@ -2241,7 +2210,7 @@ intel_pin_and_fence_fb_obj(struct drm_framebuffer *fb,
                            unsigned int rotation)
 {
         struct drm_device *dev = fb->dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         struct drm_i915_gem_object *obj = intel_fb_obj(fb);
         struct i915_ggtt_view view;
         u32 alignment;
@@ -2258,7 +2227,7 @@ intel_pin_and_fence_fb_obj(struct drm_framebuffer *fb,
          * we should always have valid PTE following the scanout preventing
          * the VT-d warning.
          */
-        if (need_vtd_wa(dev) && alignment < 256 * 1024)
+        if (intel_scanout_needs_vtd_wa(dev_priv) && alignment < 256 * 1024)
                 alignment = 256 * 1024;
 
         /*
@@ -2309,7 +2278,7 @@ err_pm:
         return ret;
 }
 
-static void intel_unpin_fb_obj(struct drm_framebuffer *fb, unsigned int rotation)
+void intel_unpin_fb_obj(struct drm_framebuffer *fb, unsigned int rotation)
 {
         struct drm_i915_gem_object *obj = intel_fb_obj(fb);
         struct i915_ggtt_view view;
@@ -2543,7 +2512,7 @@ intel_find_initial_plane_obj(struct intel_crtc *intel_crtc,
                              struct intel_initial_plane_config *plane_config)
 {
         struct drm_device *dev = intel_crtc->base.dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         struct drm_crtc *c;
         struct intel_crtc *i;
         struct drm_i915_gem_object *obj;
@@ -2639,7 +2608,7 @@ static void i9xx_update_primary_plane(struct drm_plane *primary,
                                       const struct intel_plane_state *plane_state)
 {
         struct drm_device *dev = primary->dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         struct intel_crtc *intel_crtc = to_intel_crtc(crtc_state->base.crtc);
         struct drm_framebuffer *fb = plane_state->base.fb;
         struct drm_i915_gem_object *obj = intel_fb_obj(fb);
@@ -2752,7 +2721,7 @@ static void i9xx_disable_primary_plane(struct drm_plane *primary,
                                        struct drm_crtc *crtc)
 {
         struct drm_device *dev = crtc->dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
         int plane = intel_crtc->plane;
 
@@ -2769,7 +2738,7 @@ static void ironlake_update_primary_plane(struct drm_plane *primary,
                                           const struct intel_plane_state *plane_state)
 {
         struct drm_device *dev = primary->dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         struct intel_crtc *intel_crtc = to_intel_crtc(crtc_state->base.crtc);
         struct drm_framebuffer *fb = plane_state->base.fb;
         struct drm_i915_gem_object *obj = intel_fb_obj(fb);
@@ -2897,7 +2866,7 @@ u32 intel_plane_obj_offset(struct intel_plane *intel_plane,
 static void skl_detach_scaler(struct intel_crtc *intel_crtc, int id)
 {
         struct drm_device *dev = intel_crtc->base.dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
 
         I915_WRITE(SKL_PS_CTRL(intel_crtc->pipe, id), 0);
         I915_WRITE(SKL_PS_WIN_POS(intel_crtc->pipe, id), 0);
@@ -3007,7 +2976,7 @@ static void skylake_update_primary_plane(struct drm_plane *plane,
                                          const struct intel_plane_state *plane_state)
 {
         struct drm_device *dev = plane->dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         struct intel_crtc *intel_crtc = to_intel_crtc(crtc_state->base.crtc);
         struct drm_framebuffer *fb = plane_state->base.fb;
         struct drm_i915_gem_object *obj = intel_fb_obj(fb);
@@ -3091,7 +3060,7 @@ static void skylake_disable_primary_plane(struct drm_plane *primary,
                                           struct drm_crtc *crtc)
 {
         struct drm_device *dev = crtc->dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         int pipe = to_intel_crtc(crtc)->pipe;
 
         I915_WRITE(PLANE_CTL(pipe, 0), 0);
@@ -3110,17 +3079,12 @@ intel_pipe_set_base_atomic(struct drm_crtc *crtc, struct drm_framebuffer *fb,
         return -ENODEV;
 }
 
-static void intel_complete_page_flips(struct drm_device *dev)
+static void intel_complete_page_flips(struct drm_i915_private *dev_priv)
 {
-        struct drm_crtc *crtc;
-
-        for_each_crtc(dev, crtc) {
-                struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
-                enum plane plane = intel_crtc->plane;
+        struct intel_crtc *crtc;
 
-                intel_prepare_page_flip(dev, plane);
-                intel_finish_page_flip_plane(dev, plane);
-        }
+        for_each_intel_crtc(&dev_priv->drm, crtc)
+                intel_finish_page_flip_cs(dev_priv, crtc->pipe);
 }
 
 static void intel_update_primary_planes(struct drm_device *dev)
@@ -3143,41 +3107,39 @@ static void intel_update_primary_planes(struct drm_device *dev)
         }
 }
 
-void intel_prepare_reset(struct drm_device *dev)
+void intel_prepare_reset(struct drm_i915_private *dev_priv)
 {
         /* no reset support for gen2 */
-        if (IS_GEN2(dev))
+        if (IS_GEN2(dev_priv))
                 return;
 
         /* reset doesn't touch the display */
-        if (INTEL_INFO(dev)->gen >= 5 || IS_G4X(dev))
+        if (INTEL_GEN(dev_priv) >= 5 || IS_G4X(dev_priv))
                 return;
 
-        drm_modeset_lock_all(dev);
+        drm_modeset_lock_all(&dev_priv->drm);
         /*
          * Disabling the crtcs gracefully seems nicer. Also the
          * g33 docs say we should at least disable all the planes.
          */
-        intel_display_suspend(dev);
+        intel_display_suspend(&dev_priv->drm);
 }
 
-void intel_finish_reset(struct drm_device *dev)
+void intel_finish_reset(struct drm_i915_private *dev_priv)
 {
-        struct drm_i915_private *dev_priv = to_i915(dev);
-
         /*
          * Flips in the rings will be nuked by the reset,
          * so complete all pending flips so that user space
          * will get its events and not get stuck.
          */
-        intel_complete_page_flips(dev);
+        intel_complete_page_flips(dev_priv);
 
         /* no reset support for gen2 */
-        if (IS_GEN2(dev))
+        if (IS_GEN2(dev_priv))
                 return;
 
         /* reset doesn't touch the display */
-        if (INTEL_INFO(dev)->gen >= 5 || IS_G4X(dev)) {
+        if (INTEL_GEN(dev_priv) >= 5 || IS_G4X(dev_priv)) {
                 /*
                  * Flips in the rings have been nuked by the reset,
                  * so update the base address of all primary
@@ -3187,7 +3149,7 @@ void intel_finish_reset(struct drm_device *dev)
                  * FIXME: Atomic will make this obsolete since we won't schedule
                  * CS-based flips (which might get lost in gpu resets) any more.
                  */
-                intel_update_primary_planes(dev);
+                intel_update_primary_planes(&dev_priv->drm);
                 return;
         }
 
@@ -3198,18 +3160,18 @@ void intel_finish_reset(struct drm_device *dev)
         intel_runtime_pm_disable_interrupts(dev_priv);
         intel_runtime_pm_enable_interrupts(dev_priv);
 
-        intel_modeset_init_hw(dev);
+        intel_modeset_init_hw(&dev_priv->drm);
 
         spin_lock_irq(&dev_priv->irq_lock);
         if (dev_priv->display.hpd_irq_setup)
-                dev_priv->display.hpd_irq_setup(dev);
+                dev_priv->display.hpd_irq_setup(dev_priv);
         spin_unlock_irq(&dev_priv->irq_lock);
 
-        intel_display_resume(dev);
+        intel_display_resume(&dev_priv->drm);
 
         intel_hpd_init(dev_priv);
 
-        drm_modeset_unlock_all(dev);
+        drm_modeset_unlock_all(&dev_priv->drm);
 }
 
 static bool intel_crtc_has_pending_flip(struct drm_crtc *crtc)
@@ -3224,7 +3186,7 @@ static bool intel_crtc_has_pending_flip(struct drm_crtc *crtc)
                 return false;
 
         spin_lock_irq(&dev->event_lock);
-        pending = to_intel_crtc(crtc)->unpin_work != NULL;
+        pending = to_intel_crtc(crtc)->flip_work != NULL;
         spin_unlock_irq(&dev->event_lock);
 
         return pending;
@@ -3234,7 +3196,7 @@ static void intel_update_pipe_config(struct intel_crtc *crtc,
                                      struct intel_crtc_state *old_crtc_state)
 {
         struct drm_device *dev = crtc->base.dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         struct intel_crtc_state *pipe_config =
                 to_intel_crtc_state(crtc->base.state);
 
@@ -3275,7 +3237,7 @@ static void intel_update_pipe_config(struct intel_crtc *crtc,
 static void intel_fdi_normal_train(struct drm_crtc *crtc)
 {
         struct drm_device *dev = crtc->dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
         int pipe = intel_crtc->pipe;
         i915_reg_t reg;
@@ -3318,7 +3280,7 @@ static void intel_fdi_normal_train(struct drm_crtc *crtc)
 static void ironlake_fdi_link_train(struct drm_crtc *crtc)
 {
         struct drm_device *dev = crtc->dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
         int pipe = intel_crtc->pipe;
         i915_reg_t reg;
@@ -3419,7 +3381,7 @@ static const int snb_b_fdi_train_param[] = {
 static void gen6_fdi_link_train(struct drm_crtc *crtc)
 {
         struct drm_device *dev = crtc->dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
         int pipe = intel_crtc->pipe;
         i915_reg_t reg;
@@ -3552,7 +3514,7 @@ static void gen6_fdi_link_train(struct drm_crtc *crtc)
 static void ivb_manual_fdi_link_train(struct drm_crtc *crtc)
 {
         struct drm_device *dev = crtc->dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
         int pipe = intel_crtc->pipe;
         i915_reg_t reg;
@@ -3671,7 +3633,7 @@ train_done:
 static void ironlake_fdi_pll_enable(struct intel_crtc *intel_crtc)
 {
         struct drm_device *dev = intel_crtc->base.dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         int pipe = intel_crtc->pipe;
         i915_reg_t reg;
         u32 temp;
@@ -3708,7 +3670,7 @@ static void ironlake_fdi_pll_enable(struct intel_crtc *intel_crtc)
 static void ironlake_fdi_pll_disable(struct intel_crtc *intel_crtc)
 {
         struct drm_device *dev = intel_crtc->base.dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         int pipe = intel_crtc->pipe;
         i915_reg_t reg;
         u32 temp;
@@ -3738,7 +3700,7 @@ static void ironlake_fdi_pll_disable(struct intel_crtc *intel_crtc)
 static void ironlake_fdi_disable(struct drm_crtc *crtc)
 {
         struct drm_device *dev = crtc->dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
         int pipe = intel_crtc->pipe;
         i915_reg_t reg;
@@ -3803,7 +3765,7 @@ bool intel_has_pending_fb_unpin(struct drm_device *dev)
                 if (atomic_read(&crtc->unpin_work_count) == 0)
                         continue;
 
-                if (crtc->unpin_work)
+                if (crtc->flip_work)
                         intel_wait_for_vblank(dev, crtc->pipe);
 
                 return true;
@@ -3815,11 +3777,9 @@ bool intel_has_pending_fb_unpin(struct drm_device *dev)
 static void page_flip_completed(struct intel_crtc *intel_crtc)
 {
         struct drm_i915_private *dev_priv = to_i915(intel_crtc->base.dev);
-        struct intel_unpin_work *work = intel_crtc->unpin_work;
+        struct intel_flip_work *work = intel_crtc->flip_work;
 
-        /* ensure that the unpin work is consistent wrt ->pending. */
-        smp_rmb();
-        intel_crtc->unpin_work = NULL;
+        intel_crtc->flip_work = NULL;
 
         if (work->event)
                 drm_crtc_send_vblank_event(&intel_crtc->base, work->event);
@@ -3827,7 +3787,7 @@ static void page_flip_completed(struct intel_crtc *intel_crtc)
         drm_crtc_vblank_put(&intel_crtc->base);
 
         wake_up_all(&dev_priv->pending_flip_queue);
-        queue_work(dev_priv->wq, &work->work);
+        queue_work(dev_priv->wq, &work->unpin_work);
 
         trace_i915_flip_complete(intel_crtc->plane,
                                  work->pending_flip_obj);
@@ -3836,7 +3796,7 @@ static void page_flip_completed(struct intel_crtc *intel_crtc)
 static int intel_crtc_wait_for_pending_flips(struct drm_crtc *crtc)
 {
         struct drm_device *dev = crtc->dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         long ret;
 
         WARN_ON(waitqueue_active(&dev_priv->pending_flip_queue));
@@ -3851,9 +3811,11 @@ static int intel_crtc_wait_for_pending_flips(struct drm_crtc *crtc)
 
         if (ret == 0) {
                 struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
+                struct intel_flip_work *work;
 
                 spin_lock_irq(&dev->event_lock);
-                if (intel_crtc->unpin_work) {
+                work = intel_crtc->flip_work;
+                if (work && !is_mmio_work(work)) {
                         WARN_ONCE(1, "Removing stuck page flip\n");
                         page_flip_completed(intel_crtc);
                 }
@@ -3997,7 +3959,7 @@ static void ironlake_pch_transcoder_set_timings(struct intel_crtc *crtc,
                                                 enum pipe pch_transcoder)
 {
         struct drm_device *dev = crtc->base.dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         enum transcoder cpu_transcoder = crtc->config->cpu_transcoder;
 
         I915_WRITE(PCH_TRANS_HTOTAL(pch_transcoder),
@@ -4019,7 +3981,7 @@ static void ironlake_pch_transcoder_set_timings(struct intel_crtc *crtc,
 
 static void cpt_set_fdi_bc_bifurcation(struct drm_device *dev, bool enable)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         uint32_t temp;
 
         temp = I915_READ(SOUTH_CHICKEN1);
@@ -4069,7 +4031,7 @@ intel_trans_dp_port_sel(struct drm_crtc *crtc)
         struct intel_encoder *encoder;
 
         for_each_encoder_on_crtc(dev, crtc, encoder) {
-                if (encoder->type == INTEL_OUTPUT_DISPLAYPORT ||
+                if (encoder->type == INTEL_OUTPUT_DP ||
                     encoder->type == INTEL_OUTPUT_EDP)
                         return enc_to_dig_port(&encoder->base)->port;
         }
@@ -4088,7 +4050,7 @@ intel_trans_dp_port_sel(struct drm_crtc *crtc)
 static void ironlake_pch_enable(struct drm_crtc *crtc)
 {
         struct drm_device *dev = crtc->dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
         int pipe = intel_crtc->pipe;
         u32 temp;
@@ -4138,7 +4100,7 @@ static void ironlake_pch_enable(struct drm_crtc *crtc)
         intel_fdi_normal_train(crtc);
 
         /* For PCH DP, enable TRANS_DP_CTL */
-        if (HAS_PCH_CPT(dev) && intel_crtc->config->has_dp_encoder) {
+        if (HAS_PCH_CPT(dev) && intel_crtc_has_dp_encoder(intel_crtc->config)) {
                 const struct drm_display_mode *adjusted_mode =
                         &intel_crtc->config->base.adjusted_mode;
                 u32 bpc = (I915_READ(PIPECONF(pipe)) & PIPECONF_BPC_MASK) >> 5;
@@ -4178,7 +4140,7 @@ static void ironlake_pch_enable(struct drm_crtc *crtc)
 static void lpt_pch_enable(struct drm_crtc *crtc)
 {
         struct drm_device *dev = crtc->dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
         enum transcoder cpu_transcoder = intel_crtc->config->cpu_transcoder;
 
@@ -4194,7 +4156,7 @@ static void lpt_pch_enable(struct drm_crtc *crtc)
 
 static void cpt_verify_modeset(struct drm_device *dev, int pipe)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         i915_reg_t dslreg = PIPEDSL(pipe);
         u32 temp;
 
@@ -4281,8 +4243,9 @@ int skl_update_scaler_crtc(struct intel_crtc_state *state)
         struct intel_crtc *intel_crtc = to_intel_crtc(state->base.crtc);
         const struct drm_display_mode *adjusted_mode = &state->base.adjusted_mode;
 
-        DRM_DEBUG_KMS("Updating scaler for [CRTC:%i] scaler_user index %u.%u\n",
-                      intel_crtc->base.base.id, intel_crtc->pipe, SKL_CRTC_INDEX);
+        DRM_DEBUG_KMS("Updating scaler for [CRTC:%d:%s] scaler_user index %u.%u\n",
+                      intel_crtc->base.base.id, intel_crtc->base.name,
+                      intel_crtc->pipe, SKL_CRTC_INDEX);
 
         return skl_update_scaler(state, !state->base.active, SKL_CRTC_INDEX,
                 &state->scaler_state.scaler_id, BIT(DRM_ROTATE_0),
@@ -4312,9 +4275,9 @@ static int skl_update_scaler_plane(struct intel_crtc_state *crtc_state,
 
         bool force_detach = !fb || !plane_state->visible;
 
-        DRM_DEBUG_KMS("Updating scaler for [PLANE:%d] scaler_user index %u.%u\n",
-                      intel_plane->base.base.id, intel_crtc->pipe,
-                      drm_plane_index(&intel_plane->base));
+        DRM_DEBUG_KMS("Updating scaler for [PLANE:%d:%s] scaler_user index %u.%u\n",
+                      intel_plane->base.base.id, intel_plane->base.name,
+                      intel_crtc->pipe, drm_plane_index(&intel_plane->base));
 
         ret = skl_update_scaler(crtc_state, force_detach,
                                 drm_plane_index(&intel_plane->base),
@@ -4330,8 +4293,9 @@ static int skl_update_scaler_plane(struct intel_crtc_state *crtc_state,
 
         /* check colorkey */
         if (plane_state->ckey.flags != I915_SET_COLORKEY_NONE) {
-                DRM_DEBUG_KMS("[PLANE:%d] scaling with color key not allowed",
-                              intel_plane->base.base.id);
+                DRM_DEBUG_KMS("[PLANE:%d:%s] scaling with color key not allowed",
+                              intel_plane->base.base.id,
+                              intel_plane->base.name);
                 return -EINVAL;
         }
 
@@ -4350,8 +4314,9 @@ static int skl_update_scaler_plane(struct intel_crtc_state *crtc_state,
         case DRM_FORMAT_VYUY:
                 break;
         default:
-                DRM_DEBUG_KMS("[PLANE:%d] FB:%d unsupported scaling format 0x%x\n",
-                        intel_plane->base.base.id, fb->base.id, fb->pixel_format);
+                DRM_DEBUG_KMS("[PLANE:%d:%s] FB:%d unsupported scaling format 0x%x\n",
+                              intel_plane->base.base.id, intel_plane->base.name,
+                              fb->base.id, fb->pixel_format);
                 return -EINVAL;
         }
 
@@ -4369,7 +4334,7 @@ static void skylake_scaler_disable(struct intel_crtc *crtc)
 static void skylake_pfit_enable(struct intel_crtc *crtc)
 {
         struct drm_device *dev = crtc->base.dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         int pipe = crtc->pipe;
         struct intel_crtc_scaler_state *scaler_state =
                 &crtc->config->scaler_state;
@@ -4397,7 +4362,7 @@ static void skylake_pfit_enable(struct intel_crtc *crtc)
 static void ironlake_pfit_enable(struct intel_crtc *crtc)
 {
         struct drm_device *dev = crtc->base.dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         int pipe = crtc->pipe;
 
         if (crtc->config->pch_pfit.enabled) {
@@ -4418,7 +4383,7 @@ static void ironlake_pfit_enable(struct intel_crtc *crtc)
 void hsw_enable_ips(struct intel_crtc *crtc)
 {
         struct drm_device *dev = crtc->base.dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
 
         if (!crtc->config->ips_enabled)
                 return;
@@ -4446,7 +4411,9 @@ void hsw_enable_ips(struct intel_crtc *crtc)
                  * and don't wait for vblanks until the end of crtc_enable, then
                  * the HW state readout code will complain that the expected
                  * IPS_CTL value is not the one we read. */
-                if (wait_for(I915_READ_NOTRACE(IPS_CTL) & IPS_ENABLE, 50))
+                if (intel_wait_for_register(dev_priv,
+                                            IPS_CTL, IPS_ENABLE, IPS_ENABLE,
+                                            50))
                         DRM_ERROR("Timed out waiting for IPS enable\n");
         }
 }
@@ -4454,7 +4421,7 @@ void hsw_enable_ips(struct intel_crtc *crtc)
 void hsw_disable_ips(struct intel_crtc *crtc)
 {
         struct drm_device *dev = crtc->base.dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
 
         if (!crtc->config->ips_enabled)
                 return;
@@ -4465,7 +4432,9 @@ void hsw_disable_ips(struct intel_crtc *crtc)
                 WARN_ON(sandybridge_pcode_write(dev_priv, DISPLAY_IPS_CONTROL, 0));
                 mutex_unlock(&dev_priv->rps.hw_lock);
                 /* wait for pcode to finish disabling IPS, which may take up to 42ms */
-                if (wait_for((I915_READ(IPS_CTL) & IPS_ENABLE) == 0, 42))
+                if (intel_wait_for_register(dev_priv,
+                                            IPS_CTL, IPS_ENABLE, 0,
+                                            42))
                         DRM_ERROR("Timed out waiting for IPS disable\n");
         } else {
                 I915_WRITE(IPS_CTL, 0);
@@ -4480,7 +4449,7 @@ static void intel_crtc_dpms_overlay_disable(struct intel_crtc *intel_crtc)
 {
         if (intel_crtc->overlay) {
                 struct drm_device *dev = intel_crtc->base.dev;
-                struct drm_i915_private *dev_priv = dev->dev_private;
+                struct drm_i915_private *dev_priv = to_i915(dev);
 
                 mutex_lock(&dev->struct_mutex);
                 dev_priv->mm.interruptible = false;
@@ -4508,7 +4477,7 @@ static void
 intel_post_enable_primary(struct drm_crtc *crtc)
 {
         struct drm_device *dev = crtc->dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
         int pipe = intel_crtc->pipe;
 
@@ -4540,7 +4509,7 @@ static void
 intel_pre_disable_primary(struct drm_crtc *crtc)
 {
         struct drm_device *dev = crtc->dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
         int pipe = intel_crtc->pipe;
 
@@ -4567,7 +4536,7 @@ static void
 intel_pre_disable_primary_noatomic(struct drm_crtc *crtc)
 {
         struct drm_device *dev = crtc->dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
         int pipe = intel_crtc->pipe;
 
@@ -4626,7 +4595,7 @@ static void intel_pre_plane_update(struct intel_crtc_state *old_crtc_state)
 {
         struct intel_crtc *crtc = to_intel_crtc(old_crtc_state->base.crtc);
         struct drm_device *dev = crtc->base.dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         struct intel_crtc_state *pipe_config =
                 to_intel_crtc_state(crtc->base.state);
         struct drm_atomic_state *old_state = old_crtc_state->base.state;
@@ -4641,14 +4610,14 @@ static void intel_pre_plane_update(struct intel_crtc_state *old_crtc_state)
                 struct intel_plane_state *old_primary_state =
                         to_intel_plane_state(old_pri_state);
 
-                intel_fbc_pre_update(crtc);
+                intel_fbc_pre_update(crtc, pipe_config, primary_state);
 
                 if (old_primary_state->visible &&
                     (modeset || !primary_state->visible))
                         intel_pre_disable_primary(&crtc->base);
         }
 
-        if (pipe_config->disable_cxsr) {
+        if (pipe_config->disable_cxsr && HAS_GMCH_DISPLAY(dev)) {
                 crtc->wm.cxsr_allowed = false;
 
                 /*
@@ -4729,7 +4698,7 @@ static void intel_crtc_disable_planes(struct drm_crtc *crtc, unsigned plane_mask
 static void ironlake_crtc_enable(struct drm_crtc *crtc)
 {
         struct drm_device *dev = crtc->dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
         struct intel_encoder *encoder;
         int pipe = intel_crtc->pipe;
@@ -4757,7 +4726,7 @@ static void ironlake_crtc_enable(struct drm_crtc *crtc)
         if (intel_crtc->config->has_pch_encoder)
                 intel_prepare_shared_dpll(intel_crtc);
 
-        if (intel_crtc->config->has_dp_encoder)
+        if (intel_crtc_has_dp_encoder(intel_crtc->config))
                 intel_dp_set_m_n(intel_crtc, M1_N1);
 
         intel_set_pipe_timings(intel_crtc);
@@ -4826,7 +4795,7 @@ static bool hsw_crtc_supports_ips(struct intel_crtc *crtc)
 static void haswell_crtc_enable(struct drm_crtc *crtc)
 {
         struct drm_device *dev = crtc->dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
         struct intel_encoder *encoder;
         int pipe = intel_crtc->pipe, hsw_workaround_pipe;
@@ -4841,13 +4810,17 @@ static void haswell_crtc_enable(struct drm_crtc *crtc)
                 intel_set_pch_fifo_underrun_reporting(dev_priv, TRANSCODER_A,
                                                       false);
 
+        for_each_encoder_on_crtc(dev, crtc, encoder)
+                if (encoder->pre_pll_enable)
+                        encoder->pre_pll_enable(encoder);
+
         if (intel_crtc->config->shared_dpll)
                 intel_enable_shared_dpll(intel_crtc);
 
-        if (intel_crtc->config->has_dp_encoder)
+        if (intel_crtc_has_dp_encoder(intel_crtc->config))
                 intel_dp_set_m_n(intel_crtc, M1_N1);
 
-        if (!intel_crtc->config->has_dsi_encoder)
+        if (!transcoder_is_dsi(cpu_transcoder))
                 intel_set_pipe_timings(intel_crtc);
 
         intel_set_pipe_src_size(intel_crtc);
@@ -4863,7 +4836,7 @@ static void haswell_crtc_enable(struct drm_crtc *crtc)
                                      &intel_crtc->config->fdi_m_n, NULL);
         }
 
-        if (!intel_crtc->config->has_dsi_encoder)
+        if (!transcoder_is_dsi(cpu_transcoder))
                 haswell_set_pipeconf(crtc);
 
         haswell_set_pipemisc(crtc);
@@ -4885,7 +4858,7 @@ static void haswell_crtc_enable(struct drm_crtc *crtc)
         if (intel_crtc->config->has_pch_encoder)
                 dev_priv->display.fdi_link_train(crtc);
 
-        if (!intel_crtc->config->has_dsi_encoder)
+        if (!transcoder_is_dsi(cpu_transcoder))
                 intel_ddi_enable_pipe_clock(intel_crtc);
 
         if (INTEL_INFO(dev)->gen >= 9)
@@ -4900,7 +4873,7 @@ static void haswell_crtc_enable(struct drm_crtc *crtc)
         intel_color_load_luts(&pipe_config->base);
 
         intel_ddi_set_pipe_settings(crtc);
-        if (!intel_crtc->config->has_dsi_encoder)
+        if (!transcoder_is_dsi(cpu_transcoder))
                 intel_ddi_enable_transcoder_func(crtc);
 
         if (dev_priv->display.initial_watermarks != NULL)
@@ -4909,7 +4882,7 @@ static void haswell_crtc_enable(struct drm_crtc *crtc)
                 intel_update_watermarks(crtc);
 
         /* XXX: Do the pipe assertions at the right place for BXT DSI. */
-        if (!intel_crtc->config->has_dsi_encoder)
+        if (!transcoder_is_dsi(cpu_transcoder))
                 intel_enable_pipe(intel_crtc);
 
         if (intel_crtc->config->has_pch_encoder)
@@ -4946,7 +4919,7 @@ static void haswell_crtc_enable(struct drm_crtc *crtc)
 static void ironlake_pfit_disable(struct intel_crtc *crtc, bool force)
 {
         struct drm_device *dev = crtc->base.dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         int pipe = crtc->pipe;
 
         /* To avoid upsetting the power well on haswell only disable the pfit if
@@ -4961,7 +4934,7 @@ static void ironlake_pfit_disable(struct intel_crtc *crtc, bool force)
 static void ironlake_crtc_disable(struct drm_crtc *crtc)
 {
         struct drm_device *dev = crtc->dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
         struct intel_encoder *encoder;
         int pipe = intel_crtc->pipe;
@@ -5024,7 +4997,7 @@ static void ironlake_crtc_disable(struct drm_crtc *crtc)
 static void haswell_crtc_disable(struct drm_crtc *crtc)
 {
         struct drm_device *dev = crtc->dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
         struct intel_encoder *encoder;
         enum transcoder cpu_transcoder = intel_crtc->config->cpu_transcoder;
@@ -5042,13 +5015,13 @@ static void haswell_crtc_disable(struct drm_crtc *crtc)
         assert_vblank_disabled(crtc);
 
         /* XXX: Do the pipe assertions at the right place for BXT DSI. */
-        if (!intel_crtc->config->has_dsi_encoder)
+        if (!transcoder_is_dsi(cpu_transcoder))
                 intel_disable_pipe(intel_crtc);
 
         if (intel_crtc->config->dp_encoder_is_mst)
                 intel_ddi_set_vc_payload_alloc(crtc, false);
 
-        if (!intel_crtc->config->has_dsi_encoder)
+        if (!transcoder_is_dsi(cpu_transcoder))
                 intel_ddi_disable_transcoder_func(dev_priv, cpu_transcoder);
 
         if (INTEL_INFO(dev)->gen >= 9)
@@ -5056,7 +5029,7 @@ static void haswell_crtc_disable(struct drm_crtc *crtc)
         else
                 ironlake_pfit_disable(intel_crtc, false);
 
-        if (!intel_crtc->config->has_dsi_encoder)
+        if (!transcoder_is_dsi(cpu_transcoder))
                 intel_ddi_disable_pipe_clock(intel_crtc);
 
         for_each_encoder_on_crtc(dev, crtc, encoder)
@@ -5076,7 +5049,7 @@ static void haswell_crtc_disable(struct drm_crtc *crtc)
 static void i9xx_pfit_enable(struct intel_crtc *crtc)
 {
         struct drm_device *dev = crtc->base.dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         struct intel_crtc_state *pipe_config = crtc->config;
 
         if (!pipe_config->gmch_pfit.control)
@@ -5146,7 +5119,7 @@ intel_display_port_power_domain(struct intel_encoder *intel_encoder)
         case INTEL_OUTPUT_UNKNOWN:
                 /* Only DDI platforms should ever use this output type */
                 WARN_ON_ONCE(!HAS_DDI(dev));
-        case INTEL_OUTPUT_DISPLAYPORT:
+        case INTEL_OUTPUT_DP:
         case INTEL_OUTPUT_HDMI:
         case INTEL_OUTPUT_EDP:
                 intel_dig_port = enc_to_dig_port(&intel_encoder->base);
@@ -5180,7 +5153,7 @@ intel_display_port_aux_power_domain(struct intel_encoder *intel_encoder)
                  * run the DP detection too.
                  */
                 WARN_ON_ONCE(!HAS_DDI(dev));
-        case INTEL_OUTPUT_DISPLAYPORT:
+        case INTEL_OUTPUT_DP:
         case INTEL_OUTPUT_EDP:
                 intel_dig_port = enc_to_dig_port(&intel_encoder->base);
                 return port_to_aux_power_domain(intel_dig_port->port);
@@ -5228,7 +5201,7 @@ static unsigned long
 modeset_get_crtc_power_domains(struct drm_crtc *crtc,
                                struct intel_crtc_state *crtc_state)
 {
-        struct drm_i915_private *dev_priv = crtc->dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(crtc->dev);
         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
         enum intel_display_power_domain domain;
         unsigned long domains, new_domains, old_domains;
@@ -5269,21 +5242,34 @@ static int intel_compute_max_dotclk(struct drm_i915_private *dev_priv)
                 return max_cdclk_freq*90/100;
 }
 
+static int skl_calc_cdclk(int max_pixclk, int vco);
+
 static void intel_update_max_cdclk(struct drm_device *dev)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
 
         if (IS_SKYLAKE(dev) || IS_KABYLAKE(dev)) {
                 u32 limit = I915_READ(SKL_DFSM) & SKL_DFSM_CDCLK_LIMIT_MASK;
+                int max_cdclk, vco;
+
+                vco = dev_priv->skl_preferred_vco_freq;
+                WARN_ON(vco != 8100000 && vco != 8640000);
 
+                /*
+                 * Use the lower (vco 8640) cdclk values as a
+                 * first guess. skl_calc_cdclk() will correct it
+                 * if the preferred vco is 8100 instead.
+                 */
                 if (limit == SKL_DFSM_CDCLK_LIMIT_675)
-                        dev_priv->max_cdclk_freq = 675000;
+                        max_cdclk = 617143;
                 else if (limit == SKL_DFSM_CDCLK_LIMIT_540)
-                        dev_priv->max_cdclk_freq = 540000;
+                        max_cdclk = 540000;
                 else if (limit == SKL_DFSM_CDCLK_LIMIT_450)
-                        dev_priv->max_cdclk_freq = 450000;
+                        max_cdclk = 432000;
                 else
-                        dev_priv->max_cdclk_freq = 337500;
+                        max_cdclk = 308571;
+
+                dev_priv->max_cdclk_freq = skl_calc_cdclk(max_cdclk, vco);
         } else if (IS_BROXTON(dev)) {
                 dev_priv->max_cdclk_freq = 624000;
         } else if (IS_BROADWELL(dev))  {
@@ -5321,267 +5307,322 @@ static void intel_update_max_cdclk(struct drm_device *dev)
 
 static void intel_update_cdclk(struct drm_device *dev)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
 
         dev_priv->cdclk_freq = dev_priv->display.get_display_clock_speed(dev);
-        DRM_DEBUG_DRIVER("Current CD clock rate: %d kHz\n",
-                         dev_priv->cdclk_freq);
+
+        if (INTEL_GEN(dev_priv) >= 9)
+                DRM_DEBUG_DRIVER("Current CD clock rate: %d kHz, VCO: %d kHz, ref: %d kHz\n",
+                                 dev_priv->cdclk_freq, dev_priv->cdclk_pll.vco,
+                                 dev_priv->cdclk_pll.ref);
+        else
+                DRM_DEBUG_DRIVER("Current CD clock rate: %d kHz\n",
+                                 dev_priv->cdclk_freq);
 
         /*
-         * Program the gmbus_freq based on the cdclk frequency.
-         * BSpec erroneously claims we should aim for 4MHz, but
-         * in fact 1MHz is the correct frequency.
+         * 9:0 CMBUS [sic] CDCLK frequency (cdfreq):
+         * Programmng [sic] note: bit[9:2] should be programmed to the number
+         * of cdclk that generates 4MHz reference clock freq which is used to
+         * generate GMBus clock. This will vary with the cdclk freq.
          */
-        if (IS_VALLEYVIEW(dev) || IS_CHERRYVIEW(dev)) {
-                /*
-                 * Program the gmbus_freq based on the cdclk frequency.
-                 * BSpec erroneously claims we should aim for 4MHz, but
-                 * in fact 1MHz is the correct frequency.
-                 */
+        if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
                 I915_WRITE(GMBUSFREQ_VLV, DIV_ROUND_UP(dev_priv->cdclk_freq, 1000));
-        }
+}
 
-        if (dev_priv->max_cdclk_freq == 0)
-                intel_update_max_cdclk(dev);
+/* convert from kHz to .1 fixpoint MHz with -1MHz offset */
+static int skl_cdclk_decimal(int cdclk)
+{
+        return DIV_ROUND_CLOSEST(cdclk - 1000, 500);
 }
 
-static void broxton_set_cdclk(struct drm_i915_private *dev_priv, int frequency)
+static int bxt_de_pll_vco(struct drm_i915_private *dev_priv, int cdclk)
 {
-        uint32_t divider;
-        uint32_t ratio;
-        uint32_t current_freq;
-        int ret;
+        int ratio;
 
-        /* frequency = 19.2MHz * ratio / 2 / div{1,1.5,2,4} */
-        switch (frequency) {
+        if (cdclk == dev_priv->cdclk_pll.ref)
+                return 0;
+
+        switch (cdclk) {
+        default:
+                MISSING_CASE(cdclk);
         case 144000:
+        case 288000:
+        case 384000:
+        case 576000:
+                ratio = 60;
+                break;
+        case 624000:
+                ratio = 65;
+                break;
+        }
+
+        return dev_priv->cdclk_pll.ref * ratio;
+}
+
+static void bxt_de_pll_disable(struct drm_i915_private *dev_priv)
+{
+        I915_WRITE(BXT_DE_PLL_ENABLE, 0);
+
+        /* Timeout 200us */
+        if (intel_wait_for_register(dev_priv,
+                                    BXT_DE_PLL_ENABLE, BXT_DE_PLL_LOCK, 0,
+                                    1))
+                DRM_ERROR("timeout waiting for DE PLL unlock\n");
+
+        dev_priv->cdclk_pll.vco = 0;
+}
+
+static void bxt_de_pll_enable(struct drm_i915_private *dev_priv, int vco)
+{
+        int ratio = DIV_ROUND_CLOSEST(vco, dev_priv->cdclk_pll.ref);
+        u32 val;
+
+        val = I915_READ(BXT_DE_PLL_CTL);
+        val &= ~BXT_DE_PLL_RATIO_MASK;
+        val |= BXT_DE_PLL_RATIO(ratio);
+        I915_WRITE(BXT_DE_PLL_CTL, val);
+
+        I915_WRITE(BXT_DE_PLL_ENABLE, BXT_DE_PLL_PLL_ENABLE);
+
+        /* Timeout 200us */
+        if (intel_wait_for_register(dev_priv,
+                                    BXT_DE_PLL_ENABLE,
+                                    BXT_DE_PLL_LOCK,
+                                    BXT_DE_PLL_LOCK,
+                                    1))
+                DRM_ERROR("timeout waiting for DE PLL lock\n");
+
+        dev_priv->cdclk_pll.vco = vco;
+}
+
+static void bxt_set_cdclk(struct drm_i915_private *dev_priv, int cdclk)
+{
+        u32 val, divider;
+        int vco, ret;
+
+        vco = bxt_de_pll_vco(dev_priv, cdclk);
+
+        DRM_DEBUG_DRIVER("Changing CDCLK to %d kHz (VCO %d kHz)\n", cdclk, vco);
+
+        /* cdclk = vco / 2 / div{1,1.5,2,4} */
+        switch (DIV_ROUND_CLOSEST(vco, cdclk)) {
+        case 8:
                 divider = BXT_CDCLK_CD2X_DIV_SEL_4;
-                ratio = BXT_DE_PLL_RATIO(60);
                 break;
-        case 288000:
+        case 4:
                 divider = BXT_CDCLK_CD2X_DIV_SEL_2;
-                ratio = BXT_DE_PLL_RATIO(60);
                 break;
-        case 384000:
+        case 3:
                 divider = BXT_CDCLK_CD2X_DIV_SEL_1_5;
-                ratio = BXT_DE_PLL_RATIO(60);
-                break;
-        case 576000:
-                divider = BXT_CDCLK_CD2X_DIV_SEL_1;
-                ratio = BXT_DE_PLL_RATIO(60);
                 break;
-        case 624000:
+        case 2:
                 divider = BXT_CDCLK_CD2X_DIV_SEL_1;
-                ratio = BXT_DE_PLL_RATIO(65);
-                break;
-        case 19200:
-                /*
-                 * Bypass frequency with DE PLL disabled. Init ratio, divider
-                 * to suppress GCC warning.
-                 */
-                ratio = 0;
-                divider = 0;
                 break;
         default:
-                DRM_ERROR("unsupported CDCLK freq %d", frequency);
+                WARN_ON(cdclk != dev_priv->cdclk_pll.ref);
+                WARN_ON(vco != 0);
 
-                return;
+                divider = BXT_CDCLK_CD2X_DIV_SEL_1;
+                break;
         }
 
-        mutex_lock(&dev_priv->rps.hw_lock);
         /* Inform power controller of upcoming frequency change */
+        mutex_lock(&dev_priv->rps.hw_lock);
         ret = sandybridge_pcode_write(dev_priv, HSW_PCODE_DE_WRITE_FREQ_REQ,
                                       0x80000000);
         mutex_unlock(&dev_priv->rps.hw_lock);
 
         if (ret) {
                 DRM_ERROR("PCode CDCLK freq change notify failed (err %d, freq %d)\n",
-                          ret, frequency);
+                          ret, cdclk);
                 return;
         }
 
-        current_freq = I915_READ(CDCLK_CTL) & CDCLK_FREQ_DECIMAL_MASK;
-        /* convert from .1 fixpoint MHz with -1MHz offset to kHz */
-        current_freq = current_freq * 500 + 1000;
+        if (dev_priv->cdclk_pll.vco != 0 &&
+            dev_priv->cdclk_pll.vco != vco)
+                bxt_de_pll_disable(dev_priv);
 
-        /*
-         * DE PLL has to be disabled when
-         * - setting to 19.2MHz (bypass, PLL isn't used)
-         * - before setting to 624MHz (PLL needs toggling)
-         * - before setting to any frequency from 624MHz (PLL needs toggling)
-         */
-        if (frequency == 19200 || frequency == 624000 ||
-            current_freq == 624000) {
-                I915_WRITE(BXT_DE_PLL_ENABLE, ~BXT_DE_PLL_PLL_ENABLE);
-                /* Timeout 200us */
-                if (wait_for(!(I915_READ(BXT_DE_PLL_ENABLE) & BXT_DE_PLL_LOCK),
-                             1))
-                        DRM_ERROR("timout waiting for DE PLL unlock\n");
-        }
-
-        if (frequency != 19200) {
-                uint32_t val;
-
-                val = I915_READ(BXT_DE_PLL_CTL);
-                val &= ~BXT_DE_PLL_RATIO_MASK;
-                val |= ratio;
-                I915_WRITE(BXT_DE_PLL_CTL, val);
-
-                I915_WRITE(BXT_DE_PLL_ENABLE, BXT_DE_PLL_PLL_ENABLE);
-                /* Timeout 200us */
-                if (wait_for(I915_READ(BXT_DE_PLL_ENABLE) & BXT_DE_PLL_LOCK, 1))
-                        DRM_ERROR("timeout waiting for DE PLL lock\n");
-
-                val = I915_READ(CDCLK_CTL);
-                val &= ~BXT_CDCLK_CD2X_DIV_SEL_MASK;
-                val |= divider;
-                /*
-                 * Disable SSA Precharge when CD clock frequency < 500 MHz,
-                 * enable otherwise.
-                 */
-                val &= ~BXT_CDCLK_SSA_PRECHARGE_ENABLE;
-                if (frequency >= 500000)
-                        val |= BXT_CDCLK_SSA_PRECHARGE_ENABLE;
+        if (dev_priv->cdclk_pll.vco != vco)
+                bxt_de_pll_enable(dev_priv, vco);
 
-                val &= ~CDCLK_FREQ_DECIMAL_MASK;
-                /* convert from kHz to .1 fixpoint MHz with -1MHz offset */
-                val |= (frequency - 1000) / 500;
-                I915_WRITE(CDCLK_CTL, val);
-        }
+        val = divider | skl_cdclk_decimal(cdclk);
+        /*
+         * FIXME if only the cd2x divider needs changing, it could be done
+         * without shutting off the pipe (if only one pipe is active).
+         */
+        val |= BXT_CDCLK_CD2X_PIPE_NONE;
+        /*
+         * Disable SSA Precharge when CD clock frequency < 500 MHz,
+         * enable otherwise.
+         */
+        if (cdclk >= 500000)
+                val |= BXT_CDCLK_SSA_PRECHARGE_ENABLE;
+        I915_WRITE(CDCLK_CTL, val);
 
         mutex_lock(&dev_priv->rps.hw_lock);
         ret = sandybridge_pcode_write(dev_priv, HSW_PCODE_DE_WRITE_FREQ_REQ,
-                                      DIV_ROUND_UP(frequency, 25000));
+                                      DIV_ROUND_UP(cdclk, 25000));
         mutex_unlock(&dev_priv->rps.hw_lock);
 
         if (ret) {
                 DRM_ERROR("PCode CDCLK freq set failed, (err %d, freq %d)\n",
-                          ret, frequency);
+                          ret, cdclk);
                 return;
         }
 
-        intel_update_cdclk(dev_priv->dev);
+        intel_update_cdclk(&dev_priv->drm);
 }
 
-static bool broxton_cdclk_is_enabled(struct drm_i915_private *dev_priv)
+static void bxt_sanitize_cdclk(struct drm_i915_private *dev_priv)
 {
-        if (!(I915_READ(BXT_DE_PLL_ENABLE) & BXT_DE_PLL_PLL_ENABLE))
-                return false;
+        u32 cdctl, expected;
 
-        /* TODO: Check for a valid CDCLK rate */
+        intel_update_cdclk(&dev_priv->drm);
 
-        if (!(I915_READ(DBUF_CTL) & DBUF_POWER_REQUEST)) {
-                DRM_DEBUG_DRIVER("CDCLK enabled, but DBUF power not requested\n");
+        if (dev_priv->cdclk_pll.vco == 0 ||
+            dev_priv->cdclk_freq == dev_priv->cdclk_pll.ref)
+                goto sanitize;
 
-                return false;
-        }
+        /* DPLL okay; verify the cdclock
+         *
+         * Some BIOS versions leave an incorrect decimal frequency value and
+         * set reserved MBZ bits in CDCLK_CTL at least during exiting from S4,
+         * so sanitize this register.
+         */
+        cdctl = I915_READ(CDCLK_CTL);
+        /*
+         * Let's ignore the pipe field, since BIOS could have configured the
+         * dividers both synching to an active pipe, or asynchronously
+         * (PIPE_NONE).
+         */
+        cdctl &= ~BXT_CDCLK_CD2X_PIPE_NONE;
 
-        if (!(I915_READ(DBUF_CTL) & DBUF_POWER_STATE)) {
-                DRM_DEBUG_DRIVER("CDCLK enabled, but DBUF power hasn't settled\n");
+        expected = (cdctl & BXT_CDCLK_CD2X_DIV_SEL_MASK) |
+                   skl_cdclk_decimal(dev_priv->cdclk_freq);
+        /*
+         * Disable SSA Precharge when CD clock frequency < 500 MHz,
+         * enable otherwise.
+         */
+        if (dev_priv->cdclk_freq >= 500000)
+                expected |= BXT_CDCLK_SSA_PRECHARGE_ENABLE;
 
-                return false;
-        }
+        if (cdctl == expected)
+                /* All well; nothing to sanitize */
+                return;
 
-        return true;
-}
+sanitize:
+        DRM_DEBUG_KMS("Sanitizing cdclk programmed by pre-os\n");
 
-bool broxton_cdclk_verify_state(struct drm_i915_private *dev_priv)
-{
-        return broxton_cdclk_is_enabled(dev_priv);
+        /* force cdclk programming */
+        dev_priv->cdclk_freq = 0;
+
+        /* force full PLL disable + enable */
+        dev_priv->cdclk_pll.vco = -1;
 }
 
-void broxton_init_cdclk(struct drm_i915_private *dev_priv)
+void bxt_init_cdclk(struct drm_i915_private *dev_priv)
 {
-        /* check if cd clock is enabled */
-        if (broxton_cdclk_is_enabled(dev_priv)) {
-                DRM_DEBUG_KMS("CDCLK already enabled, won't reprogram it\n");
-                return;
-        }
+        bxt_sanitize_cdclk(dev_priv);
 
-        DRM_DEBUG_KMS("CDCLK not enabled, enabling it\n");
+        if (dev_priv->cdclk_freq != 0 && dev_priv->cdclk_pll.vco != 0)
+                return;
 
         /*
          * FIXME:
          * - The initial CDCLK needs to be read from VBT.
          *   Need to make this change after VBT has changes for BXT.
-         * - check if setting the max (or any) cdclk freq is really necessary
-         *   here, it belongs to modeset time
          */
-        broxton_set_cdclk(dev_priv, 624000);
-
-        I915_WRITE(DBUF_CTL, I915_READ(DBUF_CTL) | DBUF_POWER_REQUEST);
-        POSTING_READ(DBUF_CTL);
+        bxt_set_cdclk(dev_priv, bxt_calc_cdclk(0));
+}
 
-        udelay(10);
+void bxt_uninit_cdclk(struct drm_i915_private *dev_priv)
+{
+        bxt_set_cdclk(dev_priv, dev_priv->cdclk_pll.ref);
+}
 
-        if (!(I915_READ(DBUF_CTL) & DBUF_POWER_STATE))
-                DRM_ERROR("DBuf power enable timeout!\n");
+static int skl_calc_cdclk(int max_pixclk, int vco)
+{
+        if (vco == 8640000) {
+                if (max_pixclk > 540000)
+                        return 617143;
+                else if (max_pixclk > 432000)
+                        return 540000;
+                else if (max_pixclk > 308571)
+                        return 432000;
+                else
+                        return 308571;
+        } else {
+                if (max_pixclk > 540000)
+                        return 675000;
+                else if (max_pixclk > 450000)
+                        return 540000;
+                else if (max_pixclk > 337500)
+                        return 450000;
+                else
+                        return 337500;
+        }
 }
 
-void broxton_uninit_cdclk(struct drm_i915_private *dev_priv)
+static void
+skl_dpll0_update(struct drm_i915_private *dev_priv)
 {
-        I915_WRITE(DBUF_CTL, I915_READ(DBUF_CTL) & ~DBUF_POWER_REQUEST);
-        POSTING_READ(DBUF_CTL);
+        u32 val;
 
-        udelay(10);
+        dev_priv->cdclk_pll.ref = 24000;
+        dev_priv->cdclk_pll.vco = 0;
 
-        if (I915_READ(DBUF_CTL) & DBUF_POWER_STATE)
-                DRM_ERROR("DBuf power disable timeout!\n");
+        val = I915_READ(LCPLL1_CTL);
+        if ((val & LCPLL_PLL_ENABLE) == 0)
+                return;
 
-        /* Set minimum (bypass) frequency, in effect turning off the DE PLL */
-        broxton_set_cdclk(dev_priv, 19200);
-}
+        if (WARN_ON((val & LCPLL_PLL_LOCK) == 0))
+                return;
 
-static const struct skl_cdclk_entry {
-        unsigned int freq;
-        unsigned int vco;
-} skl_cdclk_frequencies[] = {
-        { .freq = 308570, .vco = 8640 },
-        { .freq = 337500, .vco = 8100 },
-        { .freq = 432000, .vco = 8640 },
-        { .freq = 450000, .vco = 8100 },
-        { .freq = 540000, .vco = 8100 },
-        { .freq = 617140, .vco = 8640 },
-        { .freq = 675000, .vco = 8100 },
-};
+        val = I915_READ(DPLL_CTRL1);
 
-static unsigned int skl_cdclk_decimal(unsigned int freq)
-{
-        return (freq - 1000) / 500;
+        if (WARN_ON((val & (DPLL_CTRL1_HDMI_MODE(SKL_DPLL0) |
+                            DPLL_CTRL1_SSC(SKL_DPLL0) |
+                            DPLL_CTRL1_OVERRIDE(SKL_DPLL0))) !=
+                    DPLL_CTRL1_OVERRIDE(SKL_DPLL0)))
+                return;
+
+        switch (val & DPLL_CTRL1_LINK_RATE_MASK(SKL_DPLL0)) {
+        case DPLL_CTRL1_LINK_RATE(DPLL_CTRL1_LINK_RATE_810, SKL_DPLL0):
+        case DPLL_CTRL1_LINK_RATE(DPLL_CTRL1_LINK_RATE_1350, SKL_DPLL0):
+        case DPLL_CTRL1_LINK_RATE(DPLL_CTRL1_LINK_RATE_1620, SKL_DPLL0):
+        case DPLL_CTRL1_LINK_RATE(DPLL_CTRL1_LINK_RATE_2700, SKL_DPLL0):
+                dev_priv->cdclk_pll.vco = 8100000;
+                break;
+        case DPLL_CTRL1_LINK_RATE(DPLL_CTRL1_LINK_RATE_1080, SKL_DPLL0):
+        case DPLL_CTRL1_LINK_RATE(DPLL_CTRL1_LINK_RATE_2160, SKL_DPLL0):
+                dev_priv->cdclk_pll.vco = 8640000;
+                break;
+        default:
+                MISSING_CASE(val & DPLL_CTRL1_LINK_RATE_MASK(SKL_DPLL0));
+                break;
+        }
 }
 
-static unsigned int skl_cdclk_get_vco(unsigned int freq)
+void skl_set_preferred_cdclk_vco(struct drm_i915_private *dev_priv, int vco)
 {
-        unsigned int i;
-
-        for (i = 0; i < ARRAY_SIZE(skl_cdclk_frequencies); i++) {
-                const struct skl_cdclk_entry *e = &skl_cdclk_frequencies[i];
+        bool changed = dev_priv->skl_preferred_vco_freq != vco;
 
-                if (e->freq == freq)
-                        return e->vco;
-        }
+        dev_priv->skl_preferred_vco_freq = vco;
 
-        return 8100;
+        if (changed)
+                intel_update_max_cdclk(&dev_priv->drm);
 }
 
 static void
-skl_dpll0_enable(struct drm_i915_private *dev_priv, unsigned int required_vco)
+skl_dpll0_enable(struct drm_i915_private *dev_priv, int vco)
 {
-        unsigned int min_freq;
+        int min_cdclk = skl_calc_cdclk(0, vco);
         u32 val;
 
-        /* select the minimum CDCLK before enabling DPLL 0 */
-        val = I915_READ(CDCLK_CTL);
-        val &= ~CDCLK_FREQ_SEL_MASK | ~CDCLK_FREQ_DECIMAL_MASK;
-        val |= CDCLK_FREQ_337_308;
-
-        if (required_vco == 8640)
-                min_freq = 308570;
-        else
-                min_freq = 337500;
-
-        val = CDCLK_FREQ_337_308 | skl_cdclk_decimal(min_freq);
+        WARN_ON(vco != 8100000 && vco != 8640000);
 
+        /* select the minimum CDCLK before enabling DPLL 0 */
+        val = CDCLK_FREQ_337_308 | skl_cdclk_decimal(min_cdclk);
         I915_WRITE(CDCLK_CTL, val);
         POSTING_READ(CDCLK_CTL);
 
@@ -5592,14 +5633,14 @@ skl_dpll0_enable(struct drm_i915_private *dev_priv, unsigned int required_vco)
          * 8100 while the eDP 1.4 alternate link rates need a VCO of 8640.
          * The modeset code is responsible for the selection of the exact link
          * rate later on, with the constraint of choosing a frequency that
-         * works with required_vco.
+         * works with vco.
          */
         val = I915_READ(DPLL_CTRL1);
 
         val &= ~(DPLL_CTRL1_HDMI_MODE(SKL_DPLL0) | DPLL_CTRL1_SSC(SKL_DPLL0) |
                  DPLL_CTRL1_LINK_RATE_MASK(SKL_DPLL0));
         val |= DPLL_CTRL1_OVERRIDE(SKL_DPLL0);
-        if (required_vco == 8640)
+        if (vco == 8640000)
                 val |= DPLL_CTRL1_LINK_RATE(DPLL_CTRL1_LINK_RATE_1080,
                                             SKL_DPLL0);
         else
@@ -5611,8 +5652,27 @@ skl_dpll0_enable(struct drm_i915_private *dev_priv, unsigned int required_vco)
 
         I915_WRITE(LCPLL1_CTL, I915_READ(LCPLL1_CTL) | LCPLL_PLL_ENABLE);
 
-        if (wait_for(I915_READ(LCPLL1_CTL) & LCPLL_PLL_LOCK, 5))
+        if (intel_wait_for_register(dev_priv,
+                                    LCPLL1_CTL, LCPLL_PLL_LOCK, LCPLL_PLL_LOCK,
+                                    5))
                 DRM_ERROR("DPLL0 not locked\n");
+
+        dev_priv->cdclk_pll.vco = vco;
+
+        /* We'll want to keep using the current vco from now on. */
+        skl_set_preferred_cdclk_vco(dev_priv, vco);
+}
+
+static void
+skl_dpll0_disable(struct drm_i915_private *dev_priv)
+{
+        I915_WRITE(LCPLL1_CTL, I915_READ(LCPLL1_CTL) & ~LCPLL_PLL_ENABLE);
+        if (intel_wait_for_register(dev_priv,
+                                   LCPLL1_CTL, LCPLL_PLL_LOCK, 0,
+                                   1))
+                DRM_ERROR("Couldn't disable DPLL0\n");
+
+        dev_priv->cdclk_pll.vco = 0;
 }
 
 static bool skl_cdclk_pcu_ready(struct drm_i915_private *dev_priv)
@@ -5642,12 +5702,14 @@ static bool skl_cdclk_wait_for_pcu_ready(struct drm_i915_private *dev_priv)
         return false;
 }
 
-static void skl_set_cdclk(struct drm_i915_private *dev_priv, unsigned int freq)
+static void skl_set_cdclk(struct drm_i915_private *dev_priv, int cdclk, int vco)
 {
-        struct drm_device *dev = dev_priv->dev;
+        struct drm_device *dev = &dev_priv->drm;
         u32 freq_select, pcu_ack;
 
-        DRM_DEBUG_DRIVER("Changing CDCLK to %dKHz\n", freq);
+        WARN_ON((cdclk == 24000) != (vco == 0));
+
+        DRM_DEBUG_DRIVER("Changing CDCLK to %d kHz (VCO %d kHz)\n", cdclk, vco);
 
         if (!skl_cdclk_wait_for_pcu_ready(dev_priv)) {
                 DRM_ERROR("failed to inform PCU about cdclk change\n");
@@ -5655,7 +5717,7 @@ static void skl_set_cdclk(struct drm_i915_private *dev_priv, unsigned int freq)
         }
 
         /* set CDCLK_CTL */
-        switch(freq) {
+        switch (cdclk) {
         case 450000:
         case 432000:
                 freq_select = CDCLK_FREQ_450_432;
@@ -5665,20 +5727,27 @@ static void skl_set_cdclk(struct drm_i915_private *dev_priv, unsigned int freq)
                 freq_select = CDCLK_FREQ_540;
                 pcu_ack = 2;
                 break;
-        case 308570:
+        case 308571:
         case 337500:
         default:
                 freq_select = CDCLK_FREQ_337_308;
                 pcu_ack = 0;
                 break;
-        case 617140:
+        case 617143:
         case 675000:
                 freq_select = CDCLK_FREQ_675_617;
                 pcu_ack = 3;
                 break;
         }
 
-        I915_WRITE(CDCLK_CTL, freq_select | skl_cdclk_decimal(freq));
+        if (dev_priv->cdclk_pll.vco != 0 &&
+            dev_priv->cdclk_pll.vco != vco)
+                skl_dpll0_disable(dev_priv);
+
+        if (dev_priv->cdclk_pll.vco != vco)
+                skl_dpll0_enable(dev_priv, vco);
+
+        I915_WRITE(CDCLK_CTL, freq_select | skl_cdclk_decimal(cdclk));
         POSTING_READ(CDCLK_CTL);
 
         /* inform PCU of the change */
@@ -5689,52 +5758,41 @@ static void skl_set_cdclk(struct drm_i915_private *dev_priv, unsigned int freq)
         intel_update_cdclk(dev);
 }
 
+static void skl_sanitize_cdclk(struct drm_i915_private *dev_priv);
+
 void skl_uninit_cdclk(struct drm_i915_private *dev_priv)
 {
-        /* disable DBUF power */
-        I915_WRITE(DBUF_CTL, I915_READ(DBUF_CTL) & ~DBUF_POWER_REQUEST);
-        POSTING_READ(DBUF_CTL);
-
-        udelay(10);
-
-        if (I915_READ(DBUF_CTL) & DBUF_POWER_STATE)
-                DRM_ERROR("DBuf power disable timeout\n");
-
-        /* disable DPLL0 */
-        I915_WRITE(LCPLL1_CTL, I915_READ(LCPLL1_CTL) & ~LCPLL_PLL_ENABLE);
-        if (wait_for(!(I915_READ(LCPLL1_CTL) & LCPLL_PLL_LOCK), 1))
-                DRM_ERROR("Couldn't disable DPLL0\n");
+        skl_set_cdclk(dev_priv, dev_priv->cdclk_pll.ref, 0);
 }
 
 void skl_init_cdclk(struct drm_i915_private *dev_priv)
 {
-        unsigned int required_vco;
+        int cdclk, vco;
 
-        /* DPLL0 not enabled (happens on early BIOS versions) */
-        if (!(I915_READ(LCPLL1_CTL) & LCPLL_PLL_ENABLE)) {
-                /* enable DPLL0 */
-                required_vco = skl_cdclk_get_vco(dev_priv->skl_boot_cdclk);
-                skl_dpll0_enable(dev_priv, required_vco);
-        }
+        skl_sanitize_cdclk(dev_priv);
 
-        /* set CDCLK to the frequency the BIOS chose */
-        skl_set_cdclk(dev_priv, dev_priv->skl_boot_cdclk);
-
-        /* enable DBUF power */
-        I915_WRITE(DBUF_CTL, I915_READ(DBUF_CTL) | DBUF_POWER_REQUEST);
-        POSTING_READ(DBUF_CTL);
+        if (dev_priv->cdclk_freq != 0 && dev_priv->cdclk_pll.vco != 0) {
+                /*
+                 * Use the current vco as our initial
+                 * guess as to what the preferred vco is.
+                 */
+                if (dev_priv->skl_preferred_vco_freq == 0)
+                        skl_set_preferred_cdclk_vco(dev_priv,
+                                                    dev_priv->cdclk_pll.vco);
+                return;
+        }
 
-        udelay(10);
+        vco = dev_priv->skl_preferred_vco_freq;
+        if (vco == 0)
+                vco = 8100000;
+        cdclk = skl_calc_cdclk(0, vco);
 
-        if (!(I915_READ(DBUF_CTL) & DBUF_POWER_STATE))
-                DRM_ERROR("DBuf power enable timeout\n");
+        skl_set_cdclk(dev_priv, cdclk, vco);
 }
 
-int skl_sanitize_cdclk(struct drm_i915_private *dev_priv)
+static void skl_sanitize_cdclk(struct drm_i915_private *dev_priv)
 {
-        uint32_t lcpll1 = I915_READ(LCPLL1_CTL);
-        uint32_t cdctl = I915_READ(CDCLK_CTL);
-        int freq = dev_priv->skl_boot_cdclk;
+        uint32_t cdctl, expected;
 
         /*
          * check if the pre-os intialized the display
@@ -5744,8 +5802,10 @@ int skl_sanitize_cdclk(struct drm_i915_private *dev_priv)
         if ((I915_READ(SWF_ILK(0x18)) & 0x00FFFFFF) == 0)
                 goto sanitize;
 
+        intel_update_cdclk(&dev_priv->drm);
         /* Is PLL enabled and locked ? */
-        if (!((lcpll1 & LCPLL_PLL_ENABLE) && (lcpll1 & LCPLL_PLL_LOCK)))
+        if (dev_priv->cdclk_pll.vco == 0 ||
+            dev_priv->cdclk_freq == dev_priv->cdclk_pll.ref)
                 goto sanitize;
 
         /* DPLL okay; verify the cdclock
@@ -5754,25 +5814,26 @@ int skl_sanitize_cdclk(struct drm_i915_private *dev_priv)
          * decimal part is programmed wrong from BIOS where pre-os does not
          * enable display. Verify the same as well.
          */
-        if (cdctl == ((cdctl & CDCLK_FREQ_SEL_MASK) | skl_cdclk_decimal(freq)))
+        cdctl = I915_READ(CDCLK_CTL);
+        expected = (cdctl & CDCLK_FREQ_SEL_MASK) |
+                skl_cdclk_decimal(dev_priv->cdclk_freq);
+        if (cdctl == expected)
                 /* All well; nothing to sanitize */
-                return false;
+                return;
+
 sanitize:
-        /*
-         * As of now initialize with max cdclk till
-         * we get dynamic cdclk support
-         * */
-        dev_priv->skl_boot_cdclk = dev_priv->max_cdclk_freq;
-        skl_init_cdclk(dev_priv);
+        DRM_DEBUG_KMS("Sanitizing cdclk programmed by pre-os\n");
 
-        /* we did have to sanitize */
-        return true;
+        /* force cdclk programming */
+        dev_priv->cdclk_freq = 0;
+        /* force full PLL disable + enable */
+        dev_priv->cdclk_pll.vco = -1;
 }
 
 /* Adjust CDclk dividers to allow high res or save power if possible */
 static void valleyview_set_cdclk(struct drm_device *dev, int cdclk)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         u32 val, cmd;
 
         WARN_ON(dev_priv->display.get_display_clock_speed(dev)
@@ -5837,7 +5898,7 @@ static void valleyview_set_cdclk(struct drm_device *dev, int cdclk)
 
 static void cherryview_set_cdclk(struct drm_device *dev, int cdclk)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         u32 val, cmd;
 
         WARN_ON(dev_priv->display.get_display_clock_speed(dev)
@@ -5906,21 +5967,15 @@ static int valleyview_calc_cdclk(struct drm_i915_private *dev_priv,
                 return 200000;
 }
 
-static int broxton_calc_cdclk(struct drm_i915_private *dev_priv,
-                              int max_pixclk)
+static int bxt_calc_cdclk(int max_pixclk)
 {
-        /*
-         * FIXME:
-         * - remove the guardband, it's not needed on BXT
-         * - set 19.2MHz bypass frequency if there are no active pipes
-         */
-        if (max_pixclk > 576000*9/10)
+        if (max_pixclk > 576000)
                 return 624000;
-        else if (max_pixclk > 384000*9/10)
+        else if (max_pixclk > 384000)
                 return 576000;
-        else if (max_pixclk > 288000*9/10)
+        else if (max_pixclk > 288000)
                 return 384000;
-        else if (max_pixclk > 144000*9/10)
+        else if (max_pixclk > 144000)
                 return 288000;
         else
                 return 144000;
@@ -5931,7 +5986,7 @@ static int intel_mode_max_pixclk(struct drm_device *dev,
                                  struct drm_atomic_state *state)
 {
         struct intel_atomic_state *intel_state = to_intel_atomic_state(state);
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         struct drm_crtc *crtc;
         struct drm_crtc_state *crtc_state;
         unsigned max_pixclk = 0, i;
@@ -5958,14 +6013,11 @@ static int intel_mode_max_pixclk(struct drm_device *dev,
 static int valleyview_modeset_calc_cdclk(struct drm_atomic_state *state)
 {
         struct drm_device *dev = state->dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         int max_pixclk = intel_mode_max_pixclk(dev, state);
         struct intel_atomic_state *intel_state =
                 to_intel_atomic_state(state);
 
-        if (max_pixclk < 0)
-                return max_pixclk;
-
         intel_state->cdclk = intel_state->dev_cdclk =
                 valleyview_calc_cdclk(dev_priv, max_pixclk);
 
@@ -5975,22 +6027,17 @@ static int valleyview_modeset_calc_cdclk(struct drm_atomic_state *state)
         return 0;
 }
 
-static int broxton_modeset_calc_cdclk(struct drm_atomic_state *state)
+static int bxt_modeset_calc_cdclk(struct drm_atomic_state *state)
 {
-        struct drm_device *dev = state->dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
-        int max_pixclk = intel_mode_max_pixclk(dev, state);
+        int max_pixclk = ilk_max_pixel_rate(state);
         struct intel_atomic_state *intel_state =
                 to_intel_atomic_state(state);
 
-        if (max_pixclk < 0)
-                return max_pixclk;
-
         intel_state->cdclk = intel_state->dev_cdclk =
-                broxton_calc_cdclk(dev_priv, max_pixclk);
+                bxt_calc_cdclk(max_pixclk);
 
         if (!intel_state->active_crtcs)
-                intel_state->dev_cdclk = broxton_calc_cdclk(dev_priv, 0);
+                intel_state->dev_cdclk = bxt_calc_cdclk(0);
 
         return 0;
 }
@@ -6034,7 +6081,7 @@ static void vlv_program_pfi_credits(struct drm_i915_private *dev_priv)
 static void valleyview_modeset_commit_cdclk(struct drm_atomic_state *old_state)
 {
         struct drm_device *dev = old_state->dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         struct intel_atomic_state *old_intel_state =
                 to_intel_atomic_state(old_state);
         unsigned req_cdclk = old_intel_state->dev_cdclk;
@@ -6073,14 +6120,14 @@ static void valleyview_crtc_enable(struct drm_crtc *crtc)
         if (WARN_ON(intel_crtc->active))
                 return;
 
-        if (intel_crtc->config->has_dp_encoder)
+        if (intel_crtc_has_dp_encoder(intel_crtc->config))
                 intel_dp_set_m_n(intel_crtc, M1_N1);
 
         intel_set_pipe_timings(intel_crtc);
         intel_set_pipe_src_size(intel_crtc);
 
         if (IS_CHERRYVIEW(dev) && pipe == PIPE_B) {
-                struct drm_i915_private *dev_priv = dev->dev_private;
+                struct drm_i915_private *dev_priv = to_i915(dev);
 
                 I915_WRITE(CHV_BLEND(pipe), CHV_BLEND_LEGACY);
                 I915_WRITE(CHV_CANVAS(pipe), 0);
@@ -6125,7 +6172,7 @@ static void valleyview_crtc_enable(struct drm_crtc *crtc)
 static void i9xx_set_pll_dividers(struct intel_crtc *crtc)
 {
         struct drm_device *dev = crtc->base.dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
 
         I915_WRITE(FP0(crtc->pipe), crtc->config->dpll_hw_state.fp0);
         I915_WRITE(FP1(crtc->pipe), crtc->config->dpll_hw_state.fp1);
@@ -6146,7 +6193,7 @@ static void i9xx_crtc_enable(struct drm_crtc *crtc)
 
         i9xx_set_pll_dividers(intel_crtc);
 
-        if (intel_crtc->config->has_dp_encoder)
+        if (intel_crtc_has_dp_encoder(intel_crtc->config))
                 intel_dp_set_m_n(intel_crtc, M1_N1);
 
         intel_set_pipe_timings(intel_crtc);
@@ -6182,7 +6229,7 @@ static void i9xx_crtc_enable(struct drm_crtc *crtc)
 static void i9xx_pfit_disable(struct intel_crtc *crtc)
 {
         struct drm_device *dev = crtc->base.dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
 
         if (!crtc->config->gmch_pfit.control)
                 return;
@@ -6197,7 +6244,7 @@ static void i9xx_pfit_disable(struct intel_crtc *crtc)
 static void i9xx_crtc_disable(struct drm_crtc *crtc)
 {
         struct drm_device *dev = crtc->dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
         struct intel_encoder *encoder;
         int pipe = intel_crtc->pipe;
@@ -6223,7 +6270,7 @@ static void i9xx_crtc_disable(struct drm_crtc *crtc)
                 if (encoder->post_disable)
                         encoder->post_disable(encoder);
 
-        if (!intel_crtc->config->has_dsi_encoder) {
+        if (!intel_crtc_has_type(intel_crtc->config, INTEL_OUTPUT_DSI)) {
                 if (IS_CHERRYVIEW(dev))
                         chv_disable_pll(dev_priv, pipe);
                 else if (IS_VALLEYVIEW(dev))
@@ -6252,7 +6299,7 @@ static void intel_crtc_disable_noatomic(struct drm_crtc *crtc)
                 return;
 
         if (to_intel_plane_state(crtc->primary->state)->visible) {
-                WARN_ON(intel_crtc->unpin_work);
+                WARN_ON(intel_crtc->flip_work);
 
                 intel_pre_disable_primary_noatomic(crtc);
 
@@ -6262,8 +6309,8 @@ static void intel_crtc_disable_noatomic(struct drm_crtc *crtc)
 
         dev_priv->display.crtc_disable(crtc);
 
-        DRM_DEBUG_KMS("[CRTC:%d] hw state adjusted, was enabled, now disabled\n",
-                      crtc->base.id);
+        DRM_DEBUG_KMS("[CRTC:%d:%s] hw state adjusted, was enabled, now disabled\n",
+                      crtc->base.id, crtc->name);
 
         WARN_ON(drm_atomic_set_mode_for_crtc(crtc->state, NULL) < 0);
         crtc->state->active = false;
@@ -6541,7 +6588,7 @@ static void hsw_compute_ips_config(struct intel_crtc *crtc,
                                    struct intel_crtc_state *pipe_config)
 {
         struct drm_device *dev = crtc->base.dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
 
         pipe_config->ips_enabled = i915.enable_ips &&
                 hsw_crtc_supports_ips(crtc) &&
@@ -6561,12 +6608,12 @@ static int intel_crtc_compute_config(struct intel_crtc *crtc,
                                      struct intel_crtc_state *pipe_config)
 {
         struct drm_device *dev = crtc->base.dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         const struct drm_display_mode *adjusted_mode = &pipe_config->base.adjusted_mode;
+        int clock_limit = dev_priv->max_dotclk_freq;
 
-        /* FIXME should check pixel clock limits on all platforms */
         if (INTEL_INFO(dev)->gen < 4) {
-                int clock_limit = dev_priv->max_cdclk_freq * 9 / 10;
+                clock_limit = dev_priv->max_cdclk_freq * 9 / 10;
 
                 /*
                  * Enable double wide mode when the dot clock
@@ -6574,16 +6621,16 @@ static int intel_crtc_compute_config(struct intel_crtc *crtc,
                  */
                 if (intel_crtc_supports_double_wide(crtc) &&
                     adjusted_mode->crtc_clock > clock_limit) {
-                        clock_limit *= 2;
+                        clock_limit = dev_priv->max_dotclk_freq;
                         pipe_config->double_wide = true;
                 }
+        }
 
-                if (adjusted_mode->crtc_clock > clock_limit) {
-                        DRM_DEBUG_KMS("requested pixel clock (%d kHz) too high (max: %d kHz, double wide: %s)\n",
-                                      adjusted_mode->crtc_clock, clock_limit,
-                                      yesno(pipe_config->double_wide));
-                        return -EINVAL;
-                }
+        if (adjusted_mode->crtc_clock > clock_limit) {
+                DRM_DEBUG_KMS("requested pixel clock (%d kHz) too high (max: %d kHz, double wide: %s)\n",
+                              adjusted_mode->crtc_clock, clock_limit,
+                              yesno(pipe_config->double_wide));
+                return -EINVAL;
         }
 
         /*
@@ -6592,7 +6639,7 @@ static int intel_crtc_compute_config(struct intel_crtc *crtc,
          * - LVDS dual channel mode
          * - Double wide pipe
          */
-        if ((intel_pipe_will_have_type(pipe_config, INTEL_OUTPUT_LVDS) &&
+        if ((intel_crtc_has_type(pipe_config, INTEL_OUTPUT_LVDS) &&
              intel_is_dual_link_lvds(dev)) || pipe_config->double_wide)
                 pipe_config->pipe_src_w &= ~1;
 
@@ -6615,81 +6662,103 @@ static int intel_crtc_compute_config(struct intel_crtc *crtc,
 static int skylake_get_display_clock_speed(struct drm_device *dev)
 {
         struct drm_i915_private *dev_priv = to_i915(dev);
-        uint32_t lcpll1 = I915_READ(LCPLL1_CTL);
-        uint32_t cdctl = I915_READ(CDCLK_CTL);
-        uint32_t linkrate;
+        uint32_t cdctl;
 
-        if (!(lcpll1 & LCPLL_PLL_ENABLE))
-                return 24000; /* 24MHz is the cd freq with NSSC ref */
+        skl_dpll0_update(dev_priv);
 
-        if ((cdctl & CDCLK_FREQ_SEL_MASK) == CDCLK_FREQ_540)
-                return 540000;
+        if (dev_priv->cdclk_pll.vco == 0)
+                return dev_priv->cdclk_pll.ref;
 
-        linkrate = (I915_READ(DPLL_CTRL1) &
-                    DPLL_CTRL1_LINK_RATE_MASK(SKL_DPLL0)) >> 1;
+        cdctl = I915_READ(CDCLK_CTL);
 
-        if (linkrate == DPLL_CTRL1_LINK_RATE_2160 ||
-            linkrate == DPLL_CTRL1_LINK_RATE_1080) {
-                /* vco 8640 */
+        if (dev_priv->cdclk_pll.vco == 8640000) {
                 switch (cdctl & CDCLK_FREQ_SEL_MASK) {
                 case CDCLK_FREQ_450_432:
                         return 432000;
                 case CDCLK_FREQ_337_308:
-                        return 308570;
+                        return 308571;
+                case CDCLK_FREQ_540:
+                        return 540000;
                 case CDCLK_FREQ_675_617:
-                        return 617140;
+                        return 617143;
                 default:
-                        WARN(1, "Unknown cd freq selection\n");
+                        MISSING_CASE(cdctl & CDCLK_FREQ_SEL_MASK);
                 }
         } else {
-                /* vco 8100 */
                 switch (cdctl & CDCLK_FREQ_SEL_MASK) {
                 case CDCLK_FREQ_450_432:
                         return 450000;
                 case CDCLK_FREQ_337_308:
                         return 337500;
+                case CDCLK_FREQ_540:
+                        return 540000;
                 case CDCLK_FREQ_675_617:
                         return 675000;
                 default:
-                        WARN(1, "Unknown cd freq selection\n");
+                        MISSING_CASE(cdctl & CDCLK_FREQ_SEL_MASK);
                 }
         }
 
-        /* error case, do as if DPLL0 isn't enabled */
-        return 24000;
+        return dev_priv->cdclk_pll.ref;
+}
+
+static void bxt_de_pll_update(struct drm_i915_private *dev_priv)
+{
+        u32 val;
+
+        dev_priv->cdclk_pll.ref = 19200;
+        dev_priv->cdclk_pll.vco = 0;
+
+        val = I915_READ(BXT_DE_PLL_ENABLE);
+        if ((val & BXT_DE_PLL_PLL_ENABLE) == 0)
+                return;
+
+        if (WARN_ON((val & BXT_DE_PLL_LOCK) == 0))
+                return;
+
+        val = I915_READ(BXT_DE_PLL_CTL);
+        dev_priv->cdclk_pll.vco = (val & BXT_DE_PLL_RATIO_MASK) *
+                dev_priv->cdclk_pll.ref;
 }
 
 static int broxton_get_display_clock_speed(struct drm_device *dev)
 {
         struct drm_i915_private *dev_priv = to_i915(dev);
-        uint32_t cdctl = I915_READ(CDCLK_CTL);
-        uint32_t pll_ratio = I915_READ(BXT_DE_PLL_CTL) & BXT_DE_PLL_RATIO_MASK;
-        uint32_t pll_enab = I915_READ(BXT_DE_PLL_ENABLE);
-        int cdclk;
+        u32 divider;
+        int div, vco;
+
+        bxt_de_pll_update(dev_priv);
 
-        if (!(pll_enab & BXT_DE_PLL_PLL_ENABLE))
-                return 19200;
+        vco = dev_priv->cdclk_pll.vco;
+        if (vco == 0)
+                return dev_priv->cdclk_pll.ref;
 
-        cdclk = 19200 * pll_ratio / 2;
+        divider = I915_READ(CDCLK_CTL) & BXT_CDCLK_CD2X_DIV_SEL_MASK;
 
-        switch (cdctl & BXT_CDCLK_CD2X_DIV_SEL_MASK) {
+        switch (divider) {
         case BXT_CDCLK_CD2X_DIV_SEL_1:
-                return cdclk;  /* 576MHz or 624MHz */
+                div = 2;
+                break;
         case BXT_CDCLK_CD2X_DIV_SEL_1_5:
-                return cdclk * 2 / 3; /* 384MHz */
+                div = 3;
+                break;
         case BXT_CDCLK_CD2X_DIV_SEL_2:
-                return cdclk / 2; /* 288MHz */
+                div = 4;
+                break;
         case BXT_CDCLK_CD2X_DIV_SEL_4:
-                return cdclk / 4; /* 144MHz */
+                div = 8;
+                break;
+        default:
+                MISSING_CASE(divider);
+                return dev_priv->cdclk_pll.ref;
         }
 
-        /* error case, do as if DE PLL isn't enabled */
-        return 19200;
+        return DIV_ROUND_CLOSEST(vco, div);
 }
 
 static int broadwell_get_display_clock_speed(struct drm_device *dev)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         uint32_t lcpll = I915_READ(LCPLL_CTL);
         uint32_t freq = lcpll & LCPLL_CLK_FREQ_MASK;
 
@@ -6709,7 +6778,7 @@ static int broadwell_get_display_clock_speed(struct drm_device *dev)
 
 static int haswell_get_display_clock_speed(struct drm_device *dev)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         uint32_t lcpll = I915_READ(LCPLL_CTL);
         uint32_t freq = lcpll & LCPLL_CLK_FREQ_MASK;
 
@@ -6843,7 +6912,7 @@ static int i830_get_display_clock_speed(struct drm_device *dev)
 
 static unsigned int intel_hpll_vco(struct drm_device *dev)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         static const unsigned int blb_vco[8] = {
                 [0] = 3200000,
                 [1] = 4000000,
@@ -7063,7 +7132,7 @@ static uint32_t i9xx_dpll_compute_fp(struct dpll *dpll)
 
 static void i9xx_update_pll_dividers(struct intel_crtc *crtc,
                                      struct intel_crtc_state *crtc_state,
-                                     intel_clock_t *reduced_clock)
+                                     struct dpll *reduced_clock)
 {
         struct drm_device *dev = crtc->base.dev;
         u32 fp, fp2 = 0;
@@ -7081,7 +7150,7 @@ static void i9xx_update_pll_dividers(struct intel_crtc *crtc,
         crtc_state->dpll_hw_state.fp0 = fp;
 
         crtc->lowfreq_avail = false;
-        if (intel_pipe_will_have_type(crtc_state, INTEL_OUTPUT_LVDS) &&
+        if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_LVDS) &&
             reduced_clock) {
                 crtc_state->dpll_hw_state.fp1 = fp2;
                 crtc->lowfreq_avail = true;
@@ -7123,7 +7192,7 @@ static void intel_pch_transcoder_set_m_n(struct intel_crtc *crtc,
                                          struct intel_link_m_n *m_n)
 {
         struct drm_device *dev = crtc->base.dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         int pipe = crtc->pipe;
 
         I915_WRITE(PCH_TRANS_DATA_M1(pipe), TU_SIZE(m_n->tu) | m_n->gmch_m);
@@ -7137,7 +7206,7 @@ static void intel_cpu_transcoder_set_m_n(struct intel_crtc *crtc,
                                          struct intel_link_m_n *m2_n2)
 {
         struct drm_device *dev = crtc->base.dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         int pipe = crtc->pipe;
         enum transcoder transcoder = crtc->config->cpu_transcoder;
 
@@ -7200,7 +7269,7 @@ static void vlv_compute_dpll(struct intel_crtc *crtc,
                 pipe_config->dpll_hw_state.dpll |= DPLL_INTEGRATED_CRI_CLK_VLV;
 
         /* DPLL not used with DSI, but still need the rest set up */
-        if (!pipe_config->has_dsi_encoder)
+        if (!intel_crtc_has_type(pipe_config, INTEL_OUTPUT_DSI))
                 pipe_config->dpll_hw_state.dpll |= DPLL_VCO_ENABLE |
                         DPLL_EXT_BUFFER_ENABLE_VLV;
 
@@ -7217,7 +7286,7 @@ static void chv_compute_dpll(struct intel_crtc *crtc,
                 pipe_config->dpll_hw_state.dpll |= DPLL_INTEGRATED_CRI_CLK_VLV;
 
         /* DPLL not used with DSI, but still need the rest set up */
-        if (!pipe_config->has_dsi_encoder)
+        if (!intel_crtc_has_type(pipe_config, INTEL_OUTPUT_DSI))
                 pipe_config->dpll_hw_state.dpll |= DPLL_VCO_ENABLE;
 
         pipe_config->dpll_hw_state.dpll_md =
@@ -7228,7 +7297,7 @@ static void vlv_prepare_pll(struct intel_crtc *crtc,
                             const struct intel_crtc_state *pipe_config)
 {
         struct drm_device *dev = crtc->base.dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         enum pipe pipe = crtc->pipe;
         u32 mdiv;
         u32 bestn, bestm1, bestm2, bestp1, bestp2;
@@ -7287,15 +7356,15 @@ static void vlv_prepare_pll(struct intel_crtc *crtc,
 
         /* Set HBR and RBR LPF coefficients */
         if (pipe_config->port_clock == 162000 ||
-            intel_pipe_has_type(crtc, INTEL_OUTPUT_ANALOG) ||
-            intel_pipe_has_type(crtc, INTEL_OUTPUT_HDMI))
+            intel_crtc_has_type(crtc->config, INTEL_OUTPUT_ANALOG) ||
+            intel_crtc_has_type(crtc->config, INTEL_OUTPUT_HDMI))
                 vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW10(pipe),
                                  0x009f0003);
         else
                 vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW10(pipe),
                                  0x00d0000f);
 
-        if (pipe_config->has_dp_encoder) {
+        if (intel_crtc_has_dp_encoder(pipe_config)) {
                 /* Use SSC source */
                 if (pipe == PIPE_A)
                         vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW5(pipe),
@@ -7315,8 +7384,7 @@ static void vlv_prepare_pll(struct intel_crtc *crtc,
 
         coreclk = vlv_dpio_read(dev_priv, pipe, VLV_PLL_DW7(pipe));
         coreclk = (coreclk & 0x0000ff00) | 0x01c00000;
-        if (intel_pipe_has_type(crtc, INTEL_OUTPUT_DISPLAYPORT) ||
-            intel_pipe_has_type(crtc, INTEL_OUTPUT_EDP))
+        if (intel_crtc_has_dp_encoder(crtc->config))
                 coreclk |= 0x01000000;
         vlv_dpio_write(dev_priv, pipe, VLV_PLL_DW7(pipe), coreclk);
 
@@ -7328,7 +7396,7 @@ static void chv_prepare_pll(struct intel_crtc *crtc,
                             const struct intel_crtc_state *pipe_config)
 {
         struct drm_device *dev = crtc->base.dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         enum pipe pipe = crtc->pipe;
         enum dpio_channel port = vlv_pipe_to_channel(pipe);
         u32 loopfilter, tribuf_calcntr;
@@ -7487,22 +7555,18 @@ void vlv_force_pll_off(struct drm_device *dev, enum pipe pipe)
 
 static void i9xx_compute_dpll(struct intel_crtc *crtc,
                               struct intel_crtc_state *crtc_state,
-                              intel_clock_t *reduced_clock)
+                              struct dpll *reduced_clock)
 {
         struct drm_device *dev = crtc->base.dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         u32 dpll;
-        bool is_sdvo;
         struct dpll *clock = &crtc_state->dpll;
 
         i9xx_update_pll_dividers(crtc, crtc_state, reduced_clock);
 
-        is_sdvo = intel_pipe_will_have_type(crtc_state, INTEL_OUTPUT_SDVO) ||
-                intel_pipe_will_have_type(crtc_state, INTEL_OUTPUT_HDMI);
-
         dpll = DPLL_VGA_MODE_DIS;
 
-        if (intel_pipe_will_have_type(crtc_state, INTEL_OUTPUT_LVDS))
+        if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_LVDS))
                 dpll |= DPLLB_MODE_LVDS;
         else
                 dpll |= DPLLB_MODE_DAC_SERIAL;
@@ -7512,10 +7576,11 @@ static void i9xx_compute_dpll(struct intel_crtc *crtc,
                         << SDVO_MULTIPLIER_SHIFT_HIRES;
         }
 
-        if (is_sdvo)
+        if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_SDVO) ||
+            intel_crtc_has_type(crtc_state, INTEL_OUTPUT_HDMI))
                 dpll |= DPLL_SDVO_HIGH_SPEED;
 
-        if (crtc_state->has_dp_encoder)
+        if (intel_crtc_has_dp_encoder(crtc_state))
                 dpll |= DPLL_SDVO_HIGH_SPEED;
 
         /* compute bitmask from p1 value */
@@ -7545,7 +7610,7 @@ static void i9xx_compute_dpll(struct intel_crtc *crtc,
 
         if (crtc_state->sdvo_tv_clock)
                 dpll |= PLL_REF_INPUT_TVCLKINBC;
-        else if (intel_pipe_will_have_type(crtc_state, INTEL_OUTPUT_LVDS) &&
+        else if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_LVDS) &&
                  intel_panel_use_ssc(dev_priv))
                 dpll |= PLLB_REF_INPUT_SPREADSPECTRUMIN;
         else
@@ -7563,10 +7628,10 @@ static void i9xx_compute_dpll(struct intel_crtc *crtc,
 
 static void i8xx_compute_dpll(struct intel_crtc *crtc,
                               struct intel_crtc_state *crtc_state,
-                              intel_clock_t *reduced_clock)
+                              struct dpll *reduced_clock)
 {
         struct drm_device *dev = crtc->base.dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         u32 dpll;
         struct dpll *clock = &crtc_state->dpll;
 
@@ -7574,7 +7639,7 @@ static void i8xx_compute_dpll(struct intel_crtc *crtc,
 
         dpll = DPLL_VGA_MODE_DIS;
 
-        if (intel_pipe_will_have_type(crtc_state, INTEL_OUTPUT_LVDS)) {
+        if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_LVDS)) {
                 dpll |= (1 << (clock->p1 - 1)) << DPLL_FPA01_P1_POST_DIV_SHIFT;
         } else {
                 if (clock->p1 == 2)
@@ -7585,10 +7650,10 @@ static void i8xx_compute_dpll(struct intel_crtc *crtc,
                         dpll |= PLL_P2_DIVIDE_BY_4;
         }
 
-        if (!IS_I830(dev) && intel_pipe_will_have_type(crtc_state, INTEL_OUTPUT_DVO))
+        if (!IS_I830(dev) && intel_crtc_has_type(crtc_state, INTEL_OUTPUT_DVO))
                 dpll |= DPLL_DVO_2X_MODE;
 
-        if (intel_pipe_will_have_type(crtc_state, INTEL_OUTPUT_LVDS) &&
+        if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_LVDS) &&
             intel_panel_use_ssc(dev_priv))
                 dpll |= PLLB_REF_INPUT_SPREADSPECTRUMIN;
         else
@@ -7601,7 +7666,7 @@ static void i8xx_compute_dpll(struct intel_crtc *crtc,
 static void intel_set_pipe_timings(struct intel_crtc *intel_crtc)
 {
         struct drm_device *dev = intel_crtc->base.dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         enum pipe pipe = intel_crtc->pipe;
         enum transcoder cpu_transcoder = intel_crtc->config->cpu_transcoder;
         const struct drm_display_mode *adjusted_mode = &intel_crtc->config->base.adjusted_mode;
@@ -7618,7 +7683,7 @@ static void intel_set_pipe_timings(struct intel_crtc *intel_crtc)
                 crtc_vtotal -= 1;
                 crtc_vblank_end -= 1;
 
-                if (intel_pipe_has_type(intel_crtc, INTEL_OUTPUT_SDVO))
+                if (intel_crtc_has_type(intel_crtc->config, INTEL_OUTPUT_SDVO))
                         vsyncshift = (adjusted_mode->crtc_htotal - 1) / 2;
                 else
                         vsyncshift = adjusted_mode->crtc_hsync_start -
@@ -7663,7 +7728,7 @@ static void intel_set_pipe_timings(struct intel_crtc *intel_crtc)
 static void intel_set_pipe_src_size(struct intel_crtc *intel_crtc)
 {
         struct drm_device *dev = intel_crtc->base.dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         enum pipe pipe = intel_crtc->pipe;
 
         /* pipesrc controls the size that is scaled from, which should
@@ -7678,7 +7743,7 @@ static void intel_get_pipe_timings(struct intel_crtc *crtc,
                                    struct intel_crtc_state *pipe_config)
 {
         struct drm_device *dev = crtc->base.dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         enum transcoder cpu_transcoder = pipe_config->cpu_transcoder;
         uint32_t tmp;
 
@@ -7713,7 +7778,7 @@ static void intel_get_pipe_src_size(struct intel_crtc *crtc,
                                     struct intel_crtc_state *pipe_config)
 {
         struct drm_device *dev = crtc->base.dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         u32 tmp;
 
         tmp = I915_READ(PIPESRC(crtc->pipe));
@@ -7751,7 +7816,7 @@ void intel_mode_from_pipe_config(struct drm_display_mode *mode,
 static void i9xx_set_pipeconf(struct intel_crtc *intel_crtc)
 {
         struct drm_device *dev = intel_crtc->base.dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         uint32_t pipeconf;
 
         pipeconf = 0;
@@ -7797,7 +7862,7 @@ static void i9xx_set_pipeconf(struct intel_crtc *intel_crtc)
 
         if (intel_crtc->config->base.adjusted_mode.flags & DRM_MODE_FLAG_INTERLACE) {
                 if (INTEL_INFO(dev)->gen < 4 ||
-                    intel_pipe_has_type(intel_crtc, INTEL_OUTPUT_SDVO))
+                    intel_crtc_has_type(intel_crtc->config, INTEL_OUTPUT_SDVO))
                         pipeconf |= PIPECONF_INTERLACE_W_FIELD_INDICATION;
                 else
                         pipeconf |= PIPECONF_INTERLACE_W_SYNC_SHIFT;
@@ -7816,21 +7881,21 @@ static int i8xx_crtc_compute_clock(struct intel_crtc *crtc,
                                    struct intel_crtc_state *crtc_state)
 {
         struct drm_device *dev = crtc->base.dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
-        const intel_limit_t *limit;
+        struct drm_i915_private *dev_priv = to_i915(dev);
+        const struct intel_limit *limit;
         int refclk = 48000;
 
         memset(&crtc_state->dpll_hw_state, 0,
                sizeof(crtc_state->dpll_hw_state));
 
-        if (intel_pipe_will_have_type(crtc_state, INTEL_OUTPUT_LVDS)) {
+        if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_LVDS)) {
                 if (intel_panel_use_ssc(dev_priv)) {
                         refclk = dev_priv->vbt.lvds_ssc_freq;
                         DRM_DEBUG_KMS("using SSC reference clock of %d kHz\n", refclk);
                 }
 
                 limit = &intel_limits_i8xx_lvds;
-        } else if (intel_pipe_will_have_type(crtc_state, INTEL_OUTPUT_DVO)) {
+        } else if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_DVO)) {
                 limit = &intel_limits_i8xx_dvo;
         } else {
                 limit = &intel_limits_i8xx_dac;
@@ -7852,14 +7917,14 @@ static int g4x_crtc_compute_clock(struct intel_crtc *crtc,
                                   struct intel_crtc_state *crtc_state)
 {
         struct drm_device *dev = crtc->base.dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
-        const intel_limit_t *limit;
+        struct drm_i915_private *dev_priv = to_i915(dev);
+        const struct intel_limit *limit;
         int refclk = 96000;
 
         memset(&crtc_state->dpll_hw_state, 0,
                sizeof(crtc_state->dpll_hw_state));
 
-        if (intel_pipe_will_have_type(crtc_state, INTEL_OUTPUT_LVDS)) {
+        if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_LVDS)) {
                 if (intel_panel_use_ssc(dev_priv)) {
                         refclk = dev_priv->vbt.lvds_ssc_freq;
                         DRM_DEBUG_KMS("using SSC reference clock of %d kHz\n", refclk);
@@ -7869,10 +7934,10 @@ static int g4x_crtc_compute_clock(struct intel_crtc *crtc,
                         limit = &intel_limits_g4x_dual_channel_lvds;
                 else
                         limit = &intel_limits_g4x_single_channel_lvds;
-        } else if (intel_pipe_will_have_type(crtc_state, INTEL_OUTPUT_HDMI) ||
-                   intel_pipe_will_have_type(crtc_state, INTEL_OUTPUT_ANALOG)) {
+        } else if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_HDMI) ||
+                   intel_crtc_has_type(crtc_state, INTEL_OUTPUT_ANALOG)) {
                 limit = &intel_limits_g4x_hdmi;
-        } else if (intel_pipe_will_have_type(crtc_state, INTEL_OUTPUT_SDVO)) {
+        } else if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_SDVO)) {
                 limit = &intel_limits_g4x_sdvo;
         } else {
                 /* The option is for other outputs */
@@ -7895,14 +7960,14 @@ static int pnv_crtc_compute_clock(struct intel_crtc *crtc,
                                   struct intel_crtc_state *crtc_state)
 {
         struct drm_device *dev = crtc->base.dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
-        const intel_limit_t *limit;
+        struct drm_i915_private *dev_priv = to_i915(dev);
+        const struct intel_limit *limit;
         int refclk = 96000;
 
         memset(&crtc_state->dpll_hw_state, 0,
                sizeof(crtc_state->dpll_hw_state));
 
-        if (intel_pipe_will_have_type(crtc_state, INTEL_OUTPUT_LVDS)) {
+        if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_LVDS)) {
                 if (intel_panel_use_ssc(dev_priv)) {
                         refclk = dev_priv->vbt.lvds_ssc_freq;
                         DRM_DEBUG_KMS("using SSC reference clock of %d kHz\n", refclk);
@@ -7929,14 +7994,14 @@ static int i9xx_crtc_compute_clock(struct intel_crtc *crtc,
                                    struct intel_crtc_state *crtc_state)
 {
         struct drm_device *dev = crtc->base.dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
-        const intel_limit_t *limit;
+        struct drm_i915_private *dev_priv = to_i915(dev);
+        const struct intel_limit *limit;
         int refclk = 96000;
 
         memset(&crtc_state->dpll_hw_state, 0,
                sizeof(crtc_state->dpll_hw_state));
 
-        if (intel_pipe_will_have_type(crtc_state, INTEL_OUTPUT_LVDS)) {
+        if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_LVDS)) {
                 if (intel_panel_use_ssc(dev_priv)) {
                         refclk = dev_priv->vbt.lvds_ssc_freq;
                         DRM_DEBUG_KMS("using SSC reference clock of %d kHz\n", refclk);
@@ -7963,7 +8028,7 @@ static int chv_crtc_compute_clock(struct intel_crtc *crtc,
                                   struct intel_crtc_state *crtc_state)
 {
         int refclk = 100000;
-        const intel_limit_t *limit = &intel_limits_chv;
+        const struct intel_limit *limit = &intel_limits_chv;
 
         memset(&crtc_state->dpll_hw_state, 0,
                sizeof(crtc_state->dpll_hw_state));
@@ -7984,7 +8049,7 @@ static int vlv_crtc_compute_clock(struct intel_crtc *crtc,
                                   struct intel_crtc_state *crtc_state)
 {
         int refclk = 100000;
-        const intel_limit_t *limit = &intel_limits_vlv;
+        const struct intel_limit *limit = &intel_limits_vlv;
 
         memset(&crtc_state->dpll_hw_state, 0,
                sizeof(crtc_state->dpll_hw_state));
@@ -8005,7 +8070,7 @@ static void i9xx_get_pfit_config(struct intel_crtc *crtc,
                                  struct intel_crtc_state *pipe_config)
 {
         struct drm_device *dev = crtc->base.dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         uint32_t tmp;
 
         if (INTEL_INFO(dev)->gen <= 3 && (IS_I830(dev) || !IS_MOBILE(dev)))
@@ -8032,9 +8097,9 @@ static void vlv_crtc_clock_get(struct intel_crtc *crtc,
                                struct intel_crtc_state *pipe_config)
 {
         struct drm_device *dev = crtc->base.dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         int pipe = pipe_config->cpu_transcoder;
-        intel_clock_t clock;
+        struct dpll clock;
         u32 mdiv;
         int refclk = 100000;
 
@@ -8060,7 +8125,7 @@ i9xx_get_initial_plane_config(struct intel_crtc *crtc,
                               struct intel_initial_plane_config *plane_config)
 {
         struct drm_device *dev = crtc->base.dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         u32 val, base, offset;
         int pipe = crtc->pipe, plane = crtc->plane;
         int fourcc, pixel_format;
@@ -8128,10 +8193,10 @@ static void chv_crtc_clock_get(struct intel_crtc *crtc,
                                struct intel_crtc_state *pipe_config)
 {
         struct drm_device *dev = crtc->base.dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         int pipe = pipe_config->cpu_transcoder;
         enum dpio_channel port = vlv_pipe_to_channel(pipe);
-        intel_clock_t clock;
+        struct dpll clock;
         u32 cmn_dw13, pll_dw0, pll_dw1, pll_dw2, pll_dw3;
         int refclk = 100000;
 
@@ -8162,7 +8227,7 @@ static bool i9xx_get_pipe_config(struct intel_crtc *crtc,
                                  struct intel_crtc_state *pipe_config)
 {
         struct drm_device *dev = crtc->base.dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         enum intel_display_power_domain power_domain;
         uint32_t tmp;
         bool ret;
@@ -8273,7 +8338,7 @@ out:
 
 static void ironlake_init_pch_refclk(struct drm_device *dev)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         struct intel_encoder *encoder;
         int i;
         u32 val, final;
@@ -8544,7 +8609,7 @@ static void lpt_program_fdi_mphy(struct drm_i915_private *dev_priv)
 static void lpt_enable_clkout_dp(struct drm_device *dev, bool with_spread,
                                  bool with_fdi)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         uint32_t reg, tmp;
 
         if (WARN(with_fdi && !with_spread, "FDI requires downspread\n"))
@@ -8583,7 +8648,7 @@ static void lpt_enable_clkout_dp(struct drm_device *dev, bool with_spread,
 /* Sequence to disable CLKOUT_DP */
 static void lpt_disable_clkout_dp(struct drm_device *dev)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         uint32_t reg, tmp;
 
         mutex_lock(&dev_priv->sb_lock);
@@ -8704,7 +8769,7 @@ void intel_init_pch_refclk(struct drm_device *dev)
 
 static void ironlake_set_pipeconf(struct drm_crtc *crtc)
 {
-        struct drm_i915_private *dev_priv = crtc->dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(crtc->dev);
         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
         int pipe = intel_crtc->pipe;
         uint32_t val;
@@ -8746,7 +8811,7 @@ static void ironlake_set_pipeconf(struct drm_crtc *crtc)
 
 static void haswell_set_pipeconf(struct drm_crtc *crtc)
 {
-        struct drm_i915_private *dev_priv = crtc->dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(crtc->dev);
         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
         enum transcoder cpu_transcoder = intel_crtc->config->cpu_transcoder;
         u32 val = 0;
@@ -8765,7 +8830,7 @@ static void haswell_set_pipeconf(struct drm_crtc *crtc)
 
 static void haswell_set_pipemisc(struct drm_crtc *crtc)
 {
-        struct drm_i915_private *dev_priv = crtc->dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(crtc->dev);
         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
 
         if (IS_BROADWELL(dev_priv) || INTEL_INFO(dev_priv)->gen >= 9) {
@@ -8814,41 +8879,17 @@ static bool ironlake_needs_fb_cb_tune(struct dpll *dpll, int factor)
 
 static void ironlake_compute_dpll(struct intel_crtc *intel_crtc,
                                   struct intel_crtc_state *crtc_state,
-                                  intel_clock_t *reduced_clock)
+                                  struct dpll *reduced_clock)
 {
         struct drm_crtc *crtc = &intel_crtc->base;
         struct drm_device *dev = crtc->dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
-        struct drm_atomic_state *state = crtc_state->base.state;
-        struct drm_connector *connector;
-        struct drm_connector_state *connector_state;
-        struct intel_encoder *encoder;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         u32 dpll, fp, fp2;
-        int factor, i;
-        bool is_lvds = false, is_sdvo = false;
-
-        for_each_connector_in_state(state, connector, connector_state, i) {
-                if (connector_state->crtc != crtc_state->base.crtc)
-                        continue;
-
-                encoder = to_intel_encoder(connector_state->best_encoder);
-
-                switch (encoder->type) {
-                case INTEL_OUTPUT_LVDS:
-                        is_lvds = true;
-                        break;
-                case INTEL_OUTPUT_SDVO:
-                case INTEL_OUTPUT_HDMI:
-                        is_sdvo = true;
-                        break;
-                default:
-                        break;
-                }
-        }
+        int factor;
 
         /* Enable autotuning of the PLL clock (if permissible) */
         factor = 21;
-        if (is_lvds) {
+        if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_LVDS)) {
                 if ((intel_panel_use_ssc(dev_priv) &&
                      dev_priv->vbt.lvds_ssc_freq == 100000) ||
                     (HAS_PCH_IBX(dev) && intel_is_dual_link_lvds(dev)))
@@ -8872,7 +8913,7 @@ static void ironlake_compute_dpll(struct intel_crtc *intel_crtc,
 
         dpll = 0;
 
-        if (is_lvds)
+        if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_LVDS))
                 dpll |= DPLLB_MODE_LVDS;
         else
                 dpll |= DPLLB_MODE_DAC_SERIAL;
@@ -8880,9 +8921,11 @@ static void ironlake_compute_dpll(struct intel_crtc *intel_crtc,
         dpll |= (crtc_state->pixel_multiplier - 1)
                 << PLL_REF_SDVO_HDMI_MULTIPLIER_SHIFT;
 
-        if (is_sdvo)
+        if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_SDVO) ||
+            intel_crtc_has_type(crtc_state, INTEL_OUTPUT_HDMI))
                 dpll |= DPLL_SDVO_HIGH_SPEED;
-        if (crtc_state->has_dp_encoder)
+
+        if (intel_crtc_has_dp_encoder(crtc_state))
                 dpll |= DPLL_SDVO_HIGH_SPEED;
 
         /* compute bitmask from p1 value */
@@ -8905,7 +8948,8 @@ static void ironlake_compute_dpll(struct intel_crtc *intel_crtc,
                 break;
         }
 
-        if (is_lvds && intel_panel_use_ssc(dev_priv))
+        if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_LVDS) &&
+            intel_panel_use_ssc(dev_priv))
                 dpll |= PLLB_REF_INPUT_SPREADSPECTRUMIN;
         else
                 dpll |= PLL_REF_INPUT_DREFCLK;
@@ -8921,11 +8965,11 @@ static int ironlake_crtc_compute_clock(struct intel_crtc *crtc,
                                        struct intel_crtc_state *crtc_state)
 {
         struct drm_device *dev = crtc->base.dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
-        intel_clock_t reduced_clock;
+        struct drm_i915_private *dev_priv = to_i915(dev);
+        struct dpll reduced_clock;
         bool has_reduced_clock = false;
         struct intel_shared_dpll *pll;
-        const intel_limit_t *limit;
+        const struct intel_limit *limit;
         int refclk = 120000;
 
         memset(&crtc_state->dpll_hw_state, 0,
@@ -8937,7 +8981,7 @@ static int ironlake_crtc_compute_clock(struct intel_crtc *crtc,
         if (!crtc_state->has_pch_encoder)
                 return 0;
 
-        if (intel_pipe_will_have_type(crtc_state, INTEL_OUTPUT_LVDS)) {
+        if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_LVDS)) {
                 if (intel_panel_use_ssc(dev_priv)) {
                         DRM_DEBUG_KMS("using SSC reference clock of %d kHz\n",
                                       dev_priv->vbt.lvds_ssc_freq);
@@ -8976,7 +9020,7 @@ static int ironlake_crtc_compute_clock(struct intel_crtc *crtc,
                 return -EINVAL;
         }
 
-        if (intel_pipe_will_have_type(crtc_state, INTEL_OUTPUT_LVDS) &&
+        if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_LVDS) &&
             has_reduced_clock)
                 crtc->lowfreq_avail = true;
 
@@ -8987,7 +9031,7 @@ static void intel_pch_transcoder_get_m_n(struct intel_crtc *crtc,
                                          struct intel_link_m_n *m_n)
 {
         struct drm_device *dev = crtc->base.dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         enum pipe pipe = crtc->pipe;
 
         m_n->link_m = I915_READ(PCH_TRANS_LINK_M1(pipe));
@@ -9005,7 +9049,7 @@ static void intel_cpu_transcoder_get_m_n(struct intel_crtc *crtc,
                                          struct intel_link_m_n *m2_n2)
 {
         struct drm_device *dev = crtc->base.dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         enum pipe pipe = crtc->pipe;
 
         if (INTEL_INFO(dev)->gen >= 5) {
@@ -9063,7 +9107,7 @@ static void skylake_get_pfit_config(struct intel_crtc *crtc,
                                     struct intel_crtc_state *pipe_config)
 {
         struct drm_device *dev = crtc->base.dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         struct intel_crtc_scaler_state *scaler_state = &pipe_config->scaler_state;
         uint32_t ps_ctrl = 0;
         int id = -1;
@@ -9094,7 +9138,7 @@ skylake_get_initial_plane_config(struct intel_crtc *crtc,
                                  struct intel_initial_plane_config *plane_config)
 {
         struct drm_device *dev = crtc->base.dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         u32 val, base, offset, stride_mult, tiling;
         int pipe = crtc->pipe;
         int fourcc, pixel_format;
@@ -9177,7 +9221,7 @@ static void ironlake_get_pfit_config(struct intel_crtc *crtc,
                                      struct intel_crtc_state *pipe_config)
 {
         struct drm_device *dev = crtc->base.dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         uint32_t tmp;
 
         tmp = I915_READ(PF_CTL(crtc->pipe));
@@ -9202,7 +9246,7 @@ ironlake_get_initial_plane_config(struct intel_crtc *crtc,
                                   struct intel_initial_plane_config *plane_config)
 {
         struct drm_device *dev = crtc->base.dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         u32 val, base, offset;
         int pipe = crtc->pipe;
         int fourcc, pixel_format;
@@ -9270,7 +9314,7 @@ static bool ironlake_get_pipe_config(struct intel_crtc *crtc,
                                      struct intel_crtc_state *pipe_config)
 {
         struct drm_device *dev = crtc->base.dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         enum intel_display_power_domain power_domain;
         uint32_t tmp;
         bool ret;
@@ -9320,6 +9364,10 @@ static bool ironlake_get_pipe_config(struct intel_crtc *crtc,
                 ironlake_get_fdi_m_n_config(crtc, pipe_config);
 
                 if (HAS_PCH_IBX(dev_priv)) {
+                        /*
+                         * The pipe->pch transcoder and pch transcoder->pll
+                         * mapping is fixed.
+                         */
                         pll_id = (enum intel_dpll_id) crtc->pipe;
                 } else {
                         tmp = I915_READ(PCH_DPLL_SEL);
@@ -9361,7 +9409,7 @@ out:
 
 static void assert_can_disable_lcpll(struct drm_i915_private *dev_priv)
 {
-        struct drm_device *dev = dev_priv->dev;
+        struct drm_device *dev = &dev_priv->drm;
         struct intel_crtc *crtc;
 
         for_each_intel_crtc(dev, crtc)
@@ -9395,7 +9443,7 @@ static void assert_can_disable_lcpll(struct drm_i915_private *dev_priv)
 
 static uint32_t hsw_read_dcomp(struct drm_i915_private *dev_priv)
 {
-        struct drm_device *dev = dev_priv->dev;
+        struct drm_device *dev = &dev_priv->drm;
 
         if (IS_HASWELL(dev))
                 return I915_READ(D_COMP_HSW);
@@ -9405,7 +9453,7 @@ static uint32_t hsw_read_dcomp(struct drm_i915_private *dev_priv)
 
 static void hsw_write_dcomp(struct drm_i915_private *dev_priv, uint32_t val)
 {
-        struct drm_device *dev = dev_priv->dev;
+        struct drm_device *dev = &dev_priv->drm;
 
         if (IS_HASWELL(dev)) {
                 mutex_lock(&dev_priv->rps.hw_lock);
@@ -9451,7 +9499,7 @@ static void hsw_disable_lcpll(struct drm_i915_private *dev_priv,
         I915_WRITE(LCPLL_CTL, val);
         POSTING_READ(LCPLL_CTL);
 
-        if (wait_for((I915_READ(LCPLL_CTL) & LCPLL_PLL_LOCK) == 0, 1))
+        if (intel_wait_for_register(dev_priv, LCPLL_CTL, LCPLL_PLL_LOCK, 0, 1))
                 DRM_ERROR("LCPLL still locked\n");
 
         val = hsw_read_dcomp(dev_priv);
@@ -9506,7 +9554,9 @@ static void hsw_restore_lcpll(struct drm_i915_private *dev_priv)
         val &= ~LCPLL_PLL_DISABLE;
         I915_WRITE(LCPLL_CTL, val);
 
-        if (wait_for(I915_READ(LCPLL_CTL) & LCPLL_PLL_LOCK, 5))
+        if (intel_wait_for_register(dev_priv,
+                                    LCPLL_CTL, LCPLL_PLL_LOCK, LCPLL_PLL_LOCK,
+                                    5))
                 DRM_ERROR("LCPLL not locked yet\n");
 
         if (val & LCPLL_CD_SOURCE_FCLK) {
@@ -9520,7 +9570,7 @@ static void hsw_restore_lcpll(struct drm_i915_private *dev_priv)
         }
 
         intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
-        intel_update_cdclk(dev_priv->dev);
+        intel_update_cdclk(&dev_priv->drm);
 }
 
 /*
@@ -9548,7 +9598,7 @@ static void hsw_restore_lcpll(struct drm_i915_private *dev_priv)
  */
 void hsw_enable_pc8(struct drm_i915_private *dev_priv)
 {
-        struct drm_device *dev = dev_priv->dev;
+        struct drm_device *dev = &dev_priv->drm;
         uint32_t val;
 
         DRM_DEBUG_KMS("Enabling package C8+\n");
@@ -9565,7 +9615,7 @@ void hsw_enable_pc8(struct drm_i915_private *dev_priv)
 
 void hsw_disable_pc8(struct drm_i915_private *dev_priv)
 {
-        struct drm_device *dev = dev_priv->dev;
+        struct drm_device *dev = &dev_priv->drm;
         uint32_t val;
 
         DRM_DEBUG_KMS("Disabling package C8+\n");
@@ -9580,21 +9630,21 @@ void hsw_disable_pc8(struct drm_i915_private *dev_priv)
         }
 }
 
-static void broxton_modeset_commit_cdclk(struct drm_atomic_state *old_state)
+static void bxt_modeset_commit_cdclk(struct drm_atomic_state *old_state)
 {
         struct drm_device *dev = old_state->dev;
         struct intel_atomic_state *old_intel_state =
                 to_intel_atomic_state(old_state);
         unsigned int req_cdclk = old_intel_state->dev_cdclk;
 
-        broxton_set_cdclk(to_i915(dev), req_cdclk);
+        bxt_set_cdclk(to_i915(dev), req_cdclk);
 }
 
 /* compute the max rate for new configuration */
 static int ilk_max_pixel_rate(struct drm_atomic_state *state)
 {
         struct intel_atomic_state *intel_state = to_intel_atomic_state(state);
-        struct drm_i915_private *dev_priv = state->dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(state->dev);
         struct drm_crtc *crtc;
         struct drm_crtc_state *cstate;
         struct intel_crtc_state *crtc_state;
@@ -9630,7 +9680,7 @@ static int ilk_max_pixel_rate(struct drm_atomic_state *state)
 
 static void broadwell_set_cdclk(struct drm_device *dev, int cdclk)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         uint32_t val, data;
         int ret;
 
@@ -9707,6 +9757,18 @@ static void broadwell_set_cdclk(struct drm_device *dev, int cdclk)
              cdclk, dev_priv->cdclk_freq);
 }
 
+static int broadwell_calc_cdclk(int max_pixclk)
+{
+        if (max_pixclk > 540000)
+                return 675000;
+        else if (max_pixclk > 450000)
+                return 540000;
+        else if (max_pixclk > 337500)
+                return 450000;
+        else
+                return 337500;
+}
+
 static int broadwell_modeset_calc_cdclk(struct drm_atomic_state *state)
 {
         struct drm_i915_private *dev_priv = to_i915(state->dev);
@@ -9718,14 +9780,7 @@ static int broadwell_modeset_calc_cdclk(struct drm_atomic_state *state)
          * FIXME should also account for plane ratio
          * once 64bpp pixel formats are supported.
          */
-        if (max_pixclk > 540000)
-                cdclk = 675000;
-        else if (max_pixclk > 450000)
-                cdclk = 540000;
-        else if (max_pixclk > 337500)
-                cdclk = 450000;
-        else
-                cdclk = 337500;
+        cdclk = broadwell_calc_cdclk(max_pixclk);
 
         if (cdclk > dev_priv->max_cdclk_freq) {
                 DRM_DEBUG_KMS("requested cdclk (%d kHz) exceeds max (%d kHz)\n",
@@ -9735,7 +9790,7 @@ static int broadwell_modeset_calc_cdclk(struct drm_atomic_state *state)
 
         intel_state->cdclk = intel_state->dev_cdclk = cdclk;
         if (!intel_state->active_crtcs)
-                intel_state->dev_cdclk = 337500;
+                intel_state->dev_cdclk = broadwell_calc_cdclk(0);
 
         return 0;
 }
@@ -9750,13 +9805,51 @@ static void broadwell_modeset_commit_cdclk(struct drm_atomic_state *old_state)
         broadwell_set_cdclk(dev, req_cdclk);
 }
 
+static int skl_modeset_calc_cdclk(struct drm_atomic_state *state)
+{
+        struct intel_atomic_state *intel_state = to_intel_atomic_state(state);
+        struct drm_i915_private *dev_priv = to_i915(state->dev);
+        const int max_pixclk = ilk_max_pixel_rate(state);
+        int vco = intel_state->cdclk_pll_vco;
+        int cdclk;
+
+        /*
+         * FIXME should also account for plane ratio
+         * once 64bpp pixel formats are supported.
+         */
+        cdclk = skl_calc_cdclk(max_pixclk, vco);
+
+        /*
+         * FIXME move the cdclk caclulation to
+         * compute_config() so we can fail gracegully.
+         */
+        if (cdclk > dev_priv->max_cdclk_freq) {
+                DRM_ERROR("requested cdclk (%d kHz) exceeds max (%d kHz)\n",
+                          cdclk, dev_priv->max_cdclk_freq);
+                cdclk = dev_priv->max_cdclk_freq;
+        }
+
+        intel_state->cdclk = intel_state->dev_cdclk = cdclk;
+        if (!intel_state->active_crtcs)
+                intel_state->dev_cdclk = skl_calc_cdclk(0, vco);
+
+        return 0;
+}
+
+static void skl_modeset_commit_cdclk(struct drm_atomic_state *old_state)
+{
+        struct drm_i915_private *dev_priv = to_i915(old_state->dev);
+        struct intel_atomic_state *intel_state = to_intel_atomic_state(old_state);
+        unsigned int req_cdclk = intel_state->dev_cdclk;
+        unsigned int req_vco = intel_state->cdclk_pll_vco;
+
+        skl_set_cdclk(dev_priv, req_cdclk, req_vco);
+}
+
 static int haswell_crtc_compute_clock(struct intel_crtc *crtc,
                                       struct intel_crtc_state *crtc_state)
 {
-        struct intel_encoder *intel_encoder =
-                intel_ddi_get_crtc_new_encoder(crtc_state);
-
-        if (intel_encoder->type != INTEL_OUTPUT_DSI) {
+        if (!intel_crtc_has_type(crtc_state, INTEL_OUTPUT_DSI)) {
                 if (!intel_ddi_pll_select(crtc, crtc_state))
                         return -EINVAL;
         }
@@ -9866,10 +9959,14 @@ static bool hsw_get_transcoder_state(struct intel_crtc *crtc,
                                      unsigned long *power_domain_mask)
 {
         struct drm_device *dev = crtc->base.dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         enum intel_display_power_domain power_domain;
         u32 tmp;
 
+        /*
+         * The pipe->transcoder mapping is fixed with the exception of the eDP
+         * transcoder handled below.
+         */
         pipe_config->cpu_transcoder = (enum transcoder) crtc->pipe;
 
         /*
@@ -9913,14 +10010,12 @@ static bool bxt_get_dsi_transcoder_state(struct intel_crtc *crtc,
                                          unsigned long *power_domain_mask)
 {
         struct drm_device *dev = crtc->base.dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         enum intel_display_power_domain power_domain;
         enum port port;
         enum transcoder cpu_transcoder;
         u32 tmp;
 
-        pipe_config->has_dsi_encoder = false;
-
         for_each_port_masked(port, BIT(PORT_A) | BIT(PORT_C)) {
                 if (port == PORT_A)
                         cpu_transcoder = TRANSCODER_DSI_A;
@@ -9952,18 +10047,17 @@ static bool bxt_get_dsi_transcoder_state(struct intel_crtc *crtc,
                         continue;
 
                 pipe_config->cpu_transcoder = cpu_transcoder;
-                pipe_config->has_dsi_encoder = true;
                 break;
         }
 
-        return pipe_config->has_dsi_encoder;
+        return transcoder_is_dsi(pipe_config->cpu_transcoder);
 }
 
 static void haswell_get_ddi_port_state(struct intel_crtc *crtc,
                                        struct intel_crtc_state *pipe_config)
 {
         struct drm_device *dev = crtc->base.dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         struct intel_shared_dpll *pll;
         enum port port;
         uint32_t tmp;
@@ -10006,7 +10100,7 @@ static bool haswell_get_pipe_config(struct intel_crtc *crtc,
                                     struct intel_crtc_state *pipe_config)
 {
         struct drm_device *dev = crtc->base.dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         enum intel_display_power_domain power_domain;
         unsigned long power_domain_mask;
         bool active;
@@ -10020,18 +10114,16 @@ static bool haswell_get_pipe_config(struct intel_crtc *crtc,
 
         active = hsw_get_transcoder_state(crtc, pipe_config, &power_domain_mask);
 
-        if (IS_BROXTON(dev_priv)) {
-                bxt_get_dsi_transcoder_state(crtc, pipe_config,
-                                             &power_domain_mask);
-                WARN_ON(active && pipe_config->has_dsi_encoder);
-                if (pipe_config->has_dsi_encoder)
-                        active = true;
+        if (IS_BROXTON(dev_priv) &&
+            bxt_get_dsi_transcoder_state(crtc, pipe_config, &power_domain_mask)) {
+                WARN_ON(active);
+                active = true;
         }
 
         if (!active)
                 goto out;
 
-        if (!pipe_config->has_dsi_encoder) {
+        if (!transcoder_is_dsi(pipe_config->cpu_transcoder)) {
                 haswell_get_ddi_port_state(crtc, pipe_config);
                 intel_get_pipe_timings(crtc, pipe_config);
         }
@@ -10082,7 +10174,7 @@ static void i845_update_cursor(struct drm_crtc *crtc, u32 base,
                                const struct intel_plane_state *plane_state)
 {
         struct drm_device *dev = crtc->dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
         uint32_t cntl = 0, size = 0;
 
@@ -10145,7 +10237,7 @@ static void i9xx_update_cursor(struct drm_crtc *crtc, u32 base,
                                const struct intel_plane_state *plane_state)
 {
         struct drm_device *dev = crtc->dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
         int pipe = intel_crtc->pipe;
         uint32_t cntl = 0;
@@ -10193,7 +10285,7 @@ static void intel_crtc_update_cursor(struct drm_crtc *crtc,
                                      const struct intel_plane_state *plane_state)
 {
         struct drm_device *dev = crtc->dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
         int pipe = intel_crtc->pipe;
         u32 base = intel_crtc->cursor_addr;
@@ -10337,10 +10429,10 @@ intel_framebuffer_create_for_mode(struct drm_device *dev,
         struct drm_i915_gem_object *obj;
         struct drm_mode_fb_cmd2 mode_cmd = { 0 };
 
-        obj = i915_gem_alloc_object(dev,
+        obj = i915_gem_object_create(dev,
                                     intel_framebuffer_size_for_mode(mode, bpp));
-        if (obj == NULL)
-                return ERR_PTR(-ENOMEM);
+        if (IS_ERR(obj))
+                return ERR_CAST(obj);
 
         mode_cmd.width = mode->hdisplay;
         mode_cmd.height = mode->vdisplay;
@@ -10360,7 +10452,7 @@ mode_fits_in_fbdev(struct drm_device *dev,
                    struct drm_display_mode *mode)
 {
 #ifdef CONFIG_DRM_FBDEV_EMULATION
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         struct drm_i915_gem_object *obj;
         struct drm_framebuffer *fb;
 
@@ -10630,7 +10722,7 @@ void intel_release_load_detect_pipe(struct drm_connector *connector,
 static int i9xx_pll_refclk(struct drm_device *dev,
                            const struct intel_crtc_state *pipe_config)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         u32 dpll = pipe_config->dpll_hw_state.dpll;
 
         if ((dpll & PLL_REF_INPUT_MASK) == PLLB_REF_INPUT_SPREADSPECTRUMIN)
@@ -10648,11 +10740,11 @@ static void i9xx_crtc_clock_get(struct intel_crtc *crtc,
                                 struct intel_crtc_state *pipe_config)
 {
         struct drm_device *dev = crtc->base.dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         int pipe = pipe_config->cpu_transcoder;
         u32 dpll = pipe_config->dpll_hw_state.dpll;
         u32 fp;
-        intel_clock_t clock;
+        struct dpll clock;
         int port_clock;
         int refclk = i9xx_pll_refclk(dev, pipe_config);
 
@@ -10774,7 +10866,7 @@ static void ironlake_pch_clock_get(struct intel_crtc *crtc,
 struct drm_display_mode *intel_crtc_mode_get(struct drm_device *dev,
                                              struct drm_crtc *crtc)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
         enum transcoder cpu_transcoder = intel_crtc->config->cpu_transcoder;
         struct drm_display_mode *mode;
@@ -10826,48 +10918,20 @@ struct drm_display_mode *intel_crtc_mode_get(struct drm_device *dev,
         return mode;
 }
 
-void intel_mark_busy(struct drm_device *dev)
-{
-        struct drm_i915_private *dev_priv = dev->dev_private;
-
-        if (dev_priv->mm.busy)
-                return;
-
-        intel_runtime_pm_get(dev_priv);
-        i915_update_gfx_val(dev_priv);
-        if (INTEL_INFO(dev)->gen >= 6)
-                gen6_rps_busy(dev_priv);
-        dev_priv->mm.busy = true;
-}
-
-void intel_mark_idle(struct drm_device *dev)
-{
-        struct drm_i915_private *dev_priv = dev->dev_private;
-
-        if (!dev_priv->mm.busy)
-                return;
-
-        dev_priv->mm.busy = false;
-
-        if (INTEL_INFO(dev)->gen >= 6)
-                gen6_rps_idle(dev->dev_private);
-
-        intel_runtime_pm_put(dev_priv);
-}
-
 static void intel_crtc_destroy(struct drm_crtc *crtc)
 {
         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
         struct drm_device *dev = crtc->dev;
-        struct intel_unpin_work *work;
+        struct intel_flip_work *work;
 
         spin_lock_irq(&dev->event_lock);
-        work = intel_crtc->unpin_work;
-        intel_crtc->unpin_work = NULL;
+        work = intel_crtc->flip_work;
+        intel_crtc->flip_work = NULL;
         spin_unlock_irq(&dev->event_lock);
 
         if (work) {
-                cancel_work_sync(&work->work);
+                cancel_work_sync(&work->mmio_work);
+                cancel_work_sync(&work->unpin_work);
                 kfree(work);
         }
 
@@ -10878,12 +10942,15 @@ static void intel_crtc_destroy(struct drm_crtc *crtc)
 
 static void intel_unpin_work_fn(struct work_struct *__work)
 {
-        struct intel_unpin_work *work =
-                container_of(__work, struct intel_unpin_work, work);
+        struct intel_flip_work *work =
+                container_of(__work, struct intel_flip_work, unpin_work);
         struct intel_crtc *crtc = to_intel_crtc(work->crtc);
         struct drm_device *dev = crtc->base.dev;
         struct drm_plane *primary = crtc->base.primary;
 
+        if (is_mmio_work(work))
+                flush_work(&work->mmio_work);
+
         mutex_lock(&dev->struct_mutex);
         intel_unpin_fb_obj(work->old_fb, primary->state->rotation);
         drm_gem_object_unreference(&work->pending_flip_obj->base);
@@ -10902,63 +10969,17 @@ static void intel_unpin_work_fn(struct work_struct *__work)
         kfree(work);
 }
 
-static void do_intel_finish_page_flip(struct drm_device *dev,
-                                      struct drm_crtc *crtc)
-{
-        struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
-        struct intel_unpin_work *work;
-        unsigned long flags;
-
-        /* Ignore early vblank irqs */
-        if (intel_crtc == NULL)
-                return;
-
-        /*
-         * This is called both by irq handlers and the reset code (to complete
-         * lost pageflips) so needs the full irqsave spinlocks.
-         */
-        spin_lock_irqsave(&dev->event_lock, flags);
-        work = intel_crtc->unpin_work;
-
-        /* Ensure we don't miss a work->pending update ... */
-        smp_rmb();
-
-        if (work == NULL || atomic_read(&work->pending) < INTEL_FLIP_COMPLETE) {
-                spin_unlock_irqrestore(&dev->event_lock, flags);
-                return;
-        }
-
-        page_flip_completed(intel_crtc);
-
-        spin_unlock_irqrestore(&dev->event_lock, flags);
-}
-
-void intel_finish_page_flip(struct drm_device *dev, int pipe)
-{
-        struct drm_i915_private *dev_priv = dev->dev_private;
-        struct drm_crtc *crtc = dev_priv->pipe_to_crtc_mapping[pipe];
-
-        do_intel_finish_page_flip(dev, crtc);
-}
-
-void intel_finish_page_flip_plane(struct drm_device *dev, int plane)
-{
-        struct drm_i915_private *dev_priv = dev->dev_private;
-        struct drm_crtc *crtc = dev_priv->plane_to_crtc_mapping[plane];
-
-        do_intel_finish_page_flip(dev, crtc);
-}
-
 /* Is 'a' after or equal to 'b'? */
 static bool g4x_flip_count_after_eq(u32 a, u32 b)
 {
         return !((a - b) & 0x80000000);
 }
 
-static bool page_flip_finished(struct intel_crtc *crtc)
+static bool __pageflip_finished_cs(struct intel_crtc *crtc,
+                                   struct intel_flip_work *work)
 {
         struct drm_device *dev = crtc->base.dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         unsigned reset_counter;
 
         reset_counter = i915_reset_counter(&dev_priv->gpu_error);
@@ -10997,40 +11018,103 @@ static bool page_flip_finished(struct intel_crtc *crtc)
          * anyway, we don't really care.
          */
         return (I915_READ(DSPSURFLIVE(crtc->plane)) & ~0xfff) ==
-                crtc->unpin_work->gtt_offset &&
+                crtc->flip_work->gtt_offset &&
                 g4x_flip_count_after_eq(I915_READ(PIPE_FLIPCOUNT_G4X(crtc->pipe)),
-                                    crtc->unpin_work->flip_count);
+                                    crtc->flip_work->flip_count);
 }
 
-void intel_prepare_page_flip(struct drm_device *dev, int plane)
+static bool
+__pageflip_finished_mmio(struct intel_crtc *crtc,
+                               struct intel_flip_work *work)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
-        struct intel_crtc *intel_crtc =
-                to_intel_crtc(dev_priv->plane_to_crtc_mapping[plane]);
+        /*
+         * MMIO work completes when vblank is different from
+         * flip_queued_vblank.
+         *
+         * Reset counter value doesn't matter, this is handled by
+         * i915_wait_request finishing early, so no need to handle
+         * reset here.
+         */
+        return intel_crtc_get_vblank_counter(crtc) != work->flip_queued_vblank;
+}
+
+
+static bool pageflip_finished(struct intel_crtc *crtc,
+                              struct intel_flip_work *work)
+{
+        if (!atomic_read(&work->pending))
+                return false;
+
+        smp_rmb();
+
+        if (is_mmio_work(work))
+                return __pageflip_finished_mmio(crtc, work);
+        else
+                return __pageflip_finished_cs(crtc, work);
+}
+
+void intel_finish_page_flip_cs(struct drm_i915_private *dev_priv, int pipe)
+{
+        struct drm_device *dev = &dev_priv->drm;
+        struct drm_crtc *crtc = dev_priv->pipe_to_crtc_mapping[pipe];
+        struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
+        struct intel_flip_work *work;
         unsigned long flags;
 
+        /* Ignore early vblank irqs */
+        if (!crtc)
+                return;
 
         /*
          * This is called both by irq handlers and the reset code (to complete
          * lost pageflips) so needs the full irqsave spinlocks.
-         *
-         * NB: An MMIO update of the plane base pointer will also
-         * generate a page-flip completion irq, i.e. every modeset
-         * is also accompanied by a spurious intel_prepare_page_flip().
          */
         spin_lock_irqsave(&dev->event_lock, flags);
-        if (intel_crtc->unpin_work && page_flip_finished(intel_crtc))
-                atomic_inc_not_zero(&intel_crtc->unpin_work->pending);
+        work = intel_crtc->flip_work;
+
+        if (work != NULL &&
+            !is_mmio_work(work) &&
+            pageflip_finished(intel_crtc, work))
+                page_flip_completed(intel_crtc);
+
         spin_unlock_irqrestore(&dev->event_lock, flags);
 }
 
-static inline void intel_mark_page_flip_active(struct intel_unpin_work *work)
+void intel_finish_page_flip_mmio(struct drm_i915_private *dev_priv, int pipe)
 {
+        struct drm_device *dev = &dev_priv->drm;
+        struct drm_crtc *crtc = dev_priv->pipe_to_crtc_mapping[pipe];
+        struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
+        struct intel_flip_work *work;
+        unsigned long flags;
+
+        /* Ignore early vblank irqs */
+        if (!crtc)
+                return;
+
+        /*
+         * This is called both by irq handlers and the reset code (to complete
+         * lost pageflips) so needs the full irqsave spinlocks.
+         */
+        spin_lock_irqsave(&dev->event_lock, flags);
+        work = intel_crtc->flip_work;
+
+        if (work != NULL &&
+            is_mmio_work(work) &&
+            pageflip_finished(intel_crtc, work))
+                page_flip_completed(intel_crtc);
+
+        spin_unlock_irqrestore(&dev->event_lock, flags);
+}
+
+static inline void intel_mark_page_flip_active(struct intel_crtc *crtc,
+                                               struct intel_flip_work *work)
+{
+        work->flip_queued_vblank = intel_crtc_get_vblank_counter(crtc);
+
         /* Ensure that the work item is consistent when activating it ... */
-        smp_wmb();
-        atomic_set(&work->pending, INTEL_FLIP_PENDING);
-        /* and that it is marked active as soon as the irq could fire. */
-        smp_wmb();
+        smp_mb__before_atomic();
+        atomic_set(&work->pending, 1);
 }
 
 static int intel_gen2_queue_flip(struct drm_device *dev,
@@ -11061,10 +11145,9 @@ static int intel_gen2_queue_flip(struct drm_device *dev,
         intel_ring_emit(engine, MI_DISPLAY_FLIP |
                         MI_DISPLAY_FLIP_PLANE(intel_crtc->plane));
         intel_ring_emit(engine, fb->pitches[0]);
-        intel_ring_emit(engine, intel_crtc->unpin_work->gtt_offset);
+        intel_ring_emit(engine, intel_crtc->flip_work->gtt_offset);
         intel_ring_emit(engine, 0); /* aux display base address, unused */
 
-        intel_mark_page_flip_active(intel_crtc->unpin_work);
         return 0;
 }
 
@@ -11093,10 +11176,9 @@ static int intel_gen3_queue_flip(struct drm_device *dev,
         intel_ring_emit(engine, MI_DISPLAY_FLIP_I915 |
                         MI_DISPLAY_FLIP_PLANE(intel_crtc->plane));
         intel_ring_emit(engine, fb->pitches[0]);
-        intel_ring_emit(engine, intel_crtc->unpin_work->gtt_offset);
+        intel_ring_emit(engine, intel_crtc->flip_work->gtt_offset);
         intel_ring_emit(engine, MI_NOOP);
 
-        intel_mark_page_flip_active(intel_crtc->unpin_work);
         return 0;
 }
 
@@ -11108,7 +11190,7 @@ static int intel_gen4_queue_flip(struct drm_device *dev,
                                  uint32_t flags)
 {
         struct intel_engine_cs *engine = req->engine;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
         uint32_t pf, pipesrc;
         int ret;
@@ -11124,7 +11206,7 @@ static int intel_gen4_queue_flip(struct drm_device *dev,
         intel_ring_emit(engine, MI_DISPLAY_FLIP |
                         MI_DISPLAY_FLIP_PLANE(intel_crtc->plane));
         intel_ring_emit(engine, fb->pitches[0]);
-        intel_ring_emit(engine, intel_crtc->unpin_work->gtt_offset |
+        intel_ring_emit(engine, intel_crtc->flip_work->gtt_offset |
                         obj->tiling_mode);
 
         /* XXX Enabling the panel-fitter across page-flip is so far
@@ -11135,7 +11217,6 @@ static int intel_gen4_queue_flip(struct drm_device *dev,
         pipesrc = I915_READ(PIPESRC(intel_crtc->pipe)) & 0x0fff0fff;
         intel_ring_emit(engine, pf | pipesrc);
 
-        intel_mark_page_flip_active(intel_crtc->unpin_work);
         return 0;
 }
 
@@ -11147,7 +11228,7 @@ static int intel_gen6_queue_flip(struct drm_device *dev,
                                  uint32_t flags)
 {
         struct intel_engine_cs *engine = req->engine;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
         uint32_t pf, pipesrc;
         int ret;
@@ -11159,7 +11240,7 @@ static int intel_gen6_queue_flip(struct drm_device *dev,
         intel_ring_emit(engine, MI_DISPLAY_FLIP |
                         MI_DISPLAY_FLIP_PLANE(intel_crtc->plane));
         intel_ring_emit(engine, fb->pitches[0] | obj->tiling_mode);
-        intel_ring_emit(engine, intel_crtc->unpin_work->gtt_offset);
+        intel_ring_emit(engine, intel_crtc->flip_work->gtt_offset);
 
         /* Contrary to the suggestions in the documentation,
          * "Enable Panel Fitter" does not seem to be required when page
@@ -11171,7 +11252,6 @@ static int intel_gen6_queue_flip(struct drm_device *dev,
         pipesrc = I915_READ(PIPESRC(intel_crtc->pipe)) & 0x0fff0fff;
         intel_ring_emit(engine, pf | pipesrc);
 
-        intel_mark_page_flip_active(intel_crtc->unpin_work);
         return 0;
 }
 
@@ -11263,16 +11343,17 @@ static int intel_gen7_queue_flip(struct drm_device *dev,
 
         intel_ring_emit(engine, MI_DISPLAY_FLIP_I915 | plane_bit);
         intel_ring_emit(engine, (fb->pitches[0] | obj->tiling_mode));
-        intel_ring_emit(engine, intel_crtc->unpin_work->gtt_offset);
+        intel_ring_emit(engine, intel_crtc->flip_work->gtt_offset);
         intel_ring_emit(engine, (MI_NOOP));
 
-        intel_mark_page_flip_active(intel_crtc->unpin_work);
         return 0;
 }
 
 static bool use_mmio_flip(struct intel_engine_cs *engine,
                           struct drm_i915_gem_object *obj)
 {
+        struct reservation_object *resv;
+
         /*
          * This is not being used for older platforms, because
          * non-availability of flip done interrupt forces us to use
@@ -11284,7 +11365,7 @@ static bool use_mmio_flip(struct intel_engine_cs *engine,
         if (engine == NULL)
                 return true;
 
-        if (INTEL_INFO(engine->dev)->gen < 5)
+        if (INTEL_GEN(engine->i915) < 5)
                 return false;
 
         if (i915.use_mmio_flip < 0)
@@ -11293,20 +11374,20 @@ static bool use_mmio_flip(struct intel_engine_cs *engine,
                 return true;
         else if (i915.enable_execlists)
                 return true;
-        else if (obj->base.dma_buf &&
-                 !reservation_object_test_signaled_rcu(obj->base.dma_buf->resv,
-                                                       false))
+
+        resv = i915_gem_object_get_dmabuf_resv(obj);
+        if (resv && !reservation_object_test_signaled_rcu(resv, false))
                 return true;
-        else
-                return engine != i915_gem_request_get_engine(obj->last_write_req);
+
+        return engine != i915_gem_request_get_engine(obj->last_write_req);
 }
 
 static void skl_do_mmio_flip(struct intel_crtc *intel_crtc,
                              unsigned int rotation,
-                             struct intel_unpin_work *work)
+                             struct intel_flip_work *work)
 {
         struct drm_device *dev = intel_crtc->base.dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         struct drm_framebuffer *fb = intel_crtc->base.primary->fb;
         const enum pipe pipe = intel_crtc->pipe;
         u32 ctl, stride, tile_height;
@@ -11355,10 +11436,10 @@ static void skl_do_mmio_flip(struct intel_crtc *intel_crtc,
 }
 
 static void ilk_do_mmio_flip(struct intel_crtc *intel_crtc,
-                             struct intel_unpin_work *work)
+                             struct intel_flip_work *work)
 {
         struct drm_device *dev = intel_crtc->base.dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         struct intel_framebuffer *intel_fb =
                 to_intel_framebuffer(intel_crtc->base.primary->fb);
         struct drm_i915_gem_object *obj = intel_fb->obj;
@@ -11378,78 +11459,37 @@ static void ilk_do_mmio_flip(struct intel_crtc *intel_crtc,
         POSTING_READ(DSPSURF(intel_crtc->plane));
 }
 
-/*
- * XXX: This is the temporary way to update the plane registers until we get
- * around to using the usual plane update functions for MMIO flips
- */
-static void intel_do_mmio_flip(struct intel_mmio_flip *mmio_flip)
-{
-        struct intel_crtc *crtc = mmio_flip->crtc;
-        struct intel_unpin_work *work;
-
-        spin_lock_irq(&crtc->base.dev->event_lock);
-        work = crtc->unpin_work;
-        spin_unlock_irq(&crtc->base.dev->event_lock);
-        if (work == NULL)
-                return;
-
-        intel_mark_page_flip_active(work);
-
-        intel_pipe_update_start(crtc);
-
-        if (INTEL_INFO(mmio_flip->i915)->gen >= 9)
-                skl_do_mmio_flip(crtc, mmio_flip->rotation, work);
-        else
-                /* use_mmio_flip() retricts MMIO flips to ilk+ */
-                ilk_do_mmio_flip(crtc, work);
-
-        intel_pipe_update_end(crtc);
-}
-
-static void intel_mmio_flip_work_func(struct work_struct *work)
+static void intel_mmio_flip_work_func(struct work_struct *w)
 {
-        struct intel_mmio_flip *mmio_flip =
-                container_of(work, struct intel_mmio_flip, work);
+        struct intel_flip_work *work =
+                container_of(w, struct intel_flip_work, mmio_work);
+        struct intel_crtc *crtc = to_intel_crtc(work->crtc);
+        struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
         struct intel_framebuffer *intel_fb =
-                to_intel_framebuffer(mmio_flip->crtc->base.primary->fb);
+                to_intel_framebuffer(crtc->base.primary->fb);
         struct drm_i915_gem_object *obj = intel_fb->obj;
+        struct reservation_object *resv;
 
-        if (mmio_flip->req) {
-                WARN_ON(__i915_wait_request(mmio_flip->req,
+        if (work->flip_queued_req)
+                WARN_ON(__i915_wait_request(work->flip_queued_req,
                                             false, NULL,
-                                            &mmio_flip->i915->rps.mmioflips));
-                i915_gem_request_unreference__unlocked(mmio_flip->req);
-        }
+                                            &dev_priv->rps.mmioflips));
 
         /* For framebuffer backed by dmabuf, wait for fence */
-        if (obj->base.dma_buf)
-                WARN_ON(reservation_object_wait_timeout_rcu(obj->base.dma_buf->resv,
-                                                            false, false,
+        resv = i915_gem_object_get_dmabuf_resv(obj);
+        if (resv)
+                WARN_ON(reservation_object_wait_timeout_rcu(resv, false, false,
                                                             MAX_SCHEDULE_TIMEOUT) < 0);
 
-        intel_do_mmio_flip(mmio_flip);
-        kfree(mmio_flip);
-}
-
-static int intel_queue_mmio_flip(struct drm_device *dev,
-                                 struct drm_crtc *crtc,
-                                 struct drm_i915_gem_object *obj)
-{
-        struct intel_mmio_flip *mmio_flip;
-
-        mmio_flip = kmalloc(sizeof(*mmio_flip), GFP_KERNEL);
-        if (mmio_flip == NULL)
-                return -ENOMEM;
-
-        mmio_flip->i915 = to_i915(dev);
-        mmio_flip->req = i915_gem_request_reference(obj->last_write_req);
-        mmio_flip->crtc = to_intel_crtc(crtc);
-        mmio_flip->rotation = crtc->primary->state->rotation;
+        intel_pipe_update_start(crtc);
 
-        INIT_WORK(&mmio_flip->work, intel_mmio_flip_work_func);
-        schedule_work(&mmio_flip->work);
+        if (INTEL_GEN(dev_priv) >= 9)
+                skl_do_mmio_flip(crtc, work->rotation, work);
+        else
+                /* use_mmio_flip() retricts MMIO flips to ilk+ */
+                ilk_do_mmio_flip(crtc, work);
 
-        return 0;
+        intel_pipe_update_end(crtc, work);
 }
 
 static int intel_default_queue_flip(struct drm_device *dev,
@@ -11462,37 +11502,32 @@ static int intel_default_queue_flip(struct drm_device *dev,
         return -ENODEV;
 }
 
-static bool __intel_pageflip_stall_check(struct drm_device *dev,
-                                         struct drm_crtc *crtc)
+static bool __pageflip_stall_check_cs(struct drm_i915_private *dev_priv,
+                                      struct intel_crtc *intel_crtc,
+                                      struct intel_flip_work *work)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
-        struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
-        struct intel_unpin_work *work = intel_crtc->unpin_work;
-        u32 addr;
-
-        if (atomic_read(&work->pending) >= INTEL_FLIP_COMPLETE)
-                return true;
+        u32 addr, vblank;
 
-        if (atomic_read(&work->pending) < INTEL_FLIP_PENDING)
+        if (!atomic_read(&work->pending))
                 return false;
 
-        if (!work->enable_stall_check)
-                return false;
+        smp_rmb();
 
+        vblank = intel_crtc_get_vblank_counter(intel_crtc);
         if (work->flip_ready_vblank == 0) {
                 if (work->flip_queued_req &&
-                    !i915_gem_request_completed(work->flip_queued_req, true))
+                    !i915_gem_request_completed(work->flip_queued_req))
                         return false;
 
-                work->flip_ready_vblank = drm_crtc_vblank_count(crtc);
+                work->flip_ready_vblank = vblank;
         }
 
-        if (drm_crtc_vblank_count(crtc) - work->flip_ready_vblank < 3)
+        if (vblank - work->flip_ready_vblank < 3)
                 return false;
 
         /* Potential stall - if we see that the flip has happened,
          * assume a missed interrupt. */
-        if (INTEL_INFO(dev)->gen >= 4)
+        if (INTEL_GEN(dev_priv) >= 4)
                 addr = I915_HI_DISPBASE(I915_READ(DSPSURF(intel_crtc->plane)));
         else
                 addr = I915_READ(DSPADDR(intel_crtc->plane));
@@ -11504,12 +11539,12 @@ static bool __intel_pageflip_stall_check(struct drm_device *dev,
         return addr == work->gtt_offset;
 }
 
-void intel_check_page_flip(struct drm_device *dev, int pipe)
+void intel_check_page_flip(struct drm_i915_private *dev_priv, int pipe)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_device *dev = &dev_priv->drm;
         struct drm_crtc *crtc = dev_priv->pipe_to_crtc_mapping[pipe];
         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
-        struct intel_unpin_work *work;
+        struct intel_flip_work *work;
 
         WARN_ON(!in_interrupt());
 
@@ -11517,16 +11552,20 @@ void intel_check_page_flip(struct drm_device *dev, int pipe)
                 return;
 
         spin_lock(&dev->event_lock);
-        work = intel_crtc->unpin_work;
-        if (work != NULL && __intel_pageflip_stall_check(dev, crtc)) {
-                WARN_ONCE(1, "Kicking stuck page flip: queued at %d, now %d\n",
-                         work->flip_queued_vblank, drm_vblank_count(dev, pipe));
+        work = intel_crtc->flip_work;
+
+        if (work != NULL && !is_mmio_work(work) &&
+            __pageflip_stall_check_cs(dev_priv, intel_crtc, work)) {
+                WARN_ONCE(1,
+                          "Kicking stuck page flip: queued at %d, now %d\n",
+                        work->flip_queued_vblank, intel_crtc_get_vblank_counter(intel_crtc));
                 page_flip_completed(intel_crtc);
                 work = NULL;
         }
-        if (work != NULL &&
-            drm_vblank_count(dev, pipe) - work->flip_queued_vblank > 1)
-                intel_queue_rps_boost_for_request(dev, work->flip_queued_req);
+
+        if (work != NULL && !is_mmio_work(work) &&
+            intel_crtc_get_vblank_counter(intel_crtc) - work->flip_queued_vblank > 1)
+                intel_queue_rps_boost_for_request(work->flip_queued_req);
         spin_unlock(&dev->event_lock);
 }
 
@@ -11536,13 +11575,13 @@ static int intel_crtc_page_flip(struct drm_crtc *crtc,
                                 uint32_t page_flip_flags)
 {
         struct drm_device *dev = crtc->dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         struct drm_framebuffer *old_fb = crtc->primary->fb;
         struct drm_i915_gem_object *obj = intel_fb_obj(fb);
         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
         struct drm_plane *primary = crtc->primary;
         enum pipe pipe = intel_crtc->pipe;
-        struct intel_unpin_work *work;
+        struct intel_flip_work *work;
         struct intel_engine_cs *engine;
         bool mmio_flip;
         struct drm_i915_gem_request *request = NULL;
@@ -11579,19 +11618,19 @@ static int intel_crtc_page_flip(struct drm_crtc *crtc,
         work->event = event;
         work->crtc = crtc;
         work->old_fb = old_fb;
-        INIT_WORK(&work->work, intel_unpin_work_fn);
+        INIT_WORK(&work->unpin_work, intel_unpin_work_fn);
 
         ret = drm_crtc_vblank_get(crtc);
         if (ret)
                 goto free_work;
 
-        /* We borrow the event spin lock for protecting unpin_work */
+        /* We borrow the event spin lock for protecting flip_work */
         spin_lock_irq(&dev->event_lock);
-        if (intel_crtc->unpin_work) {
+        if (intel_crtc->flip_work) {
                 /* Before declaring the flip queue wedged, check if
                  * the hardware completed the operation behind our backs.
                  */
-                if (__intel_pageflip_stall_check(dev, crtc)) {
+                if (pageflip_finished(intel_crtc, intel_crtc->flip_work)) {
                         DRM_DEBUG_DRIVER("flip queue: previous flip completed, continuing\n");
                         page_flip_completed(intel_crtc);
                 } else {
@@ -11603,7 +11642,7 @@ static int intel_crtc_page_flip(struct drm_crtc *crtc,
                         return -EBUSY;
                 }
         }
-        intel_crtc->unpin_work = work;
+        intel_crtc->flip_work = work;
         spin_unlock_irq(&dev->event_lock);
 
         if (atomic_read(&intel_crtc->unpin_work_count) >= 2)
@@ -11615,7 +11654,9 @@ static int intel_crtc_page_flip(struct drm_crtc *crtc,
 
         crtc->primary->fb = fb;
         update_state_fb(crtc->primary);
-        intel_fbc_pre_update(intel_crtc);
+
+        intel_fbc_pre_update(intel_crtc, intel_crtc->config,
+                             to_intel_plane_state(primary->state));
 
         work->pending_flip_obj = obj;
 
@@ -11658,6 +11699,11 @@ static int intel_crtc_page_flip(struct drm_crtc *crtc,
          */
         if (!mmio_flip) {
                 ret = i915_gem_object_sync(obj, engine, &request);
+                if (!ret && !request) {
+                        request = i915_gem_request_alloc(engine, NULL);
+                        ret = PTR_ERR_OR_ZERO(request);
+                }
+
                 if (ret)
                         goto cleanup_pending;
         }
@@ -11669,38 +11715,28 @@ static int intel_crtc_page_flip(struct drm_crtc *crtc,
         work->gtt_offset = intel_plane_obj_offset(to_intel_plane(primary),
                                                   obj, 0);
         work->gtt_offset += intel_crtc->dspaddr_offset;
+        work->rotation = crtc->primary->state->rotation;
 
         if (mmio_flip) {
-                ret = intel_queue_mmio_flip(dev, crtc, obj);
-                if (ret)
-                        goto cleanup_unpin;
+                INIT_WORK(&work->mmio_work, intel_mmio_flip_work_func);
 
                 i915_gem_request_assign(&work->flip_queued_req,
                                         obj->last_write_req);
-        } else {
-                if (!request) {
-                        request = i915_gem_request_alloc(engine, NULL);
-                        if (IS_ERR(request)) {
-                                ret = PTR_ERR(request);
-                                goto cleanup_unpin;
-                        }
-                }
 
+                schedule_work(&work->mmio_work);
+        } else {
+                i915_gem_request_assign(&work->flip_queued_req, request);
                 ret = dev_priv->display.queue_flip(dev, crtc, fb, obj, request,
                                                    page_flip_flags);
                 if (ret)
                         goto cleanup_unpin;
 
-                i915_gem_request_assign(&work->flip_queued_req, request);
-        }
+                intel_mark_page_flip_active(intel_crtc, work);
 
-        if (request)
                 i915_add_request_no_flush(request);
+        }
 
-        work->flip_queued_vblank = drm_crtc_vblank_count(crtc);
-        work->enable_stall_check = true;
-
-        i915_gem_track_fb(intel_fb_obj(work->old_fb), obj,
+        i915_gem_track_fb(intel_fb_obj(old_fb), obj,
                           to_intel_plane(primary)->frontbuffer_bit);
         mutex_unlock(&dev->struct_mutex);
 
@@ -11726,7 +11762,7 @@ cleanup:
         drm_framebuffer_unreference(work->old_fb);
 
         spin_lock_irq(&dev->event_lock);
-        intel_crtc->unpin_work = NULL;
+        intel_crtc->flip_work = NULL;
         spin_unlock_irq(&dev->event_lock);
 
         drm_crtc_vblank_put(crtc);
@@ -11828,15 +11864,14 @@ int intel_plane_atomic_calc_changes(struct drm_crtc_state *crtc_state,
         struct drm_i915_private *dev_priv = to_i915(dev);
         struct intel_plane_state *old_plane_state =
                 to_intel_plane_state(plane->state);
-        int idx = intel_crtc->base.base.id, ret;
         bool mode_changed = needs_modeset(crtc_state);
         bool was_crtc_enabled = crtc->state->active;
         bool is_crtc_enabled = crtc_state->active;
         bool turn_off, turn_on, visible, was_visible;
         struct drm_framebuffer *fb = plane_state->fb;
+        int ret;
 
-        if (crtc_state && INTEL_INFO(dev)->gen >= 9 &&
-            plane->type != DRM_PLANE_TYPE_CURSOR) {
+        if (INTEL_GEN(dev) >= 9 && plane->type != DRM_PLANE_TYPE_CURSOR) {
                 ret = skl_update_scaler_plane(
                         to_intel_crtc_state(crtc_state),
                         to_intel_plane_state(plane_state));
@@ -11854,6 +11889,11 @@ int intel_plane_atomic_calc_changes(struct drm_crtc_state *crtc_state,
          * Visibility is calculated as if the crtc was on, but
          * after scaler setup everything depends on it being off
          * when the crtc isn't active.
+         *
+         * FIXME this is wrong for watermarks. Watermarks should also
+         * be computed as if the pipe would be active. Perhaps move
+         * per-plane wm computation to the .check_plane() hook, and
+         * only combine the results from all planes in the current place?
          */
         if (!is_crtc_enabled)
                 to_intel_plane_state(plane_state)->visible = visible = false;
@@ -11867,11 +11907,15 @@ int intel_plane_atomic_calc_changes(struct drm_crtc_state *crtc_state,
         turn_off = was_visible && (!visible || mode_changed);
         turn_on = visible && (!was_visible || mode_changed);
 
-        DRM_DEBUG_ATOMIC("[CRTC:%i] has [PLANE:%i] with fb %i\n", idx,
-                         plane->base.id, fb ? fb->base.id : -1);
+        DRM_DEBUG_ATOMIC("[CRTC:%d:%s] has [PLANE:%d:%s] with fb %i\n",
+                         intel_crtc->base.base.id,
+                         intel_crtc->base.name,
+                         plane->base.id, plane->name,
+                         fb ? fb->base.id : -1);
 
-        DRM_DEBUG_ATOMIC("[PLANE:%i] visible %i -> %i, off %i, on %i, ms %i\n",
-                         plane->base.id, was_visible, visible,
+        DRM_DEBUG_ATOMIC("[PLANE:%d:%s] visible %i -> %i, off %i, on %i, ms %i\n",
+                         plane->base.id, plane->name,
+                         was_visible, visible,
                          turn_off, turn_on, mode_changed);
 
         if (turn_on) {
@@ -11944,31 +11988,11 @@ static bool check_single_encoder_cloning(struct drm_atomic_state *state,
         return true;
 }
 
-static bool check_encoder_cloning(struct drm_atomic_state *state,
-                                  struct intel_crtc *crtc)
-{
-        struct intel_encoder *encoder;
-        struct drm_connector *connector;
-        struct drm_connector_state *connector_state;
-        int i;
-
-        for_each_connector_in_state(state, connector, connector_state, i) {
-                if (connector_state->crtc != &crtc->base)
-                        continue;
-
-                encoder = to_intel_encoder(connector_state->best_encoder);
-                if (!check_single_encoder_cloning(state, crtc, encoder))
-                        return false;
-        }
-
-        return true;
-}
-
 static int intel_crtc_atomic_check(struct drm_crtc *crtc,
                                    struct drm_crtc_state *crtc_state)
 {
         struct drm_device *dev = crtc->dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
         struct intel_crtc_state *pipe_config =
                 to_intel_crtc_state(crtc_state);
@@ -11976,11 +12000,6 @@ static int intel_crtc_atomic_check(struct drm_crtc *crtc,
         int ret;
         bool mode_changed = needs_modeset(crtc_state);
 
-        if (mode_changed && !check_encoder_cloning(state, intel_crtc)) {
-                DRM_DEBUG_KMS("rejecting invalid cloning configuration\n");
-                return -EINVAL;
-        }
-
         if (mode_changed && !crtc_state->active)
                 pipe_config->update_wm_post = true;
 
@@ -12033,7 +12052,7 @@ static int intel_crtc_atomic_check(struct drm_crtc *crtc,
                 }
         } else if (dev_priv->display.compute_intermediate_wm) {
                 if (HAS_PCH_SPLIT(dev_priv) && INTEL_GEN(dev_priv) < 9)
-                        pipe_config->wm.intermediate = pipe_config->wm.optimal.ilk;
+                        pipe_config->wm.ilk.intermediate = pipe_config->wm.ilk.optimal;
         }
 
         if (INTEL_INFO(dev)->gen >= 9) {
@@ -12168,7 +12187,8 @@ static void intel_dump_pipe_config(struct intel_crtc *crtc,
         struct intel_plane_state *state;
         struct drm_framebuffer *fb;
 
-        DRM_DEBUG_KMS("[CRTC:%d]%s config %p for pipe %c\n", crtc->base.base.id,
+        DRM_DEBUG_KMS("[CRTC:%d:%s]%s config %p for pipe %c\n",
+                      crtc->base.base.id, crtc->base.name,
                       context, pipe_config, pipe_name(crtc->pipe));
 
         DRM_DEBUG_KMS("cpu_transcoder: %s\n", transcoder_name(pipe_config->cpu_transcoder));
@@ -12181,14 +12201,14 @@ static void intel_dump_pipe_config(struct intel_crtc *crtc,
                       pipe_config->fdi_m_n.link_m, pipe_config->fdi_m_n.link_n,
                       pipe_config->fdi_m_n.tu);
         DRM_DEBUG_KMS("dp: %i, lanes: %i, gmch_m: %u, gmch_n: %u, link_m: %u, link_n: %u, tu: %u\n",
-                      pipe_config->has_dp_encoder,
+                      intel_crtc_has_dp_encoder(pipe_config),
                       pipe_config->lane_count,
                       pipe_config->dp_m_n.gmch_m, pipe_config->dp_m_n.gmch_n,
                       pipe_config->dp_m_n.link_m, pipe_config->dp_m_n.link_n,
                       pipe_config->dp_m_n.tu);
 
         DRM_DEBUG_KMS("dp: %i, lanes: %i, gmch_m2: %u, gmch_n2: %u, link_m2: %u, link_n2: %u, tu2: %u\n",
-                      pipe_config->has_dp_encoder,
+                      intel_crtc_has_dp_encoder(pipe_config),
                       pipe_config->lane_count,
                       pipe_config->dp_m2_n2.gmch_m,
                       pipe_config->dp_m2_n2.gmch_n,
@@ -12269,29 +12289,24 @@ static void intel_dump_pipe_config(struct intel_crtc *crtc,
                 state = to_intel_plane_state(plane->state);
                 fb = state->base.fb;
                 if (!fb) {
-                        DRM_DEBUG_KMS("%s PLANE:%d plane: %u.%u idx: %d "
-                                "disabled, scaler_id = %d\n",
-                                plane->type == DRM_PLANE_TYPE_CURSOR ? "CURSOR" : "STANDARD",
-                                plane->base.id, intel_plane->pipe,
-                                (crtc->base.primary == plane) ? 0 : intel_plane->plane + 1,
-                                drm_plane_index(plane), state->scaler_id);
+                        DRM_DEBUG_KMS("[PLANE:%d:%s] disabled, scaler_id = %d\n",
+                                      plane->base.id, plane->name, state->scaler_id);
                         continue;
                 }
 
-                DRM_DEBUG_KMS("%s PLANE:%d plane: %u.%u idx: %d enabled",
-                        plane->type == DRM_PLANE_TYPE_CURSOR ? "CURSOR" : "STANDARD",
-                        plane->base.id, intel_plane->pipe,
-                        crtc->base.primary == plane ? 0 : intel_plane->plane + 1,
-                        drm_plane_index(plane));
-                DRM_DEBUG_KMS("\tFB:%d, fb = %ux%u format = 0x%x",
-                        fb->base.id, fb->width, fb->height, fb->pixel_format);
-                DRM_DEBUG_KMS("\tscaler:%d src (%u, %u) %ux%u dst (%u, %u) %ux%u\n",
-                        state->scaler_id,
-                        state->src.x1 >> 16, state->src.y1 >> 16,
-                        drm_rect_width(&state->src) >> 16,
-                        drm_rect_height(&state->src) >> 16,
-                        state->dst.x1, state->dst.y1,
-                        drm_rect_width(&state->dst), drm_rect_height(&state->dst));
+                DRM_DEBUG_KMS("[PLANE:%d:%s] enabled",
+                              plane->base.id, plane->name);
+                DRM_DEBUG_KMS("\tFB:%d, fb = %ux%u format = %s",
+                              fb->base.id, fb->width, fb->height,
+                              drm_get_format_name(fb->pixel_format));
+                DRM_DEBUG_KMS("\tscaler:%d src %dx%d+%d+%d dst %dx%d+%d+%d\n",
+                              state->scaler_id,
+                              state->src.x1 >> 16, state->src.y1 >> 16,
+                              drm_rect_width(&state->src) >> 16,
+                              drm_rect_height(&state->src) >> 16,
+                              state->dst.x1, state->dst.y1,
+                              drm_rect_width(&state->dst),
+                              drm_rect_height(&state->dst));
         }
 }
 
@@ -12326,7 +12341,7 @@ static bool check_digital_port_conflicts(struct drm_atomic_state *state)
                 case INTEL_OUTPUT_UNKNOWN:
                         if (WARN_ON(!HAS_DDI(dev)))
                                 break;
-                case INTEL_OUTPUT_DISPLAYPORT:
+                case INTEL_OUTPUT_DP:
                 case INTEL_OUTPUT_HDMI:
                 case INTEL_OUTPUT_EDP:
                         port_mask = 1 << enc_to_dig_port(&encoder->base)->port;
@@ -12423,6 +12438,24 @@ intel_modeset_pipe_config(struct drm_crtc *crtc,
                                &pipe_config->pipe_src_w,
                                &pipe_config->pipe_src_h);
 
+        for_each_connector_in_state(state, connector, connector_state, i) {
+                if (connector_state->crtc != crtc)
+                        continue;
+
+                encoder = to_intel_encoder(connector_state->best_encoder);
+
+                if (!check_single_encoder_cloning(state, to_intel_crtc(crtc), encoder)) {
+                        DRM_DEBUG_KMS("rejecting invalid cloning configuration\n");
+                        goto fail;
+                }
+
+                /*
+                 * Determine output_types before calling the .compute_config()
+                 * hooks so that the hooks can use this information safely.
+                 */
+                pipe_config->output_types |= 1 << encoder->type;
+        }
+
 encoder_retry:
         /* Ensure the port clock defaults are reset when retrying. */
         pipe_config->port_clock = 0;
@@ -12708,8 +12741,8 @@ intel_pipe_config_compare(struct drm_device *dev,
         PIPE_CONF_CHECK_I(fdi_lanes);
         PIPE_CONF_CHECK_M_N(fdi_m_n);
 
-        PIPE_CONF_CHECK_I(has_dp_encoder);
         PIPE_CONF_CHECK_I(lane_count);
+        PIPE_CONF_CHECK_X(lane_lat_optim_mask);
 
         if (INTEL_INFO(dev)->gen < 8) {
                 PIPE_CONF_CHECK_M_N(dp_m_n);
@@ -12719,7 +12752,7 @@ intel_pipe_config_compare(struct drm_device *dev,
         } else
                 PIPE_CONF_CHECK_M_N_ALT(dp_m_n, dp_m2_n2);
 
-        PIPE_CONF_CHECK_I(has_dsi_encoder);
+        PIPE_CONF_CHECK_X(output_types);
 
         PIPE_CONF_CHECK_I(base.adjusted_mode.crtc_hdisplay);
         PIPE_CONF_CHECK_I(base.adjusted_mode.crtc_htotal);
@@ -12838,7 +12871,7 @@ static void verify_wm_state(struct drm_crtc *crtc,
                             struct drm_crtc_state *new_state)
 {
         struct drm_device *dev = crtc->dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         struct skl_ddb_allocation hw_ddb, *sw_ddb;
         struct skl_ddb_entry *hw_entry, *sw_entry;
         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
@@ -12944,7 +12977,7 @@ verify_crtc_state(struct drm_crtc *crtc,
                   struct drm_crtc_state *new_crtc_state)
 {
         struct drm_device *dev = crtc->dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         struct intel_encoder *encoder;
         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
         struct intel_crtc_state *pipe_config, *sw_config;
@@ -12958,7 +12991,7 @@ verify_crtc_state(struct drm_crtc *crtc,
         pipe_config->base.crtc = crtc;
         pipe_config->base.state = old_state;
 
-        DRM_DEBUG_KMS("[CRTC:%d]\n", crtc->base.id);
+        DRM_DEBUG_KMS("[CRTC:%d:%s]\n", crtc->base.id, crtc->name);
 
         active = dev_priv->display.get_pipe_config(intel_crtc, pipe_config);
 
@@ -12987,8 +13020,10 @@ verify_crtc_state(struct drm_crtc *crtc,
                                 "Encoder connected to wrong pipe %c\n",
                                 pipe_name(pipe));
 
-                if (active)
+                if (active) {
+                        pipe_config->output_types |= 1 << encoder->type;
                         encoder->get_config(encoder, pipe_config);
+                }
         }
 
         if (!new_crtc_state->active)
@@ -13067,7 +13102,7 @@ verify_shared_dpll_state(struct drm_device *dev, struct drm_crtc *crtc,
                          struct drm_crtc_state *old_crtc_state,
                          struct drm_crtc_state *new_crtc_state)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         struct intel_crtc_state *old_state = to_intel_crtc_state(old_crtc_state);
         struct intel_crtc_state *new_state = to_intel_crtc_state(new_crtc_state);
 
@@ -13106,7 +13141,7 @@ intel_modeset_verify_crtc(struct drm_crtc *crtc,
 static void
 verify_disabled_dpll_state(struct drm_device *dev)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         int i;
 
         for (i = 0; i < dev_priv->num_shared_dpll; i++)
@@ -13153,7 +13188,7 @@ static void update_scanline_offset(struct intel_crtc *crtc)
 
                 crtc->scanline_offset = vtotal - 1;
         } else if (HAS_DDI(dev) &&
-                   intel_pipe_has_type(crtc, INTEL_OUTPUT_HDMI)) {
+                   intel_crtc_has_type(crtc->config, INTEL_OUTPUT_HDMI)) {
                 crtc->scanline_offset = 2;
         } else
                 crtc->scanline_offset = 1;
@@ -13288,7 +13323,7 @@ static int intel_modeset_all_pipes(struct drm_atomic_state *state)
 static int intel_modeset_checks(struct drm_atomic_state *state)
 {
         struct intel_atomic_state *intel_state = to_intel_atomic_state(state);
-        struct drm_i915_private *dev_priv = state->dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(state->dev);
         struct drm_crtc *crtc;
         struct drm_crtc_state *crtc_state;
         int ret = 0, i;
@@ -13306,6 +13341,9 @@ static int intel_modeset_checks(struct drm_atomic_state *state)
                         intel_state->active_crtcs |= 1 << i;
                 else
                         intel_state->active_crtcs &= ~(1 << i);
+
+                if (crtc_state->active != crtc->state->active)
+                        intel_state->active_pipe_changes |= drm_crtc_mask(crtc);
         }
 
         /*
@@ -13316,9 +13354,17 @@ static int intel_modeset_checks(struct drm_atomic_state *state)
          * adjusted_mode bits in the crtc directly.
          */
         if (dev_priv->display.modeset_calc_cdclk) {
+                if (!intel_state->cdclk_pll_vco)
+                        intel_state->cdclk_pll_vco = dev_priv->cdclk_pll.vco;
+                if (!intel_state->cdclk_pll_vco)
+                        intel_state->cdclk_pll_vco = dev_priv->skl_preferred_vco_freq;
+
                 ret = dev_priv->display.modeset_calc_cdclk(state);
+                if (ret < 0)
+                        return ret;
 
-                if (!ret && intel_state->dev_cdclk != dev_priv->cdclk_freq)
+                if (intel_state->dev_cdclk != dev_priv->cdclk_freq ||
+                    intel_state->cdclk_pll_vco != dev_priv->cdclk_pll.vco)
                         ret = intel_modeset_all_pipes(state);
 
                 if (ret < 0)
@@ -13342,38 +13388,16 @@ static int intel_modeset_checks(struct drm_atomic_state *state)
  * phase.  The code here should be run after the per-crtc and per-plane 'check'
  * handlers to ensure that all derived state has been updated.
  */
-static void calc_watermark_data(struct drm_atomic_state *state)
+static int calc_watermark_data(struct drm_atomic_state *state)
 {
         struct drm_device *dev = state->dev;
-        struct intel_atomic_state *intel_state = to_intel_atomic_state(state);
-        struct drm_crtc *crtc;
-        struct drm_crtc_state *cstate;
-        struct drm_plane *plane;
-        struct drm_plane_state *pstate;
-
-        /*
-         * Calculate watermark configuration details now that derived
-         * plane/crtc state is all properly updated.
-         */
-        drm_for_each_crtc(crtc, dev) {
-                cstate = drm_atomic_get_existing_crtc_state(state, crtc) ?:
-                        crtc->state;
-
-                if (cstate->active)
-                        intel_state->wm_config.num_pipes_active++;
-        }
-        drm_for_each_legacy_plane(plane, dev) {
-                pstate = drm_atomic_get_existing_plane_state(state, plane) ?:
-                        plane->state;
+        struct drm_i915_private *dev_priv = to_i915(dev);
 
-                if (!to_intel_plane_state(pstate)->visible)
-                        continue;
+        /* Is there platform-specific watermark information to calculate? */
+        if (dev_priv->display.compute_global_watermarks)
+                return dev_priv->display.compute_global_watermarks(state);
 
-                intel_state->wm_config.sprites_enabled = true;
-                if (pstate->crtc_w != pstate->src_w >> 16 ||
-                    pstate->crtc_h != pstate->src_h >> 16)
-                        intel_state->wm_config.sprites_scaled = true;
-        }
+        return 0;
 }
 
 /**
@@ -13403,14 +13427,13 @@ static int intel_atomic_check(struct drm_device *dev,
                 if (crtc_state->mode.private_flags != crtc->state->mode.private_flags)
                         crtc_state->mode_changed = true;
 
-                if (!crtc_state->enable) {
-                        if (needs_modeset(crtc_state))
-                                any_ms = true;
+                if (!needs_modeset(crtc_state))
                         continue;
-                }
 
-                if (!needs_modeset(crtc_state))
+                if (!crtc_state->enable) {
+                        any_ms = true;
                         continue;
+                }
 
                 /* FIXME: For only active_changed we shouldn't need to do any
                  * state recomputation at all. */
@@ -13420,8 +13443,11 @@ static int intel_atomic_check(struct drm_device *dev,
                         return ret;
 
                 ret = intel_modeset_pipe_config(crtc, pipe_config);
-                if (ret)
+                if (ret) {
+                        intel_dump_pipe_config(to_intel_crtc(crtc),
+                                               pipe_config, "[failed]");
                         return ret;
+                }
 
                 if (i915.fastboot &&
                     intel_pipe_config_compare(dev,
@@ -13431,13 +13457,12 @@ static int intel_atomic_check(struct drm_device *dev,
                         to_intel_crtc_state(crtc_state)->update_pipe = true;
                 }
 
-                if (needs_modeset(crtc_state)) {
+                if (needs_modeset(crtc_state))
                         any_ms = true;
 
-                        ret = drm_atomic_add_affected_planes(state, crtc);
-                        if (ret)
-                                return ret;
-                }
+                ret = drm_atomic_add_affected_planes(state, crtc);
+                if (ret)
+                        return ret;
 
                 intel_dump_pipe_config(to_intel_crtc(crtc), pipe_config,
                                        needs_modeset(crtc_state) ?
@@ -13457,27 +13482,20 @@ static int intel_atomic_check(struct drm_device *dev,
                 return ret;
 
         intel_fbc_choose_crtc(dev_priv, state);
-        calc_watermark_data(state);
-
-        return 0;
+        return calc_watermark_data(state);
 }
 
 static int intel_atomic_prepare_commit(struct drm_device *dev,
                                        struct drm_atomic_state *state,
                                        bool nonblock)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         struct drm_plane_state *plane_state;
         struct drm_crtc_state *crtc_state;
         struct drm_plane *plane;
         struct drm_crtc *crtc;
         int i, ret;
 
-        if (nonblock) {
-                DRM_DEBUG_KMS("i915 does not yet support nonblocking commit\n");
-                return -EINVAL;
-        }
-
         for_each_crtc_in_state(state, crtc, crtc_state, i) {
                 if (state->legacy_cursor_update)
                         continue;
@@ -13521,6 +13539,16 @@ static int intel_atomic_prepare_commit(struct drm_device *dev,
         return ret;
 }
 
+u32 intel_crtc_get_vblank_counter(struct intel_crtc *crtc)
+{
+        struct drm_device *dev = crtc->base.dev;
+
+        if (!dev->max_vblank_count)
+                return drm_accurate_vblank_count(&crtc->base);
+
+        return dev->driver->get_vblank_counter(dev, crtc->pipe);
+}
+
 static void intel_atomic_wait_for_vblanks(struct drm_device *dev,
                                           struct drm_i915_private *dev_priv,
                                           unsigned crtc_mask)
@@ -13586,45 +13614,36 @@ static bool needs_vblank_wait(struct intel_crtc_state *crtc_state)
         return false;
 }
 
-/**
- * intel_atomic_commit - commit validated state object
- * @dev: DRM device
- * @state: the top-level driver state object
- * @nonblock: nonblocking commit
- *
- * This function commits a top-level state object that has been validated
- * with drm_atomic_helper_check().
- *
- * FIXME:  Atomic modeset support for i915 is not yet complete.  At the moment
- * we can only handle plane-related operations and do not yet support
- * nonblocking commit.
- *
- * RETURNS
- * Zero for success or -errno.
- */
-static int intel_atomic_commit(struct drm_device *dev,
-                               struct drm_atomic_state *state,
-                               bool nonblock)
+static void intel_atomic_commit_tail(struct drm_atomic_state *state)
 {
+        struct drm_device *dev = state->dev;
         struct intel_atomic_state *intel_state = to_intel_atomic_state(state);
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         struct drm_crtc_state *old_crtc_state;
         struct drm_crtc *crtc;
         struct intel_crtc_state *intel_cstate;
-        int ret = 0, i;
+        struct drm_plane *plane;
+        struct drm_plane_state *plane_state;
         bool hw_check = intel_state->modeset;
         unsigned long put_domains[I915_MAX_PIPES] = {};
         unsigned crtc_vblank_mask = 0;
+        int i, ret;
 
-        ret = intel_atomic_prepare_commit(dev, state, nonblock);
-        if (ret) {
-                DRM_DEBUG_ATOMIC("Preparing state failed with %i\n", ret);
-                return ret;
+        for_each_plane_in_state(state, plane, plane_state, i) {
+                struct intel_plane_state *intel_plane_state =
+                        to_intel_plane_state(plane_state);
+
+                if (!intel_plane_state->wait_req)
+                        continue;
+
+                ret = __i915_wait_request(intel_plane_state->wait_req,
+                                          true, NULL, NULL);
+                /* EIO should be eaten, and we can't get interrupted in the
+                 * worker, and blocking commits have waited already. */
+                WARN_ON(ret);
         }
 
-        drm_atomic_helper_swap_state(dev, state);
-        dev_priv->wm.config = intel_state->wm_config;
-        intel_shared_dpll_commit(state);
+        drm_atomic_helper_wait_for_dependencies(state);
 
         if (intel_state->modeset) {
                 memcpy(dev_priv->min_pixclk, intel_state->min_pixclk,
@@ -13679,7 +13698,8 @@ static int intel_atomic_commit(struct drm_device *dev,
                 drm_atomic_helper_update_legacy_modeset_state(state->dev, state);
 
                 if (dev_priv->display.modeset_commit_cdclk &&
-                    intel_state->dev_cdclk != dev_priv->cdclk_freq)
+                    (intel_state->dev_cdclk != dev_priv->cdclk_freq ||
+                     intel_state->cdclk_pll_vco != dev_priv->cdclk_pll.vco))
                         dev_priv->display.modeset_commit_cdclk(state);
 
                 intel_modeset_verify_disabled(dev);
@@ -13691,30 +13711,44 @@ static int intel_atomic_commit(struct drm_device *dev,
                 bool modeset = needs_modeset(crtc->state);
                 struct intel_crtc_state *pipe_config =
                         to_intel_crtc_state(crtc->state);
-                bool update_pipe = !modeset && pipe_config->update_pipe;
 
                 if (modeset && crtc->state->active) {
                         update_scanline_offset(to_intel_crtc(crtc));
                         dev_priv->display.crtc_enable(crtc);
                 }
 
+                /* Complete events for now disable pipes here. */
+                if (modeset && !crtc->state->active && crtc->state->event) {
+                        spin_lock_irq(&dev->event_lock);
+                        drm_crtc_send_vblank_event(crtc, crtc->state->event);
+                        spin_unlock_irq(&dev->event_lock);
+
+                        crtc->state->event = NULL;
+                }
+
                 if (!modeset)
                         intel_pre_plane_update(to_intel_crtc_state(old_crtc_state));
 
                 if (crtc->state->active &&
                     drm_atomic_get_existing_plane_state(state, crtc->primary))
-                        intel_fbc_enable(intel_crtc);
+                        intel_fbc_enable(intel_crtc, pipe_config, to_intel_plane_state(crtc->primary->state));
 
-                if (crtc->state->active &&
-                    (crtc->state->planes_changed || update_pipe))
+                if (crtc->state->active)
                         drm_atomic_helper_commit_planes_on_crtc(old_crtc_state);
 
                 if (pipe_config->base.active && needs_vblank_wait(pipe_config))
                         crtc_vblank_mask |= 1 << i;
         }
 
-        /* FIXME: add subpixel order */
-
+        /* FIXME: We should call drm_atomic_helper_commit_hw_done() here
+         * already, but still need the state for the delayed optimization. To
+         * fix this:
+         * - wrap the optimization/post_plane_update stuff into a per-crtc work.
+         * - schedule that vblank worker _before_ calling hw_done
+         * - at the start of commit_tail, cancel it _synchrously
+         * - switch over to the vblank wait helper in the core after that since
+         *   we don't need out special handling any more.
+         */
         if (!state->legacy_cursor_update)
                 intel_atomic_wait_for_vblanks(dev, dev_priv, crtc_vblank_mask);
 
@@ -13741,6 +13775,8 @@ static int intel_atomic_commit(struct drm_device *dev,
                 intel_modeset_verify_crtc(crtc, old_crtc_state, crtc->state);
         }
 
+        drm_atomic_helper_commit_hw_done(state);
+
         if (intel_state->modeset)
                 intel_display_power_put(dev_priv, POWER_DOMAIN_MODESET);
 
@@ -13748,6 +13784,8 @@ static int intel_atomic_commit(struct drm_device *dev,
         drm_atomic_helper_cleanup_planes(dev, state);
         mutex_unlock(&dev->struct_mutex);
 
+        drm_atomic_helper_commit_cleanup_done(state);
+
         drm_atomic_state_free(state);
 
         /* As one of the primary mmio accessors, KMS has a high likelihood
@@ -13762,6 +13800,86 @@ static int intel_atomic_commit(struct drm_device *dev,
          * can happen also when the device is completely off.
          */
         intel_uncore_arm_unclaimed_mmio_detection(dev_priv);
+}
+
+static void intel_atomic_commit_work(struct work_struct *work)
+{
+        struct drm_atomic_state *state = container_of(work,
+                                                      struct drm_atomic_state,
+                                                      commit_work);
+        intel_atomic_commit_tail(state);
+}
+
+static void intel_atomic_track_fbs(struct drm_atomic_state *state)
+{
+        struct drm_plane_state *old_plane_state;
+        struct drm_plane *plane;
+        struct drm_i915_gem_object *obj, *old_obj;
+        struct intel_plane *intel_plane;
+        int i;
+
+        mutex_lock(&state->dev->struct_mutex);
+        for_each_plane_in_state(state, plane, old_plane_state, i) {
+                obj = intel_fb_obj(plane->state->fb);
+                old_obj = intel_fb_obj(old_plane_state->fb);
+                intel_plane = to_intel_plane(plane);
+
+                i915_gem_track_fb(old_obj, obj, intel_plane->frontbuffer_bit);
+        }
+        mutex_unlock(&state->dev->struct_mutex);
+}
+
+/**
+ * intel_atomic_commit - commit validated state object
+ * @dev: DRM device
+ * @state: the top-level driver state object
+ * @nonblock: nonblocking commit
+ *
+ * This function commits a top-level state object that has been validated
+ * with drm_atomic_helper_check().
+ *
+ * FIXME:  Atomic modeset support for i915 is not yet complete.  At the moment
+ * nonblocking commits are only safe for pure plane updates. Everything else
+ * should work though.
+ *
+ * RETURNS
+ * Zero for success or -errno.
+ */
+static int intel_atomic_commit(struct drm_device *dev,
+                               struct drm_atomic_state *state,
+                               bool nonblock)
+{
+        struct intel_atomic_state *intel_state = to_intel_atomic_state(state);
+        struct drm_i915_private *dev_priv = to_i915(dev);
+        int ret = 0;
+
+        if (intel_state->modeset && nonblock) {
+                DRM_DEBUG_KMS("nonblocking commit for modeset not yet implemented.\n");
+                return -EINVAL;
+        }
+
+        ret = drm_atomic_helper_setup_commit(state, nonblock);
+        if (ret)
+                return ret;
+
+        INIT_WORK(&state->commit_work, intel_atomic_commit_work);
+
+        ret = intel_atomic_prepare_commit(dev, state, nonblock);
+        if (ret) {
+                DRM_DEBUG_ATOMIC("Preparing state failed with %i\n", ret);
+                return ret;
+        }
+
+        drm_atomic_helper_swap_state(state, true);
+        dev_priv->wm.distrust_bios_wm = false;
+        dev_priv->wm.skl_results = intel_state->wm_results;
+        intel_shared_dpll_commit(state);
+        intel_atomic_track_fbs(state);
+
+        if (nonblock)
+                queue_work(system_unbound_wq, &state->commit_work);
+        else
+                intel_atomic_commit_tail(state);
 
         return 0;
 }
@@ -13775,8 +13893,8 @@ void intel_crtc_restore_mode(struct drm_crtc *crtc)
 
         state = drm_atomic_state_alloc(dev);
         if (!state) {
-                DRM_DEBUG_KMS("[CRTC:%d] crtc restore failed, out of memory",
-                              crtc->base.id);
+                DRM_DEBUG_KMS("[CRTC:%d:%s] crtc restore failed, out of memory",
+                              crtc->base.id, crtc->name);
                 return;
         }
 
@@ -13806,8 +13924,50 @@ out:
 
 #undef for_each_intel_crtc_masked
 
+/*
+ * FIXME: Remove this once i915 is fully DRIVER_ATOMIC by calling
+ *        drm_atomic_helper_legacy_gamma_set() directly.
+ */
+static int intel_atomic_legacy_gamma_set(struct drm_crtc *crtc,
+                                         u16 *red, u16 *green, u16 *blue,
+                                         uint32_t size)
+{
+        struct drm_device *dev = crtc->dev;
+        struct drm_mode_config *config = &dev->mode_config;
+        struct drm_crtc_state *state;
+        int ret;
+
+        ret = drm_atomic_helper_legacy_gamma_set(crtc, red, green, blue, size);
+        if (ret)
+                return ret;
+
+        /*
+         * Make sure we update the legacy properties so this works when
+         * atomic is not enabled.
+         */
+
+        state = crtc->state;
+
+        drm_object_property_set_value(&crtc->base,
+                                      config->degamma_lut_property,
+                                      (state->degamma_lut) ?
+                                      state->degamma_lut->base.id : 0);
+
+        drm_object_property_set_value(&crtc->base,
+                                      config->ctm_property,
+                                      (state->ctm) ?
+                                      state->ctm->base.id : 0);
+
+        drm_object_property_set_value(&crtc->base,
+                                      config->gamma_lut_property,
+                                      (state->gamma_lut) ?
+                                      state->gamma_lut->base.id : 0);
+
+        return 0;
+}
+
 static const struct drm_crtc_funcs intel_crtc_funcs = {
-        .gamma_set = drm_atomic_helper_legacy_gamma_set,
+        .gamma_set = intel_atomic_legacy_gamma_set,
         .set_config = drm_atomic_helper_set_config,
         .set_property = drm_atomic_helper_crtc_set_property,
         .destroy = intel_crtc_destroy,
@@ -13836,9 +13996,9 @@ intel_prepare_plane_fb(struct drm_plane *plane,
 {
         struct drm_device *dev = plane->dev;
         struct drm_framebuffer *fb = new_state->fb;
-        struct intel_plane *intel_plane = to_intel_plane(plane);
         struct drm_i915_gem_object *obj = intel_fb_obj(fb);
         struct drm_i915_gem_object *old_obj = intel_fb_obj(plane->state->fb);
+        struct reservation_object *resv;
         int ret = 0;
 
         if (!obj && !old_obj)
@@ -13868,12 +14028,15 @@ intel_prepare_plane_fb(struct drm_plane *plane,
                 }
         }
 
+        if (!obj)
+                return 0;
+
         /* For framebuffer backed by dmabuf, wait for fence */
-        if (obj && obj->base.dma_buf) {
+        resv = i915_gem_object_get_dmabuf_resv(obj);
+        if (resv) {
                 long lret;
 
-                lret = reservation_object_wait_timeout_rcu(obj->base.dma_buf->resv,
-                                                           false, true,
+                lret = reservation_object_wait_timeout_rcu(resv, false, true,
                                                            MAX_SCHEDULE_TIMEOUT);
                 if (lret == -ERESTARTSYS)
                         return lret;
@@ -13881,9 +14044,7 @@ intel_prepare_plane_fb(struct drm_plane *plane,
                 WARN(lret < 0, "waiting returns %li\n", lret);
         }
 
-        if (!obj) {
-                ret = 0;
-        } else if (plane->type == DRM_PLANE_TYPE_CURSOR &&
+        if (plane->type == DRM_PLANE_TYPE_CURSOR &&
             INTEL_INFO(dev)->cursor_needs_physical) {
                 int align = IS_I830(dev) ? 16 * 1024 : 256;
                 ret = i915_gem_object_attach_phys(obj, align);
@@ -13894,15 +14055,11 @@ intel_prepare_plane_fb(struct drm_plane *plane,
         }
 
         if (ret == 0) {
-                if (obj) {
-                        struct intel_plane_state *plane_state =
-                                to_intel_plane_state(new_state);
+                struct intel_plane_state *plane_state =
+                        to_intel_plane_state(new_state);
 
-                        i915_gem_request_assign(&plane_state->wait_req,
-                                                obj->last_write_req);
-                }
-
-                i915_gem_track_fb(old_obj, obj, intel_plane->frontbuffer_bit);
+                i915_gem_request_assign(&plane_state->wait_req,
+                                        obj->last_write_req);
         }
 
         return ret;
@@ -13922,7 +14079,6 @@ intel_cleanup_plane_fb(struct drm_plane *plane,
                        const struct drm_plane_state *old_state)
 {
         struct drm_device *dev = plane->dev;
-        struct intel_plane *intel_plane = to_intel_plane(plane);
         struct intel_plane_state *old_intel_state;
         struct drm_i915_gem_object *old_obj = intel_fb_obj(old_state->fb);
         struct drm_i915_gem_object *obj = intel_fb_obj(plane->state->fb);
@@ -13936,11 +14092,6 @@ intel_cleanup_plane_fb(struct drm_plane *plane,
             !INTEL_INFO(dev)->cursor_needs_physical))
                 intel_unpin_fb_obj(old_state->fb, old_state->rotation);
 
-        /* prepare_fb aborted? */
-        if ((old_obj && (old_obj->frontbuffer_bits & intel_plane->frontbuffer_bit)) ||
-            (obj && !(obj->frontbuffer_bits & intel_plane->frontbuffer_bit)))
-                i915_gem_track_fb(old_obj, obj, intel_plane->frontbuffer_bit);
-
         i915_gem_request_assign(&old_intel_state->wait_req, NULL);
 }
 
@@ -13948,15 +14099,11 @@ int
 skl_max_scale(struct intel_crtc *intel_crtc, struct intel_crtc_state *crtc_state)
 {
         int max_scale;
-        struct drm_device *dev;
-        struct drm_i915_private *dev_priv;
         int crtc_clock, cdclk;
 
         if (!intel_crtc || !crtc_state->base.enable)
                 return DRM_PLANE_HELPER_NO_SCALING;
 
-        dev = intel_crtc->base.dev;
-        dev_priv = dev->dev_private;
         crtc_clock = crtc_state->base.adjusted_mode.crtc_clock;
         cdclk = to_intel_atomic_state(crtc_state->base.state)->cdclk;
 
@@ -13996,6 +14143,7 @@ intel_check_primary_plane(struct drm_plane *plane,
 
         return drm_plane_helper_check_update(plane, crtc, fb, &state->src,
                                              &state->dst, &state->clip,
+                                             state->base.rotation,
                                              min_scale, max_scale,
                                              can_position, true,
                                              &state->visible);
@@ -14032,7 +14180,7 @@ static void intel_finish_crtc_commit(struct drm_crtc *crtc,
 {
         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
 
-        intel_pipe_update_end(intel_crtc);
+        intel_pipe_update_end(intel_crtc, NULL);
 }
 
 /**
@@ -14044,9 +14192,11 @@ static void intel_finish_crtc_commit(struct drm_crtc *crtc,
  */
 void intel_plane_destroy(struct drm_plane *plane)
 {
-        struct intel_plane *intel_plane = to_intel_plane(plane);
+        if (!plane)
+                return;
+
         drm_plane_cleanup(plane);
-        kfree(intel_plane);
+        kfree(to_intel_plane(plane));
 }
 
 const struct drm_plane_funcs intel_plane_funcs = {
@@ -14118,10 +14268,24 @@ static struct drm_plane *intel_primary_plane_create(struct drm_device *dev,
                 primary->disable_plane = i9xx_disable_primary_plane;
         }
 
-        ret = drm_universal_plane_init(dev, &primary->base, 0,
-                                       &intel_plane_funcs,
-                                       intel_primary_formats, num_formats,
-                                       DRM_PLANE_TYPE_PRIMARY, NULL);
+        if (INTEL_INFO(dev)->gen >= 9)
+                ret = drm_universal_plane_init(dev, &primary->base, 0,
+                                               &intel_plane_funcs,
+                                               intel_primary_formats, num_formats,
+                                               DRM_PLANE_TYPE_PRIMARY,
+                                               "plane 1%c", pipe_name(pipe));
+        else if (INTEL_INFO(dev)->gen >= 5 || IS_G4X(dev))
+                ret = drm_universal_plane_init(dev, &primary->base, 0,
+                                               &intel_plane_funcs,
+                                               intel_primary_formats, num_formats,
+                                               DRM_PLANE_TYPE_PRIMARY,
+                                               "primary %c", pipe_name(pipe));
+        else
+                ret = drm_universal_plane_init(dev, &primary->base, 0,
+                                               &intel_plane_funcs,
+                                               intel_primary_formats, num_formats,
+                                               DRM_PLANE_TYPE_PRIMARY,
+                                               "plane %c", plane_name(primary->plane));
         if (ret)
                 goto fail;
 
@@ -14171,6 +14335,7 @@ intel_check_cursor_plane(struct drm_plane *plane,
 
         ret = drm_plane_helper_check_update(plane, crtc, fb, &state->src,
                                             &state->dst, &state->clip,
+                                            state->base.rotation,
                                             DRM_PLANE_HELPER_NO_SCALING,
                                             DRM_PLANE_HELPER_NO_SCALING,
                                             true, true, &state->visible);
@@ -14279,7 +14444,8 @@ static struct drm_plane *intel_cursor_plane_create(struct drm_device *dev,
                                        &intel_plane_funcs,
                                        intel_cursor_formats,
                                        ARRAY_SIZE(intel_cursor_formats),
-                                       DRM_PLANE_TYPE_CURSOR, NULL);
+                                       DRM_PLANE_TYPE_CURSOR,
+                                       "cursor %c", pipe_name(pipe));
         if (ret)
                 goto fail;
 
@@ -14327,7 +14493,7 @@ static void skl_init_scalers(struct drm_device *dev, struct intel_crtc *intel_cr
 
 static void intel_crtc_init(struct drm_device *dev, int pipe)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         struct intel_crtc *intel_crtc;
         struct intel_crtc_state *crtc_state = NULL;
         struct drm_plane *primary = NULL;
@@ -14364,7 +14530,8 @@ static void intel_crtc_init(struct drm_device *dev, int pipe)
                 goto fail;
 
         ret = drm_crtc_init_with_planes(dev, &intel_crtc->base, primary,
-                                        cursor, &intel_crtc_funcs, NULL);
+                                        cursor, &intel_crtc_funcs,
+                                        "pipe %c", pipe_name(pipe));
         if (ret)
                 goto fail;
 
@@ -14398,10 +14565,8 @@ static void intel_crtc_init(struct drm_device *dev, int pipe)
         return;
 
 fail:
-        if (primary)
-                drm_plane_cleanup(primary);
-        if (cursor)
-                drm_plane_cleanup(cursor);
+        intel_plane_destroy(primary);
+        intel_plane_destroy(cursor);
         kfree(crtc_state);
         kfree(intel_crtc);
 }
@@ -14427,11 +14592,8 @@ int intel_get_pipe_from_crtc_id(struct drm_device *dev, void *data,
         struct intel_crtc *crtc;
 
         drmmode_crtc = drm_crtc_find(dev, pipe_from_crtc_id->crtc_id);
-
-        if (!drmmode_crtc) {
-                DRM_ERROR("no such CRTC id\n");
+        if (!drmmode_crtc)
                 return -ENOENT;
-        }
 
         crtc = to_intel_crtc(drmmode_crtc);
         pipe_from_crtc_id->pipe = crtc->pipe;
@@ -14458,7 +14620,7 @@ static int intel_encoder_clones(struct intel_encoder *encoder)
 
 static bool has_edp_a(struct drm_device *dev)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
 
         if (!IS_MOBILE(dev))
                 return false;
@@ -14474,7 +14636,7 @@ static bool has_edp_a(struct drm_device *dev)
 
 static bool intel_crt_present(struct drm_device *dev)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
 
         if (INTEL_INFO(dev)->gen >= 9)
                 return false;
@@ -14500,10 +14662,15 @@ static bool intel_crt_present(struct drm_device *dev)
 
 static void intel_setup_outputs(struct drm_device *dev)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         struct intel_encoder *encoder;
         bool dpd_is_edp = false;
 
+        /*
+         * intel_edp_init_connector() depends on this completing first, to
+         * prevent the registeration of both eDP and LVDS and the incorrect
+         * sharing of the PPS.
+         */
         intel_lvds_init(dev);
 
         if (intel_crt_present(dev))
@@ -15088,12 +15255,13 @@ void intel_init_display_hooks(struct drm_i915_private *dev_priv)
                 dev_priv->display.fdi_link_train = ivb_manual_fdi_link_train;
         } else if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) {
                 dev_priv->display.fdi_link_train = hsw_fdi_link_train;
-                if (IS_BROADWELL(dev_priv)) {
-                        dev_priv->display.modeset_commit_cdclk =
-                                broadwell_modeset_commit_cdclk;
-                        dev_priv->display.modeset_calc_cdclk =
-                                broadwell_modeset_calc_cdclk;
-                }
+        }
+
+        if (IS_BROADWELL(dev_priv)) {
+                dev_priv->display.modeset_commit_cdclk =
+                        broadwell_modeset_commit_cdclk;
+                dev_priv->display.modeset_calc_cdclk =
+                        broadwell_modeset_calc_cdclk;
         } else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) {
                 dev_priv->display.modeset_commit_cdclk =
                         valleyview_modeset_commit_cdclk;
@@ -15101,9 +15269,14 @@ void intel_init_display_hooks(struct drm_i915_private *dev_priv)
                         valleyview_modeset_calc_cdclk;
         } else if (IS_BROXTON(dev_priv)) {
                 dev_priv->display.modeset_commit_cdclk =
-                        broxton_modeset_commit_cdclk;
+                        bxt_modeset_commit_cdclk;
                 dev_priv->display.modeset_calc_cdclk =
-                        broxton_modeset_calc_cdclk;
+                        bxt_modeset_calc_cdclk;
+        } else if (IS_SKYLAKE(dev_priv) || IS_KABYLAKE(dev_priv)) {
+                dev_priv->display.modeset_commit_cdclk =
+                        skl_modeset_commit_cdclk;
+                dev_priv->display.modeset_calc_cdclk =
+                        skl_modeset_calc_cdclk;
         }
 
         switch (INTEL_INFO(dev_priv)->gen) {
@@ -15142,7 +15315,7 @@ void intel_init_display_hooks(struct drm_i915_private *dev_priv)
  */
 static void quirk_pipea_force(struct drm_device *dev)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
 
         dev_priv->quirks |= QUIRK_PIPEA_FORCE;
         DRM_INFO("applying pipe a force quirk\n");
@@ -15150,7 +15323,7 @@ static void quirk_pipea_force(struct drm_device *dev)
 
 static void quirk_pipeb_force(struct drm_device *dev)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
 
         dev_priv->quirks |= QUIRK_PIPEB_FORCE;
         DRM_INFO("applying pipe b force quirk\n");
@@ -15161,7 +15334,7 @@ static void quirk_pipeb_force(struct drm_device *dev)
  */
 static void quirk_ssc_force_disable(struct drm_device *dev)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         dev_priv->quirks |= QUIRK_LVDS_SSC_DISABLE;
         DRM_INFO("applying lvds SSC disable quirk\n");
 }
@@ -15172,7 +15345,7 @@ static void quirk_ssc_force_disable(struct drm_device *dev)
  */
 static void quirk_invert_brightness(struct drm_device *dev)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         dev_priv->quirks |= QUIRK_INVERT_BRIGHTNESS;
         DRM_INFO("applying inverted panel brightness quirk\n");
 }
@@ -15180,7 +15353,7 @@ static void quirk_invert_brightness(struct drm_device *dev)
 /* Some VBT's incorrectly indicate no backlight is present */
 static void quirk_backlight_present(struct drm_device *dev)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         dev_priv->quirks |= QUIRK_BACKLIGHT_PRESENT;
         DRM_INFO("applying backlight present quirk\n");
 }
@@ -15306,7 +15479,7 @@ static void intel_init_quirks(struct drm_device *dev)
 /* Disable the VGA plane that we never use */
 static void i915_disable_vga(struct drm_device *dev)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         u8 sr1;
         i915_reg_t vga_reg = i915_vgacntrl_reg(dev);
 
@@ -15324,14 +15497,14 @@ static void i915_disable_vga(struct drm_device *dev)
 
 void intel_modeset_init_hw(struct drm_device *dev)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
 
         intel_update_cdclk(dev);
 
         dev_priv->atomic_cdclk_freq = dev_priv->cdclk_freq;
 
         intel_init_clock_gating(dev);
-        intel_enable_gt_powersave(dev);
+        intel_enable_gt_powersave(dev_priv);
 }
 
 /*
@@ -15401,7 +15574,6 @@ retry:
         }
 
         /* Write calculated watermark values back */
-        to_i915(dev)->wm.config = to_intel_atomic_state(state)->wm_config;
         for_each_crtc_in_state(state, crtc, cstate, i) {
                 struct intel_crtc_state *cs = to_intel_crtc_state(cstate);
 
@@ -15499,11 +15671,13 @@ void intel_modeset_init(struct drm_device *dev)
         }
 
         intel_update_czclk(dev_priv);
-        intel_update_rawclk(dev_priv);
         intel_update_cdclk(dev);
 
         intel_shared_dpll_init(dev);
 
+        if (dev_priv->max_cdclk_freq == 0)
+                intel_update_max_cdclk(dev);
+
         /* Just disable it once at startup */
         i915_disable_vga(dev);
         intel_setup_outputs(dev);
@@ -15571,7 +15745,7 @@ static bool
 intel_check_plane_mapping(struct intel_crtc *crtc)
 {
         struct drm_device *dev = crtc->base.dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         u32 val;
 
         if (INTEL_INFO(dev)->num_pipes == 1)
@@ -15611,7 +15785,7 @@ static bool intel_encoder_has_connectors(struct intel_encoder *encoder)
 static void intel_sanitize_crtc(struct intel_crtc *crtc)
 {
         struct drm_device *dev = crtc->base.dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         enum transcoder cpu_transcoder = crtc->config->cpu_transcoder;
 
         /* Clear any frame start delays used for debugging left by the BIOS */
@@ -15644,8 +15818,8 @@ static void intel_sanitize_crtc(struct intel_crtc *crtc)
         if (INTEL_INFO(dev)->gen < 4 && !intel_check_plane_mapping(crtc)) {
                 bool plane;
 
-                DRM_DEBUG_KMS("[CRTC:%d] wrong plane connection detected!\n",
-                              crtc->base.base.id);
+                DRM_DEBUG_KMS("[CRTC:%d:%s] wrong plane connection detected!\n",
+                              crtc->base.base.id, crtc->base.name);
 
                 /* Pipe has the wrong plane attached and the plane is active.
                  * Temporarily change the plane mapping and disable everything
@@ -15736,7 +15910,7 @@ static void intel_sanitize_encoder(struct intel_encoder *encoder)
 
 void i915_redisable_vga_power_on(struct drm_device *dev)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         i915_reg_t vga_reg = i915_vgacntrl_reg(dev);
 
         if (!(I915_READ(vga_reg) & VGA_DISP_DISABLE)) {
@@ -15747,7 +15921,7 @@ void i915_redisable_vga_power_on(struct drm_device *dev)
 
 void i915_redisable_vga(struct drm_device *dev)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
 
         /* This function can be called both from intel_modeset_setup_hw_state or
          * at a very early point in our resume sequence, where the power well
@@ -15787,7 +15961,7 @@ static void readout_plane_state(struct intel_crtc *crtc)
 
 static void intel_modeset_readout_hw_state(struct drm_device *dev)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         enum pipe pipe;
         struct intel_crtc *crtc;
         struct intel_encoder *encoder;
@@ -15813,26 +15987,24 @@ static void intel_modeset_readout_hw_state(struct drm_device *dev)
                 if (crtc_state->base.active) {
                         dev_priv->active_crtcs |= 1 << crtc->pipe;
 
-                        if (IS_BROADWELL(dev_priv)) {
+                        if (INTEL_GEN(dev_priv) >= 9 || IS_BROADWELL(dev_priv))
                                 pixclk = ilk_pipe_pixel_rate(crtc_state);
-
-                                /* pixel rate mustn't exceed 95% of cdclk with IPS on BDW */
-                                if (crtc_state->ips_enabled)
-                                        pixclk = DIV_ROUND_UP(pixclk * 100, 95);
-                        } else if (IS_VALLEYVIEW(dev_priv) ||
-                                   IS_CHERRYVIEW(dev_priv) ||
-                                   IS_BROXTON(dev_priv))
+                        else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
                                 pixclk = crtc_state->base.adjusted_mode.crtc_clock;
                         else
                                 WARN_ON(dev_priv->display.modeset_calc_cdclk);
+
+                        /* pixel rate mustn't exceed 95% of cdclk with IPS on BDW */
+                        if (IS_BROADWELL(dev_priv) && crtc_state->ips_enabled)
+                                pixclk = DIV_ROUND_UP(pixclk * 100, 95);
                 }
 
                 dev_priv->min_pixclk[crtc->pipe] = pixclk;
 
                 readout_plane_state(crtc);
 
-                DRM_DEBUG_KMS("[CRTC:%d] hw state readout: %s\n",
-                              crtc->base.base.id,
+                DRM_DEBUG_KMS("[CRTC:%d:%s] hw state readout: %s\n",
+                              crtc->base.base.id, crtc->base.name,
                               crtc->active ? "enabled" : "disabled");
         }
 
@@ -15858,6 +16030,7 @@ static void intel_modeset_readout_hw_state(struct drm_device *dev)
                 if (encoder->get_hw_state(encoder, &pipe)) {
                         crtc = to_intel_crtc(dev_priv->pipe_to_crtc_mapping[pipe]);
                         encoder->base.crtc = &crtc->base;
+                        crtc->config->output_types |= 1 << encoder->type;
                         encoder->get_config(encoder, crtc->config);
                 } else {
                         encoder->base.crtc = NULL;
@@ -15942,7 +16115,7 @@ static void intel_modeset_readout_hw_state(struct drm_device *dev)
 static void
 intel_modeset_setup_hw_state(struct drm_device *dev)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         enum pipe pipe;
         struct intel_crtc *crtc;
         struct intel_encoder *encoder;
@@ -16063,15 +16236,16 @@ retry:
 
 void intel_modeset_gem_init(struct drm_device *dev)
 {
+        struct drm_i915_private *dev_priv = to_i915(dev);
         struct drm_crtc *c;
         struct drm_i915_gem_object *obj;
         int ret;
 
-        intel_init_gt_powersave(dev);
+        intel_init_gt_powersave(dev_priv);
 
         intel_modeset_init_hw(dev);
 
-        intel_setup_overlay(dev);
+        intel_setup_overlay(dev_priv);
 
         /*
          * Make sure any fbs we allocated at startup are properly
@@ -16097,26 +16271,36 @@ void intel_modeset_gem_init(struct drm_device *dev)
                         c->state->plane_mask &= ~(1 << drm_plane_index(c->primary));
                 }
         }
+}
+
+int intel_connector_register(struct drm_connector *connector)
+{
+        struct intel_connector *intel_connector = to_intel_connector(connector);
+        int ret;
+
+        ret = intel_backlight_device_register(intel_connector);
+        if (ret)
+                goto err;
 
-        intel_backlight_register(dev);
+        return 0;
+
+err:
+        return ret;
 }
 
-void intel_connector_unregister(struct intel_connector *intel_connector)
+void intel_connector_unregister(struct drm_connector *connector)
 {
-        struct drm_connector *connector = &intel_connector->base;
+        struct intel_connector *intel_connector = to_intel_connector(connector);
 
+        intel_backlight_device_unregister(intel_connector);
         intel_panel_destroy_backlight(connector);
-        drm_connector_unregister(connector);
 }
 
 void intel_modeset_cleanup(struct drm_device *dev)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
-        struct intel_connector *connector;
-
-        intel_disable_gt_powersave(dev);
+        struct drm_i915_private *dev_priv = to_i915(dev);
 
-        intel_backlight_unregister(dev);
+        intel_disable_gt_powersave(dev_priv);
 
         /*
          * Interrupts and polling as the first thing to avoid creating havoc.
@@ -16138,27 +16322,15 @@ void intel_modeset_cleanup(struct drm_device *dev)
         /* flush any delayed tasks or pending work */
         flush_scheduled_work();
 
-        /* destroy the backlight and sysfs files before encoders/connectors */
-        for_each_intel_connector(dev, connector)
-                connector->unregister(connector);
-
         drm_mode_config_cleanup(dev);
 
-        intel_cleanup_overlay(dev);
+        intel_cleanup_overlay(dev_priv);
 
-        intel_cleanup_gt_powersave(dev);
+        intel_cleanup_gt_powersave(dev_priv);
 
         intel_teardown_gmbus(dev);
 }
 
-/*
- * Return which encoder is currently attached for connector.
- */
-struct drm_encoder *intel_best_encoder(struct drm_connector *connector)
-{
-        return &intel_attached_encoder(connector)->base;
-}
-
 void intel_connector_attach_encoder(struct intel_connector *connector,
                                     struct intel_encoder *encoder)
 {
@@ -16172,7 +16344,7 @@ void intel_connector_attach_encoder(struct intel_connector *connector,
  */
 int intel_modeset_vga_set_state(struct drm_device *dev, bool state)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         unsigned reg = INTEL_INFO(dev)->gen >= 6 ? SNB_GMCH_CTRL : INTEL_GMCH_CTRL;
         u16 gmch_ctrl;
 
@@ -16242,9 +16414,8 @@ struct intel_display_error_state {
 };
 
 struct intel_display_error_state *
-intel_display_capture_error_state(struct drm_device *dev)
+intel_display_capture_error_state(struct drm_i915_private *dev_priv)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
         struct intel_display_error_state *error;
         int transcoders[] = {
                 TRANSCODER_A,
@@ -16254,14 +16425,14 @@ intel_display_capture_error_state(struct drm_device *dev)
         };
         int i;
 
-        if (INTEL_INFO(dev)->num_pipes == 0)
+        if (INTEL_INFO(dev_priv)->num_pipes == 0)
                 return NULL;
 
         error = kzalloc(sizeof(*error), GFP_ATOMIC);
         if (error == NULL)
                 return NULL;
 
-        if (IS_HASWELL(dev) || IS_BROADWELL(dev))
+        if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
                 error->power_well_driver = I915_READ(HSW_PWR_WELL_DRIVER);
 
         for_each_pipe(dev_priv, i) {
@@ -16277,25 +16448,25 @@ intel_display_capture_error_state(struct drm_device *dev)
 
                 error->plane[i].control = I915_READ(DSPCNTR(i));
                 error->plane[i].stride = I915_READ(DSPSTRIDE(i));
-                if (INTEL_INFO(dev)->gen <= 3) {
+                if (INTEL_GEN(dev_priv) <= 3) {
                         error->plane[i].size = I915_READ(DSPSIZE(i));
                         error->plane[i].pos = I915_READ(DSPPOS(i));
                 }
-                if (INTEL_INFO(dev)->gen <= 7 && !IS_HASWELL(dev))
+                if (INTEL_GEN(dev_priv) <= 7 && !IS_HASWELL(dev_priv))
                         error->plane[i].addr = I915_READ(DSPADDR(i));
-                if (INTEL_INFO(dev)->gen >= 4) {
+                if (INTEL_GEN(dev_priv) >= 4) {
                         error->plane[i].surface = I915_READ(DSPSURF(i));
                         error->plane[i].tile_offset = I915_READ(DSPTILEOFF(i));
                 }
 
                 error->pipe[i].source = I915_READ(PIPESRC(i));
 
-                if (HAS_GMCH_DISPLAY(dev))
+                if (HAS_GMCH_DISPLAY(dev_priv))
                         error->pipe[i].stat = I915_READ(PIPESTAT(i));
         }
 
         /* Note: this does not include DSI transcoders. */
-        error->num_transcoders = INTEL_INFO(dev)->num_pipes;
+        error->num_transcoders = INTEL_INFO(dev_priv)->num_pipes;
         if (HAS_DDI(dev_priv))
                 error->num_transcoders++; /* Account for eDP. */
 
@@ -16329,7 +16500,7 @@ intel_display_print_error_state(struct drm_i915_error_state_buf *m,
                                 struct drm_device *dev,
                                 struct intel_display_error_state *error)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         int i;
 
         if (!error)
diff --git a/drivers/gpu/drm/i915/intel_dp.c b/drivers/gpu/drm/i915/intel_dp.c
index 891107f92d9f..21b04c3eda41 100644
--- a/drivers/gpu/drm/i915/intel_dp.c
+++ b/drivers/gpu/drm/i915/intel_dp.c
@@ -131,11 +131,6 @@ static void vlv_steal_power_sequencer(struct drm_device *dev,
                                       enum pipe pipe);
 static void intel_dp_unset_edid(struct intel_dp *intel_dp);
 
-static unsigned int intel_dp_unused_lane_mask(int lane_count)
-{
-        return ~((1 << lane_count) - 1) & 0xf;
-}
-
 static int
 intel_dp_max_link_bw(struct intel_dp  *intel_dp)
 {
@@ -267,7 +262,7 @@ static void pps_lock(struct intel_dp *intel_dp)
         struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp);
         struct intel_encoder *encoder = &intel_dig_port->base;
         struct drm_device *dev = encoder->base.dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         enum intel_display_power_domain power_domain;
 
         /*
@@ -285,7 +280,7 @@ static void pps_unlock(struct intel_dp *intel_dp)
         struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp);
         struct intel_encoder *encoder = &intel_dig_port->base;
         struct drm_device *dev = encoder->base.dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         enum intel_display_power_domain power_domain;
 
         mutex_unlock(&dev_priv->pps_mutex);
@@ -299,7 +294,7 @@ vlv_power_sequencer_kick(struct intel_dp *intel_dp)
 {
         struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp);
         struct drm_device *dev = intel_dig_port->base.base.dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         enum pipe pipe = intel_dp->pps_pipe;
         bool pll_enabled, release_cl_override = false;
         enum dpio_phy phy = DPIO_PHY(pipe);
@@ -373,7 +368,7 @@ vlv_power_sequencer_pipe(struct intel_dp *intel_dp)
 {
         struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp);
         struct drm_device *dev = intel_dig_port->base.base.dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         struct intel_encoder *encoder;
         unsigned int pipes = (1 << PIPE_A) | (1 << PIPE_B);
         enum pipe pipe;
@@ -431,6 +426,37 @@ vlv_power_sequencer_pipe(struct intel_dp *intel_dp)
         return intel_dp->pps_pipe;
 }
 
+static int
+bxt_power_sequencer_idx(struct intel_dp *intel_dp)
+{
+        struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp);
+        struct drm_device *dev = intel_dig_port->base.base.dev;
+        struct drm_i915_private *dev_priv = to_i915(dev);
+
+        lockdep_assert_held(&dev_priv->pps_mutex);
+
+        /* We should never land here with regular DP ports */
+        WARN_ON(!is_edp(intel_dp));
+
+        /*
+         * TODO: BXT has 2 PPS instances. The correct port->PPS instance
+         * mapping needs to be retrieved from VBT, for now just hard-code to
+         * use instance #0 always.
+         */
+        if (!intel_dp->pps_reset)
+                return 0;
+
+        intel_dp->pps_reset = false;
+
+        /*
+         * Only the HW needs to be reprogrammed, the SW state is fixed and
+         * has been setup during connector init.
+         */
+        intel_dp_init_panel_power_sequencer_registers(dev, intel_dp);
+
+        return 0;
+}
+
 typedef bool (*vlv_pipe_check)(struct drm_i915_private *dev_priv,
                                enum pipe pipe);
 
@@ -480,7 +506,7 @@ vlv_initial_power_sequencer_setup(struct intel_dp *intel_dp)
 {
         struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp);
         struct drm_device *dev = intel_dig_port->base.base.dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         enum port port = intel_dig_port->port;
 
         lockdep_assert_held(&dev_priv->pps_mutex);
@@ -512,12 +538,13 @@ vlv_initial_power_sequencer_setup(struct intel_dp *intel_dp)
         intel_dp_init_panel_power_sequencer_registers(dev, intel_dp);
 }
 
-void vlv_power_sequencer_reset(struct drm_i915_private *dev_priv)
+void intel_power_sequencer_reset(struct drm_i915_private *dev_priv)
 {
-        struct drm_device *dev = dev_priv->dev;
+        struct drm_device *dev = &dev_priv->drm;
         struct intel_encoder *encoder;
 
-        if (WARN_ON(!IS_VALLEYVIEW(dev) && !IS_CHERRYVIEW(dev)))
+        if (WARN_ON(!IS_VALLEYVIEW(dev) && !IS_CHERRYVIEW(dev) &&
+                    !IS_BROXTON(dev)))
                 return;
 
         /*
@@ -537,34 +564,71 @@ void vlv_power_sequencer_reset(struct drm_i915_private *dev_priv)
                         continue;
 
                 intel_dp = enc_to_intel_dp(&encoder->base);
-                intel_dp->pps_pipe = INVALID_PIPE;
+                if (IS_BROXTON(dev))
+                        intel_dp->pps_reset = true;
+                else
+                        intel_dp->pps_pipe = INVALID_PIPE;
+        }
+}
+
+struct pps_registers {
+        i915_reg_t pp_ctrl;
+        i915_reg_t pp_stat;
+        i915_reg_t pp_on;
+        i915_reg_t pp_off;
+        i915_reg_t pp_div;
+};
+
+static void intel_pps_get_registers(struct drm_i915_private *dev_priv,
+                                    struct intel_dp *intel_dp,
+                                    struct pps_registers *regs)
+{
+        memset(regs, 0, sizeof(*regs));
+
+        if (IS_BROXTON(dev_priv)) {
+                int idx = bxt_power_sequencer_idx(intel_dp);
+
+                regs->pp_ctrl = BXT_PP_CONTROL(idx);
+                regs->pp_stat = BXT_PP_STATUS(idx);
+                regs->pp_on = BXT_PP_ON_DELAYS(idx);
+                regs->pp_off = BXT_PP_OFF_DELAYS(idx);
+        } else if (HAS_PCH_SPLIT(dev_priv)) {
+                regs->pp_ctrl = PCH_PP_CONTROL;
+                regs->pp_stat = PCH_PP_STATUS;
+                regs->pp_on = PCH_PP_ON_DELAYS;
+                regs->pp_off = PCH_PP_OFF_DELAYS;
+                regs->pp_div = PCH_PP_DIVISOR;
+        } else {
+                enum pipe pipe = vlv_power_sequencer_pipe(intel_dp);
+
+                regs->pp_ctrl = VLV_PIPE_PP_CONTROL(pipe);
+                regs->pp_stat = VLV_PIPE_PP_STATUS(pipe);
+                regs->pp_on = VLV_PIPE_PP_ON_DELAYS(pipe);
+                regs->pp_off = VLV_PIPE_PP_OFF_DELAYS(pipe);
+                regs->pp_div = VLV_PIPE_PP_DIVISOR(pipe);
         }
 }
 
 static i915_reg_t
 _pp_ctrl_reg(struct intel_dp *intel_dp)
 {
-        struct drm_device *dev = intel_dp_to_dev(intel_dp);
+        struct pps_registers regs;
 
-        if (IS_BROXTON(dev))
-                return BXT_PP_CONTROL(0);
-        else if (HAS_PCH_SPLIT(dev))
-                return PCH_PP_CONTROL;
-        else
-                return VLV_PIPE_PP_CONTROL(vlv_power_sequencer_pipe(intel_dp));
+        intel_pps_get_registers(to_i915(intel_dp_to_dev(intel_dp)), intel_dp,
+                                &regs);
+
+        return regs.pp_ctrl;
 }
 
 static i915_reg_t
 _pp_stat_reg(struct intel_dp *intel_dp)
 {
-        struct drm_device *dev = intel_dp_to_dev(intel_dp);
+        struct pps_registers regs;
 
-        if (IS_BROXTON(dev))
-                return BXT_PP_STATUS(0);
-        else if (HAS_PCH_SPLIT(dev))
-                return PCH_PP_STATUS;
-        else
-                return VLV_PIPE_PP_STATUS(vlv_power_sequencer_pipe(intel_dp));
+        intel_pps_get_registers(to_i915(intel_dp_to_dev(intel_dp)), intel_dp,
+                                &regs);
+
+        return regs.pp_stat;
 }
 
 /* Reboot notifier handler to shutdown panel power to guarantee T12 timing
@@ -575,7 +639,7 @@ static int edp_notify_handler(struct notifier_block *this, unsigned long code,
         struct intel_dp *intel_dp = container_of(this, typeof(* intel_dp),
                                                  edp_notifier);
         struct drm_device *dev = intel_dp_to_dev(intel_dp);
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
 
         if (!is_edp(intel_dp) || code != SYS_RESTART)
                 return 0;
@@ -606,7 +670,7 @@ static int edp_notify_handler(struct notifier_block *this, unsigned long code,
 static bool edp_have_panel_power(struct intel_dp *intel_dp)
 {
         struct drm_device *dev = intel_dp_to_dev(intel_dp);
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
 
         lockdep_assert_held(&dev_priv->pps_mutex);
 
@@ -620,7 +684,7 @@ static bool edp_have_panel_power(struct intel_dp *intel_dp)
 static bool edp_have_panel_vdd(struct intel_dp *intel_dp)
 {
         struct drm_device *dev = intel_dp_to_dev(intel_dp);
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
 
         lockdep_assert_held(&dev_priv->pps_mutex);
 
@@ -635,7 +699,7 @@ static void
 intel_dp_check_edp(struct intel_dp *intel_dp)
 {
         struct drm_device *dev = intel_dp_to_dev(intel_dp);
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
 
         if (!is_edp(intel_dp))
                 return;
@@ -653,7 +717,7 @@ intel_dp_aux_wait_done(struct intel_dp *intel_dp, bool has_aux_irq)
 {
         struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp);
         struct drm_device *dev = intel_dig_port->base.base.dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         i915_reg_t ch_ctl = intel_dp->aux_ch_ctl_reg;
         uint32_t status;
         bool done;
@@ -775,6 +839,7 @@ static uint32_t skl_get_aux_send_ctl(struct intel_dp *intel_dp,
                DP_AUX_CH_CTL_TIME_OUT_1600us |
                DP_AUX_CH_CTL_RECEIVE_ERROR |
                (send_bytes << DP_AUX_CH_CTL_MESSAGE_SIZE_SHIFT) |
+               DP_AUX_CH_CTL_FW_SYNC_PULSE_SKL(32) |
                DP_AUX_CH_CTL_SYNC_PULSE_SKL(32);
 }
 
@@ -785,7 +850,7 @@ intel_dp_aux_ch(struct intel_dp *intel_dp,
 {
         struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp);
         struct drm_device *dev = intel_dig_port->base.base.dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         i915_reg_t ch_ctl = intel_dp->aux_ch_ctl_reg;
         uint32_t aux_clock_divider;
         int i, ret, recv_bytes;
@@ -1181,48 +1246,21 @@ static void intel_aux_reg_init(struct intel_dp *intel_dp)
 static void
 intel_dp_aux_fini(struct intel_dp *intel_dp)
 {
-        drm_dp_aux_unregister(&intel_dp->aux);
         kfree(intel_dp->aux.name);
 }
 
-static int
+static void
 intel_dp_aux_init(struct intel_dp *intel_dp, struct intel_connector *connector)
 {
         struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp);
         enum port port = intel_dig_port->port;
-        int ret;
 
         intel_aux_reg_init(intel_dp);
+        drm_dp_aux_init(&intel_dp->aux);
 
+        /* Failure to allocate our preferred name is not critical */
         intel_dp->aux.name = kasprintf(GFP_KERNEL, "DPDDC-%c", port_name(port));
-        if (!intel_dp->aux.name)
-                return -ENOMEM;
-
-        intel_dp->aux.dev = connector->base.kdev;
         intel_dp->aux.transfer = intel_dp_aux_transfer;
-
-        DRM_DEBUG_KMS("registering %s bus for %s\n",
-                      intel_dp->aux.name,
-                      connector->base.kdev->kobj.name);
-
-        ret = drm_dp_aux_register(&intel_dp->aux);
-        if (ret < 0) {
-                DRM_ERROR("drm_dp_aux_register() for %s failed (%d)\n",
-                          intel_dp->aux.name, ret);
-                kfree(intel_dp->aux.name);
-                return ret;
-        }
-
-        return 0;
-}
-
-static void
-intel_dp_connector_unregister(struct intel_connector *intel_connector)
-{
-        struct intel_dp *intel_dp = intel_attached_dp(&intel_connector->base);
-
-        intel_dp_aux_fini(intel_dp);
-        intel_connector_unregister(intel_connector);
 }
 
 static int
@@ -1435,7 +1473,7 @@ intel_dp_compute_config(struct intel_encoder *encoder,
                         struct intel_crtc_state *pipe_config)
 {
         struct drm_device *dev = encoder->base.dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         struct drm_display_mode *adjusted_mode = &pipe_config->base.adjusted_mode;
         struct intel_dp *intel_dp = enc_to_intel_dp(&encoder->base);
         enum port port = dp_to_dig_port(intel_dp)->port;
@@ -1463,7 +1501,6 @@ intel_dp_compute_config(struct intel_encoder *encoder,
         if (HAS_PCH_SPLIT(dev) && !HAS_DDI(dev) && port != PORT_A)
                 pipe_config->has_pch_encoder = true;
 
-        pipe_config->has_dp_encoder = true;
         pipe_config->has_drrs = false;
         pipe_config->has_audio = intel_dp->has_audio && port != PORT_A;
 
@@ -1582,6 +1619,27 @@ found:
                                 &pipe_config->dp_m2_n2);
         }
 
+        /*
+         * DPLL0 VCO may need to be adjusted to get the correct
+         * clock for eDP. This will affect cdclk as well.
+         */
+        if (is_edp(intel_dp) &&
+            (IS_SKYLAKE(dev_priv) || IS_KABYLAKE(dev_priv))) {
+                int vco;
+
+                switch (pipe_config->port_clock / 2) {
+                case 108000:
+                case 216000:
+                        vco = 8640000;
+                        break;
+                default:
+                        vco = 8100000;
+                        break;
+                }
+
+                to_intel_atomic_state(pipe_config->base.state)->cdclk_pll_vco = vco;
+        }
+
         if (!HAS_DDI(dev))
                 intel_dp_set_clock(encoder, pipe_config);
 
@@ -1598,7 +1656,7 @@ void intel_dp_set_link_params(struct intel_dp *intel_dp,
 static void intel_dp_prepare(struct intel_encoder *encoder)
 {
         struct drm_device *dev = encoder->base.dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         struct intel_dp *intel_dp = enc_to_intel_dp(&encoder->base);
         enum port port = dp_to_dig_port(intel_dp)->port;
         struct intel_crtc *crtc = to_intel_crtc(encoder->base.crtc);
@@ -1686,16 +1744,21 @@ static void intel_dp_prepare(struct intel_encoder *encoder)
 #define IDLE_CYCLE_MASK         (PP_ON | PP_SEQUENCE_MASK | PP_CYCLE_DELAY_ACTIVE | PP_SEQUENCE_STATE_MASK)
 #define IDLE_CYCLE_VALUE        (0     | PP_SEQUENCE_NONE | 0                     | PP_SEQUENCE_STATE_OFF_IDLE)
 
+static void intel_pps_verify_state(struct drm_i915_private *dev_priv,
+                                   struct intel_dp *intel_dp);
+
 static void wait_panel_status(struct intel_dp *intel_dp,
                                        u32 mask,
                                        u32 value)
 {
         struct drm_device *dev = intel_dp_to_dev(intel_dp);
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         i915_reg_t pp_stat_reg, pp_ctrl_reg;
 
         lockdep_assert_held(&dev_priv->pps_mutex);
 
+        intel_pps_verify_state(dev_priv, intel_dp);
+
         pp_stat_reg = _pp_stat_reg(intel_dp);
         pp_ctrl_reg = _pp_ctrl_reg(intel_dp);
 
@@ -1704,8 +1767,9 @@ static void wait_panel_status(struct intel_dp *intel_dp,
                         I915_READ(pp_stat_reg),
                         I915_READ(pp_ctrl_reg));
 
-        if (_wait_for((I915_READ(pp_stat_reg) & mask) == value,
-                      5 * USEC_PER_SEC, 10 * USEC_PER_MSEC))
+        if (intel_wait_for_register(dev_priv,
+                                    pp_stat_reg, mask, value,
+                                    5000))
                 DRM_ERROR("Panel status timeout: status %08x control %08x\n",
                                 I915_READ(pp_stat_reg),
                                 I915_READ(pp_ctrl_reg));
@@ -1765,7 +1829,7 @@ static void edp_wait_backlight_off(struct intel_dp *intel_dp)
 static  u32 ironlake_get_pp_control(struct intel_dp *intel_dp)
 {
         struct drm_device *dev = intel_dp_to_dev(intel_dp);
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         u32 control;
 
         lockdep_assert_held(&dev_priv->pps_mutex);
@@ -1788,7 +1852,7 @@ static bool edp_panel_vdd_on(struct intel_dp *intel_dp)
         struct drm_device *dev = intel_dp_to_dev(intel_dp);
         struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp);
         struct intel_encoder *intel_encoder = &intel_dig_port->base;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         enum intel_display_power_domain power_domain;
         u32 pp;
         i915_reg_t pp_stat_reg, pp_ctrl_reg;
@@ -1861,7 +1925,7 @@ void intel_edp_panel_vdd_on(struct intel_dp *intel_dp)
 static void edp_panel_vdd_off_sync(struct intel_dp *intel_dp)
 {
         struct drm_device *dev = intel_dp_to_dev(intel_dp);
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         struct intel_digital_port *intel_dig_port =
                 dp_to_dig_port(intel_dp);
         struct intel_encoder *intel_encoder = &intel_dig_port->base;
@@ -1930,8 +1994,7 @@ static void edp_panel_vdd_schedule_off(struct intel_dp *intel_dp)
  */
 static void edp_panel_vdd_off(struct intel_dp *intel_dp, bool sync)
 {
-        struct drm_i915_private *dev_priv =
-                intel_dp_to_dev(intel_dp)->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(intel_dp_to_dev(intel_dp));
 
         lockdep_assert_held(&dev_priv->pps_mutex);
 
@@ -1952,7 +2015,7 @@ static void edp_panel_vdd_off(struct intel_dp *intel_dp, bool sync)
 static void edp_panel_on(struct intel_dp *intel_dp)
 {
         struct drm_device *dev = intel_dp_to_dev(intel_dp);
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         u32 pp;
         i915_reg_t pp_ctrl_reg;
 
@@ -2013,7 +2076,7 @@ static void edp_panel_off(struct intel_dp *intel_dp)
         struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp);
         struct intel_encoder *intel_encoder = &intel_dig_port->base;
         struct drm_device *dev = intel_dp_to_dev(intel_dp);
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         enum intel_display_power_domain power_domain;
         u32 pp;
         i915_reg_t pp_ctrl_reg;
@@ -2065,7 +2128,7 @@ static void _intel_edp_backlight_on(struct intel_dp *intel_dp)
 {
         struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp);
         struct drm_device *dev = intel_dig_port->base.base.dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         u32 pp;
         i915_reg_t pp_ctrl_reg;
 
@@ -2106,7 +2169,7 @@ void intel_edp_backlight_on(struct intel_dp *intel_dp)
 static void _intel_edp_backlight_off(struct intel_dp *intel_dp)
 {
         struct drm_device *dev = intel_dp_to_dev(intel_dp);
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         u32 pp;
         i915_reg_t pp_ctrl_reg;
 
@@ -2222,7 +2285,7 @@ static void ironlake_edp_pll_on(struct intel_dp *intel_dp)
          * 2. Program DP PLL enable
          */
         if (IS_GEN5(dev_priv))
-                intel_wait_for_vblank_if_active(dev_priv->dev, !crtc->pipe);
+                intel_wait_for_vblank_if_active(&dev_priv->drm, !crtc->pipe);
 
         intel_dp->DP |= DP_PLL_ENABLE;
 
@@ -2287,7 +2350,7 @@ static bool intel_dp_get_hw_state(struct intel_encoder *encoder,
         struct intel_dp *intel_dp = enc_to_intel_dp(&encoder->base);
         enum port port = dp_to_dig_port(intel_dp)->port;
         struct drm_device *dev = encoder->base.dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         enum intel_display_power_domain power_domain;
         u32 tmp;
         bool ret;
@@ -2340,7 +2403,7 @@ static void intel_dp_get_config(struct intel_encoder *encoder,
         struct intel_dp *intel_dp = enc_to_intel_dp(&encoder->base);
         u32 tmp, flags = 0;
         struct drm_device *dev = encoder->base.dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         enum port port = dp_to_dig_port(intel_dp)->port;
         struct intel_crtc *crtc = to_intel_crtc(encoder->base.crtc);
 
@@ -2378,8 +2441,6 @@ static void intel_dp_get_config(struct intel_encoder *encoder,
             !IS_CHERRYVIEW(dev) && tmp & DP_COLOR_RANGE_16_235)
                 pipe_config->limited_color_range = true;
 
-        pipe_config->has_dp_encoder = true;
-
         pipe_config->lane_count =
                 ((tmp & DP_PORT_WIDTH_MASK) >> DP_PORT_WIDTH_SHIFT) + 1;
 
@@ -2460,55 +2521,11 @@ static void vlv_post_disable_dp(struct intel_encoder *encoder)
         intel_dp_link_down(intel_dp);
 }
 
-static void chv_data_lane_soft_reset(struct intel_encoder *encoder,
-                                     bool reset)
-{
-        struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
-        enum dpio_channel ch = vlv_dport_to_channel(enc_to_dig_port(&encoder->base));
-        struct intel_crtc *crtc = to_intel_crtc(encoder->base.crtc);
-        enum pipe pipe = crtc->pipe;
-        uint32_t val;
-
-        val = vlv_dpio_read(dev_priv, pipe, VLV_PCS01_DW0(ch));
-        if (reset)
-                val &= ~(DPIO_PCS_TX_LANE2_RESET | DPIO_PCS_TX_LANE1_RESET);
-        else
-                val |= DPIO_PCS_TX_LANE2_RESET | DPIO_PCS_TX_LANE1_RESET;
-        vlv_dpio_write(dev_priv, pipe, VLV_PCS01_DW0(ch), val);
-
-        if (crtc->config->lane_count > 2) {
-                val = vlv_dpio_read(dev_priv, pipe, VLV_PCS23_DW0(ch));
-                if (reset)
-                        val &= ~(DPIO_PCS_TX_LANE2_RESET | DPIO_PCS_TX_LANE1_RESET);
-                else
-                        val |= DPIO_PCS_TX_LANE2_RESET | DPIO_PCS_TX_LANE1_RESET;
-                vlv_dpio_write(dev_priv, pipe, VLV_PCS23_DW0(ch), val);
-        }
-
-        val = vlv_dpio_read(dev_priv, pipe, VLV_PCS01_DW1(ch));
-        val |= CHV_PCS_REQ_SOFTRESET_EN;
-        if (reset)
-                val &= ~DPIO_PCS_CLK_SOFT_RESET;
-        else
-                val |= DPIO_PCS_CLK_SOFT_RESET;
-        vlv_dpio_write(dev_priv, pipe, VLV_PCS01_DW1(ch), val);
-
-        if (crtc->config->lane_count > 2) {
-                val = vlv_dpio_read(dev_priv, pipe, VLV_PCS23_DW1(ch));
-                val |= CHV_PCS_REQ_SOFTRESET_EN;
-                if (reset)
-                        val &= ~DPIO_PCS_CLK_SOFT_RESET;
-                else
-                        val |= DPIO_PCS_CLK_SOFT_RESET;
-                vlv_dpio_write(dev_priv, pipe, VLV_PCS23_DW1(ch), val);
-        }
-}
-
 static void chv_post_disable_dp(struct intel_encoder *encoder)
 {
         struct intel_dp *intel_dp = enc_to_intel_dp(&encoder->base);
         struct drm_device *dev = encoder->base.dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
 
         intel_dp_link_down(intel_dp);
 
@@ -2527,7 +2544,7 @@ _intel_dp_set_link_train(struct intel_dp *intel_dp,
 {
         struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp);
         struct drm_device *dev = intel_dig_port->base.base.dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         enum port port = intel_dig_port->port;
 
         if (HAS_DDI(dev)) {
@@ -2607,7 +2624,7 @@ _intel_dp_set_link_train(struct intel_dp *intel_dp,
 static void intel_dp_enable_port(struct intel_dp *intel_dp)
 {
         struct drm_device *dev = intel_dp_to_dev(intel_dp);
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         struct intel_crtc *crtc =
                 to_intel_crtc(dp_to_dig_port(intel_dp)->base.base.crtc);
 
@@ -2636,7 +2653,7 @@ static void intel_enable_dp(struct intel_encoder *encoder)
 {
         struct intel_dp *intel_dp = enc_to_intel_dp(&encoder->base);
         struct drm_device *dev = encoder->base.dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         struct intel_crtc *crtc = to_intel_crtc(encoder->base.crtc);
         uint32_t dp_reg = I915_READ(intel_dp->output_reg);
         enum pipe pipe = crtc->pipe;
@@ -2709,7 +2726,7 @@ static void g4x_pre_enable_dp(struct intel_encoder *encoder)
 static void vlv_detach_power_sequencer(struct intel_dp *intel_dp)
 {
         struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp);
-        struct drm_i915_private *dev_priv = intel_dig_port->base.base.dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(intel_dig_port->base.base.dev);
         enum pipe pipe = intel_dp->pps_pipe;
         i915_reg_t pp_on_reg = VLV_PIPE_PP_ON_DELAYS(pipe);
 
@@ -2735,7 +2752,7 @@ static void vlv_detach_power_sequencer(struct intel_dp *intel_dp)
 static void vlv_steal_power_sequencer(struct drm_device *dev,
                                       enum pipe pipe)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         struct intel_encoder *encoder;
 
         lockdep_assert_held(&dev_priv->pps_mutex);
@@ -2773,7 +2790,7 @@ static void vlv_init_panel_power_sequencer(struct intel_dp *intel_dp)
         struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp);
         struct intel_encoder *encoder = &intel_dig_port->base;
         struct drm_device *dev = encoder->base.dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         struct intel_crtc *crtc = to_intel_crtc(encoder->base.crtc);
 
         lockdep_assert_held(&dev_priv->pps_mutex);
@@ -2811,266 +2828,38 @@ static void vlv_init_panel_power_sequencer(struct intel_dp *intel_dp)
 
 static void vlv_pre_enable_dp(struct intel_encoder *encoder)
 {
-        struct intel_dp *intel_dp = enc_to_intel_dp(&encoder->base);
-        struct intel_digital_port *dport = dp_to_dig_port(intel_dp);
-        struct drm_device *dev = encoder->base.dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
-        struct intel_crtc *intel_crtc = to_intel_crtc(encoder->base.crtc);
-        enum dpio_channel port = vlv_dport_to_channel(dport);
-        int pipe = intel_crtc->pipe;
-        u32 val;
-
-        mutex_lock(&dev_priv->sb_lock);
-
-        val = vlv_dpio_read(dev_priv, pipe, VLV_PCS01_DW8(port));
-        val = 0;
-        if (pipe)
-                val |= (1<<21);
-        else
-                val &= ~(1<<21);
-        val |= 0x001000c4;
-        vlv_dpio_write(dev_priv, pipe, VLV_PCS_DW8(port), val);
-        vlv_dpio_write(dev_priv, pipe, VLV_PCS_DW14(port), 0x00760018);
-        vlv_dpio_write(dev_priv, pipe, VLV_PCS_DW23(port), 0x00400888);
-
-        mutex_unlock(&dev_priv->sb_lock);
+        vlv_phy_pre_encoder_enable(encoder);
 
         intel_enable_dp(encoder);
 }
 
 static void vlv_dp_pre_pll_enable(struct intel_encoder *encoder)
 {
-        struct intel_digital_port *dport = enc_to_dig_port(&encoder->base);
-        struct drm_device *dev = encoder->base.dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
-        struct intel_crtc *intel_crtc =
-                to_intel_crtc(encoder->base.crtc);
-        enum dpio_channel port = vlv_dport_to_channel(dport);
-        int pipe = intel_crtc->pipe;
-
         intel_dp_prepare(encoder);
 
-        /* Program Tx lane resets to default */
-        mutex_lock(&dev_priv->sb_lock);
-        vlv_dpio_write(dev_priv, pipe, VLV_PCS_DW0(port),
-                         DPIO_PCS_TX_LANE2_RESET |
-                         DPIO_PCS_TX_LANE1_RESET);
-        vlv_dpio_write(dev_priv, pipe, VLV_PCS_DW1(port),
-                         DPIO_PCS_CLK_CRI_RXEB_EIOS_EN |
-                         DPIO_PCS_CLK_CRI_RXDIGFILTSG_EN |
-                         (1<<DPIO_PCS_CLK_DATAWIDTH_SHIFT) |
-                                 DPIO_PCS_CLK_SOFT_RESET);
-
-        /* Fix up inter-pair skew failure */
-        vlv_dpio_write(dev_priv, pipe, VLV_PCS_DW12(port), 0x00750f00);
-        vlv_dpio_write(dev_priv, pipe, VLV_TX_DW11(port), 0x00001500);
-        vlv_dpio_write(dev_priv, pipe, VLV_TX_DW14(port), 0x40400000);
-        mutex_unlock(&dev_priv->sb_lock);
+        vlv_phy_pre_pll_enable(encoder);
 }
 
 static void chv_pre_enable_dp(struct intel_encoder *encoder)
 {
-        struct intel_dp *intel_dp = enc_to_intel_dp(&encoder->base);
-        struct intel_digital_port *dport = dp_to_dig_port(intel_dp);
-        struct drm_device *dev = encoder->base.dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
-        struct intel_crtc *intel_crtc =
-                to_intel_crtc(encoder->base.crtc);
-        enum dpio_channel ch = vlv_dport_to_channel(dport);
-        int pipe = intel_crtc->pipe;
-        int data, i, stagger;
-        u32 val;
-
-        mutex_lock(&dev_priv->sb_lock);
-
-        /* allow hardware to manage TX FIFO reset source */
-        val = vlv_dpio_read(dev_priv, pipe, VLV_PCS01_DW11(ch));
-        val &= ~DPIO_LANEDESKEW_STRAP_OVRD;
-        vlv_dpio_write(dev_priv, pipe, VLV_PCS01_DW11(ch), val);
-
-        if (intel_crtc->config->lane_count > 2) {
-                val = vlv_dpio_read(dev_priv, pipe, VLV_PCS23_DW11(ch));
-                val &= ~DPIO_LANEDESKEW_STRAP_OVRD;
-                vlv_dpio_write(dev_priv, pipe, VLV_PCS23_DW11(ch), val);
-        }
-
-        /* Program Tx lane latency optimal setting*/
-        for (i = 0; i < intel_crtc->config->lane_count; i++) {
-                /* Set the upar bit */
-                if (intel_crtc->config->lane_count == 1)
-                        data = 0x0;
-                else
-                        data = (i == 1) ? 0x0 : 0x1;
-                vlv_dpio_write(dev_priv, pipe, CHV_TX_DW14(ch, i),
-                                data << DPIO_UPAR_SHIFT);
-        }
-
-        /* Data lane stagger programming */
-        if (intel_crtc->config->port_clock > 270000)
-                stagger = 0x18;
-        else if (intel_crtc->config->port_clock > 135000)
-                stagger = 0xd;
-        else if (intel_crtc->config->port_clock > 67500)
-                stagger = 0x7;
-        else if (intel_crtc->config->port_clock > 33750)
-                stagger = 0x4;
-        else
-                stagger = 0x2;
-
-        val = vlv_dpio_read(dev_priv, pipe, VLV_PCS01_DW11(ch));
-        val |= DPIO_TX2_STAGGER_MASK(0x1f);
-        vlv_dpio_write(dev_priv, pipe, VLV_PCS01_DW11(ch), val);
-
-        if (intel_crtc->config->lane_count > 2) {
-                val = vlv_dpio_read(dev_priv, pipe, VLV_PCS23_DW11(ch));
-                val |= DPIO_TX2_STAGGER_MASK(0x1f);
-                vlv_dpio_write(dev_priv, pipe, VLV_PCS23_DW11(ch), val);
-        }
-
-        vlv_dpio_write(dev_priv, pipe, VLV_PCS01_DW12(ch),
-                       DPIO_LANESTAGGER_STRAP(stagger) |
-                       DPIO_LANESTAGGER_STRAP_OVRD |
-                       DPIO_TX1_STAGGER_MASK(0x1f) |
-                       DPIO_TX1_STAGGER_MULT(6) |
-                       DPIO_TX2_STAGGER_MULT(0));
-
-        if (intel_crtc->config->lane_count > 2) {
-                vlv_dpio_write(dev_priv, pipe, VLV_PCS23_DW12(ch),
-                               DPIO_LANESTAGGER_STRAP(stagger) |
-                               DPIO_LANESTAGGER_STRAP_OVRD |
-                               DPIO_TX1_STAGGER_MASK(0x1f) |
-                               DPIO_TX1_STAGGER_MULT(7) |
-                               DPIO_TX2_STAGGER_MULT(5));
-        }
-
-        /* Deassert data lane reset */
-        chv_data_lane_soft_reset(encoder, false);
-
-        mutex_unlock(&dev_priv->sb_lock);
+        chv_phy_pre_encoder_enable(encoder);
 
         intel_enable_dp(encoder);
 
         /* Second common lane will stay alive on its own now */
-        if (dport->release_cl2_override) {
-                chv_phy_powergate_ch(dev_priv, DPIO_PHY0, DPIO_CH1, false);
-                dport->release_cl2_override = false;
-        }
+        chv_phy_release_cl2_override(encoder);
 }
 
 static void chv_dp_pre_pll_enable(struct intel_encoder *encoder)
 {
-        struct intel_digital_port *dport = enc_to_dig_port(&encoder->base);
-        struct drm_device *dev = encoder->base.dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
-        struct intel_crtc *intel_crtc =
-                to_intel_crtc(encoder->base.crtc);
-        enum dpio_channel ch = vlv_dport_to_channel(dport);
-        enum pipe pipe = intel_crtc->pipe;
-        unsigned int lane_mask =
-                intel_dp_unused_lane_mask(intel_crtc->config->lane_count);
-        u32 val;
-
         intel_dp_prepare(encoder);
 
-        /*
-         * Must trick the second common lane into life.
-         * Otherwise we can't even access the PLL.
-         */
-        if (ch == DPIO_CH0 && pipe == PIPE_B)
-                dport->release_cl2_override =
-                        !chv_phy_powergate_ch(dev_priv, DPIO_PHY0, DPIO_CH1, true);
-
-        chv_phy_powergate_lanes(encoder, true, lane_mask);
-
-        mutex_lock(&dev_priv->sb_lock);
-
-        /* Assert data lane reset */
-        chv_data_lane_soft_reset(encoder, true);
-
-        /* program left/right clock distribution */
-        if (pipe != PIPE_B) {
-                val = vlv_dpio_read(dev_priv, pipe, _CHV_CMN_DW5_CH0);
-                val &= ~(CHV_BUFLEFTENA1_MASK | CHV_BUFRIGHTENA1_MASK);
-                if (ch == DPIO_CH0)
-                        val |= CHV_BUFLEFTENA1_FORCE;
-                if (ch == DPIO_CH1)
-                        val |= CHV_BUFRIGHTENA1_FORCE;
-                vlv_dpio_write(dev_priv, pipe, _CHV_CMN_DW5_CH0, val);
-        } else {
-                val = vlv_dpio_read(dev_priv, pipe, _CHV_CMN_DW1_CH1);
-                val &= ~(CHV_BUFLEFTENA2_MASK | CHV_BUFRIGHTENA2_MASK);
-                if (ch == DPIO_CH0)
-                        val |= CHV_BUFLEFTENA2_FORCE;
-                if (ch == DPIO_CH1)
-                        val |= CHV_BUFRIGHTENA2_FORCE;
-                vlv_dpio_write(dev_priv, pipe, _CHV_CMN_DW1_CH1, val);
-        }
-
-        /* program clock channel usage */
-        val = vlv_dpio_read(dev_priv, pipe, VLV_PCS01_DW8(ch));
-        val |= CHV_PCS_USEDCLKCHANNEL_OVRRIDE;
-        if (pipe != PIPE_B)
-                val &= ~CHV_PCS_USEDCLKCHANNEL;
-        else
-                val |= CHV_PCS_USEDCLKCHANNEL;
-        vlv_dpio_write(dev_priv, pipe, VLV_PCS01_DW8(ch), val);
-
-        if (intel_crtc->config->lane_count > 2) {
-                val = vlv_dpio_read(dev_priv, pipe, VLV_PCS23_DW8(ch));
-                val |= CHV_PCS_USEDCLKCHANNEL_OVRRIDE;
-                if (pipe != PIPE_B)
-                        val &= ~CHV_PCS_USEDCLKCHANNEL;
-                else
-                        val |= CHV_PCS_USEDCLKCHANNEL;
-                vlv_dpio_write(dev_priv, pipe, VLV_PCS23_DW8(ch), val);
-        }
-
-        /*
-         * This a a bit weird since generally CL
-         * matches the pipe, but here we need to
-         * pick the CL based on the port.
-         */
-        val = vlv_dpio_read(dev_priv, pipe, CHV_CMN_DW19(ch));
-        if (pipe != PIPE_B)
-                val &= ~CHV_CMN_USEDCLKCHANNEL;
-        else
-                val |= CHV_CMN_USEDCLKCHANNEL;
-        vlv_dpio_write(dev_priv, pipe, CHV_CMN_DW19(ch), val);
-
-        mutex_unlock(&dev_priv->sb_lock);
+        chv_phy_pre_pll_enable(encoder);
 }
 
 static void chv_dp_post_pll_disable(struct intel_encoder *encoder)
 {
-        struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
-        enum pipe pipe = to_intel_crtc(encoder->base.crtc)->pipe;
-        u32 val;
-
-        mutex_lock(&dev_priv->sb_lock);
-
-        /* disable left/right clock distribution */
-        if (pipe != PIPE_B) {
-                val = vlv_dpio_read(dev_priv, pipe, _CHV_CMN_DW5_CH0);
-                val &= ~(CHV_BUFLEFTENA1_MASK | CHV_BUFRIGHTENA1_MASK);
-                vlv_dpio_write(dev_priv, pipe, _CHV_CMN_DW5_CH0, val);
-        } else {
-                val = vlv_dpio_read(dev_priv, pipe, _CHV_CMN_DW1_CH1);
-                val &= ~(CHV_BUFLEFTENA2_MASK | CHV_BUFRIGHTENA2_MASK);
-                vlv_dpio_write(dev_priv, pipe, _CHV_CMN_DW1_CH1, val);
-        }
-
-        mutex_unlock(&dev_priv->sb_lock);
-
-        /*
-         * Leave the power down bit cleared for at least one
-         * lane so that chv_powergate_phy_ch() will power
-         * on something when the channel is otherwise unused.
-         * When the port is off and the override is removed
-         * the lanes power down anyway, so otherwise it doesn't
-         * really matter what the state of power down bits is
-         * after this.
-         */
-        chv_phy_powergate_lanes(encoder, false, 0x0);
+        chv_phy_post_pll_disable(encoder);
 }
 
 /*
@@ -3089,7 +2878,7 @@ uint8_t
 intel_dp_voltage_max(struct intel_dp *intel_dp)
 {
         struct drm_device *dev = intel_dp_to_dev(intel_dp);
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         enum port port = dp_to_dig_port(intel_dp)->port;
 
         if (IS_BROXTON(dev))
@@ -3178,16 +2967,10 @@ intel_dp_pre_emphasis_max(struct intel_dp *intel_dp, uint8_t voltage_swing)
 
 static uint32_t vlv_signal_levels(struct intel_dp *intel_dp)
 {
-        struct drm_device *dev = intel_dp_to_dev(intel_dp);
-        struct drm_i915_private *dev_priv = dev->dev_private;
-        struct intel_digital_port *dport = dp_to_dig_port(intel_dp);
-        struct intel_crtc *intel_crtc =
-                to_intel_crtc(dport->base.base.crtc);
+        struct intel_encoder *encoder = &dp_to_dig_port(intel_dp)->base;
         unsigned long demph_reg_value, preemph_reg_value,
                 uniqtranscale_reg_value;
         uint8_t train_set = intel_dp->train_set[0];
-        enum dpio_channel port = vlv_dport_to_channel(dport);
-        int pipe = intel_crtc->pipe;
 
         switch (train_set & DP_TRAIN_PRE_EMPHASIS_MASK) {
         case DP_TRAIN_PRE_EMPH_LEVEL_0:
@@ -3262,37 +3045,18 @@ static uint32_t vlv_signal_levels(struct intel_dp *intel_dp)
                 return 0;
         }
 
-        mutex_lock(&dev_priv->sb_lock);
-        vlv_dpio_write(dev_priv, pipe, VLV_TX_DW5(port), 0x00000000);
-        vlv_dpio_write(dev_priv, pipe, VLV_TX_DW4(port), demph_reg_value);
-        vlv_dpio_write(dev_priv, pipe, VLV_TX_DW2(port),
-                         uniqtranscale_reg_value);
-        vlv_dpio_write(dev_priv, pipe, VLV_TX_DW3(port), 0x0C782040);
-        vlv_dpio_write(dev_priv, pipe, VLV_PCS_DW11(port), 0x00030000);
-        vlv_dpio_write(dev_priv, pipe, VLV_PCS_DW9(port), preemph_reg_value);
-        vlv_dpio_write(dev_priv, pipe, VLV_TX_DW5(port), 0x80000000);
-        mutex_unlock(&dev_priv->sb_lock);
+        vlv_set_phy_signal_level(encoder, demph_reg_value, preemph_reg_value,
+                                 uniqtranscale_reg_value, 0);
 
         return 0;
 }
 
-static bool chv_need_uniq_trans_scale(uint8_t train_set)
-{
-        return (train_set & DP_TRAIN_PRE_EMPHASIS_MASK) == DP_TRAIN_PRE_EMPH_LEVEL_0 &&
-                (train_set & DP_TRAIN_VOLTAGE_SWING_MASK) == DP_TRAIN_VOLTAGE_SWING_LEVEL_3;
-}
-
 static uint32_t chv_signal_levels(struct intel_dp *intel_dp)
 {
-        struct drm_device *dev = intel_dp_to_dev(intel_dp);
-        struct drm_i915_private *dev_priv = dev->dev_private;
-        struct intel_digital_port *dport = dp_to_dig_port(intel_dp);
-        struct intel_crtc *intel_crtc = to_intel_crtc(dport->base.base.crtc);
-        u32 deemph_reg_value, margin_reg_value, val;
+        struct intel_encoder *encoder = &dp_to_dig_port(intel_dp)->base;
+        u32 deemph_reg_value, margin_reg_value;
+        bool uniq_trans_scale = false;
         uint8_t train_set = intel_dp->train_set[0];
-        enum dpio_channel ch = vlv_dport_to_channel(dport);
-        enum pipe pipe = intel_crtc->pipe;
-        int i;
 
         switch (train_set & DP_TRAIN_PRE_EMPHASIS_MASK) {
         case DP_TRAIN_PRE_EMPH_LEVEL_0:
@@ -3312,7 +3076,7 @@ static uint32_t chv_signal_levels(struct intel_dp *intel_dp)
                 case DP_TRAIN_VOLTAGE_SWING_LEVEL_3:
                         deemph_reg_value = 128;
                         margin_reg_value = 154;
-                        /* FIXME extra to set for 1200 */
+                        uniq_trans_scale = true;
                         break;
                 default:
                         return 0;
@@ -3364,88 +3128,8 @@ static uint32_t chv_signal_levels(struct intel_dp *intel_dp)
                 return 0;
         }
 
-        mutex_lock(&dev_priv->sb_lock);
-
-        /* Clear calc init */
-        val = vlv_dpio_read(dev_priv, pipe, VLV_PCS01_DW10(ch));
-        val &= ~(DPIO_PCS_SWING_CALC_TX0_TX2 | DPIO_PCS_SWING_CALC_TX1_TX3);
-        val &= ~(DPIO_PCS_TX1DEEMP_MASK | DPIO_PCS_TX2DEEMP_MASK);
-        val |= DPIO_PCS_TX1DEEMP_9P5 | DPIO_PCS_TX2DEEMP_9P5;
-        vlv_dpio_write(dev_priv, pipe, VLV_PCS01_DW10(ch), val);
-
-        if (intel_crtc->config->lane_count > 2) {
-                val = vlv_dpio_read(dev_priv, pipe, VLV_PCS23_DW10(ch));
-                val &= ~(DPIO_PCS_SWING_CALC_TX0_TX2 | DPIO_PCS_SWING_CALC_TX1_TX3);
-                val &= ~(DPIO_PCS_TX1DEEMP_MASK | DPIO_PCS_TX2DEEMP_MASK);
-                val |= DPIO_PCS_TX1DEEMP_9P5 | DPIO_PCS_TX2DEEMP_9P5;
-                vlv_dpio_write(dev_priv, pipe, VLV_PCS23_DW10(ch), val);
-        }
-
-        val = vlv_dpio_read(dev_priv, pipe, VLV_PCS01_DW9(ch));
-        val &= ~(DPIO_PCS_TX1MARGIN_MASK | DPIO_PCS_TX2MARGIN_MASK);
-        val |= DPIO_PCS_TX1MARGIN_000 | DPIO_PCS_TX2MARGIN_000;
-        vlv_dpio_write(dev_priv, pipe, VLV_PCS01_DW9(ch), val);
-
-        if (intel_crtc->config->lane_count > 2) {
-                val = vlv_dpio_read(dev_priv, pipe, VLV_PCS23_DW9(ch));
-                val &= ~(DPIO_PCS_TX1MARGIN_MASK | DPIO_PCS_TX2MARGIN_MASK);
-                val |= DPIO_PCS_TX1MARGIN_000 | DPIO_PCS_TX2MARGIN_000;
-                vlv_dpio_write(dev_priv, pipe, VLV_PCS23_DW9(ch), val);
-        }
-
-        /* Program swing deemph */
-        for (i = 0; i < intel_crtc->config->lane_count; i++) {
-                val = vlv_dpio_read(dev_priv, pipe, CHV_TX_DW4(ch, i));
-                val &= ~DPIO_SWING_DEEMPH9P5_MASK;
-                val |= deemph_reg_value << DPIO_SWING_DEEMPH9P5_SHIFT;
-                vlv_dpio_write(dev_priv, pipe, CHV_TX_DW4(ch, i), val);
-        }
-
-        /* Program swing margin */
-        for (i = 0; i < intel_crtc->config->lane_count; i++) {
-                val = vlv_dpio_read(dev_priv, pipe, CHV_TX_DW2(ch, i));
-
-                val &= ~DPIO_SWING_MARGIN000_MASK;
-                val |= margin_reg_value << DPIO_SWING_MARGIN000_SHIFT;
-
-                /*
-                 * Supposedly this value shouldn't matter when unique transition
-                 * scale is disabled, but in fact it does matter. Let's just
-                 * always program the same value and hope it's OK.
-                 */
-                val &= ~(0xff << DPIO_UNIQ_TRANS_SCALE_SHIFT);
-                val |= 0x9a << DPIO_UNIQ_TRANS_SCALE_SHIFT;
-
-                vlv_dpio_write(dev_priv, pipe, CHV_TX_DW2(ch, i), val);
-        }
-
-        /*
-         * The document said it needs to set bit 27 for ch0 and bit 26
-         * for ch1. Might be a typo in the doc.
-         * For now, for this unique transition scale selection, set bit
-         * 27 for ch0 and ch1.
-         */
-        for (i = 0; i < intel_crtc->config->lane_count; i++) {
-                val = vlv_dpio_read(dev_priv, pipe, CHV_TX_DW3(ch, i));
-                if (chv_need_uniq_trans_scale(train_set))
-                        val |= DPIO_TX_UNIQ_TRANS_SCALE_EN;
-                else
-                        val &= ~DPIO_TX_UNIQ_TRANS_SCALE_EN;
-                vlv_dpio_write(dev_priv, pipe, CHV_TX_DW3(ch, i), val);
-        }
-
-        /* Start swing calculation */
-        val = vlv_dpio_read(dev_priv, pipe, VLV_PCS01_DW10(ch));
-        val |= DPIO_PCS_SWING_CALC_TX0_TX2 | DPIO_PCS_SWING_CALC_TX1_TX3;
-        vlv_dpio_write(dev_priv, pipe, VLV_PCS01_DW10(ch), val);
-
-        if (intel_crtc->config->lane_count > 2) {
-                val = vlv_dpio_read(dev_priv, pipe, VLV_PCS23_DW10(ch));
-                val |= DPIO_PCS_SWING_CALC_TX0_TX2 | DPIO_PCS_SWING_CALC_TX1_TX3;
-                vlv_dpio_write(dev_priv, pipe, VLV_PCS23_DW10(ch), val);
-        }
-
-        mutex_unlock(&dev_priv->sb_lock);
+        chv_set_phy_signal_level(encoder, deemph_reg_value,
+                                 margin_reg_value, uniq_trans_scale);
 
         return 0;
 }
@@ -3612,7 +3296,7 @@ void intel_dp_set_idle_link_train(struct intel_dp *intel_dp)
 {
         struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp);
         struct drm_device *dev = intel_dig_port->base.base.dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         enum port port = intel_dig_port->port;
         uint32_t val;
 
@@ -3634,8 +3318,10 @@ void intel_dp_set_idle_link_train(struct intel_dp *intel_dp)
         if (port == PORT_A)
                 return;
 
-        if (wait_for((I915_READ(DP_TP_STATUS(port)) & DP_TP_STATUS_IDLE_DONE),
-                     1))
+        if (intel_wait_for_register(dev_priv,DP_TP_STATUS(port),
+                                    DP_TP_STATUS_IDLE_DONE,
+                                    DP_TP_STATUS_IDLE_DONE,
+                                    1))
                 DRM_ERROR("Timed out waiting for DP idle patterns\n");
 }
 
@@ -3646,7 +3332,7 @@ intel_dp_link_down(struct intel_dp *intel_dp)
         struct intel_crtc *crtc = to_intel_crtc(intel_dig_port->base.base.crtc);
         enum port port = intel_dig_port->port;
         struct drm_device *dev = intel_dig_port->base.base.dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         uint32_t DP = intel_dp->DP;
 
         if (WARN_ON(HAS_DDI(dev)))
@@ -3698,7 +3384,7 @@ intel_dp_link_down(struct intel_dp *intel_dp)
                 I915_WRITE(intel_dp->output_reg, DP);
                 POSTING_READ(intel_dp->output_reg);
 
-                intel_wait_for_vblank_if_active(dev_priv->dev, PIPE_A);
+                intel_wait_for_vblank_if_active(&dev_priv->drm, PIPE_A);
                 intel_set_cpu_fifo_underrun_reporting(dev_priv, PIPE_A, true);
                 intel_set_pch_fifo_underrun_reporting(dev_priv, PIPE_A, true);
         }
@@ -3713,8 +3399,7 @@ intel_dp_get_dpcd(struct intel_dp *intel_dp)
 {
         struct intel_digital_port *dig_port = dp_to_dig_port(intel_dp);
         struct drm_device *dev = dig_port->base.base.dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
-        uint8_t rev;
+        struct drm_i915_private *dev_priv = to_i915(dev);
 
         if (drm_dp_dpcd_read(&intel_dp->aux, 0x000, intel_dp->dpcd,
                              sizeof(intel_dp->dpcd)) < 0)
@@ -3771,6 +3456,15 @@ intel_dp_get_dpcd(struct intel_dp *intel_dp)
                         DRM_DEBUG_KMS("PSR2 %s on sink",
                                 dev_priv->psr.psr2_support ? "supported" : "not supported");
                 }
+
+                /* Read the eDP Display control capabilities registers */
+                memset(intel_dp->edp_dpcd, 0, sizeof(intel_dp->edp_dpcd));
+                if ((intel_dp->dpcd[DP_EDP_CONFIGURATION_CAP] & DP_DPCD_DISPLAY_CONTROL_CAPABLE) &&
+                                (drm_dp_dpcd_read(&intel_dp->aux, DP_EDP_DPCD_REV,
+                                                intel_dp->edp_dpcd, sizeof(intel_dp->edp_dpcd)) ==
+                                                                sizeof(intel_dp->edp_dpcd)))
+                        DRM_DEBUG_KMS("EDP DPCD : %*ph\n", (int) sizeof(intel_dp->edp_dpcd),
+                                        intel_dp->edp_dpcd);
         }
 
         DRM_DEBUG_KMS("Display Port TPS3 support: source %s, sink %s\n",
@@ -3778,10 +3472,7 @@ intel_dp_get_dpcd(struct intel_dp *intel_dp)
                       yesno(drm_dp_tps3_supported(intel_dp->dpcd)));
 
         /* Intermediate frequency support */
-        if (is_edp(intel_dp) &&
-            (intel_dp->dpcd[DP_EDP_CONFIGURATION_CAP] & DP_DPCD_DISPLAY_CONTROL_CAPABLE) &&
-            (drm_dp_dpcd_read(&intel_dp->aux, DP_EDP_DPCD_REV, &rev, 1) == 1) &&
-            (rev >= 0x03)) { /* eDp v1.4 or higher */
+        if (is_edp(intel_dp) && (intel_dp->edp_dpcd[0] >= 0x03)) { /* eDp v1.4 or higher */
                 __le16 sink_rates[DP_MAX_SUPPORTED_RATES];
                 int i;
 
@@ -4559,7 +4250,7 @@ intel_dp_long_pulse(struct intel_connector *intel_connector)
         }
 
         if (intel_encoder->type != INTEL_OUTPUT_EDP)
-                intel_encoder->type = INTEL_OUTPUT_DISPLAYPORT;
+                intel_encoder->type = INTEL_OUTPUT_DP;
 
         intel_dp_probe_oui(intel_dp);
 
@@ -4635,7 +4326,7 @@ intel_dp_detect(struct drm_connector *connector, bool force)
                 /* MST devices are disconnected from a monitor POV */
                 intel_dp_unset_edid(intel_dp);
                 if (intel_encoder->type != INTEL_OUTPUT_EDP)
-                        intel_encoder->type = INTEL_OUTPUT_DISPLAYPORT;
+                        intel_encoder->type = INTEL_OUTPUT_DP;
                 return connector_status_disconnected;
         }
 
@@ -4674,7 +4365,7 @@ intel_dp_force(struct drm_connector *connector)
         intel_display_power_put(dev_priv, power_domain);
 
         if (intel_encoder->type != INTEL_OUTPUT_EDP)
-                intel_encoder->type = INTEL_OUTPUT_DISPLAYPORT;
+                intel_encoder->type = INTEL_OUTPUT_DP;
 }
 
 static int intel_dp_get_modes(struct drm_connector *connector)
@@ -4723,7 +4414,7 @@ intel_dp_set_property(struct drm_connector *connector,
                       struct drm_property *property,
                       uint64_t val)
 {
-        struct drm_i915_private *dev_priv = connector->dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(connector->dev);
         struct intel_connector *intel_connector = to_intel_connector(connector);
         struct intel_encoder *intel_encoder = intel_attached_encoder(connector);
         struct intel_dp *intel_dp = enc_to_intel_dp(&intel_encoder->base);
@@ -4811,6 +4502,32 @@ done:
         return 0;
 }
 
+static int
+intel_dp_connector_register(struct drm_connector *connector)
+{
+        struct intel_dp *intel_dp = intel_attached_dp(connector);
+        int ret;
+
+        ret = intel_connector_register(connector);
+        if (ret)
+                return ret;
+
+        i915_debugfs_connector_add(connector);
+
+        DRM_DEBUG_KMS("registering %s bus for %s\n",
+                      intel_dp->aux.name, connector->kdev->kobj.name);
+
+        intel_dp->aux.dev = connector->kdev;
+        return drm_dp_aux_register(&intel_dp->aux);
+}
+
+static void
+intel_dp_connector_unregister(struct drm_connector *connector)
+{
+        drm_dp_aux_unregister(&intel_attached_dp(connector)->aux);
+        intel_connector_unregister(connector);
+}
+
 static void
 intel_dp_connector_destroy(struct drm_connector *connector)
 {
@@ -4851,6 +4568,9 @@ void intel_dp_encoder_destroy(struct drm_encoder *encoder)
                         intel_dp->edp_notifier.notifier_call = NULL;
                 }
         }
+
+        intel_dp_aux_fini(intel_dp);
+
         drm_encoder_cleanup(encoder);
         kfree(intel_dig_port);
 }
@@ -4876,7 +4596,7 @@ static void intel_edp_panel_vdd_sanitize(struct intel_dp *intel_dp)
 {
         struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp);
         struct drm_device *dev = intel_dig_port->base.base.dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         enum intel_display_power_domain power_domain;
 
         lockdep_assert_held(&dev_priv->pps_mutex);
@@ -4929,6 +4649,8 @@ static const struct drm_connector_funcs intel_dp_connector_funcs = {
         .fill_modes = drm_helper_probe_single_connector_modes,
         .set_property = intel_dp_set_property,
         .atomic_get_property = intel_connector_atomic_get_property,
+        .late_register = intel_dp_connector_register,
+        .early_unregister = intel_dp_connector_unregister,
         .destroy = intel_dp_connector_destroy,
         .atomic_destroy_state = drm_atomic_helper_connector_destroy_state,
         .atomic_duplicate_state = drm_atomic_helper_connector_duplicate_state,
@@ -4937,7 +4659,6 @@ static const struct drm_connector_funcs intel_dp_connector_funcs = {
 static const struct drm_connector_helper_funcs intel_dp_connector_helper_funcs = {
         .get_modes = intel_dp_get_modes,
         .mode_valid = intel_dp_mode_valid,
-        .best_encoder = intel_best_encoder,
 };
 
 static const struct drm_encoder_funcs intel_dp_enc_funcs = {
@@ -4951,13 +4672,13 @@ intel_dp_hpd_pulse(struct intel_digital_port *intel_dig_port, bool long_hpd)
         struct intel_dp *intel_dp = &intel_dig_port->dp;
         struct intel_encoder *intel_encoder = &intel_dig_port->base;
         struct drm_device *dev = intel_dig_port->base.base.dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         enum intel_display_power_domain power_domain;
         enum irqreturn ret = IRQ_NONE;
 
         if (intel_dig_port->base.type != INTEL_OUTPUT_EDP &&
             intel_dig_port->base.type != INTEL_OUTPUT_HDMI)
-                intel_dig_port->base.type = INTEL_OUTPUT_DISPLAYPORT;
+                intel_dig_port->base.type = INTEL_OUTPUT_DP;
 
         if (long_hpd && intel_dig_port->base.type == INTEL_OUTPUT_EDP) {
                 /*
@@ -5019,7 +4740,7 @@ put_power:
 /* check the VBT to see whether the eDP is on another port */
 bool intel_dp_is_edp(struct drm_device *dev, enum port port)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
 
         /*
          * eDP not supported on g4x. so bail out early just
@@ -5061,82 +4782,93 @@ static void intel_dp_init_panel_power_timestamps(struct intel_dp *intel_dp)
 }
 
 static void
-intel_dp_init_panel_power_sequencer(struct drm_device *dev,
-                                    struct intel_dp *intel_dp)
+intel_pps_readout_hw_state(struct drm_i915_private *dev_priv,
+                           struct intel_dp *intel_dp, struct edp_power_seq *seq)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
-        struct edp_power_seq cur, vbt, spec,
-                *final = &intel_dp->pps_delays;
         u32 pp_on, pp_off, pp_div = 0, pp_ctl = 0;
-        i915_reg_t pp_ctrl_reg, pp_on_reg, pp_off_reg, pp_div_reg;
-
-        lockdep_assert_held(&dev_priv->pps_mutex);
-
-        /* already initialized? */
-        if (final->t11_t12 != 0)
-                return;
-
-        if (IS_BROXTON(dev)) {
-                /*
-                 * TODO: BXT has 2 sets of PPS registers.
-                 * Correct Register for Broxton need to be identified
-                 * using VBT. hardcoding for now
-                 */
-                pp_ctrl_reg = BXT_PP_CONTROL(0);
-                pp_on_reg = BXT_PP_ON_DELAYS(0);
-                pp_off_reg = BXT_PP_OFF_DELAYS(0);
-        } else if (HAS_PCH_SPLIT(dev)) {
-                pp_ctrl_reg = PCH_PP_CONTROL;
-                pp_on_reg = PCH_PP_ON_DELAYS;
-                pp_off_reg = PCH_PP_OFF_DELAYS;
-                pp_div_reg = PCH_PP_DIVISOR;
-        } else {
-                enum pipe pipe = vlv_power_sequencer_pipe(intel_dp);
+        struct pps_registers regs;
 
-                pp_ctrl_reg = VLV_PIPE_PP_CONTROL(pipe);
-                pp_on_reg = VLV_PIPE_PP_ON_DELAYS(pipe);
-                pp_off_reg = VLV_PIPE_PP_OFF_DELAYS(pipe);
-                pp_div_reg = VLV_PIPE_PP_DIVISOR(pipe);
-        }
+        intel_pps_get_registers(dev_priv, intel_dp, &regs);
 
         /* Workaround: Need to write PP_CONTROL with the unlock key as
          * the very first thing. */
         pp_ctl = ironlake_get_pp_control(intel_dp);
 
-        pp_on = I915_READ(pp_on_reg);
-        pp_off = I915_READ(pp_off_reg);
-        if (!IS_BROXTON(dev)) {
-                I915_WRITE(pp_ctrl_reg, pp_ctl);
-                pp_div = I915_READ(pp_div_reg);
+        pp_on = I915_READ(regs.pp_on);
+        pp_off = I915_READ(regs.pp_off);
+        if (!IS_BROXTON(dev_priv)) {
+                I915_WRITE(regs.pp_ctrl, pp_ctl);
+                pp_div = I915_READ(regs.pp_div);
         }
 
         /* Pull timing values out of registers */
-        cur.t1_t3 = (pp_on & PANEL_POWER_UP_DELAY_MASK) >>
-                PANEL_POWER_UP_DELAY_SHIFT;
+        seq->t1_t3 = (pp_on & PANEL_POWER_UP_DELAY_MASK) >>
+                     PANEL_POWER_UP_DELAY_SHIFT;
 
-        cur.t8 = (pp_on & PANEL_LIGHT_ON_DELAY_MASK) >>
-                PANEL_LIGHT_ON_DELAY_SHIFT;
+        seq->t8 = (pp_on & PANEL_LIGHT_ON_DELAY_MASK) >>
+                  PANEL_LIGHT_ON_DELAY_SHIFT;
 
-        cur.t9 = (pp_off & PANEL_LIGHT_OFF_DELAY_MASK) >>
-                PANEL_LIGHT_OFF_DELAY_SHIFT;
+        seq->t9 = (pp_off & PANEL_LIGHT_OFF_DELAY_MASK) >>
+                  PANEL_LIGHT_OFF_DELAY_SHIFT;
 
-        cur.t10 = (pp_off & PANEL_POWER_DOWN_DELAY_MASK) >>
-                PANEL_POWER_DOWN_DELAY_SHIFT;
+        seq->t10 = (pp_off & PANEL_POWER_DOWN_DELAY_MASK) >>
+                   PANEL_POWER_DOWN_DELAY_SHIFT;
 
-        if (IS_BROXTON(dev)) {
+        if (IS_BROXTON(dev_priv)) {
                 u16 tmp = (pp_ctl & BXT_POWER_CYCLE_DELAY_MASK) >>
                         BXT_POWER_CYCLE_DELAY_SHIFT;
                 if (tmp > 0)
-                        cur.t11_t12 = (tmp - 1) * 1000;
+                        seq->t11_t12 = (tmp - 1) * 1000;
                 else
-                        cur.t11_t12 = 0;
+                        seq->t11_t12 = 0;
         } else {
-                cur.t11_t12 = ((pp_div & PANEL_POWER_CYCLE_DELAY_MASK) >>
+                seq->t11_t12 = ((pp_div & PANEL_POWER_CYCLE_DELAY_MASK) >>
                        PANEL_POWER_CYCLE_DELAY_SHIFT) * 1000;
         }
+}
+
+static void
+intel_pps_dump_state(const char *state_name, const struct edp_power_seq *seq)
+{
+        DRM_DEBUG_KMS("%s t1_t3 %d t8 %d t9 %d t10 %d t11_t12 %d\n",
+                      state_name,
+                      seq->t1_t3, seq->t8, seq->t9, seq->t10, seq->t11_t12);
+}
+
+static void
+intel_pps_verify_state(struct drm_i915_private *dev_priv,
+                       struct intel_dp *intel_dp)
+{
+        struct edp_power_seq hw;
+        struct edp_power_seq *sw = &intel_dp->pps_delays;
+
+        intel_pps_readout_hw_state(dev_priv, intel_dp, &hw);
+
+        if (hw.t1_t3 != sw->t1_t3 || hw.t8 != sw->t8 || hw.t9 != sw->t9 ||
+            hw.t10 != sw->t10 || hw.t11_t12 != sw->t11_t12) {
+                DRM_ERROR("PPS state mismatch\n");
+                intel_pps_dump_state("sw", sw);
+                intel_pps_dump_state("hw", &hw);
+        }
+}
+
+static void
+intel_dp_init_panel_power_sequencer(struct drm_device *dev,
+                                    struct intel_dp *intel_dp)
+{
+        struct drm_i915_private *dev_priv = to_i915(dev);
+        struct edp_power_seq cur, vbt, spec,
+                *final = &intel_dp->pps_delays;
+
+        lockdep_assert_held(&dev_priv->pps_mutex);
+
+        /* already initialized? */
+        if (final->t11_t12 != 0)
+                return;
 
-        DRM_DEBUG_KMS("cur t1_t3 %d t8 %d t9 %d t10 %d t11_t12 %d\n",
-                      cur.t1_t3, cur.t8, cur.t9, cur.t10, cur.t11_t12);
+        intel_pps_readout_hw_state(dev_priv, intel_dp, &cur);
+
+        intel_pps_dump_state("cur", &cur);
 
         vbt = dev_priv->vbt.edp.pps;
 
@@ -5152,8 +4884,7 @@ intel_dp_init_panel_power_sequencer(struct drm_device *dev,
          * too. */
         spec.t11_t12 = (510 + 100) * 10;
 
-        DRM_DEBUG_KMS("vbt t1_t3 %d t8 %d t9 %d t10 %d t11_t12 %d\n",
-                      vbt.t1_t3, vbt.t8, vbt.t9, vbt.t10, vbt.t11_t12);
+        intel_pps_dump_state("vbt", &vbt);
 
         /* Use the max of the register settings and vbt. If both are
          * unset, fall back to the spec limits. */
@@ -5181,59 +4912,41 @@ intel_dp_init_panel_power_sequencer(struct drm_device *dev,
 
         DRM_DEBUG_KMS("backlight on delay %d, off delay %d\n",
                       intel_dp->backlight_on_delay, intel_dp->backlight_off_delay);
+
+        /*
+         * We override the HW backlight delays to 1 because we do manual waits
+         * on them. For T8, even BSpec recommends doing it. For T9, if we
+         * don't do this, we'll end up waiting for the backlight off delay
+         * twice: once when we do the manual sleep, and once when we disable
+         * the panel and wait for the PP_STATUS bit to become zero.
+         */
+        final->t8 = 1;
+        final->t9 = 1;
 }
 
 static void
 intel_dp_init_panel_power_sequencer_registers(struct drm_device *dev,
                                               struct intel_dp *intel_dp)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         u32 pp_on, pp_off, pp_div, port_sel = 0;
         int div = dev_priv->rawclk_freq / 1000;
-        i915_reg_t pp_on_reg, pp_off_reg, pp_div_reg, pp_ctrl_reg;
+        struct pps_registers regs;
         enum port port = dp_to_dig_port(intel_dp)->port;
         const struct edp_power_seq *seq = &intel_dp->pps_delays;
 
         lockdep_assert_held(&dev_priv->pps_mutex);
 
-        if (IS_BROXTON(dev)) {
-                /*
-                 * TODO: BXT has 2 sets of PPS registers.
-                 * Correct Register for Broxton need to be identified
-                 * using VBT. hardcoding for now
-                 */
-                pp_ctrl_reg = BXT_PP_CONTROL(0);
-                pp_on_reg = BXT_PP_ON_DELAYS(0);
-                pp_off_reg = BXT_PP_OFF_DELAYS(0);
-
-        } else if (HAS_PCH_SPLIT(dev)) {
-                pp_on_reg = PCH_PP_ON_DELAYS;
-                pp_off_reg = PCH_PP_OFF_DELAYS;
-                pp_div_reg = PCH_PP_DIVISOR;
-        } else {
-                enum pipe pipe = vlv_power_sequencer_pipe(intel_dp);
+        intel_pps_get_registers(dev_priv, intel_dp, &regs);
 
-                pp_on_reg = VLV_PIPE_PP_ON_DELAYS(pipe);
-                pp_off_reg = VLV_PIPE_PP_OFF_DELAYS(pipe);
-                pp_div_reg = VLV_PIPE_PP_DIVISOR(pipe);
-        }
-
-        /*
-         * And finally store the new values in the power sequencer. The
-         * backlight delays are set to 1 because we do manual waits on them. For
-         * T8, even BSpec recommends doing it. For T9, if we don't do this,
-         * we'll end up waiting for the backlight off delay twice: once when we
-         * do the manual sleep, and once when we disable the panel and wait for
-         * the PP_STATUS bit to become zero.
-         */
         pp_on = (seq->t1_t3 << PANEL_POWER_UP_DELAY_SHIFT) |
-                (1 << PANEL_LIGHT_ON_DELAY_SHIFT);
-        pp_off = (1 << PANEL_LIGHT_OFF_DELAY_SHIFT) |
+                (seq->t8 << PANEL_LIGHT_ON_DELAY_SHIFT);
+        pp_off = (seq->t9 << PANEL_LIGHT_OFF_DELAY_SHIFT) |
                  (seq->t10 << PANEL_POWER_DOWN_DELAY_SHIFT);
         /* Compute the divisor for the pp clock, simply match the Bspec
          * formula. */
         if (IS_BROXTON(dev)) {
-                pp_div = I915_READ(pp_ctrl_reg);
+                pp_div = I915_READ(regs.pp_ctrl);
                 pp_div &= ~BXT_POWER_CYCLE_DELAY_MASK;
                 pp_div |= (DIV_ROUND_UP((seq->t11_t12 + 1), 1000)
                                 << BXT_POWER_CYCLE_DELAY_SHIFT);
@@ -5256,19 +4969,19 @@ intel_dp_init_panel_power_sequencer_registers(struct drm_device *dev,
 
         pp_on |= port_sel;
 
-        I915_WRITE(pp_on_reg, pp_on);
-        I915_WRITE(pp_off_reg, pp_off);
+        I915_WRITE(regs.pp_on, pp_on);
+        I915_WRITE(regs.pp_off, pp_off);
         if (IS_BROXTON(dev))
-                I915_WRITE(pp_ctrl_reg, pp_div);
+                I915_WRITE(regs.pp_ctrl, pp_div);
         else
-                I915_WRITE(pp_div_reg, pp_div);
+                I915_WRITE(regs.pp_div, pp_div);
 
         DRM_DEBUG_KMS("panel power sequencer register settings: PP_ON %#x, PP_OFF %#x, PP_DIV %#x\n",
-                      I915_READ(pp_on_reg),
-                      I915_READ(pp_off_reg),
+                      I915_READ(regs.pp_on),
+                      I915_READ(regs.pp_off),
                       IS_BROXTON(dev) ?
-                      (I915_READ(pp_ctrl_reg) & BXT_POWER_CYCLE_DELAY_MASK) :
-                      I915_READ(pp_div_reg));
+                      (I915_READ(regs.pp_ctrl) & BXT_POWER_CYCLE_DELAY_MASK) :
+                      I915_READ(regs.pp_div));
 }
 
 /**
@@ -5285,7 +4998,7 @@ intel_dp_init_panel_power_sequencer_registers(struct drm_device *dev,
  */
 static void intel_dp_set_drrs_state(struct drm_device *dev, int refresh_rate)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         struct intel_encoder *encoder;
         struct intel_digital_port *dig_port = NULL;
         struct intel_dp *intel_dp = dev_priv->drrs.dp;
@@ -5384,7 +5097,7 @@ static void intel_dp_set_drrs_state(struct drm_device *dev, int refresh_rate)
 void intel_edp_drrs_enable(struct intel_dp *intel_dp)
 {
         struct drm_device *dev = intel_dp_to_dev(intel_dp);
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         struct intel_digital_port *dig_port = dp_to_dig_port(intel_dp);
         struct drm_crtc *crtc = dig_port->base.base.crtc;
         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
@@ -5416,7 +5129,7 @@ unlock:
 void intel_edp_drrs_disable(struct intel_dp *intel_dp)
 {
         struct drm_device *dev = intel_dp_to_dev(intel_dp);
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         struct intel_digital_port *dig_port = dp_to_dig_port(intel_dp);
         struct drm_crtc *crtc = dig_port->base.base.crtc;
         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
@@ -5431,9 +5144,9 @@ void intel_edp_drrs_disable(struct intel_dp *intel_dp)
         }
 
         if (dev_priv->drrs.refresh_rate_type == DRRS_LOW_RR)
-                intel_dp_set_drrs_state(dev_priv->dev,
-                        intel_dp->attached_connector->panel.
-                        fixed_mode->vrefresh);
+                intel_dp_set_drrs_state(&dev_priv->drm,
+                                        intel_dp->attached_connector->panel.
+                                        fixed_mode->vrefresh);
 
         dev_priv->drrs.dp = NULL;
         mutex_unlock(&dev_priv->drrs.mutex);
@@ -5463,9 +5176,9 @@ static void intel_edp_drrs_downclock_work(struct work_struct *work)
                 goto unlock;
 
         if (dev_priv->drrs.refresh_rate_type != DRRS_LOW_RR)
-                intel_dp_set_drrs_state(dev_priv->dev,
-                        intel_dp->attached_connector->panel.
-                        downclock_mode->vrefresh);
+                intel_dp_set_drrs_state(&dev_priv->drm,
+                                        intel_dp->attached_connector->panel.
+                                        downclock_mode->vrefresh);
 
 unlock:
         mutex_unlock(&dev_priv->drrs.mutex);
@@ -5484,7 +5197,7 @@ unlock:
 void intel_edp_drrs_invalidate(struct drm_device *dev,
                 unsigned frontbuffer_bits)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         struct drm_crtc *crtc;
         enum pipe pipe;
 
@@ -5507,9 +5220,9 @@ void intel_edp_drrs_invalidate(struct drm_device *dev,
 
         /* invalidate means busy screen hence upclock */
         if (frontbuffer_bits && dev_priv->drrs.refresh_rate_type == DRRS_LOW_RR)
-                intel_dp_set_drrs_state(dev_priv->dev,
-                                dev_priv->drrs.dp->attached_connector->panel.
-                                fixed_mode->vrefresh);
+                intel_dp_set_drrs_state(&dev_priv->drm,
+                                        dev_priv->drrs.dp->attached_connector->panel.
+                                        fixed_mode->vrefresh);
 
         mutex_unlock(&dev_priv->drrs.mutex);
 }
@@ -5529,7 +5242,7 @@ void intel_edp_drrs_invalidate(struct drm_device *dev,
 void intel_edp_drrs_flush(struct drm_device *dev,
                 unsigned frontbuffer_bits)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         struct drm_crtc *crtc;
         enum pipe pipe;
 
@@ -5552,9 +5265,9 @@ void intel_edp_drrs_flush(struct drm_device *dev,
 
         /* flush means busy screen hence upclock */
         if (frontbuffer_bits && dev_priv->drrs.refresh_rate_type == DRRS_LOW_RR)
-                intel_dp_set_drrs_state(dev_priv->dev,
-                                dev_priv->drrs.dp->attached_connector->panel.
-                                fixed_mode->vrefresh);
+                intel_dp_set_drrs_state(&dev_priv->drm,
+                                        dev_priv->drrs.dp->attached_connector->panel.
+                                        fixed_mode->vrefresh);
 
         /*
          * flush also means no more activity hence schedule downclock, if all
@@ -5589,14 +5302,14 @@ void intel_edp_drrs_flush(struct drm_device *dev,
  *
  * DRRS saves power by switching to low RR based on usage scenarios.
  *
- * eDP DRRS:-
- *        The implementation is based on frontbuffer tracking implementation.
- * When there is a disturbance on the screen triggered by user activity or a
- * periodic system activity, DRRS is disabled (RR is changed to high RR).
- * When there is no movement on screen, after a timeout of 1 second, a switch
- * to low RR is made.
- *        For integration with frontbuffer tracking code,
- * intel_edp_drrs_invalidate() and intel_edp_drrs_flush() are called.
+ * The implementation is based on frontbuffer tracking implementation.  When
+ * there is a disturbance on the screen triggered by user activity or a periodic
+ * system activity, DRRS is disabled (RR is changed to high RR).  When there is
+ * no movement on screen, after a timeout of 1 second, a switch to low RR is
+ * made.
+ *
+ * For integration with frontbuffer tracking code, intel_edp_drrs_invalidate()
+ * and intel_edp_drrs_flush() are called.
  *
  * DRRS can be further extended to support other internal panels and also
  * the scenario of video playback wherein RR is set based on the rate
@@ -5622,7 +5335,7 @@ intel_dp_drrs_init(struct intel_connector *intel_connector,
 {
         struct drm_connector *connector = &intel_connector->base;
         struct drm_device *dev = connector->dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         struct drm_display_mode *downclock_mode = NULL;
 
         INIT_DELAYED_WORK(&dev_priv->drrs.work, intel_edp_drrs_downclock_work);
@@ -5660,7 +5373,7 @@ static bool intel_edp_init_connector(struct intel_dp *intel_dp,
         struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp);
         struct intel_encoder *intel_encoder = &intel_dig_port->base;
         struct drm_device *dev = intel_encoder->base.dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         struct drm_display_mode *fixed_mode = NULL;
         struct drm_display_mode *downclock_mode = NULL;
         bool has_dpcd;
@@ -5671,8 +5384,32 @@ static bool intel_edp_init_connector(struct intel_dp *intel_dp,
         if (!is_edp(intel_dp))
                 return true;
 
+        /*
+         * On IBX/CPT we may get here with LVDS already registered. Since the
+         * driver uses the only internal power sequencer available for both
+         * eDP and LVDS bail out early in this case to prevent interfering
+         * with an already powered-on LVDS power sequencer.
+         */
+        if (intel_get_lvds_encoder(dev)) {
+                WARN_ON(!(HAS_PCH_IBX(dev_priv) || HAS_PCH_CPT(dev_priv)));
+                DRM_INFO("LVDS was detected, not registering eDP\n");
+
+                return false;
+        }
+
         pps_lock(intel_dp);
+
+        intel_dp_init_panel_power_timestamps(intel_dp);
+
+        if (IS_VALLEYVIEW(dev) || IS_CHERRYVIEW(dev)) {
+                vlv_initial_power_sequencer_setup(intel_dp);
+        } else {
+                intel_dp_init_panel_power_sequencer(dev, intel_dp);
+                intel_dp_init_panel_power_sequencer_registers(dev, intel_dp);
+        }
+
         intel_edp_panel_vdd_sanitize(intel_dp);
+
         pps_unlock(intel_dp);
 
         /* Cache DPCD and EDID for edp. */
@@ -5686,14 +5423,9 @@ static bool intel_edp_init_connector(struct intel_dp *intel_dp,
         } else {
                 /* if this fails, presume the device is a ghost */
                 DRM_INFO("failed to retrieve link info, disabling eDP\n");
-                return false;
+                goto out_vdd_off;
         }
 
-        /* We now know it's not a ghost, init power sequence regs. */
-        pps_lock(intel_dp);
-        intel_dp_init_panel_power_sequencer_registers(dev, intel_dp);
-        pps_unlock(intel_dp);
-
         mutex_lock(&dev->mode_config.mutex);
         edid = drm_get_edid(connector, &intel_dp->aux.ddc);
         if (edid) {
@@ -5761,6 +5493,18 @@ static bool intel_edp_init_connector(struct intel_dp *intel_dp,
         intel_panel_setup_backlight(connector, pipe);
 
         return true;
+
+out_vdd_off:
+        cancel_delayed_work_sync(&intel_dp->panel_vdd_work);
+        /*
+         * vdd might still be enabled do to the delayed vdd off.
+         * Make sure vdd is actually turned off here.
+         */
+        pps_lock(intel_dp);
+        edp_panel_vdd_off_sync(intel_dp);
+        pps_unlock(intel_dp);
+
+        return false;
 }
 
 bool
@@ -5771,9 +5515,9 @@ intel_dp_init_connector(struct intel_digital_port *intel_dig_port,
         struct intel_dp *intel_dp = &intel_dig_port->dp;
         struct intel_encoder *intel_encoder = &intel_dig_port->base;
         struct drm_device *dev = intel_encoder->base.dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         enum port port = intel_dig_port->port;
-        int type, ret;
+        int type;
 
         if (WARN(intel_dig_port->max_lanes < 1,
                  "Not enough lanes (%d) for DP on port %c\n",
@@ -5832,17 +5576,17 @@ intel_dp_init_connector(struct intel_digital_port *intel_dig_port,
         connector->interlace_allowed = true;
         connector->doublescan_allowed = 0;
 
+        intel_dp_aux_init(intel_dp, intel_connector);
+
         INIT_DELAYED_WORK(&intel_dp->panel_vdd_work,
                           edp_panel_vdd_work);
 
         intel_connector_attach_encoder(intel_connector, intel_encoder);
-        drm_connector_register(connector);
 
         if (HAS_DDI(dev))
                 intel_connector->get_hw_state = intel_ddi_connector_get_hw_state;
         else
                 intel_connector->get_hw_state = intel_connector_get_hw_state;
-        intel_connector->unregister = intel_dp_connector_unregister;
 
         /* Set up the hotplug pin. */
         switch (port) {
@@ -5867,22 +5611,8 @@ intel_dp_init_connector(struct intel_digital_port *intel_dig_port,
                 BUG();
         }
 
-        if (is_edp(intel_dp)) {
-                pps_lock(intel_dp);
-                intel_dp_init_panel_power_timestamps(intel_dp);
-                if (IS_VALLEYVIEW(dev) || IS_CHERRYVIEW(dev))
-                        vlv_initial_power_sequencer_setup(intel_dp);
-                else
-                        intel_dp_init_panel_power_sequencer(dev, intel_dp);
-                pps_unlock(intel_dp);
-        }
-
-        ret = intel_dp_aux_init(intel_dp, intel_connector);
-        if (ret)
-                goto fail;
-
         /* init MST on ports that can support it */
-        if (HAS_DP_MST(dev) &&
+        if (HAS_DP_MST(dev) && !is_edp(intel_dp) &&
             (port == PORT_B || port == PORT_C || port == PORT_D))
                 intel_dp_mst_encoder_init(intel_dig_port,
                                           intel_connector->base.base.id);
@@ -5904,22 +5634,9 @@ intel_dp_init_connector(struct intel_digital_port *intel_dig_port,
                 I915_WRITE(PEG_BAND_GAP_DATA, (temp & ~0xf) | 0xd);
         }
 
-        i915_debugfs_connector_add(connector);
-
         return true;
 
 fail:
-        if (is_edp(intel_dp)) {
-                cancel_delayed_work_sync(&intel_dp->panel_vdd_work);
-                /*
-                 * vdd might still be enabled do to the delayed vdd off.
-                 * Make sure vdd is actually turned off here.
-                 */
-                pps_lock(intel_dp);
-                edp_panel_vdd_off_sync(intel_dp);
-                pps_unlock(intel_dp);
-        }
-        drm_connector_unregister(connector);
         drm_connector_cleanup(connector);
 
         return false;
@@ -5929,7 +5646,7 @@ bool intel_dp_init(struct drm_device *dev,
                    i915_reg_t output_reg,
                    enum port port)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         struct intel_digital_port *intel_dig_port;
         struct intel_encoder *intel_encoder;
         struct drm_encoder *encoder;
@@ -5947,7 +5664,7 @@ bool intel_dp_init(struct drm_device *dev,
         encoder = &intel_encoder->base;
 
         if (drm_encoder_init(dev, &intel_encoder->base, &intel_dp_enc_funcs,
-                             DRM_MODE_ENCODER_TMDS, NULL))
+                             DRM_MODE_ENCODER_TMDS, "DP %c", port_name(port)))
                 goto err_encoder_init;
 
         intel_encoder->compute_config = intel_dp_compute_config;
@@ -5977,7 +5694,7 @@ bool intel_dp_init(struct drm_device *dev,
         intel_dig_port->dp.output_reg = output_reg;
         intel_dig_port->max_lanes = 4;
 
-        intel_encoder->type = INTEL_OUTPUT_DISPLAYPORT;
+        intel_encoder->type = INTEL_OUTPUT_DP;
         if (IS_CHERRYVIEW(dev)) {
                 if (port == PORT_D)
                         intel_encoder->crtc_mask = 1 << 2;
@@ -6007,43 +5724,35 @@ err_connector_alloc:
 
 void intel_dp_mst_suspend(struct drm_device *dev)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         int i;
 
         /* disable MST */
         for (i = 0; i < I915_MAX_PORTS; i++) {
                 struct intel_digital_port *intel_dig_port = dev_priv->hotplug.irq_port[i];
-                if (!intel_dig_port)
+
+                if (!intel_dig_port || !intel_dig_port->dp.can_mst)
                         continue;
 
-                if (intel_dig_port->base.type == INTEL_OUTPUT_DISPLAYPORT) {
-                        if (!intel_dig_port->dp.can_mst)
-                                continue;
-                        if (intel_dig_port->dp.is_mst)
-                                drm_dp_mst_topology_mgr_suspend(&intel_dig_port->dp.mst_mgr);
-                }
+                if (intel_dig_port->dp.is_mst)
+                        drm_dp_mst_topology_mgr_suspend(&intel_dig_port->dp.mst_mgr);
         }
 }
 
 void intel_dp_mst_resume(struct drm_device *dev)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         int i;
 
         for (i = 0; i < I915_MAX_PORTS; i++) {
                 struct intel_digital_port *intel_dig_port = dev_priv->hotplug.irq_port[i];
-                if (!intel_dig_port)
-                        continue;
-                if (intel_dig_port->base.type == INTEL_OUTPUT_DISPLAYPORT) {
-                        int ret;
+                int ret;
 
-                        if (!intel_dig_port->dp.can_mst)
-                                continue;
+                if (!intel_dig_port || !intel_dig_port->dp.can_mst)
+                        continue;
 
-                        ret = drm_dp_mst_topology_mgr_resume(&intel_dig_port->dp.mst_mgr);
-                        if (ret != 0) {
-                                intel_dp_check_mst_status(&intel_dig_port->dp);
-                        }
-                }
+                ret = drm_dp_mst_topology_mgr_resume(&intel_dig_port->dp.mst_mgr);
+                if (ret)
+                        intel_dp_check_mst_status(&intel_dig_port->dp);
         }
 }
diff --git a/drivers/gpu/drm/i915/intel_dp_aux_backlight.c b/drivers/gpu/drm/i915/intel_dp_aux_backlight.c
new file mode 100644
index 000000000000..6532e226db29
--- /dev/null
+++ b/drivers/gpu/drm/i915/intel_dp_aux_backlight.c
@@ -0,0 +1,172 @@
+/*
+ * Copyright © 2015 Intel Corporation
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ *
+ */
+
+#include "intel_drv.h"
+
+static void set_aux_backlight_enable(struct intel_dp *intel_dp, bool enable)
+{
+        uint8_t reg_val = 0;
+
+        if (drm_dp_dpcd_readb(&intel_dp->aux, DP_EDP_DISPLAY_CONTROL_REGISTER,
+                              &reg_val) < 0) {
+                DRM_DEBUG_KMS("Failed to read DPCD register 0x%x\n",
+                              DP_EDP_DISPLAY_CONTROL_REGISTER);
+                return;
+        }
+        if (enable)
+                reg_val |= DP_EDP_BACKLIGHT_ENABLE;
+        else
+                reg_val &= ~(DP_EDP_BACKLIGHT_ENABLE);
+
+        if (drm_dp_dpcd_writeb(&intel_dp->aux, DP_EDP_DISPLAY_CONTROL_REGISTER,
+                               reg_val) != 1) {
+                DRM_DEBUG_KMS("Failed to %s aux backlight\n",
+                              enable ? "enable" : "disable");
+        }
+}
+
+/*
+ * Read the current backlight value from DPCD register(s) based
+ * on if 8-bit(MSB) or 16-bit(MSB and LSB) values are supported
+ */
+static uint32_t intel_dp_aux_get_backlight(struct intel_connector *connector)
+{
+        struct intel_dp *intel_dp = enc_to_intel_dp(&connector->encoder->base);
+        uint8_t read_val[2] = { 0x0 };
+        uint16_t level = 0;
+
+        if (drm_dp_dpcd_read(&intel_dp->aux, DP_EDP_BACKLIGHT_BRIGHTNESS_MSB,
+                             &read_val, sizeof(read_val)) < 0) {
+                DRM_DEBUG_KMS("Failed to read DPCD register 0x%x\n",
+                              DP_EDP_BACKLIGHT_BRIGHTNESS_MSB);
+                return 0;
+        }
+        level = read_val[0];
+        if (intel_dp->edp_dpcd[2] & DP_EDP_BACKLIGHT_BRIGHTNESS_BYTE_COUNT)
+                level = (read_val[0] << 8 | read_val[1]);
+
+        return level;
+}
+
+/*
+ * Sends the current backlight level over the aux channel, checking if its using
+ * 8-bit or 16 bit value (MSB and LSB)
+ */
+static void
+intel_dp_aux_set_backlight(struct intel_connector *connector, u32 level)
+{
+        struct intel_dp *intel_dp = enc_to_intel_dp(&connector->encoder->base);
+        uint8_t vals[2] = { 0x0 };
+
+        vals[0] = level;
+
+        /* Write the MSB and/or LSB */
+        if (intel_dp->edp_dpcd[2] & DP_EDP_BACKLIGHT_BRIGHTNESS_BYTE_COUNT) {
+                vals[0] = (level & 0xFF00) >> 8;
+                vals[1] = (level & 0xFF);
+        }
+        if (drm_dp_dpcd_write(&intel_dp->aux, DP_EDP_BACKLIGHT_BRIGHTNESS_MSB,
+                              vals, sizeof(vals)) < 0) {
+                DRM_DEBUG_KMS("Failed to write aux backlight level\n");
+                return;
+        }
+}
+
+static void intel_dp_aux_enable_backlight(struct intel_connector *connector)
+{
+        struct intel_dp *intel_dp = enc_to_intel_dp(&connector->encoder->base);
+        uint8_t dpcd_buf = 0;
+
+        set_aux_backlight_enable(intel_dp, true);
+
+        if ((drm_dp_dpcd_readb(&intel_dp->aux,
+                               DP_EDP_BACKLIGHT_MODE_SET_REGISTER, &dpcd_buf) == 1) &&
+            ((dpcd_buf & DP_EDP_BACKLIGHT_CONTROL_MODE_MASK) ==
+             DP_EDP_BACKLIGHT_CONTROL_MODE_PRESET))
+                drm_dp_dpcd_writeb(&intel_dp->aux, DP_EDP_BACKLIGHT_MODE_SET_REGISTER,
+                                   (dpcd_buf | DP_EDP_BACKLIGHT_CONTROL_MODE_DPCD));
+}
+
+static void intel_dp_aux_disable_backlight(struct intel_connector *connector)
+{
+        set_aux_backlight_enable(enc_to_intel_dp(&connector->encoder->base), false);
+}
+
+static int intel_dp_aux_setup_backlight(struct intel_connector *connector,
+                                        enum pipe pipe)
+{
+        struct intel_dp *intel_dp = enc_to_intel_dp(&connector->encoder->base);
+        struct intel_panel *panel = &connector->panel;
+
+        intel_dp_aux_enable_backlight(connector);
+
+        if (intel_dp->edp_dpcd[2] & DP_EDP_BACKLIGHT_BRIGHTNESS_BYTE_COUNT)
+                panel->backlight.max = 0xFFFF;
+        else
+                panel->backlight.max = 0xFF;
+
+        panel->backlight.min = 0;
+        panel->backlight.level = intel_dp_aux_get_backlight(connector);
+
+        panel->backlight.enabled = panel->backlight.level != 0;
+
+        return 0;
+}
+
+static bool
+intel_dp_aux_display_control_capable(struct intel_connector *connector)
+{
+        struct intel_dp *intel_dp = enc_to_intel_dp(&connector->encoder->base);
+
+        /* Check the  eDP Display control capabilities registers to determine if
+         * the panel can support backlight control over the aux channel
+         */
+        if (intel_dp->edp_dpcd[1] & DP_EDP_TCON_BACKLIGHT_ADJUSTMENT_CAP &&
+            (intel_dp->edp_dpcd[1] & DP_EDP_BACKLIGHT_AUX_ENABLE_CAP) &&
+            !((intel_dp->edp_dpcd[1] & DP_EDP_BACKLIGHT_PIN_ENABLE_CAP) ||
+              (intel_dp->edp_dpcd[2] & DP_EDP_BACKLIGHT_BRIGHTNESS_PWM_PIN_CAP))) {
+                DRM_DEBUG_KMS("AUX Backlight Control Supported!\n");
+                return true;
+        }
+        return false;
+}
+
+int intel_dp_aux_init_backlight_funcs(struct intel_connector *intel_connector)
+{
+        struct intel_panel *panel = &intel_connector->panel;
+
+        if (!i915.enable_dpcd_backlight)
+                return -ENODEV;
+
+        if (!intel_dp_aux_display_control_capable(intel_connector))
+                return -ENODEV;
+
+        panel->backlight.setup = intel_dp_aux_setup_backlight;
+        panel->backlight.enable = intel_dp_aux_enable_backlight;
+        panel->backlight.disable = intel_dp_aux_disable_backlight;
+        panel->backlight.set = intel_dp_aux_set_backlight;
+        panel->backlight.get = intel_dp_aux_get_backlight;
+
+        return 0;
+}
diff --git a/drivers/gpu/drm/i915/intel_dp_mst.c b/drivers/gpu/drm/i915/intel_dp_mst.c
index 7a34090cef34..68a005d729e9 100644
--- a/drivers/gpu/drm/i915/intel_dp_mst.c
+++ b/drivers/gpu/drm/i915/intel_dp_mst.c
@@ -47,7 +47,6 @@ static bool intel_dp_mst_compute_config(struct intel_encoder *encoder,
 
         pipe_config->dp_encoder_is_mst = true;
         pipe_config->has_pch_encoder = false;
-        pipe_config->has_dp_encoder = true;
         bpp = 24;
         /*
          * for MST we always configure max link bw - the spec doesn't
@@ -140,7 +139,7 @@ static void intel_mst_pre_enable_dp(struct intel_encoder *encoder)
         struct intel_digital_port *intel_dig_port = intel_mst->primary;
         struct intel_dp *intel_dp = &intel_dig_port->dp;
         struct drm_device *dev = encoder->base.dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         enum port port = intel_dig_port->port;
         int ret;
         uint32_t temp;
@@ -207,14 +206,17 @@ static void intel_mst_enable_dp(struct intel_encoder *encoder)
         struct intel_digital_port *intel_dig_port = intel_mst->primary;
         struct intel_dp *intel_dp = &intel_dig_port->dp;
         struct drm_device *dev = intel_dig_port->base.base.dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         enum port port = intel_dig_port->port;
         int ret;
 
         DRM_DEBUG_KMS("%d\n", intel_dp->active_mst_links);
 
-        if (wait_for((I915_READ(DP_TP_STATUS(port)) & DP_TP_STATUS_ACT_SENT),
-                     1))
+        if (intel_wait_for_register(dev_priv,
+                                    DP_TP_STATUS(port),
+                                    DP_TP_STATUS_ACT_SENT,
+                                    DP_TP_STATUS_ACT_SENT,
+                                    1))
                 DRM_ERROR("Timed out waiting for ACT sent\n");
 
         ret = drm_dp_check_act_status(&intel_dp->mst_mgr);
@@ -239,12 +241,10 @@ static void intel_dp_mst_enc_get_config(struct intel_encoder *encoder,
         struct intel_digital_port *intel_dig_port = intel_mst->primary;
         struct intel_crtc *crtc = to_intel_crtc(encoder->base.crtc);
         struct drm_device *dev = encoder->base.dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         enum transcoder cpu_transcoder = pipe_config->cpu_transcoder;
         u32 temp, flags = 0;
 
-        pipe_config->has_dp_encoder = true;
-
         temp = I915_READ(TRANS_DDI_FUNC_CTL(cpu_transcoder));
         if (temp & TRANS_DDI_PHSYNC)
                 flags |= DRM_MODE_FLAG_PHSYNC;
@@ -336,6 +336,8 @@ static const struct drm_connector_funcs intel_dp_mst_connector_funcs = {
         .fill_modes = drm_helper_probe_single_connector_modes,
         .set_property = intel_dp_mst_set_property,
         .atomic_get_property = intel_connector_atomic_get_property,
+        .late_register = intel_connector_register,
+        .early_unregister = intel_connector_unregister,
         .destroy = intel_dp_mst_connector_destroy,
         .atomic_destroy_state = drm_atomic_helper_connector_destroy_state,
         .atomic_duplicate_state = drm_atomic_helper_connector_duplicate_state,
@@ -455,7 +457,6 @@ static struct drm_connector *intel_dp_add_mst_connector(struct drm_dp_mst_topolo
         drm_connector_init(dev, connector, &intel_dp_mst_connector_funcs, DRM_MODE_CONNECTOR_DisplayPort);
         drm_connector_helper_add(connector, &intel_dp_mst_connector_helper_funcs);
 
-        intel_connector->unregister = intel_connector_unregister;
         intel_connector->get_hw_state = intel_dp_mst_get_hw_state;
         intel_connector->mst_port = intel_dp;
         intel_connector->port = port;
@@ -477,9 +478,11 @@ static void intel_dp_register_mst_connector(struct drm_connector *connector)
 {
         struct intel_connector *intel_connector = to_intel_connector(connector);
         struct drm_device *dev = connector->dev;
+
         drm_modeset_lock_all(dev);
         intel_connector_add_to_fbdev(intel_connector);
         drm_modeset_unlock_all(dev);
+
         drm_connector_register(&intel_connector->base);
 }
 
@@ -489,7 +492,7 @@ static void intel_dp_destroy_mst_connector(struct drm_dp_mst_topology_mgr *mgr,
         struct intel_connector *intel_connector = to_intel_connector(connector);
         struct drm_device *dev = connector->dev;
 
-        intel_connector->unregister(intel_connector);
+        drm_connector_unregister(connector);
 
         /* need to nuke the connector */
         drm_modeset_lock_all(dev);
@@ -534,7 +537,7 @@ intel_dp_create_fake_mst_encoder(struct intel_digital_port *intel_dig_port, enum
         intel_mst->primary = intel_dig_port;
 
         drm_encoder_init(dev, &intel_encoder->base, &intel_dp_mst_enc_funcs,
-                         DRM_MODE_ENCODER_DPMST, NULL);
+                         DRM_MODE_ENCODER_DPMST, "DP-MST %c", pipe_name(pipe));
 
         intel_encoder->type = INTEL_OUTPUT_DP_MST;
         intel_encoder->crtc_mask = 0x7;
diff --git a/drivers/gpu/drm/i915/intel_dpio_phy.c b/drivers/gpu/drm/i915/intel_dpio_phy.c
new file mode 100644
index 000000000000..047f48748944
--- /dev/null
+++ b/drivers/gpu/drm/i915/intel_dpio_phy.c
@@ -0,0 +1,470 @@
+/*
+ * Copyright © 2014-2016 Intel Corporation
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#include "intel_drv.h"
+
+void chv_set_phy_signal_level(struct intel_encoder *encoder,
+                              u32 deemph_reg_value, u32 margin_reg_value,
+                              bool uniq_trans_scale)
+{
+        struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
+        struct intel_digital_port *dport = enc_to_dig_port(&encoder->base);
+        struct intel_crtc *intel_crtc = to_intel_crtc(dport->base.base.crtc);
+        enum dpio_channel ch = vlv_dport_to_channel(dport);
+        enum pipe pipe = intel_crtc->pipe;
+        u32 val;
+        int i;
+
+        mutex_lock(&dev_priv->sb_lock);
+
+        /* Clear calc init */
+        val = vlv_dpio_read(dev_priv, pipe, VLV_PCS01_DW10(ch));
+        val &= ~(DPIO_PCS_SWING_CALC_TX0_TX2 | DPIO_PCS_SWING_CALC_TX1_TX3);
+        val &= ~(DPIO_PCS_TX1DEEMP_MASK | DPIO_PCS_TX2DEEMP_MASK);
+        val |= DPIO_PCS_TX1DEEMP_9P5 | DPIO_PCS_TX2DEEMP_9P5;
+        vlv_dpio_write(dev_priv, pipe, VLV_PCS01_DW10(ch), val);
+
+        if (intel_crtc->config->lane_count > 2) {
+                val = vlv_dpio_read(dev_priv, pipe, VLV_PCS23_DW10(ch));
+                val &= ~(DPIO_PCS_SWING_CALC_TX0_TX2 | DPIO_PCS_SWING_CALC_TX1_TX3);
+                val &= ~(DPIO_PCS_TX1DEEMP_MASK | DPIO_PCS_TX2DEEMP_MASK);
+                val |= DPIO_PCS_TX1DEEMP_9P5 | DPIO_PCS_TX2DEEMP_9P5;
+                vlv_dpio_write(dev_priv, pipe, VLV_PCS23_DW10(ch), val);
+        }
+
+        val = vlv_dpio_read(dev_priv, pipe, VLV_PCS01_DW9(ch));
+        val &= ~(DPIO_PCS_TX1MARGIN_MASK | DPIO_PCS_TX2MARGIN_MASK);
+        val |= DPIO_PCS_TX1MARGIN_000 | DPIO_PCS_TX2MARGIN_000;
+        vlv_dpio_write(dev_priv, pipe, VLV_PCS01_DW9(ch), val);
+
+        if (intel_crtc->config->lane_count > 2) {
+                val = vlv_dpio_read(dev_priv, pipe, VLV_PCS23_DW9(ch));
+                val &= ~(DPIO_PCS_TX1MARGIN_MASK | DPIO_PCS_TX2MARGIN_MASK);
+                val |= DPIO_PCS_TX1MARGIN_000 | DPIO_PCS_TX2MARGIN_000;
+                vlv_dpio_write(dev_priv, pipe, VLV_PCS23_DW9(ch), val);
+        }
+
+        /* Program swing deemph */
+        for (i = 0; i < intel_crtc->config->lane_count; i++) {
+                val = vlv_dpio_read(dev_priv, pipe, CHV_TX_DW4(ch, i));
+                val &= ~DPIO_SWING_DEEMPH9P5_MASK;
+                val |= deemph_reg_value << DPIO_SWING_DEEMPH9P5_SHIFT;
+                vlv_dpio_write(dev_priv, pipe, CHV_TX_DW4(ch, i), val);
+        }
+
+        /* Program swing margin */
+        for (i = 0; i < intel_crtc->config->lane_count; i++) {
+                val = vlv_dpio_read(dev_priv, pipe, CHV_TX_DW2(ch, i));
+
+                val &= ~DPIO_SWING_MARGIN000_MASK;
+                val |= margin_reg_value << DPIO_SWING_MARGIN000_SHIFT;
+
+                /*
+                 * Supposedly this value shouldn't matter when unique transition
+                 * scale is disabled, but in fact it does matter. Let's just
+                 * always program the same value and hope it's OK.
+                 */
+                val &= ~(0xff << DPIO_UNIQ_TRANS_SCALE_SHIFT);
+                val |= 0x9a << DPIO_UNIQ_TRANS_SCALE_SHIFT;
+
+                vlv_dpio_write(dev_priv, pipe, CHV_TX_DW2(ch, i), val);
+        }
+
+        /*
+         * The document said it needs to set bit 27 for ch0 and bit 26
+         * for ch1. Might be a typo in the doc.
+         * For now, for this unique transition scale selection, set bit
+         * 27 for ch0 and ch1.
+         */
+        for (i = 0; i < intel_crtc->config->lane_count; i++) {
+                val = vlv_dpio_read(dev_priv, pipe, CHV_TX_DW3(ch, i));
+                if (uniq_trans_scale)
+                        val |= DPIO_TX_UNIQ_TRANS_SCALE_EN;
+                else
+                        val &= ~DPIO_TX_UNIQ_TRANS_SCALE_EN;
+                vlv_dpio_write(dev_priv, pipe, CHV_TX_DW3(ch, i), val);
+        }
+
+        /* Start swing calculation */
+        val = vlv_dpio_read(dev_priv, pipe, VLV_PCS01_DW10(ch));
+        val |= DPIO_PCS_SWING_CALC_TX0_TX2 | DPIO_PCS_SWING_CALC_TX1_TX3;
+        vlv_dpio_write(dev_priv, pipe, VLV_PCS01_DW10(ch), val);
+
+        if (intel_crtc->config->lane_count > 2) {
+                val = vlv_dpio_read(dev_priv, pipe, VLV_PCS23_DW10(ch));
+                val |= DPIO_PCS_SWING_CALC_TX0_TX2 | DPIO_PCS_SWING_CALC_TX1_TX3;
+                vlv_dpio_write(dev_priv, pipe, VLV_PCS23_DW10(ch), val);
+        }
+
+        mutex_unlock(&dev_priv->sb_lock);
+
+}
+
+void chv_data_lane_soft_reset(struct intel_encoder *encoder,
+                              bool reset)
+{
+        struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
+        enum dpio_channel ch = vlv_dport_to_channel(enc_to_dig_port(&encoder->base));
+        struct intel_crtc *crtc = to_intel_crtc(encoder->base.crtc);
+        enum pipe pipe = crtc->pipe;
+        uint32_t val;
+
+        val = vlv_dpio_read(dev_priv, pipe, VLV_PCS01_DW0(ch));
+        if (reset)
+                val &= ~(DPIO_PCS_TX_LANE2_RESET | DPIO_PCS_TX_LANE1_RESET);
+        else
+                val |= DPIO_PCS_TX_LANE2_RESET | DPIO_PCS_TX_LANE1_RESET;
+        vlv_dpio_write(dev_priv, pipe, VLV_PCS01_DW0(ch), val);
+
+        if (crtc->config->lane_count > 2) {
+                val = vlv_dpio_read(dev_priv, pipe, VLV_PCS23_DW0(ch));
+                if (reset)
+                        val &= ~(DPIO_PCS_TX_LANE2_RESET | DPIO_PCS_TX_LANE1_RESET);
+                else
+                        val |= DPIO_PCS_TX_LANE2_RESET | DPIO_PCS_TX_LANE1_RESET;
+                vlv_dpio_write(dev_priv, pipe, VLV_PCS23_DW0(ch), val);
+        }
+
+        val = vlv_dpio_read(dev_priv, pipe, VLV_PCS01_DW1(ch));
+        val |= CHV_PCS_REQ_SOFTRESET_EN;
+        if (reset)
+                val &= ~DPIO_PCS_CLK_SOFT_RESET;
+        else
+                val |= DPIO_PCS_CLK_SOFT_RESET;
+        vlv_dpio_write(dev_priv, pipe, VLV_PCS01_DW1(ch), val);
+
+        if (crtc->config->lane_count > 2) {
+                val = vlv_dpio_read(dev_priv, pipe, VLV_PCS23_DW1(ch));
+                val |= CHV_PCS_REQ_SOFTRESET_EN;
+                if (reset)
+                        val &= ~DPIO_PCS_CLK_SOFT_RESET;
+                else
+                        val |= DPIO_PCS_CLK_SOFT_RESET;
+                vlv_dpio_write(dev_priv, pipe, VLV_PCS23_DW1(ch), val);
+        }
+}
+
+void chv_phy_pre_pll_enable(struct intel_encoder *encoder)
+{
+        struct intel_digital_port *dport = enc_to_dig_port(&encoder->base);
+        struct drm_device *dev = encoder->base.dev;
+        struct drm_i915_private *dev_priv = to_i915(dev);
+        struct intel_crtc *intel_crtc =
+                to_intel_crtc(encoder->base.crtc);
+        enum dpio_channel ch = vlv_dport_to_channel(dport);
+        enum pipe pipe = intel_crtc->pipe;
+        unsigned int lane_mask =
+                intel_dp_unused_lane_mask(intel_crtc->config->lane_count);
+        u32 val;
+
+        /*
+         * Must trick the second common lane into life.
+         * Otherwise we can't even access the PLL.
+         */
+        if (ch == DPIO_CH0 && pipe == PIPE_B)
+                dport->release_cl2_override =
+                        !chv_phy_powergate_ch(dev_priv, DPIO_PHY0, DPIO_CH1, true);
+
+        chv_phy_powergate_lanes(encoder, true, lane_mask);
+
+        mutex_lock(&dev_priv->sb_lock);
+
+        /* Assert data lane reset */
+        chv_data_lane_soft_reset(encoder, true);
+
+        /* program left/right clock distribution */
+        if (pipe != PIPE_B) {
+                val = vlv_dpio_read(dev_priv, pipe, _CHV_CMN_DW5_CH0);
+                val &= ~(CHV_BUFLEFTENA1_MASK | CHV_BUFRIGHTENA1_MASK);
+                if (ch == DPIO_CH0)
+                        val |= CHV_BUFLEFTENA1_FORCE;
+                if (ch == DPIO_CH1)
+                        val |= CHV_BUFRIGHTENA1_FORCE;
+                vlv_dpio_write(dev_priv, pipe, _CHV_CMN_DW5_CH0, val);
+        } else {
+                val = vlv_dpio_read(dev_priv, pipe, _CHV_CMN_DW1_CH1);
+                val &= ~(CHV_BUFLEFTENA2_MASK | CHV_BUFRIGHTENA2_MASK);
+                if (ch == DPIO_CH0)
+                        val |= CHV_BUFLEFTENA2_FORCE;
+                if (ch == DPIO_CH1)
+                        val |= CHV_BUFRIGHTENA2_FORCE;
+                vlv_dpio_write(dev_priv, pipe, _CHV_CMN_DW1_CH1, val);
+        }
+
+        /* program clock channel usage */
+        val = vlv_dpio_read(dev_priv, pipe, VLV_PCS01_DW8(ch));
+        val |= CHV_PCS_USEDCLKCHANNEL_OVRRIDE;
+        if (pipe != PIPE_B)
+                val &= ~CHV_PCS_USEDCLKCHANNEL;
+        else
+                val |= CHV_PCS_USEDCLKCHANNEL;
+        vlv_dpio_write(dev_priv, pipe, VLV_PCS01_DW8(ch), val);
+
+        if (intel_crtc->config->lane_count > 2) {
+                val = vlv_dpio_read(dev_priv, pipe, VLV_PCS23_DW8(ch));
+                val |= CHV_PCS_USEDCLKCHANNEL_OVRRIDE;
+                if (pipe != PIPE_B)
+                        val &= ~CHV_PCS_USEDCLKCHANNEL;
+                else
+                        val |= CHV_PCS_USEDCLKCHANNEL;
+                vlv_dpio_write(dev_priv, pipe, VLV_PCS23_DW8(ch), val);
+        }
+
+        /*
+         * This a a bit weird since generally CL
+         * matches the pipe, but here we need to
+         * pick the CL based on the port.
+         */
+        val = vlv_dpio_read(dev_priv, pipe, CHV_CMN_DW19(ch));
+        if (pipe != PIPE_B)
+                val &= ~CHV_CMN_USEDCLKCHANNEL;
+        else
+                val |= CHV_CMN_USEDCLKCHANNEL;
+        vlv_dpio_write(dev_priv, pipe, CHV_CMN_DW19(ch), val);
+
+        mutex_unlock(&dev_priv->sb_lock);
+}
+
+void chv_phy_pre_encoder_enable(struct intel_encoder *encoder)
+{
+        struct intel_dp *intel_dp = enc_to_intel_dp(&encoder->base);
+        struct intel_digital_port *dport = dp_to_dig_port(intel_dp);
+        struct drm_device *dev = encoder->base.dev;
+        struct drm_i915_private *dev_priv = to_i915(dev);
+        struct intel_crtc *intel_crtc =
+                to_intel_crtc(encoder->base.crtc);
+        enum dpio_channel ch = vlv_dport_to_channel(dport);
+        int pipe = intel_crtc->pipe;
+        int data, i, stagger;
+        u32 val;
+
+        mutex_lock(&dev_priv->sb_lock);
+
+        /* allow hardware to manage TX FIFO reset source */
+        val = vlv_dpio_read(dev_priv, pipe, VLV_PCS01_DW11(ch));
+        val &= ~DPIO_LANEDESKEW_STRAP_OVRD;
+        vlv_dpio_write(dev_priv, pipe, VLV_PCS01_DW11(ch), val);
+
+        if (intel_crtc->config->lane_count > 2) {
+                val = vlv_dpio_read(dev_priv, pipe, VLV_PCS23_DW11(ch));
+                val &= ~DPIO_LANEDESKEW_STRAP_OVRD;
+                vlv_dpio_write(dev_priv, pipe, VLV_PCS23_DW11(ch), val);
+        }
+
+        /* Program Tx lane latency optimal setting*/
+        for (i = 0; i < intel_crtc->config->lane_count; i++) {
+                /* Set the upar bit */
+                if (intel_crtc->config->lane_count == 1)
+                        data = 0x0;
+                else
+                        data = (i == 1) ? 0x0 : 0x1;
+                vlv_dpio_write(dev_priv, pipe, CHV_TX_DW14(ch, i),
+                                data << DPIO_UPAR_SHIFT);
+        }
+
+        /* Data lane stagger programming */
+        if (intel_crtc->config->port_clock > 270000)
+                stagger = 0x18;
+        else if (intel_crtc->config->port_clock > 135000)
+                stagger = 0xd;
+        else if (intel_crtc->config->port_clock > 67500)
+                stagger = 0x7;
+        else if (intel_crtc->config->port_clock > 33750)
+                stagger = 0x4;
+        else
+                stagger = 0x2;
+
+        val = vlv_dpio_read(dev_priv, pipe, VLV_PCS01_DW11(ch));
+        val |= DPIO_TX2_STAGGER_MASK(0x1f);
+        vlv_dpio_write(dev_priv, pipe, VLV_PCS01_DW11(ch), val);
+
+        if (intel_crtc->config->lane_count > 2) {
+                val = vlv_dpio_read(dev_priv, pipe, VLV_PCS23_DW11(ch));
+                val |= DPIO_TX2_STAGGER_MASK(0x1f);
+                vlv_dpio_write(dev_priv, pipe, VLV_PCS23_DW11(ch), val);
+        }
+
+        vlv_dpio_write(dev_priv, pipe, VLV_PCS01_DW12(ch),
+                       DPIO_LANESTAGGER_STRAP(stagger) |
+                       DPIO_LANESTAGGER_STRAP_OVRD |
+                       DPIO_TX1_STAGGER_MASK(0x1f) |
+                       DPIO_TX1_STAGGER_MULT(6) |
+                       DPIO_TX2_STAGGER_MULT(0));
+
+        if (intel_crtc->config->lane_count > 2) {
+                vlv_dpio_write(dev_priv, pipe, VLV_PCS23_DW12(ch),
+                               DPIO_LANESTAGGER_STRAP(stagger) |
+                               DPIO_LANESTAGGER_STRAP_OVRD |
+                               DPIO_TX1_STAGGER_MASK(0x1f) |
+                               DPIO_TX1_STAGGER_MULT(7) |
+                               DPIO_TX2_STAGGER_MULT(5));
+        }
+
+        /* Deassert data lane reset */
+        chv_data_lane_soft_reset(encoder, false);
+
+        mutex_unlock(&dev_priv->sb_lock);
+}
+
+void chv_phy_release_cl2_override(struct intel_encoder *encoder)
+{
+        struct intel_digital_port *dport = enc_to_dig_port(&encoder->base);
+        struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
+
+        if (dport->release_cl2_override) {
+                chv_phy_powergate_ch(dev_priv, DPIO_PHY0, DPIO_CH1, false);
+                dport->release_cl2_override = false;
+        }
+}
+
+void chv_phy_post_pll_disable(struct intel_encoder *encoder)
+{
+        struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
+        enum pipe pipe = to_intel_crtc(encoder->base.crtc)->pipe;
+        u32 val;
+
+        mutex_lock(&dev_priv->sb_lock);
+
+        /* disable left/right clock distribution */
+        if (pipe != PIPE_B) {
+                val = vlv_dpio_read(dev_priv, pipe, _CHV_CMN_DW5_CH0);
+                val &= ~(CHV_BUFLEFTENA1_MASK | CHV_BUFRIGHTENA1_MASK);
+                vlv_dpio_write(dev_priv, pipe, _CHV_CMN_DW5_CH0, val);
+        } else {
+                val = vlv_dpio_read(dev_priv, pipe, _CHV_CMN_DW1_CH1);
+                val &= ~(CHV_BUFLEFTENA2_MASK | CHV_BUFRIGHTENA2_MASK);
+                vlv_dpio_write(dev_priv, pipe, _CHV_CMN_DW1_CH1, val);
+        }
+
+        mutex_unlock(&dev_priv->sb_lock);
+
+        /*
+         * Leave the power down bit cleared for at least one
+         * lane so that chv_powergate_phy_ch() will power
+         * on something when the channel is otherwise unused.
+         * When the port is off and the override is removed
+         * the lanes power down anyway, so otherwise it doesn't
+         * really matter what the state of power down bits is
+         * after this.
+         */
+        chv_phy_powergate_lanes(encoder, false, 0x0);
+}
+
+void vlv_set_phy_signal_level(struct intel_encoder *encoder,
+                              u32 demph_reg_value, u32 preemph_reg_value,
+                              u32 uniqtranscale_reg_value, u32 tx3_demph)
+{
+        struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
+        struct intel_crtc *intel_crtc = to_intel_crtc(encoder->base.crtc);
+        struct intel_digital_port *dport = enc_to_dig_port(&encoder->base);
+        enum dpio_channel port = vlv_dport_to_channel(dport);
+        int pipe = intel_crtc->pipe;
+
+        mutex_lock(&dev_priv->sb_lock);
+        vlv_dpio_write(dev_priv, pipe, VLV_TX_DW5(port), 0x00000000);
+        vlv_dpio_write(dev_priv, pipe, VLV_TX_DW4(port), demph_reg_value);
+        vlv_dpio_write(dev_priv, pipe, VLV_TX_DW2(port),
+                         uniqtranscale_reg_value);
+        vlv_dpio_write(dev_priv, pipe, VLV_TX_DW3(port), 0x0C782040);
+
+        if (tx3_demph)
+                vlv_dpio_write(dev_priv, pipe, VLV_TX3_DW4(port), tx3_demph);
+
+        vlv_dpio_write(dev_priv, pipe, VLV_PCS_DW11(port), 0x00030000);
+        vlv_dpio_write(dev_priv, pipe, VLV_PCS_DW9(port), preemph_reg_value);
+        vlv_dpio_write(dev_priv, pipe, VLV_TX_DW5(port), DPIO_TX_OCALINIT_EN);
+        mutex_unlock(&dev_priv->sb_lock);
+}
+
+void vlv_phy_pre_pll_enable(struct intel_encoder *encoder)
+{
+        struct intel_digital_port *dport = enc_to_dig_port(&encoder->base);
+        struct drm_device *dev = encoder->base.dev;
+        struct drm_i915_private *dev_priv = to_i915(dev);
+        struct intel_crtc *intel_crtc =
+                to_intel_crtc(encoder->base.crtc);
+        enum dpio_channel port = vlv_dport_to_channel(dport);
+        int pipe = intel_crtc->pipe;
+
+        /* Program Tx lane resets to default */
+        mutex_lock(&dev_priv->sb_lock);
+        vlv_dpio_write(dev_priv, pipe, VLV_PCS_DW0(port),
+                         DPIO_PCS_TX_LANE2_RESET |
+                         DPIO_PCS_TX_LANE1_RESET);
+        vlv_dpio_write(dev_priv, pipe, VLV_PCS_DW1(port),
+                         DPIO_PCS_CLK_CRI_RXEB_EIOS_EN |
+                         DPIO_PCS_CLK_CRI_RXDIGFILTSG_EN |
+                         (1<<DPIO_PCS_CLK_DATAWIDTH_SHIFT) |
+                                 DPIO_PCS_CLK_SOFT_RESET);
+
+        /* Fix up inter-pair skew failure */
+        vlv_dpio_write(dev_priv, pipe, VLV_PCS_DW12(port), 0x00750f00);
+        vlv_dpio_write(dev_priv, pipe, VLV_TX_DW11(port), 0x00001500);
+        vlv_dpio_write(dev_priv, pipe, VLV_TX_DW14(port), 0x40400000);
+        mutex_unlock(&dev_priv->sb_lock);
+}
+
+void vlv_phy_pre_encoder_enable(struct intel_encoder *encoder)
+{
+        struct intel_dp *intel_dp = enc_to_intel_dp(&encoder->base);
+        struct intel_digital_port *dport = dp_to_dig_port(intel_dp);
+        struct drm_device *dev = encoder->base.dev;
+        struct drm_i915_private *dev_priv = to_i915(dev);
+        struct intel_crtc *intel_crtc = to_intel_crtc(encoder->base.crtc);
+        enum dpio_channel port = vlv_dport_to_channel(dport);
+        int pipe = intel_crtc->pipe;
+        u32 val;
+
+        mutex_lock(&dev_priv->sb_lock);
+
+        /* Enable clock channels for this port */
+        val = vlv_dpio_read(dev_priv, pipe, VLV_PCS01_DW8(port));
+        val = 0;
+        if (pipe)
+                val |= (1<<21);
+        else
+                val &= ~(1<<21);
+        val |= 0x001000c4;
+        vlv_dpio_write(dev_priv, pipe, VLV_PCS_DW8(port), val);
+
+        /* Program lane clock */
+        vlv_dpio_write(dev_priv, pipe, VLV_PCS_DW14(port), 0x00760018);
+        vlv_dpio_write(dev_priv, pipe, VLV_PCS_DW23(port), 0x00400888);
+
+        mutex_unlock(&dev_priv->sb_lock);
+}
+
+void vlv_phy_reset_lanes(struct intel_encoder *encoder)
+{
+        struct intel_digital_port *dport = enc_to_dig_port(&encoder->base);
+        struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
+        struct intel_crtc *intel_crtc =
+                to_intel_crtc(encoder->base.crtc);
+        enum dpio_channel port = vlv_dport_to_channel(dport);
+        int pipe = intel_crtc->pipe;
+
+        mutex_lock(&dev_priv->sb_lock);
+        vlv_dpio_write(dev_priv, pipe, VLV_PCS_DW0(port), 0x00000000);
+        vlv_dpio_write(dev_priv, pipe, VLV_PCS_DW1(port), 0x00e00060);
+        mutex_unlock(&dev_priv->sb_lock);
+}
diff --git a/drivers/gpu/drm/i915/intel_dpll_mgr.c b/drivers/gpu/drm/i915/intel_dpll_mgr.c
index 58f60b27837e..5c1f2d235ffa 100644
--- a/drivers/gpu/drm/i915/intel_dpll_mgr.c
+++ b/drivers/gpu/drm/i915/intel_dpll_mgr.c
@@ -83,7 +83,7 @@ void assert_shared_dpll(struct drm_i915_private *dev_priv,
 void intel_prepare_shared_dpll(struct intel_crtc *crtc)
 {
         struct drm_device *dev = crtc->base.dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         struct intel_shared_dpll *pll = crtc->config->shared_dpll;
 
         if (WARN_ON(pll == NULL))
@@ -112,7 +112,7 @@ void intel_prepare_shared_dpll(struct intel_crtc *crtc)
 void intel_enable_shared_dpll(struct intel_crtc *crtc)
 {
         struct drm_device *dev = crtc->base.dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         struct intel_shared_dpll *pll = crtc->config->shared_dpll;
         unsigned crtc_mask = 1 << drm_crtc_index(&crtc->base);
         unsigned old_mask;
@@ -151,7 +151,7 @@ out:
 void intel_disable_shared_dpll(struct intel_crtc *crtc)
 {
         struct drm_device *dev = crtc->base.dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         struct intel_shared_dpll *pll = crtc->config->shared_dpll;
         unsigned crtc_mask = 1 << drm_crtc_index(&crtc->base);
 
@@ -191,7 +191,7 @@ intel_find_shared_dpll(struct intel_crtc *crtc,
                        enum intel_dpll_id range_min,
                        enum intel_dpll_id range_max)
 {
-        struct drm_i915_private *dev_priv = crtc->base.dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
         struct intel_shared_dpll *pll;
         struct intel_shared_dpll_config *shared_dpll;
         enum intel_dpll_id i;
@@ -208,8 +208,8 @@ intel_find_shared_dpll(struct intel_crtc *crtc,
                 if (memcmp(&crtc_state->dpll_hw_state,
                            &shared_dpll[i].hw_state,
                            sizeof(crtc_state->dpll_hw_state)) == 0) {
-                        DRM_DEBUG_KMS("CRTC:%d sharing existing %s (crtc mask 0x%08x, active %x)\n",
-                                      crtc->base.base.id, pll->name,
+                        DRM_DEBUG_KMS("[CRTC:%d:%s] sharing existing %s (crtc mask 0x%08x, active %x)\n",
+                                      crtc->base.base.id, crtc->base.name, pll->name,
                                       shared_dpll[i].crtc_mask,
                                       pll->active_mask);
                         return pll;
@@ -220,8 +220,8 @@ intel_find_shared_dpll(struct intel_crtc *crtc,
         for (i = range_min; i <= range_max; i++) {
                 pll = &dev_priv->shared_dplls[i];
                 if (shared_dpll[i].crtc_mask == 0) {
-                        DRM_DEBUG_KMS("CRTC:%d allocated %s\n",
-                                      crtc->base.base.id, pll->name);
+                        DRM_DEBUG_KMS("[CRTC:%d:%s] allocated %s\n",
+                                      crtc->base.base.id, crtc->base.name, pll->name);
                         return pll;
                 }
         }
@@ -331,7 +331,7 @@ static void ibx_pch_dpll_enable(struct drm_i915_private *dev_priv,
 static void ibx_pch_dpll_disable(struct drm_i915_private *dev_priv,
                                  struct intel_shared_dpll *pll)
 {
-        struct drm_device *dev = dev_priv->dev;
+        struct drm_device *dev = &dev_priv->drm;
         struct intel_crtc *crtc;
 
         /* Make sure no transcoder isn't still depending on us. */
@@ -358,8 +358,8 @@ ibx_get_dpll(struct intel_crtc *crtc, struct intel_crtc_state *crtc_state,
                 i = (enum intel_dpll_id) crtc->pipe;
                 pll = &dev_priv->shared_dplls[i];
 
-                DRM_DEBUG_KMS("CRTC:%d using pre-allocated %s\n",
-                              crtc->base.base.id, pll->name);
+                DRM_DEBUG_KMS("[CRTC:%d:%s] using pre-allocated %s\n",
+                              crtc->base.base.id, crtc->base.name, pll->name);
         } else {
                 pll = intel_find_shared_dpll(crtc, crtc_state,
                                              DPLL_ID_PCH_PLL_A,
@@ -713,7 +713,7 @@ hsw_get_dpll(struct intel_crtc *crtc, struct intel_crtc_state *crtc_state,
                 pll = intel_find_shared_dpll(crtc, crtc_state,
                                              DPLL_ID_WRPLL1, DPLL_ID_WRPLL2);
 
-        } else if (encoder->type == INTEL_OUTPUT_DISPLAYPORT ||
+        } else if (encoder->type == INTEL_OUTPUT_DP ||
                    encoder->type == INTEL_OUTPUT_DP_MST ||
                    encoder->type == INTEL_OUTPUT_EDP) {
                 enum intel_dpll_id pll_id;
@@ -856,7 +856,11 @@ static void skl_ddi_pll_enable(struct drm_i915_private *dev_priv,
         I915_WRITE(regs[pll->id].ctl,
                    I915_READ(regs[pll->id].ctl) | LCPLL_PLL_ENABLE);
 
-        if (wait_for(I915_READ(DPLL_STATUS) & DPLL_LOCK(pll->id), 5))
+        if (intel_wait_for_register(dev_priv,
+                                    DPLL_STATUS,
+                                    DPLL_LOCK(pll->id),
+                                    DPLL_LOCK(pll->id),
+                                    5))
                 DRM_ERROR("DPLL %d not locked\n", pll->id);
 }
 
@@ -1222,7 +1226,7 @@ skl_get_dpll(struct intel_crtc *crtc, struct intel_crtc_state *crtc_state,
                          DPLL_CFGCR2_KDIV(wrpll_params.kdiv) |
                          DPLL_CFGCR2_PDIV(wrpll_params.pdiv) |
                          wrpll_params.central_freq;
-        } else if (encoder->type == INTEL_OUTPUT_DISPLAYPORT ||
+        } else if (encoder->type == INTEL_OUTPUT_DP ||
                    encoder->type == INTEL_OUTPUT_DP_MST ||
                    encoder->type == INTEL_OUTPUT_EDP) {
                 switch (crtc_state->port_clock / 2) {
@@ -1239,9 +1243,6 @@ skl_get_dpll(struct intel_crtc *crtc, struct intel_crtc_state *crtc_state,
                 case 162000:
                         ctrl1 |= DPLL_CTRL1_LINK_RATE(DPLL_CTRL1_LINK_RATE_1620, 0);
                         break;
-                /* TBD: For DP link rates 2.16 GHz and 4.32 GHz, VCO is 8640 which
-                results in CDCLK change. Need to handle the change of CDCLK by
-                disabling pipes and re-enabling them */
                 case 108000:
                         ctrl1 |= DPLL_CTRL1_LINK_RATE(DPLL_CTRL1_LINK_RATE_1080, 0);
                         break;
@@ -1511,7 +1512,7 @@ bxt_get_dpll(struct intel_crtc *crtc, struct intel_crtc_state *crtc_state,
         int clock = crtc_state->port_clock;
 
         if (encoder->type == INTEL_OUTPUT_HDMI) {
-                intel_clock_t best_clock;
+                struct dpll best_clock;
 
                 /* Calculate HDMI div */
                 /*
@@ -1533,7 +1534,7 @@ bxt_get_dpll(struct intel_crtc *crtc, struct intel_crtc_state *crtc_state,
                 clk_div.m2_frac_en = clk_div.m2_frac != 0;
 
                 vco = best_clock.vco;
-        } else if (encoder->type == INTEL_OUTPUT_DISPLAYPORT ||
+        } else if (encoder->type == INTEL_OUTPUT_DP ||
                    encoder->type == INTEL_OUTPUT_EDP) {
                 int i;
 
@@ -1616,8 +1617,8 @@ bxt_get_dpll(struct intel_crtc *crtc, struct intel_crtc_state *crtc_state,
         i = (enum intel_dpll_id) intel_dig_port->port;
         pll = intel_get_shared_dpll_by_id(dev_priv, i);
 
-        DRM_DEBUG_KMS("CRTC:%d using pre-allocated %s\n",
-                crtc->base.base.id, pll->name);
+        DRM_DEBUG_KMS("[CRTC:%d:%s] using pre-allocated %s\n",
+                      crtc->base.base.id, crtc->base.name, pll->name);
 
         intel_reference_shared_dpll(pll, crtc_state);
 
@@ -1635,19 +1636,11 @@ static const struct intel_shared_dpll_funcs bxt_ddi_pll_funcs = {
 
 static void intel_ddi_pll_init(struct drm_device *dev)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
-        uint32_t val = I915_READ(LCPLL_CTL);
-
-        if (IS_SKYLAKE(dev) || IS_KABYLAKE(dev)) {
-                int cdclk_freq;
-
-                cdclk_freq = dev_priv->display.get_display_clock_speed(dev);
-                dev_priv->skl_boot_cdclk = cdclk_freq;
-                if (skl_sanitize_cdclk(dev_priv))
-                        DRM_DEBUG_KMS("Sanitized cdclk programmed by pre-os\n");
-                if (!(I915_READ(LCPLL1_CTL) & LCPLL_PLL_ENABLE))
-                        DRM_ERROR("LCPLL1 is disabled\n");
-        } else if (!IS_BROXTON(dev_priv)) {
+        struct drm_i915_private *dev_priv = to_i915(dev);
+
+        if (INTEL_GEN(dev_priv) < 9) {
+                uint32_t val = I915_READ(LCPLL_CTL);
+
                 /*
                  * The LCPLL register should be turned on by the BIOS. For now
                  * let's just check its state and print errors in case
@@ -1730,7 +1723,7 @@ static const struct intel_dpll_mgr bxt_pll_mgr = {
 
 void intel_shared_dpll_init(struct drm_device *dev)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         const struct intel_dpll_mgr *dpll_mgr = NULL;
         const struct dpll_info *dpll_info;
         int i;
diff --git a/drivers/gpu/drm/i915/intel_drv.h b/drivers/gpu/drm/i915/intel_drv.h
index f7f0f01814f6..3329fc6a95f4 100644
--- a/drivers/gpu/drm/i915/intel_drv.h
+++ b/drivers/gpu/drm/i915/intel_drv.h
@@ -69,39 +69,63 @@
 })
 
 #define wait_for(COND, MS)              _wait_for((COND), (MS) * 1000, 1000)
-#define wait_for_us(COND, US)           _wait_for((COND), (US), 1)
 
 /* If CONFIG_PREEMPT_COUNT is disabled, in_atomic() always reports false. */
 #if defined(CONFIG_DRM_I915_DEBUG) && defined(CONFIG_PREEMPT_COUNT)
-# define _WAIT_FOR_ATOMIC_CHECK WARN_ON_ONCE(!in_atomic())
+# define _WAIT_FOR_ATOMIC_CHECK(ATOMIC) WARN_ON_ONCE((ATOMIC) && !in_atomic())
 #else
-# define _WAIT_FOR_ATOMIC_CHECK do { } while (0)
+# define _WAIT_FOR_ATOMIC_CHECK(ATOMIC) do { } while (0)
 #endif
 
-#define _wait_for_atomic(COND, US) ({ \
-        unsigned long end__; \
-        int ret__ = 0; \
-        _WAIT_FOR_ATOMIC_CHECK; \
+#define _wait_for_atomic(COND, US, ATOMIC) \
+({ \
+        int cpu, ret, timeout = (US) * 1000; \
+        u64 base; \
+        _WAIT_FOR_ATOMIC_CHECK(ATOMIC); \
         BUILD_BUG_ON((US) > 50000); \
-        end__ = (local_clock() >> 10) + (US) + 1; \
-        while (!(COND)) { \
-                if (time_after((unsigned long)(local_clock() >> 10), end__)) { \
-                        /* Unlike the regular wait_for(), this atomic variant \
-                         * cannot be preempted (and we'll just ignore the issue\
-                         * of irq interruptions) and so we know that no time \
-                         * has passed since the last check of COND and can \
-                         * immediately report the timeout. \
-                         */ \
-                        ret__ = -ETIMEDOUT; \
+        if (!(ATOMIC)) { \
+                preempt_disable(); \
+                cpu = smp_processor_id(); \
+        } \
+        base = local_clock(); \
+        for (;;) { \
+                u64 now = local_clock(); \
+                if (!(ATOMIC)) \
+                        preempt_enable(); \
+                if (COND) { \
+                        ret = 0; \
+                        break; \
+                } \
+                if (now - base >= timeout) { \
+                        ret = -ETIMEDOUT; \
                         break; \
                 } \
                 cpu_relax(); \
+                if (!(ATOMIC)) { \
+                        preempt_disable(); \
+                        if (unlikely(cpu != smp_processor_id())) { \
+                                timeout -= now - base; \
+                                cpu = smp_processor_id(); \
+                                base = local_clock(); \
+                        } \
+                } \
         } \
+        ret; \
+})
+
+#define wait_for_us(COND, US) \
+({ \
+        int ret__; \
+        BUILD_BUG_ON(!__builtin_constant_p(US)); \
+        if ((US) > 10) \
+                ret__ = _wait_for((COND), (US), 10); \
+        else \
+                ret__ = _wait_for_atomic((COND), (US), 0); \
         ret__; \
 })
 
-#define wait_for_atomic(COND, MS)       _wait_for_atomic((COND), (MS) * 1000)
-#define wait_for_atomic_us(COND, US)    _wait_for_atomic((COND), (US))
+#define wait_for_atomic(COND, MS)       _wait_for_atomic((COND), (MS) * 1000, 1)
+#define wait_for_atomic_us(COND, US)    _wait_for_atomic((COND), (US), 1)
 
 #define KHz(x) (1000 * (x))
 #define MHz(x) KHz(1000 * (x))
@@ -135,7 +159,7 @@ enum intel_output_type {
         INTEL_OUTPUT_LVDS = 4,
         INTEL_OUTPUT_TVOUT = 5,
         INTEL_OUTPUT_HDMI = 6,
-        INTEL_OUTPUT_DISPLAYPORT = 7,
+        INTEL_OUTPUT_DP = 7,
         INTEL_OUTPUT_EDP = 8,
         INTEL_OUTPUT_DSI = 9,
         INTEL_OUTPUT_UNKNOWN = 10,
@@ -159,6 +183,7 @@ struct intel_framebuffer {
 struct intel_fbdev {
         struct drm_fb_helper helper;
         struct intel_framebuffer *fb;
+        async_cookie_t cookie;
         int preferred_bpp;
 };
 
@@ -242,14 +267,6 @@ struct intel_connector {
          * and active (i.e. dpms ON state). */
         bool (*get_hw_state)(struct intel_connector *);
 
-        /*
-         * Removes all interfaces through which the connector is accessible
-         * - like sysfs, debugfs entries -, so that no new operations can be
-         * started on the connector. Also makes sure all currently pending
-         * operations finish before returing.
-         */
-        void (*unregister)(struct intel_connector *);
-
         /* Panel info for eDP and LVDS */
         struct intel_panel panel;
 
@@ -266,7 +283,7 @@ struct intel_connector {
         struct intel_dp *mst_port;
 };
 
-typedef struct dpll {
+struct dpll {
         /* given values */
         int n;
         int m1, m2;
@@ -276,7 +293,7 @@ typedef struct dpll {
         int     vco;
         int     m;
         int     p;
-} intel_clock_t;
+};
 
 struct intel_atomic_state {
         struct drm_atomic_state base;
@@ -291,17 +308,32 @@ struct intel_atomic_state {
 
         bool dpll_set, modeset;
 
+        /*
+         * Does this transaction change the pipes that are active?  This mask
+         * tracks which CRTC's have changed their active state at the end of
+         * the transaction (not counting the temporary disable during modesets).
+         * This mask should only be non-zero when intel_state->modeset is true,
+         * but the converse is not necessarily true; simply changing a mode may
+         * not flip the final active status of any CRTC's
+         */
+        unsigned int active_pipe_changes;
+
         unsigned int active_crtcs;
         unsigned int min_pixclk[I915_MAX_PIPES];
 
+        /* SKL/KBL Only */
+        unsigned int cdclk_pll_vco;
+
         struct intel_shared_dpll_config shared_dpll[I915_NUM_PLLS];
-        struct intel_wm_config wm_config;
 
         /*
          * Current watermarks can't be trusted during hardware readout, so
          * don't bother calculating intermediate watermarks.
          */
         bool skip_intermediate_wm;
+
+        /* Gen9+ only */
+        struct skl_wm_values wm_results;
 };
 
 struct intel_plane_state {
@@ -405,6 +437,48 @@ struct skl_pipe_wm {
         uint32_t linetime;
 };
 
+struct intel_crtc_wm_state {
+        union {
+                struct {
+                        /*
+                         * Intermediate watermarks; these can be
+                         * programmed immediately since they satisfy
+                         * both the current configuration we're
+                         * switching away from and the new
+                         * configuration we're switching to.
+                         */
+                        struct intel_pipe_wm intermediate;
+
+                        /*
+                         * Optimal watermarks, programmed post-vblank
+                         * when this state is committed.
+                         */
+                        struct intel_pipe_wm optimal;
+                } ilk;
+
+                struct {
+                        /* gen9+ only needs 1-step wm programming */
+                        struct skl_pipe_wm optimal;
+
+                        /* cached plane data rate */
+                        unsigned plane_data_rate[I915_MAX_PLANES];
+                        unsigned plane_y_data_rate[I915_MAX_PLANES];
+
+                        /* minimum block allocation */
+                        uint16_t minimum_blocks[I915_MAX_PLANES];
+                        uint16_t minimum_y_blocks[I915_MAX_PLANES];
+                } skl;
+        };
+
+        /*
+         * Platforms with two-step watermark programming will need to
+         * update watermark programming post-vblank to switch from the
+         * safe intermediate watermarks to the optimal final
+         * watermarks.
+         */
+        bool need_postvbl_update;
+};
+
 struct intel_crtc_state {
         struct drm_crtc_state base;
 
@@ -448,12 +522,10 @@ struct intel_crtc_state {
          */
         bool limited_color_range;
 
-        /* DP has a bunch of special case unfortunately, so mark the pipe
-         * accordingly. */
-        bool has_dp_encoder;
-
-        /* DSI has special cases */
-        bool has_dsi_encoder;
+        /* Bitmask of encoder types (enum intel_output_type)
+         * driven by the pipe.
+         */
+        unsigned int output_types;
 
         /* Whether we should send NULL infoframes. Required for audio. */
         bool has_hdmi_sink;
@@ -522,6 +594,12 @@ struct intel_crtc_state {
 
         uint8_t lane_count;
 
+        /*
+         * Used by platforms having DP/HDMI PHY with programmable lane
+         * latency optimization.
+         */
+        uint8_t lane_lat_optim_mask;
+
         /* Panel fitter controls for gen2-gen4 + VLV */
         struct {
                 u32 control;
@@ -558,32 +636,7 @@ struct intel_crtc_state {
         /* IVB sprite scaling w/a (WaCxSRDisabledForSpriteScaling:ivb) */
         bool disable_lp_wm;
 
-        struct {
-                /*
-                 * Optimal watermarks, programmed post-vblank when this state
-                 * is committed.
-                 */
-                union {
-                        struct intel_pipe_wm ilk;
-                        struct skl_pipe_wm skl;
-                } optimal;
-
-                /*
-                 * Intermediate watermarks; these can be programmed immediately
-                 * since they satisfy both the current configuration we're
-                 * switching away from and the new configuration we're switching
-                 * to.
-                 */
-                struct intel_pipe_wm intermediate;
-
-                /*
-                 * Platforms with two-step watermark programming will need to
-                 * update watermark programming post-vblank to switch from the
-                 * safe intermediate watermarks to the optimal final
-                 * watermarks.
-                 */
-                bool need_postvbl_update;
-        } wm;
+        struct intel_crtc_wm_state wm;
 
         /* Gamma mode programmed on the pipe */
         uint32_t gamma_mode;
@@ -598,14 +651,6 @@ struct vlv_wm_state {
         bool cxsr;
 };
 
-struct intel_mmio_flip {
-        struct work_struct work;
-        struct drm_i915_private *i915;
-        struct drm_i915_gem_request *req;
-        struct intel_crtc *crtc;
-        unsigned int rotation;
-};
-
 struct intel_crtc {
         struct drm_crtc base;
         enum pipe pipe;
@@ -620,7 +665,7 @@ struct intel_crtc {
         unsigned long enabled_power_domains;
         bool lowfreq_avail;
         struct intel_overlay *overlay;
-        struct intel_unpin_work *unpin_work;
+        struct intel_flip_work *flip_work;
 
         atomic_t unpin_work_count;
 
@@ -815,6 +860,7 @@ struct intel_dp {
         uint8_t dpcd[DP_RECEIVER_CAP_SIZE];
         uint8_t psr_dpcd[EDP_PSR_RECEIVER_CAP_SIZE];
         uint8_t downstream_ports[DP_MAX_DOWNSTREAM_PORTS];
+        uint8_t edp_dpcd[EDP_DISPLAY_CTL_CAP_SIZE];
         /* sink rates as reported by DP_SUPPORTED_LINK_RATES */
         uint8_t num_sink_rates;
         int sink_rates[DP_MAX_SUPPORTED_RATES];
@@ -838,6 +884,11 @@ struct intel_dp {
          * this port. Only relevant on VLV/CHV.
          */
         enum pipe pps_pipe;
+        /*
+         * Set if the sequencer may be reset due to a power transition,
+         * requiring a reinitialization. Only relevant on BXT.
+         */
+        bool pps_reset;
         struct edp_power_seq pps_delays;
 
         bool can_mst; /* this port supports mst */
@@ -934,33 +985,32 @@ vlv_pipe_to_channel(enum pipe pipe)
 static inline struct drm_crtc *
 intel_get_crtc_for_pipe(struct drm_device *dev, int pipe)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         return dev_priv->pipe_to_crtc_mapping[pipe];
 }
 
 static inline struct drm_crtc *
 intel_get_crtc_for_plane(struct drm_device *dev, int plane)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         return dev_priv->plane_to_crtc_mapping[plane];
 }
 
-struct intel_unpin_work {
-        struct work_struct work;
+struct intel_flip_work {
+        struct work_struct unpin_work;
+        struct work_struct mmio_work;
+
         struct drm_crtc *crtc;
         struct drm_framebuffer *old_fb;
         struct drm_i915_gem_object *pending_flip_obj;
         struct drm_pending_vblank_event *event;
         atomic_t pending;
-#define INTEL_FLIP_INACTIVE     0
-#define INTEL_FLIP_PENDING      1
-#define INTEL_FLIP_COMPLETE     2
         u32 flip_count;
         u32 gtt_offset;
         struct drm_i915_gem_request *flip_queued_req;
         u32 flip_queued_vblank;
         u32 flip_ready_vblank;
-        bool enable_stall_check;
+        unsigned int rotation;
 };
 
 struct intel_load_detect_pipe {
@@ -1029,9 +1079,9 @@ void gen5_enable_gt_irq(struct drm_i915_private *dev_priv, uint32_t mask);
 void gen5_disable_gt_irq(struct drm_i915_private *dev_priv, uint32_t mask);
 void gen6_enable_pm_irq(struct drm_i915_private *dev_priv, uint32_t mask);
 void gen6_disable_pm_irq(struct drm_i915_private *dev_priv, uint32_t mask);
-void gen6_reset_rps_interrupts(struct drm_device *dev);
-void gen6_enable_rps_interrupts(struct drm_device *dev);
-void gen6_disable_rps_interrupts(struct drm_device *dev);
+void gen6_reset_rps_interrupts(struct drm_i915_private *dev_priv);
+void gen6_enable_rps_interrupts(struct drm_i915_private *dev_priv);
+void gen6_disable_rps_interrupts(struct drm_i915_private *dev_priv);
 u32 gen6_sanitize_rps_pm_mask(struct drm_i915_private *dev_priv, u32 mask);
 void intel_runtime_pm_disable_interrupts(struct drm_i915_private *dev_priv);
 void intel_runtime_pm_enable_interrupts(struct drm_i915_private *dev_priv);
@@ -1052,7 +1102,7 @@ void gen8_irq_power_well_pre_disable(struct drm_i915_private *dev_priv,
 
 /* intel_crt.c */
 void intel_crt_init(struct drm_device *dev);
-
+void intel_crt_reset(struct drm_encoder *encoder);
 
 /* intel_ddi.c */
 void intel_ddi_clk_select(struct intel_encoder *encoder,
@@ -1110,14 +1160,16 @@ void i915_audio_component_init(struct drm_i915_private *dev_priv);
 void i915_audio_component_cleanup(struct drm_i915_private *dev_priv);
 
 /* intel_display.c */
+void skl_set_preferred_cdclk_vco(struct drm_i915_private *dev_priv, int vco);
+void intel_update_rawclk(struct drm_i915_private *dev_priv);
 int vlv_get_cck_clock(struct drm_i915_private *dev_priv,
                       const char *name, u32 reg, int ref_freq);
 extern const struct drm_plane_funcs intel_plane_funcs;
 void intel_init_display_hooks(struct drm_i915_private *dev_priv);
 unsigned int intel_rotation_info_size(const struct intel_rotation_info *rot_info);
 bool intel_has_pending_fb_unpin(struct drm_device *dev);
-void intel_mark_busy(struct drm_device *dev);
-void intel_mark_idle(struct drm_device *dev);
+void intel_mark_busy(struct drm_i915_private *dev_priv);
+void intel_mark_idle(struct drm_i915_private *dev_priv);
 void intel_crtc_restore_mode(struct drm_crtc *crtc);
 int intel_display_suspend(struct drm_device *dev);
 void intel_encoder_destroy(struct drm_encoder *encoder);
@@ -1126,7 +1178,6 @@ struct intel_connector *intel_connector_alloc(void);
 bool intel_connector_get_hw_state(struct intel_connector *connector);
 void intel_connector_attach_encoder(struct intel_connector *connector,
                                     struct intel_encoder *encoder);
-struct drm_encoder *intel_best_encoder(struct drm_connector *connector);
 struct drm_display_mode *intel_crtc_mode_get(struct drm_device *dev,
                                              struct drm_crtc *crtc);
 enum pipe intel_get_pipe_from_connector(struct intel_connector *connector);
@@ -1134,7 +1185,20 @@ int intel_get_pipe_from_crtc_id(struct drm_device *dev, void *data,
                                 struct drm_file *file_priv);
 enum transcoder intel_pipe_to_cpu_transcoder(struct drm_i915_private *dev_priv,
                                              enum pipe pipe);
-bool intel_pipe_has_type(struct intel_crtc *crtc, enum intel_output_type type);
+static inline bool
+intel_crtc_has_type(const struct intel_crtc_state *crtc_state,
+                    enum intel_output_type type)
+{
+        return crtc_state->output_types & (1 << type);
+}
+static inline bool
+intel_crtc_has_dp_encoder(const struct intel_crtc_state *crtc_state)
+{
+        return crtc_state->output_types &
+                ((1 << INTEL_OUTPUT_DP) |
+                 (1 << INTEL_OUTPUT_DP_MST) |
+                 (1 << INTEL_OUTPUT_EDP));
+}
 static inline void
 intel_wait_for_vblank(struct drm_device *dev, int pipe)
 {
@@ -1149,6 +1213,9 @@ intel_wait_for_vblank_if_active(struct drm_device *dev, int pipe)
         if (crtc->active)
                 intel_wait_for_vblank(dev, pipe);
 }
+
+u32 intel_crtc_get_vblank_counter(struct intel_crtc *crtc);
+
 int ironlake_get_lanes_required(int target_clock, int link_bw, int bpp);
 void vlv_wait_port_ready(struct drm_i915_private *dev_priv,
                          struct intel_digital_port *dport,
@@ -1162,14 +1229,14 @@ void intel_release_load_detect_pipe(struct drm_connector *connector,
                                     struct drm_modeset_acquire_ctx *ctx);
 int intel_pin_and_fence_fb_obj(struct drm_framebuffer *fb,
                                unsigned int rotation);
+void intel_unpin_fb_obj(struct drm_framebuffer *fb, unsigned int rotation);
 struct drm_framebuffer *
 __intel_framebuffer_create(struct drm_device *dev,
                            struct drm_mode_fb_cmd2 *mode_cmd,
                            struct drm_i915_gem_object *obj);
-void intel_prepare_page_flip(struct drm_device *dev, int plane);
-void intel_finish_page_flip(struct drm_device *dev, int pipe);
-void intel_finish_page_flip_plane(struct drm_device *dev, int plane);
-void intel_check_page_flip(struct drm_device *dev, int pipe);
+void intel_finish_page_flip_cs(struct drm_i915_private *dev_priv, int pipe);
+void intel_finish_page_flip_mmio(struct drm_i915_private *dev_priv, int pipe);
+void intel_check_page_flip(struct drm_i915_private *dev_priv, int pipe);
 int intel_prepare_plane_fb(struct drm_plane *plane,
                            const struct drm_plane_state *new_state);
 void intel_cleanup_plane_fb(struct drm_plane *plane,
@@ -1226,23 +1293,25 @@ u32 intel_compute_tile_offset(int *x, int *y,
                               const struct drm_framebuffer *fb, int plane,
                               unsigned int pitch,
                               unsigned int rotation);
-void intel_prepare_reset(struct drm_device *dev);
-void intel_finish_reset(struct drm_device *dev);
+void intel_prepare_reset(struct drm_i915_private *dev_priv);
+void intel_finish_reset(struct drm_i915_private *dev_priv);
 void hsw_enable_pc8(struct drm_i915_private *dev_priv);
 void hsw_disable_pc8(struct drm_i915_private *dev_priv);
-void broxton_init_cdclk(struct drm_i915_private *dev_priv);
-void broxton_uninit_cdclk(struct drm_i915_private *dev_priv);
-bool broxton_cdclk_verify_state(struct drm_i915_private *dev_priv);
-void broxton_ddi_phy_init(struct drm_i915_private *dev_priv);
-void broxton_ddi_phy_uninit(struct drm_i915_private *dev_priv);
-void broxton_ddi_phy_verify_state(struct drm_i915_private *dev_priv);
+void bxt_init_cdclk(struct drm_i915_private *dev_priv);
+void bxt_uninit_cdclk(struct drm_i915_private *dev_priv);
+void bxt_ddi_phy_init(struct drm_i915_private *dev_priv, enum dpio_phy phy);
+void bxt_ddi_phy_uninit(struct drm_i915_private *dev_priv, enum dpio_phy phy);
+bool bxt_ddi_phy_is_enabled(struct drm_i915_private *dev_priv,
+                            enum dpio_phy phy);
+bool bxt_ddi_phy_verify_state(struct drm_i915_private *dev_priv,
+                              enum dpio_phy phy);
 void gen9_sanitize_dc_state(struct drm_i915_private *dev_priv);
 void bxt_enable_dc9(struct drm_i915_private *dev_priv);
 void bxt_disable_dc9(struct drm_i915_private *dev_priv);
 void gen9_enable_dc5(struct drm_i915_private *dev_priv);
 void skl_init_cdclk(struct drm_i915_private *dev_priv);
-int skl_sanitize_cdclk(struct drm_i915_private *dev_priv);
 void skl_uninit_cdclk(struct drm_i915_private *dev_priv);
+unsigned int skl_cdclk_get_vco(unsigned int freq);
 void skl_enable_dc6(struct drm_i915_private *dev_priv);
 void skl_disable_dc6(struct drm_i915_private *dev_priv);
 void intel_dp_get_m_n(struct intel_crtc *crtc,
@@ -1250,8 +1319,8 @@ void intel_dp_get_m_n(struct intel_crtc *crtc,
 void intel_dp_set_m_n(struct intel_crtc *crtc, enum link_m_n_set m_n);
 int intel_dotclock_calculate(int link_freq, const struct intel_link_m_n *m_n);
 bool bxt_find_best_dpll(struct intel_crtc_state *crtc_state, int target_clock,
-                        intel_clock_t *best_clock);
-int chv_calc_dpll_params(int refclk, intel_clock_t *pll_clock);
+                        struct dpll *best_clock);
+int chv_calc_dpll_params(int refclk, struct dpll *pll_clock);
 
 bool intel_crtc_active(struct drm_crtc *crtc);
 void hsw_enable_ips(struct intel_crtc *crtc);
@@ -1310,7 +1379,7 @@ void intel_dp_mst_resume(struct drm_device *dev);
 int intel_dp_max_link_rate(struct intel_dp *intel_dp);
 int intel_dp_rate_select(struct intel_dp *intel_dp, int rate);
 void intel_dp_hot_plug(struct intel_encoder *intel_encoder);
-void vlv_power_sequencer_reset(struct drm_i915_private *dev_priv);
+void intel_power_sequencer_reset(struct drm_i915_private *dev_priv);
 uint32_t intel_dp_pack_aux(const uint8_t *src, int src_bytes);
 void intel_plane_destroy(struct drm_plane *plane);
 void intel_edp_drrs_enable(struct intel_dp *intel_dp);
@@ -1337,15 +1406,27 @@ bool intel_dp_source_supports_hbr2(struct intel_dp *intel_dp);
 bool
 intel_dp_get_link_status(struct intel_dp *intel_dp, uint8_t link_status[DP_LINK_STATUS_SIZE]);
 
+static inline unsigned int intel_dp_unused_lane_mask(int lane_count)
+{
+        return ~((1 << lane_count) - 1) & 0xf;
+}
+
+/* intel_dp_aux_backlight.c */
+int intel_dp_aux_init_backlight_funcs(struct intel_connector *intel_connector);
+
 /* intel_dp_mst.c */
 int intel_dp_mst_encoder_init(struct intel_digital_port *intel_dig_port, int conn_id);
 void intel_dp_mst_encoder_cleanup(struct intel_digital_port *intel_dig_port);
 /* intel_dsi.c */
 void intel_dsi_init(struct drm_device *dev);
 
+/* intel_dsi_dcs_backlight.c */
+int intel_dsi_dcs_init_backlight_funcs(struct intel_connector *intel_connector);
 
 /* intel_dvo.c */
 void intel_dvo_init(struct drm_device *dev);
+/* intel_hotplug.c */
+void intel_hpd_poll_init(struct drm_i915_private *dev_priv);
 
 
 /* legacy fbdev emulation in intel_fbdev.c */
@@ -1383,11 +1464,15 @@ static inline void intel_fbdev_restore_mode(struct drm_device *dev)
 void intel_fbc_choose_crtc(struct drm_i915_private *dev_priv,
                            struct drm_atomic_state *state);
 bool intel_fbc_is_active(struct drm_i915_private *dev_priv);
-void intel_fbc_pre_update(struct intel_crtc *crtc);
+void intel_fbc_pre_update(struct intel_crtc *crtc,
+                          struct intel_crtc_state *crtc_state,
+                          struct intel_plane_state *plane_state);
 void intel_fbc_post_update(struct intel_crtc *crtc);
 void intel_fbc_init(struct drm_i915_private *dev_priv);
 void intel_fbc_init_pipe_state(struct drm_i915_private *dev_priv);
-void intel_fbc_enable(struct intel_crtc *crtc);
+void intel_fbc_enable(struct intel_crtc *crtc,
+                      struct intel_crtc_state *crtc_state,
+                      struct intel_plane_state *plane_state);
 void intel_fbc_disable(struct intel_crtc *crtc);
 void intel_fbc_global_disable(struct drm_i915_private *dev_priv);
 void intel_fbc_invalidate(struct drm_i915_private *dev_priv,
@@ -1409,6 +1494,7 @@ void intel_dp_dual_mode_set_tmds_output(struct intel_hdmi *hdmi, bool enable);
 
 /* intel_lvds.c */
 void intel_lvds_init(struct drm_device *dev);
+struct intel_encoder *intel_get_lvds_encoder(struct drm_device *dev);
 bool intel_is_dual_link_lvds(struct drm_device *dev);
 
 
@@ -1422,13 +1508,13 @@ void intel_attach_aspect_ratio_property(struct drm_connector *connector);
 
 
 /* intel_overlay.c */
-void intel_setup_overlay(struct drm_device *dev);
-void intel_cleanup_overlay(struct drm_device *dev);
+void intel_setup_overlay(struct drm_i915_private *dev_priv);
+void intel_cleanup_overlay(struct drm_i915_private *dev_priv);
 int intel_overlay_switch_off(struct intel_overlay *overlay);
-int intel_overlay_put_image(struct drm_device *dev, void *data,
-                            struct drm_file *file_priv);
-int intel_overlay_attrs(struct drm_device *dev, void *data,
-                        struct drm_file *file_priv);
+int intel_overlay_put_image_ioctl(struct drm_device *dev, void *data,
+                                  struct drm_file *file_priv);
+int intel_overlay_attrs_ioctl(struct drm_device *dev, void *data,
+                              struct drm_file *file_priv);
 void intel_overlay_reset(struct drm_i915_private *dev_priv);
 
 
@@ -1447,7 +1533,8 @@ void intel_gmch_panel_fitting(struct intel_crtc *crtc,
                               int fitting_mode);
 void intel_panel_set_backlight_acpi(struct intel_connector *connector,
                                     u32 level, u32 max);
-int intel_panel_setup_backlight(struct drm_connector *connector, enum pipe pipe);
+int intel_panel_setup_backlight(struct drm_connector *connector,
+                                enum pipe pipe);
 void intel_panel_enable_backlight(struct intel_connector *connector);
 void intel_panel_disable_backlight(struct intel_connector *connector);
 void intel_panel_destroy_backlight(struct drm_connector *connector);
@@ -1456,8 +1543,19 @@ extern struct drm_display_mode *intel_find_panel_downclock(
                                 struct drm_device *dev,
                                 struct drm_display_mode *fixed_mode,
                                 struct drm_connector *connector);
-void intel_backlight_register(struct drm_device *dev);
-void intel_backlight_unregister(struct drm_device *dev);
+
+#if IS_ENABLED(CONFIG_BACKLIGHT_CLASS_DEVICE)
+int intel_backlight_device_register(struct intel_connector *connector);
+void intel_backlight_device_unregister(struct intel_connector *connector);
+#else /* CONFIG_BACKLIGHT_CLASS_DEVICE */
+static int intel_backlight_device_register(struct intel_connector *connector)
+{
+        return 0;
+}
+static inline void intel_backlight_device_unregister(struct intel_connector *connector)
+{
+}
+#endif /* CONFIG_BACKLIGHT_CLASS_DEVICE */
 
 
 /* intel_psr.c */
@@ -1599,21 +1697,20 @@ void intel_init_clock_gating_hooks(struct drm_i915_private *dev_priv);
 void intel_pm_setup(struct drm_device *dev);
 void intel_gpu_ips_init(struct drm_i915_private *dev_priv);
 void intel_gpu_ips_teardown(void);
-void intel_init_gt_powersave(struct drm_device *dev);
-void intel_cleanup_gt_powersave(struct drm_device *dev);
-void intel_enable_gt_powersave(struct drm_device *dev);
-void intel_disable_gt_powersave(struct drm_device *dev);
-void intel_suspend_gt_powersave(struct drm_device *dev);
-void intel_reset_gt_powersave(struct drm_device *dev);
-void gen6_update_ring_freq(struct drm_device *dev);
+void intel_init_gt_powersave(struct drm_i915_private *dev_priv);
+void intel_cleanup_gt_powersave(struct drm_i915_private *dev_priv);
+void intel_enable_gt_powersave(struct drm_i915_private *dev_priv);
+void intel_disable_gt_powersave(struct drm_i915_private *dev_priv);
+void intel_suspend_gt_powersave(struct drm_i915_private *dev_priv);
+void intel_reset_gt_powersave(struct drm_i915_private *dev_priv);
+void gen6_update_ring_freq(struct drm_i915_private *dev_priv);
 void gen6_rps_busy(struct drm_i915_private *dev_priv);
 void gen6_rps_reset_ei(struct drm_i915_private *dev_priv);
 void gen6_rps_idle(struct drm_i915_private *dev_priv);
 void gen6_rps_boost(struct drm_i915_private *dev_priv,
                     struct intel_rps_client *rps,
                     unsigned long submitted);
-void intel_queue_rps_boost_for_request(struct drm_device *dev,
-                                       struct drm_i915_gem_request *req);
+void intel_queue_rps_boost_for_request(struct drm_i915_gem_request *req);
 void vlv_wm_get_hw_state(struct drm_device *dev);
 void ilk_wm_get_hw_state(struct drm_device *dev);
 void skl_wm_get_hw_state(struct drm_device *dev);
@@ -1621,7 +1718,11 @@ void skl_ddb_get_hw_state(struct drm_i915_private *dev_priv,
                           struct skl_ddb_allocation *ddb /* out */);
 uint32_t ilk_pipe_pixel_rate(const struct intel_crtc_state *pipe_config);
 bool ilk_disable_lp_wm(struct drm_device *dev);
-int sanitize_rc6_option(const struct drm_device *dev, int enable_rc6);
+int sanitize_rc6_option(struct drm_i915_private *dev_priv, int enable_rc6);
+static inline int intel_enable_rc6(void)
+{
+        return i915.enable_rc6;
+}
 
 /* intel_sdvo.c */
 bool intel_sdvo_init(struct drm_device *dev,
@@ -1633,7 +1734,7 @@ int intel_plane_init(struct drm_device *dev, enum pipe pipe, int plane);
 int intel_sprite_set_colorkey(struct drm_device *dev, void *data,
                               struct drm_file *file_priv);
 void intel_pipe_update_start(struct intel_crtc *crtc);
-void intel_pipe_update_end(struct intel_crtc *crtc);
+void intel_pipe_update_end(struct intel_crtc *crtc, struct intel_flip_work *work);
 
 /* intel_tv.c */
 void intel_tv_init(struct drm_device *dev);
diff --git a/drivers/gpu/drm/i915/intel_dsi.c b/drivers/gpu/drm/i915/intel_dsi.c
index 4756ef639648..de8e9fb51595 100644
--- a/drivers/gpu/drm/i915/intel_dsi.c
+++ b/drivers/gpu/drm/i915/intel_dsi.c
@@ -84,13 +84,15 @@ static void wait_for_dsi_fifo_empty(struct intel_dsi *intel_dsi, enum port port)
 {
         struct drm_encoder *encoder = &intel_dsi->base.base;
         struct drm_device *dev = encoder->dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         u32 mask;
 
         mask = LP_CTRL_FIFO_EMPTY | HS_CTRL_FIFO_EMPTY |
                 LP_DATA_FIFO_EMPTY | HS_DATA_FIFO_EMPTY;
 
-        if (wait_for((I915_READ(MIPI_GEN_FIFO_STAT(port)) & mask) == mask, 100))
+        if (intel_wait_for_register(dev_priv,
+                                    MIPI_GEN_FIFO_STAT(port), mask, mask,
+                                    100))
                 DRM_ERROR("DPI FIFOs are not empty\n");
 }
 
@@ -129,7 +131,7 @@ static ssize_t intel_dsi_host_transfer(struct mipi_dsi_host *host,
 {
         struct intel_dsi_host *intel_dsi_host = to_intel_dsi_host(host);
         struct drm_device *dev = intel_dsi_host->intel_dsi->base.base.dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         enum port port = intel_dsi_host->port;
         struct mipi_dsi_packet packet;
         ssize_t ret;
@@ -158,8 +160,10 @@ static ssize_t intel_dsi_host_transfer(struct mipi_dsi_host *host,
 
         /* note: this is never true for reads */
         if (packet.payload_length) {
-
-                if (wait_for((I915_READ(MIPI_GEN_FIFO_STAT(port)) & data_mask) == 0, 50))
+                if (intel_wait_for_register(dev_priv,
+                                            MIPI_GEN_FIFO_STAT(port),
+                                            data_mask, 0,
+                                            50))
                         DRM_ERROR("Timeout waiting for HS/LP DATA FIFO !full\n");
 
                 write_data(dev_priv, data_reg, packet.payload,
@@ -170,7 +174,10 @@ static ssize_t intel_dsi_host_transfer(struct mipi_dsi_host *host,
                 I915_WRITE(MIPI_INTR_STAT(port), GEN_READ_DATA_AVAIL);
         }
 
-        if (wait_for((I915_READ(MIPI_GEN_FIFO_STAT(port)) & ctrl_mask) == 0, 50)) {
+        if (intel_wait_for_register(dev_priv,
+                                    MIPI_GEN_FIFO_STAT(port),
+                                    ctrl_mask, 0,
+                                    50)) {
                 DRM_ERROR("Timeout waiting for HS/LP CTRL FIFO !full\n");
         }
 
@@ -179,7 +186,10 @@ static ssize_t intel_dsi_host_transfer(struct mipi_dsi_host *host,
         /* ->rx_len is set only for reads */
         if (msg->rx_len) {
                 data_mask = GEN_READ_DATA_AVAIL;
-                if (wait_for((I915_READ(MIPI_INTR_STAT(port)) & data_mask) == data_mask, 50))
+                if (intel_wait_for_register(dev_priv,
+                                            MIPI_INTR_STAT(port),
+                                            data_mask, data_mask,
+                                            50))
                         DRM_ERROR("Timeout waiting for read data.\n");
 
                 read_data(dev_priv, data_reg, msg->rx_buf, msg->rx_len);
@@ -250,7 +260,7 @@ static int dpi_send_cmd(struct intel_dsi *intel_dsi, u32 cmd, bool hs,
 {
         struct drm_encoder *encoder = &intel_dsi->base.base;
         struct drm_device *dev = encoder->dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         u32 mask;
 
         /* XXX: pipe, hs */
@@ -269,7 +279,9 @@ static int dpi_send_cmd(struct intel_dsi *intel_dsi, u32 cmd, bool hs,
         I915_WRITE(MIPI_DPI_CONTROL(port), cmd);
 
         mask = SPL_PKT_SENT_INTERRUPT;
-        if (wait_for((I915_READ(MIPI_INTR_STAT(port)) & mask) == mask, 100))
+        if (intel_wait_for_register(dev_priv,
+                                    MIPI_INTR_STAT(port), mask, mask,
+                                    100))
                 DRM_ERROR("Video mode command 0x%08x send failed.\n", cmd);
 
         return 0;
@@ -302,7 +314,7 @@ static inline bool is_cmd_mode(struct intel_dsi *intel_dsi)
 static bool intel_dsi_compute_config(struct intel_encoder *encoder,
                                      struct intel_crtc_state *pipe_config)
 {
-        struct drm_i915_private *dev_priv = encoder->base.dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
         struct intel_dsi *intel_dsi = container_of(encoder, struct intel_dsi,
                                                    base);
         struct intel_connector *intel_connector = intel_dsi->attached_connector;
@@ -313,8 +325,6 @@ static bool intel_dsi_compute_config(struct intel_encoder *encoder,
 
         DRM_DEBUG_KMS("\n");
 
-        pipe_config->has_dsi_encoder = true;
-
         if (fixed_mode) {
                 intel_fixed_panel_mode(fixed_mode, adjusted_mode);
 
@@ -348,7 +358,7 @@ static bool intel_dsi_compute_config(struct intel_encoder *encoder,
 
 static void bxt_dsi_device_ready(struct intel_encoder *encoder)
 {
-        struct drm_i915_private *dev_priv = encoder->base.dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
         struct intel_dsi *intel_dsi = enc_to_intel_dsi(&encoder->base);
         enum port port;
         u32 val;
@@ -387,7 +397,7 @@ static void bxt_dsi_device_ready(struct intel_encoder *encoder)
 
 static void vlv_dsi_device_ready(struct intel_encoder *encoder)
 {
-        struct drm_i915_private *dev_priv = encoder->base.dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
         struct intel_dsi *intel_dsi = enc_to_intel_dsi(&encoder->base);
         enum port port;
         u32 val;
@@ -437,7 +447,7 @@ static void intel_dsi_device_ready(struct intel_encoder *encoder)
 static void intel_dsi_port_enable(struct intel_encoder *encoder)
 {
         struct drm_device *dev = encoder->base.dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         struct intel_crtc *intel_crtc = to_intel_crtc(encoder->base.crtc);
         struct intel_dsi *intel_dsi = enc_to_intel_dsi(&encoder->base);
         enum port port;
@@ -478,7 +488,7 @@ static void intel_dsi_port_enable(struct intel_encoder *encoder)
 static void intel_dsi_port_disable(struct intel_encoder *encoder)
 {
         struct drm_device *dev = encoder->base.dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         struct intel_dsi *intel_dsi = enc_to_intel_dsi(&encoder->base);
         enum port port;
 
@@ -497,7 +507,7 @@ static void intel_dsi_port_disable(struct intel_encoder *encoder)
 static void intel_dsi_enable(struct intel_encoder *encoder)
 {
         struct drm_device *dev = encoder->base.dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         struct intel_dsi *intel_dsi = enc_to_intel_dsi(&encoder->base);
         enum port port;
 
@@ -528,11 +538,10 @@ static void intel_dsi_prepare(struct intel_encoder *intel_encoder);
 static void intel_dsi_pre_enable(struct intel_encoder *encoder)
 {
         struct drm_device *dev = encoder->base.dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         struct intel_dsi *intel_dsi = enc_to_intel_dsi(&encoder->base);
         struct intel_crtc *crtc = to_intel_crtc(encoder->base.crtc);
         enum port port;
-        u32 tmp;
 
         DRM_DEBUG_KMS("\n");
 
@@ -551,11 +560,13 @@ static void intel_dsi_pre_enable(struct intel_encoder *encoder)
 
         msleep(intel_dsi->panel_on_delay);
 
-        if (IS_VALLEYVIEW(dev) || IS_CHERRYVIEW(dev)) {
+        if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) {
+                u32 val;
+
                 /* Disable DPOunit clock gating, can stall pipe */
-                tmp = I915_READ(DSPCLK_GATE_D);
-                tmp |= DPOUNIT_CLOCK_GATE_DISABLE;
-                I915_WRITE(DSPCLK_GATE_D, tmp);
+                val = I915_READ(DSPCLK_GATE_D);
+                val |= DPOUNIT_CLOCK_GATE_DISABLE;
+                I915_WRITE(DSPCLK_GATE_D, val);
         }
 
         /* put device in ready state */
@@ -601,7 +612,7 @@ static void intel_dsi_pre_disable(struct intel_encoder *encoder)
 static void intel_dsi_disable(struct intel_encoder *encoder)
 {
         struct drm_device *dev = encoder->base.dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         struct intel_dsi *intel_dsi = enc_to_intel_dsi(&encoder->base);
         enum port port;
         u32 temp;
@@ -640,7 +651,7 @@ static void intel_dsi_disable(struct intel_encoder *encoder)
 static void intel_dsi_clear_device_ready(struct intel_encoder *encoder)
 {
         struct drm_device *dev = encoder->base.dev;
-        struct drm_i915_private *dev_priv = encoder->base.dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
         struct intel_dsi *intel_dsi = enc_to_intel_dsi(&encoder->base);
         enum port port;
 
@@ -666,8 +677,9 @@ static void intel_dsi_clear_device_ready(struct intel_encoder *encoder)
                 /* Wait till Clock lanes are in LP-00 state for MIPI Port A
                  * only. MIPI Port C has no similar bit for checking
                  */
-                if (wait_for(((I915_READ(port_ctrl) & AFE_LATCHOUT)
-                                                == 0x00000), 30))
+                if (intel_wait_for_register(dev_priv,
+                                            port_ctrl, AFE_LATCHOUT, 0,
+                                            30))
                         DRM_ERROR("DSI LP not going Low\n");
 
                 /* Disable MIPI PHY transparent latch */
@@ -684,7 +696,7 @@ static void intel_dsi_clear_device_ready(struct intel_encoder *encoder)
 
 static void intel_dsi_post_disable(struct intel_encoder *encoder)
 {
-        struct drm_i915_private *dev_priv = encoder->base.dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
         struct intel_dsi *intel_dsi = enc_to_intel_dsi(&encoder->base);
 
         DRM_DEBUG_KMS("\n");
@@ -693,7 +705,7 @@ static void intel_dsi_post_disable(struct intel_encoder *encoder)
 
         intel_dsi_clear_device_ready(encoder);
 
-        if (!IS_BROXTON(dev_priv)) {
+        if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) {
                 u32 val;
 
                 val = I915_READ(DSPCLK_GATE_D);
@@ -719,7 +731,7 @@ static void intel_dsi_post_disable(struct intel_encoder *encoder)
 static bool intel_dsi_get_hw_state(struct intel_encoder *encoder,
                                    enum pipe *pipe)
 {
-        struct drm_i915_private *dev_priv = encoder->base.dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
         struct intel_dsi *intel_dsi = enc_to_intel_dsi(&encoder->base);
         struct drm_device *dev = encoder->base.dev;
         enum intel_display_power_domain power_domain;
@@ -793,7 +805,7 @@ static void bxt_dsi_get_pipe_config(struct intel_encoder *encoder,
                                  struct intel_crtc_state *pipe_config)
 {
         struct drm_device *dev = encoder->base.dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         struct drm_display_mode *adjusted_mode =
                                         &pipe_config->base.adjusted_mode;
         struct drm_display_mode *adjusted_mode_sw;
@@ -953,8 +965,6 @@ static void intel_dsi_get_config(struct intel_encoder *encoder,
         u32 pclk;
         DRM_DEBUG_KMS("\n");
 
-        pipe_config->has_dsi_encoder = true;
-
         if (IS_BROXTON(dev))
                 bxt_dsi_get_pipe_config(encoder, pipe_config);
 
@@ -1012,7 +1022,7 @@ static void set_dsi_timings(struct drm_encoder *encoder,
                             const struct drm_display_mode *adjusted_mode)
 {
         struct drm_device *dev = encoder->dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         struct intel_dsi *intel_dsi = enc_to_intel_dsi(encoder);
         enum port port;
         unsigned int bpp = mipi_dsi_pixel_format_to_bpp(intel_dsi->pixel_format);
@@ -1098,7 +1108,7 @@ static void intel_dsi_prepare(struct intel_encoder *intel_encoder)
 {
         struct drm_encoder *encoder = &intel_encoder->base;
         struct drm_device *dev = encoder->dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         struct intel_crtc *intel_crtc = to_intel_crtc(encoder->crtc);
         struct intel_dsi *intel_dsi = enc_to_intel_dsi(encoder);
         const struct drm_display_mode *adjusted_mode = &intel_crtc->config->base.adjusted_mode;
@@ -1171,6 +1181,12 @@ static void intel_dsi_prepare(struct intel_encoder *intel_encoder)
         if (intel_dsi->clock_stop)
                 tmp |= CLOCKSTOP;
 
+        if (IS_BROXTON(dev_priv)) {
+                tmp |= BXT_DPHY_DEFEATURE_EN;
+                if (!is_cmd_mode(intel_dsi))
+                        tmp |= BXT_DEFEATURE_DPI_FIFO_CTR;
+        }
+
         for_each_dsi_port(port, intel_dsi->ports) {
                 I915_WRITE(MIPI_DSI_FUNC_PRG(port), val);
 
@@ -1378,12 +1394,13 @@ static const struct drm_encoder_funcs intel_dsi_funcs = {
 static const struct drm_connector_helper_funcs intel_dsi_connector_helper_funcs = {
         .get_modes = intel_dsi_get_modes,
         .mode_valid = intel_dsi_mode_valid,
-        .best_encoder = intel_best_encoder,
 };
 
 static const struct drm_connector_funcs intel_dsi_connector_funcs = {
         .dpms = drm_atomic_helper_connector_dpms,
         .detect = intel_dsi_detect,
+        .late_register = intel_connector_register,
+        .early_unregister = intel_connector_unregister,
         .destroy = intel_dsi_connector_destroy,
         .fill_modes = drm_helper_probe_single_connector_modes,
         .set_property = intel_dsi_set_property,
@@ -1413,7 +1430,7 @@ void intel_dsi_init(struct drm_device *dev)
         struct intel_connector *intel_connector;
         struct drm_connector *connector;
         struct drm_display_mode *scan, *fixed_mode = NULL;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         enum port port;
         unsigned int i;
 
@@ -1449,7 +1466,7 @@ void intel_dsi_init(struct drm_device *dev)
         connector = &intel_connector->base;
 
         drm_encoder_init(dev, encoder, &intel_dsi_funcs, DRM_MODE_ENCODER_DSI,
-                         NULL);
+                         "DSI %c", port_name(port));
 
         intel_encoder->compute_config = intel_dsi_compute_config;
         intel_encoder->pre_enable = intel_dsi_pre_enable;
@@ -1460,7 +1477,6 @@ void intel_dsi_init(struct drm_device *dev)
         intel_encoder->get_config = intel_dsi_get_config;
 
         intel_connector->get_hw_state = intel_connector_get_hw_state;
-        intel_connector->unregister = intel_connector_unregister;
 
         /*
          * On BYT/CHV, pipe A maps to MIPI DSI port A, pipe B maps to MIPI DSI
@@ -1473,10 +1489,42 @@ void intel_dsi_init(struct drm_device *dev)
         else
                 intel_encoder->crtc_mask = BIT(PIPE_B);
 
-        if (dev_priv->vbt.dsi.config->dual_link)
+        if (dev_priv->vbt.dsi.config->dual_link) {
                 intel_dsi->ports = BIT(PORT_A) | BIT(PORT_C);
-        else
+
+                switch (dev_priv->vbt.dsi.config->dl_dcs_backlight_ports) {
+                case DL_DCS_PORT_A:
+                        intel_dsi->dcs_backlight_ports = BIT(PORT_A);
+                        break;
+                case DL_DCS_PORT_C:
+                        intel_dsi->dcs_backlight_ports = BIT(PORT_C);
+                        break;
+                default:
+                case DL_DCS_PORT_A_AND_C:
+                        intel_dsi->dcs_backlight_ports = BIT(PORT_A) | BIT(PORT_C);
+                        break;
+                }
+
+                switch (dev_priv->vbt.dsi.config->dl_dcs_cabc_ports) {
+                case DL_DCS_PORT_A:
+                        intel_dsi->dcs_cabc_ports = BIT(PORT_A);
+                        break;
+                case DL_DCS_PORT_C:
+                        intel_dsi->dcs_cabc_ports = BIT(PORT_C);
+                        break;
+                default:
+                case DL_DCS_PORT_A_AND_C:
+                        intel_dsi->dcs_cabc_ports = BIT(PORT_A) | BIT(PORT_C);
+                        break;
+                }
+        } else {
                 intel_dsi->ports = BIT(port);
+                intel_dsi->dcs_backlight_ports = BIT(port);
+                intel_dsi->dcs_cabc_ports = BIT(port);
+        }
+
+        if (!dev_priv->vbt.dsi.config->cabc_supported)
+                intel_dsi->dcs_cabc_ports = 0;
 
         /* Create a DSI host (and a device) for each port. */
         for_each_dsi_port(port, intel_dsi->ports) {
@@ -1549,13 +1597,10 @@ void intel_dsi_init(struct drm_device *dev)
         connector->display_info.height_mm = fixed_mode->height_mm;
 
         intel_panel_init(&intel_connector->panel, fixed_mode, NULL);
+        intel_panel_setup_backlight(connector, INVALID_PIPE);
 
         intel_dsi_add_properties(intel_connector);
 
-        drm_connector_register(connector);
-
-        intel_panel_setup_backlight(connector, INVALID_PIPE);
-
         return;
 
 err:
diff --git a/drivers/gpu/drm/i915/intel_dsi.h b/drivers/gpu/drm/i915/intel_dsi.h
index 61a6957fc6c2..5967ea6d6045 100644
--- a/drivers/gpu/drm/i915/intel_dsi.h
+++ b/drivers/gpu/drm/i915/intel_dsi.h
@@ -78,6 +78,10 @@ struct intel_dsi {
 
         u8 escape_clk_div;
         u8 dual_link;
+
+        u16 dcs_backlight_ports;
+        u16 dcs_cabc_ports;
+
         u8 pixel_overlap;
         u32 port_bits;
         u32 bw_timer;
diff --git a/drivers/gpu/drm/i915/intel_dsi_dcs_backlight.c b/drivers/gpu/drm/i915/intel_dsi_dcs_backlight.c
new file mode 100644
index 000000000000..ac7c6020c443
--- /dev/null
+++ b/drivers/gpu/drm/i915/intel_dsi_dcs_backlight.c
@@ -0,0 +1,179 @@
+/*
+ * Copyright © 2016 Intel Corporation
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ *
+ * Author: Deepak M <m.deepak at intel.com>
+ */
+
+#include "intel_drv.h"
+#include "intel_dsi.h"
+#include "i915_drv.h"
+#include <video/mipi_display.h>
+#include <drm/drm_mipi_dsi.h>
+
+#define CONTROL_DISPLAY_BCTRL           (1 << 5)
+#define CONTROL_DISPLAY_DD              (1 << 3)
+#define CONTROL_DISPLAY_BL              (1 << 2)
+
+#define POWER_SAVE_OFF                  (0 << 0)
+#define POWER_SAVE_LOW                  (1 << 0)
+#define POWER_SAVE_MEDIUM               (2 << 0)
+#define POWER_SAVE_HIGH                 (3 << 0)
+#define POWER_SAVE_OUTDOOR_MODE         (4 << 0)
+
+#define PANEL_PWM_MAX_VALUE             0xFF
+
+static u32 dcs_get_backlight(struct intel_connector *connector)
+{
+        struct intel_encoder *encoder = connector->encoder;
+        struct intel_dsi *intel_dsi = enc_to_intel_dsi(&encoder->base);
+        struct mipi_dsi_device *dsi_device;
+        u8 data;
+        enum port port;
+
+        /* FIXME: Need to take care of 16 bit brightness level */
+        for_each_dsi_port(port, intel_dsi->dcs_backlight_ports) {
+                dsi_device = intel_dsi->dsi_hosts[port]->device;
+                mipi_dsi_dcs_read(dsi_device, MIPI_DCS_GET_DISPLAY_BRIGHTNESS,
+                                  &data, sizeof(data));
+                break;
+        }
+
+        return data;
+}
+
+static void dcs_set_backlight(struct intel_connector *connector, u32 level)
+{
+        struct intel_encoder *encoder = connector->encoder;
+        struct intel_dsi *intel_dsi = enc_to_intel_dsi(&encoder->base);
+        struct mipi_dsi_device *dsi_device;
+        u8 data = level;
+        enum port port;
+
+        /* FIXME: Need to take care of 16 bit brightness level */
+        for_each_dsi_port(port, intel_dsi->dcs_backlight_ports) {
+                dsi_device = intel_dsi->dsi_hosts[port]->device;
+                mipi_dsi_dcs_write(dsi_device, MIPI_DCS_SET_DISPLAY_BRIGHTNESS,
+                                   &data, sizeof(data));
+        }
+}
+
+static void dcs_disable_backlight(struct intel_connector *connector)
+{
+        struct intel_encoder *encoder = connector->encoder;
+        struct intel_dsi *intel_dsi = enc_to_intel_dsi(&encoder->base);
+        struct mipi_dsi_device *dsi_device;
+        enum port port;
+
+        dcs_set_backlight(connector, 0);
+
+        for_each_dsi_port(port, intel_dsi->dcs_cabc_ports) {
+                u8 cabc = POWER_SAVE_OFF;
+
+                dsi_device = intel_dsi->dsi_hosts[port]->device;
+                mipi_dsi_dcs_write(dsi_device, MIPI_DCS_WRITE_POWER_SAVE,
+                                   &cabc, sizeof(cabc));
+        }
+
+        for_each_dsi_port(port, intel_dsi->dcs_backlight_ports) {
+                u8 ctrl = 0;
+
+                dsi_device = intel_dsi->dsi_hosts[port]->device;
+
+                mipi_dsi_dcs_read(dsi_device, MIPI_DCS_GET_CONTROL_DISPLAY,
+                                  &ctrl, sizeof(ctrl));
+
+                ctrl &= ~CONTROL_DISPLAY_BL;
+                ctrl &= ~CONTROL_DISPLAY_DD;
+                ctrl &= ~CONTROL_DISPLAY_BCTRL;
+
+                mipi_dsi_dcs_write(dsi_device, MIPI_DCS_WRITE_CONTROL_DISPLAY,
+                                   &ctrl, sizeof(ctrl));
+        }
+}
+
+static void dcs_enable_backlight(struct intel_connector *connector)
+{
+        struct intel_encoder *encoder = connector->encoder;
+        struct intel_dsi *intel_dsi = enc_to_intel_dsi(&encoder->base);
+        struct intel_panel *panel = &connector->panel;
+        struct mipi_dsi_device *dsi_device;
+        enum port port;
+
+        for_each_dsi_port(port, intel_dsi->dcs_backlight_ports) {
+                u8 ctrl = 0;
+
+                dsi_device = intel_dsi->dsi_hosts[port]->device;
+
+                mipi_dsi_dcs_read(dsi_device, MIPI_DCS_GET_CONTROL_DISPLAY,
+                                  &ctrl, sizeof(ctrl));
+
+                ctrl |= CONTROL_DISPLAY_BL;
+                ctrl |= CONTROL_DISPLAY_DD;
+                ctrl |= CONTROL_DISPLAY_BCTRL;
+
+                mipi_dsi_dcs_write(dsi_device, MIPI_DCS_WRITE_CONTROL_DISPLAY,
+                                   &ctrl, sizeof(ctrl));
+        }
+
+        for_each_dsi_port(port, intel_dsi->dcs_cabc_ports) {
+                u8 cabc = POWER_SAVE_MEDIUM;
+
+                dsi_device = intel_dsi->dsi_hosts[port]->device;
+                mipi_dsi_dcs_write(dsi_device, MIPI_DCS_WRITE_POWER_SAVE,
+                                   &cabc, sizeof(cabc));
+        }
+
+        dcs_set_backlight(connector, panel->backlight.level);
+}
+
+static int dcs_setup_backlight(struct intel_connector *connector,
+                               enum pipe unused)
+{
+        struct intel_panel *panel = &connector->panel;
+
+        panel->backlight.max = PANEL_PWM_MAX_VALUE;
+        panel->backlight.level = PANEL_PWM_MAX_VALUE;
+
+        return 0;
+}
+
+int intel_dsi_dcs_init_backlight_funcs(struct intel_connector *intel_connector)
+{
+        struct drm_device *dev = intel_connector->base.dev;
+        struct drm_i915_private *dev_priv = to_i915(dev);
+        struct intel_encoder *encoder = intel_connector->encoder;
+        struct intel_panel *panel = &intel_connector->panel;
+
+        if (dev_priv->vbt.backlight.type != INTEL_BACKLIGHT_DSI_DCS)
+                return -ENODEV;
+
+        if (WARN_ON(encoder->type != INTEL_OUTPUT_DSI))
+                return -EINVAL;
+
+        panel->backlight.setup = dcs_setup_backlight;
+        panel->backlight.enable = dcs_enable_backlight;
+        panel->backlight.disable = dcs_disable_backlight;
+        panel->backlight.set = dcs_set_backlight;
+        panel->backlight.get = dcs_get_backlight;
+
+        return 0;
+}
diff --git a/drivers/gpu/drm/i915/intel_dsi_panel_vbt.c b/drivers/gpu/drm/i915/intel_dsi_panel_vbt.c
index e498f1c3221e..cd154ce6b6c1 100644
--- a/drivers/gpu/drm/i915/intel_dsi_panel_vbt.c
+++ b/drivers/gpu/drm/i915/intel_dsi_panel_vbt.c
@@ -95,6 +95,24 @@ static struct gpio_map vlv_gpio_table[] = {
         { VLV_GPIO_NC_11_PANEL1_BKLTCTL },
 };
 
+#define CHV_GPIO_IDX_START_N            0
+#define CHV_GPIO_IDX_START_E            73
+#define CHV_GPIO_IDX_START_SW           100
+#define CHV_GPIO_IDX_START_SE           198
+
+#define CHV_VBT_MAX_PINS_PER_FMLY       15
+
+#define CHV_GPIO_PAD_CFG0(f, i)         (0x4400 + (f) * 0x400 + (i) * 8)
+#define  CHV_GPIO_GPIOEN                (1 << 15)
+#define  CHV_GPIO_GPIOCFG_GPIO          (0 << 8)
+#define  CHV_GPIO_GPIOCFG_GPO           (1 << 8)
+#define  CHV_GPIO_GPIOCFG_GPI           (2 << 8)
+#define  CHV_GPIO_GPIOCFG_HIZ           (3 << 8)
+#define  CHV_GPIO_GPIOTXSTATE(state)    ((!!(state)) << 1)
+
+#define CHV_GPIO_PAD_CFG1(f, i)         (0x4400 + (f) * 0x400 + (i) * 8 + 4)
+#define  CHV_GPIO_CFGLOCK               (1 << 31)
+
 static inline enum port intel_dsi_seq_port_to_port(u8 port)
 {
         return port ? PORT_C : PORT_A;
@@ -203,13 +221,14 @@ static void vlv_exec_gpio(struct drm_i915_private *dev_priv,
         map = &vlv_gpio_table[gpio_index];
 
         if (dev_priv->vbt.dsi.seq_version >= 3) {
-                DRM_DEBUG_KMS("GPIO element v3 not supported\n");
-                return;
+                /* XXX: this assumes vlv_gpio_table only has NC GPIOs. */
+                port = IOSF_PORT_GPIO_NC;
         } else {
                 if (gpio_source == 0) {
                         port = IOSF_PORT_GPIO_NC;
                 } else if (gpio_source == 1) {
-                        port = IOSF_PORT_GPIO_SC;
+                        DRM_DEBUG_KMS("SC gpio not supported\n");
+                        return;
                 } else {
                         DRM_DEBUG_KMS("unknown gpio source %u\n", gpio_source);
                         return;
@@ -231,10 +250,60 @@ static void vlv_exec_gpio(struct drm_i915_private *dev_priv,
         mutex_unlock(&dev_priv->sb_lock);
 }
 
+static void chv_exec_gpio(struct drm_i915_private *dev_priv,
+                          u8 gpio_source, u8 gpio_index, bool value)
+{
+        u16 cfg0, cfg1;
+        u16 family_num;
+        u8 port;
+
+        if (dev_priv->vbt.dsi.seq_version >= 3) {
+                if (gpio_index >= CHV_GPIO_IDX_START_SE) {
+                        /* XXX: it's unclear whether 255->57 is part of SE. */
+                        gpio_index -= CHV_GPIO_IDX_START_SE;
+                        port = CHV_IOSF_PORT_GPIO_SE;
+                } else if (gpio_index >= CHV_GPIO_IDX_START_SW) {
+                        gpio_index -= CHV_GPIO_IDX_START_SW;
+                        port = CHV_IOSF_PORT_GPIO_SW;
+                } else if (gpio_index >= CHV_GPIO_IDX_START_E) {
+                        gpio_index -= CHV_GPIO_IDX_START_E;
+                        port = CHV_IOSF_PORT_GPIO_E;
+                } else {
+                        port = CHV_IOSF_PORT_GPIO_N;
+                }
+        } else {
+                /* XXX: The spec is unclear about CHV GPIO on seq v2 */
+                if (gpio_source != 0) {
+                        DRM_DEBUG_KMS("unknown gpio source %u\n", gpio_source);
+                        return;
+                }
+
+                if (gpio_index >= CHV_GPIO_IDX_START_E) {
+                        DRM_DEBUG_KMS("invalid gpio index %u for GPIO N\n",
+                                      gpio_index);
+                        return;
+                }
+
+                port = CHV_IOSF_PORT_GPIO_N;
+        }
+
+        family_num = gpio_index / CHV_VBT_MAX_PINS_PER_FMLY;
+        gpio_index = gpio_index % CHV_VBT_MAX_PINS_PER_FMLY;
+
+        cfg0 = CHV_GPIO_PAD_CFG0(family_num, gpio_index);
+        cfg1 = CHV_GPIO_PAD_CFG1(family_num, gpio_index);
+
+        mutex_lock(&dev_priv->sb_lock);
+        vlv_iosf_sb_write(dev_priv, port, cfg1, 0);
+        vlv_iosf_sb_write(dev_priv, port, cfg0,
+                          CHV_GPIO_GPIOCFG_GPO | CHV_GPIO_GPIOTXSTATE(value));
+        mutex_unlock(&dev_priv->sb_lock);
+}
+
 static const u8 *mipi_exec_gpio(struct intel_dsi *intel_dsi, const u8 *data)
 {
         struct drm_device *dev = intel_dsi->base.base.dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         u8 gpio_source, gpio_index;
         bool value;
 
@@ -254,6 +323,8 @@ static const u8 *mipi_exec_gpio(struct intel_dsi *intel_dsi, const u8 *data)
 
         if (IS_VALLEYVIEW(dev_priv))
                 vlv_exec_gpio(dev_priv, gpio_source, gpio_index, value);
+        else if (IS_CHERRYVIEW(dev_priv))
+                chv_exec_gpio(dev_priv, gpio_source, gpio_index, value);
         else
                 DRM_DEBUG_KMS("GPIO element not supported on this platform\n");
 
@@ -398,7 +469,7 @@ static int vbt_panel_get_modes(struct drm_panel *panel)
         struct vbt_panel *vbt_panel = to_vbt_panel(panel);
         struct intel_dsi *intel_dsi = vbt_panel->intel_dsi;
         struct drm_device *dev = intel_dsi->base.base.dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         struct drm_display_mode *mode;
 
         if (!panel->connector)
@@ -426,7 +497,7 @@ static const struct drm_panel_funcs vbt_panel_funcs = {
 struct drm_panel *vbt_panel_init(struct intel_dsi *intel_dsi, u16 panel_id)
 {
         struct drm_device *dev = intel_dsi->base.base.dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         struct mipi_config *mipi_config = dev_priv->vbt.dsi.config;
         struct mipi_pps_data *pps = dev_priv->vbt.dsi.pps;
         struct drm_display_mode *mode = dev_priv->vbt.lfp_lvds_vbt_mode;
@@ -578,14 +649,13 @@ struct drm_panel *vbt_panel_init(struct intel_dsi *intel_dsi, u16 panel_id)
                                 );
 
         /*
-         * Exit zero  is unified val ths_zero and ths_exit
+         * Exit zero is unified val ths_zero and ths_exit
          * minimum value for ths_exit = 110ns
          * min (exit_zero_cnt * 2) = 110/UI
          * exit_zero_cnt = 55/UI
          */
-         if (exit_zero_cnt < (55 * ui_den / ui_num))
-                if ((55 * ui_den) % ui_num)
-                        exit_zero_cnt += 1;
+        if (exit_zero_cnt < (55 * ui_den / ui_num) && (55 * ui_den) % ui_num)
+                exit_zero_cnt += 1;
 
         /* clk zero count */
         clk_zero_cnt = DIV_ROUND_UP(
diff --git a/drivers/gpu/drm/i915/intel_dsi_pll.c b/drivers/gpu/drm/i915/intel_dsi_pll.c
index 1765e6e18f2c..6ab58a01b18e 100644
--- a/drivers/gpu/drm/i915/intel_dsi_pll.c
+++ b/drivers/gpu/drm/i915/intel_dsi_pll.c
@@ -55,12 +55,10 @@ static int dsi_calc_mnp(struct drm_i915_private *dev_priv,
                         struct intel_crtc_state *config,
                         int target_dsi_clk)
 {
-        unsigned int calc_m = 0, calc_p = 0;
         unsigned int m_min, m_max, p_min = 2, p_max = 6;
         unsigned int m, n, p;
-        int ref_clk;
-        int delta = target_dsi_clk;
-        u32 m_seed;
+        unsigned int calc_m, calc_p;
+        int delta, ref_clk;
 
         /* target_dsi_clk is expected in kHz */
         if (target_dsi_clk < 300000 || target_dsi_clk > 1150000) {
@@ -80,6 +78,10 @@ static int dsi_calc_mnp(struct drm_i915_private *dev_priv,
                 m_max = 92;
         }
 
+        calc_p = p_min;
+        calc_m = m_min;
+        delta = abs(target_dsi_clk - (m_min * ref_clk) / (p_min * n));
+
         for (m = m_min; m <= m_max && delta; m++) {
                 for (p = p_min; p <= p_max && delta; p++) {
                         /*
@@ -97,11 +99,10 @@ static int dsi_calc_mnp(struct drm_i915_private *dev_priv,
         }
 
         /* register has log2(N1), this works fine for powers of two */
-        n = ffs(n) - 1;
-        m_seed = lfsr_converts[calc_m - 62];
         config->dsi_pll.ctrl = 1 << (DSI_PLL_P1_POST_DIV_SHIFT + calc_p - 2);
-        config->dsi_pll.div = n << DSI_PLL_N1_DIV_SHIFT |
-                m_seed << DSI_PLL_M1_DIV_SHIFT;
+        config->dsi_pll.div =
+                (ffs(n) - 1) << DSI_PLL_N1_DIV_SHIFT |
+                (u32)lfsr_converts[calc_m - 62] << DSI_PLL_M1_DIV_SHIFT;
 
         return 0;
 }
@@ -113,7 +114,7 @@ static int dsi_calc_mnp(struct drm_i915_private *dev_priv,
 static int vlv_compute_dsi_pll(struct intel_encoder *encoder,
                                struct intel_crtc_state *config)
 {
-        struct drm_i915_private *dev_priv = encoder->base.dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
         struct intel_dsi *intel_dsi = enc_to_intel_dsi(&encoder->base);
         int ret;
         u32 dsi_clk;
@@ -234,8 +235,11 @@ static void bxt_disable_dsi_pll(struct intel_encoder *encoder)
          * PLL lock should deassert within 200us.
          * Wait up to 1ms before timing out.
          */
-        if (wait_for((I915_READ(BXT_DSI_PLL_ENABLE)
-                                        & BXT_DSI_PLL_LOCKED) == 0, 1))
+        if (intel_wait_for_register(dev_priv,
+                                    BXT_DSI_PLL_ENABLE,
+                                    BXT_DSI_PLL_LOCKED,
+                                    0,
+                                    1))
                 DRM_ERROR("Timeout waiting for PLL lock deassertion\n");
 }
 
@@ -321,7 +325,7 @@ static u32 bxt_dsi_get_pclk(struct intel_encoder *encoder, int pipe_bpp,
         u32 dsi_clk;
         u32 dsi_ratio;
         struct intel_dsi *intel_dsi = enc_to_intel_dsi(&encoder->base);
-        struct drm_i915_private *dev_priv = encoder->base.dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
 
         /* Divide by zero */
         if (!pipe_bpp) {
@@ -356,7 +360,7 @@ u32 intel_dsi_get_pclk(struct intel_encoder *encoder, int pipe_bpp,
 static void vlv_dsi_reset_clocks(struct intel_encoder *encoder, enum port port)
 {
         u32 temp;
-        struct drm_i915_private *dev_priv = encoder->base.dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
         struct intel_dsi *intel_dsi = enc_to_intel_dsi(&encoder->base);
 
         temp = I915_READ(MIPI_CTRL(port));
@@ -370,7 +374,7 @@ static void vlv_dsi_reset_clocks(struct intel_encoder *encoder, enum port port)
 static void bxt_dsi_program_clocks(struct drm_device *dev, enum port port,
                                    const struct intel_crtc_state *config)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         u32 tmp;
         u32 dsi_rate = 0;
         u32 pll_ratio = 0;
@@ -465,7 +469,7 @@ static int bxt_compute_dsi_pll(struct intel_encoder *encoder,
 static void bxt_enable_dsi_pll(struct intel_encoder *encoder,
                                const struct intel_crtc_state *config)
 {
-        struct drm_i915_private *dev_priv = encoder->base.dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
         struct intel_dsi *intel_dsi = enc_to_intel_dsi(&encoder->base);
         enum port port;
         u32 val;
@@ -486,7 +490,11 @@ static void bxt_enable_dsi_pll(struct intel_encoder *encoder,
         I915_WRITE(BXT_DSI_PLL_ENABLE, val);
 
         /* Timeout and fail if PLL not locked */
-        if (wait_for(I915_READ(BXT_DSI_PLL_ENABLE) & BXT_DSI_PLL_LOCKED, 1)) {
+        if (intel_wait_for_register(dev_priv,
+                                    BXT_DSI_PLL_ENABLE,
+                                    BXT_DSI_PLL_LOCKED,
+                                    BXT_DSI_PLL_LOCKED,
+                                    1)) {
                 DRM_ERROR("Timed out waiting for DSI PLL to lock\n");
                 return;
         }
@@ -542,7 +550,7 @@ static void bxt_dsi_reset_clocks(struct intel_encoder *encoder, enum port port)
 {
         u32 tmp;
         struct drm_device *dev = encoder->base.dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
 
         /* Clear old configurations */
         tmp = I915_READ(BXT_MIPI_CLOCK_CTL);
diff --git a/drivers/gpu/drm/i915/intel_dvo.c b/drivers/gpu/drm/i915/intel_dvo.c
index 286baec979c8..47bdf9dad0d3 100644
--- a/drivers/gpu/drm/i915/intel_dvo.c
+++ b/drivers/gpu/drm/i915/intel_dvo.c
@@ -122,7 +122,7 @@ static struct intel_dvo *intel_attached_dvo(struct drm_connector *connector)
 static bool intel_dvo_connector_get_hw_state(struct intel_connector *connector)
 {
         struct drm_device *dev = connector->base.dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         struct intel_dvo *intel_dvo = intel_attached_dvo(&connector->base);
         u32 tmp;
 
@@ -138,7 +138,7 @@ static bool intel_dvo_get_hw_state(struct intel_encoder *encoder,
                                    enum pipe *pipe)
 {
         struct drm_device *dev = encoder->base.dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         struct intel_dvo *intel_dvo = enc_to_dvo(encoder);
         u32 tmp;
 
@@ -155,7 +155,7 @@ static bool intel_dvo_get_hw_state(struct intel_encoder *encoder,
 static void intel_dvo_get_config(struct intel_encoder *encoder,
                                  struct intel_crtc_state *pipe_config)
 {
-        struct drm_i915_private *dev_priv = encoder->base.dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
         struct intel_dvo *intel_dvo = enc_to_dvo(encoder);
         u32 tmp, flags = 0;
 
@@ -176,7 +176,7 @@ static void intel_dvo_get_config(struct intel_encoder *encoder,
 
 static void intel_disable_dvo(struct intel_encoder *encoder)
 {
-        struct drm_i915_private *dev_priv = encoder->base.dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
         struct intel_dvo *intel_dvo = enc_to_dvo(encoder);
         i915_reg_t dvo_reg = intel_dvo->dev.dvo_reg;
         u32 temp = I915_READ(dvo_reg);
@@ -188,7 +188,7 @@ static void intel_disable_dvo(struct intel_encoder *encoder)
 
 static void intel_enable_dvo(struct intel_encoder *encoder)
 {
-        struct drm_i915_private *dev_priv = encoder->base.dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
         struct intel_dvo *intel_dvo = enc_to_dvo(encoder);
         struct intel_crtc *crtc = to_intel_crtc(encoder->base.crtc);
         i915_reg_t dvo_reg = intel_dvo->dev.dvo_reg;
@@ -256,7 +256,7 @@ static bool intel_dvo_compute_config(struct intel_encoder *encoder,
 static void intel_dvo_pre_enable(struct intel_encoder *encoder)
 {
         struct drm_device *dev = encoder->base.dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         struct intel_crtc *crtc = to_intel_crtc(encoder->base.crtc);
         const struct drm_display_mode *adjusted_mode = &crtc->config->base.adjusted_mode;
         struct intel_dvo *intel_dvo = enc_to_dvo(encoder);
@@ -305,7 +305,7 @@ intel_dvo_detect(struct drm_connector *connector, bool force)
 
 static int intel_dvo_get_modes(struct drm_connector *connector)
 {
-        struct drm_i915_private *dev_priv = connector->dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(connector->dev);
         const struct drm_display_mode *fixed_mode =
                 to_intel_connector(connector)->panel.fixed_mode;
 
@@ -341,6 +341,8 @@ static void intel_dvo_destroy(struct drm_connector *connector)
 static const struct drm_connector_funcs intel_dvo_connector_funcs = {
         .dpms = drm_atomic_helper_connector_dpms,
         .detect = intel_dvo_detect,
+        .late_register = intel_connector_register,
+        .early_unregister = intel_connector_unregister,
         .destroy = intel_dvo_destroy,
         .fill_modes = drm_helper_probe_single_connector_modes,
         .atomic_get_property = intel_connector_atomic_get_property,
@@ -351,7 +353,6 @@ static const struct drm_connector_funcs intel_dvo_connector_funcs = {
 static const struct drm_connector_helper_funcs intel_dvo_connector_helper_funcs = {
         .mode_valid = intel_dvo_mode_valid,
         .get_modes = intel_dvo_get_modes,
-        .best_encoder = intel_best_encoder,
 };
 
 static void intel_dvo_enc_destroy(struct drm_encoder *encoder)
@@ -378,7 +379,7 @@ static struct drm_display_mode *
 intel_dvo_get_current_mode(struct drm_connector *connector)
 {
         struct drm_device *dev = connector->dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         struct intel_dvo *intel_dvo = intel_attached_dvo(connector);
         uint32_t dvo_val = I915_READ(intel_dvo->dev.dvo_reg);
         struct drm_display_mode *mode = NULL;
@@ -406,9 +407,21 @@ intel_dvo_get_current_mode(struct drm_connector *connector)
         return mode;
 }
 
+static char intel_dvo_port_name(i915_reg_t dvo_reg)
+{
+        if (i915_mmio_reg_equal(dvo_reg, DVOA))
+                return 'A';
+        else if (i915_mmio_reg_equal(dvo_reg, DVOB))
+                return 'B';
+        else if (i915_mmio_reg_equal(dvo_reg, DVOC))
+                return 'C';
+        else
+                return '?';
+}
+
 void intel_dvo_init(struct drm_device *dev)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         struct intel_encoder *intel_encoder;
         struct intel_dvo *intel_dvo;
         struct intel_connector *intel_connector;
@@ -428,8 +441,6 @@ void intel_dvo_init(struct drm_device *dev)
         intel_dvo->attached_connector = intel_connector;
 
         intel_encoder = &intel_dvo->base;
-        drm_encoder_init(dev, &intel_encoder->base,
-                         &intel_dvo_enc_funcs, encoder_type, NULL);
 
         intel_encoder->disable = intel_disable_dvo;
         intel_encoder->enable = intel_enable_dvo;
@@ -438,7 +449,6 @@ void intel_dvo_init(struct drm_device *dev)
         intel_encoder->compute_config = intel_dvo_compute_config;
         intel_encoder->pre_enable = intel_dvo_pre_enable;
         intel_connector->get_hw_state = intel_dvo_connector_get_hw_state;
-        intel_connector->unregister = intel_connector_unregister;
 
         /* Now, try to find a controller */
         for (i = 0; i < ARRAY_SIZE(intel_dvo_devices); i++) {
@@ -496,6 +506,10 @@ void intel_dvo_init(struct drm_device *dev)
                 if (!dvoinit)
                         continue;
 
+                drm_encoder_init(dev, &intel_encoder->base,
+                                 &intel_dvo_enc_funcs, encoder_type,
+                                 "DVO %c", intel_dvo_port_name(dvo->dvo_reg));
+
                 intel_encoder->type = INTEL_OUTPUT_DVO;
                 intel_encoder->crtc_mask = (1 << 0) | (1 << 1);
                 switch (dvo->type) {
@@ -537,7 +551,6 @@ void intel_dvo_init(struct drm_device *dev)
                         intel_dvo->panel_wants_dither = true;
                 }
 
-                drm_connector_register(connector);
                 return;
         }
 
diff --git a/drivers/gpu/drm/i915/intel_fbc.c b/drivers/gpu/drm/i915/intel_fbc.c
index 647127f3aaff..6a7ad3ed1463 100644
--- a/drivers/gpu/drm/i915/intel_fbc.c
+++ b/drivers/gpu/drm/i915/intel_fbc.c
@@ -124,7 +124,9 @@ static void i8xx_fbc_deactivate(struct drm_i915_private *dev_priv)
         I915_WRITE(FBC_CONTROL, fbc_ctl);
 
         /* Wait for compressing bit to clear */
-        if (wait_for((I915_READ(FBC_STATUS) & FBC_STAT_COMPRESSING) == 0, 10)) {
+        if (intel_wait_for_register(dev_priv,
+                                    FBC_STATUS, FBC_STAT_COMPRESSING, 0,
+                                    10)) {
                 DRM_DEBUG_KMS("FBC idle timed out\n");
                 return;
         }
@@ -374,8 +376,9 @@ static void intel_fbc_hw_deactivate(struct drm_i915_private *dev_priv)
  * @dev_priv: i915 device instance
  *
  * This function is used to verify the current state of FBC.
+ *
  * FIXME: This should be tracked in the plane config eventually
- *        instead of queried at runtime for most callers.
+ * instead of queried at runtime for most callers.
  */
 bool intel_fbc_is_active(struct drm_i915_private *dev_priv)
 {
@@ -389,7 +392,7 @@ static void intel_fbc_work_fn(struct work_struct *__work)
         struct intel_fbc *fbc = &dev_priv->fbc;
         struct intel_fbc_work *work = &fbc->work;
         struct intel_crtc *crtc = fbc->crtc;
-        struct drm_vblank_crtc *vblank = &dev_priv->dev->vblank[crtc->pipe];
+        struct drm_vblank_crtc *vblank = &dev_priv->drm.vblank[crtc->pipe];
 
         if (drm_crtc_vblank_get(&crtc->base)) {
                 DRM_ERROR("vblank not available for FBC on pipe %c\n",
@@ -442,7 +445,7 @@ out:
 
 static void intel_fbc_schedule_activation(struct intel_crtc *crtc)
 {
-        struct drm_i915_private *dev_priv = crtc->base.dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
         struct intel_fbc *fbc = &dev_priv->fbc;
         struct intel_fbc_work *work = &fbc->work;
 
@@ -480,10 +483,10 @@ static void intel_fbc_deactivate(struct drm_i915_private *dev_priv)
                 intel_fbc_hw_deactivate(dev_priv);
 }
 
-static bool multiple_pipes_ok(struct intel_crtc *crtc)
+static bool multiple_pipes_ok(struct intel_crtc *crtc,
+                              struct intel_plane_state *plane_state)
 {
-        struct drm_i915_private *dev_priv = crtc->base.dev->dev_private;
-        struct drm_plane *primary = crtc->base.primary;
+        struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
         struct intel_fbc *fbc = &dev_priv->fbc;
         enum pipe pipe = crtc->pipe;
 
@@ -491,9 +494,7 @@ static bool multiple_pipes_ok(struct intel_crtc *crtc)
         if (!no_fbc_on_multiple_pipes(dev_priv))
                 return true;
 
-        WARN_ON(!drm_modeset_is_locked(&primary->mutex));
-
-        if (to_intel_plane_state(primary->state)->visible)
+        if (plane_state->visible)
                 fbc->visible_pipes_mask |= (1 << pipe);
         else
                 fbc->visible_pipes_mask &= ~(1 << pipe);
@@ -554,7 +555,7 @@ again:
 
 static int intel_fbc_alloc_cfb(struct intel_crtc *crtc)
 {
-        struct drm_i915_private *dev_priv = crtc->base.dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
         struct intel_fbc *fbc = &dev_priv->fbc;
         struct drm_mm_node *uninitialized_var(compressed_llb);
         int size, fb_cpp, ret;
@@ -685,7 +686,7 @@ static bool pixel_format_is_valid(struct drm_i915_private *dev_priv,
  */
 static bool intel_fbc_hw_tracking_covers_screen(struct intel_crtc *crtc)
 {
-        struct drm_i915_private *dev_priv = crtc->base.dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
         struct intel_fbc *fbc = &dev_priv->fbc;
         unsigned int effective_w, effective_h, max_w, max_h;
 
@@ -708,21 +709,16 @@ static bool intel_fbc_hw_tracking_covers_screen(struct intel_crtc *crtc)
         return effective_w <= max_w && effective_h <= max_h;
 }
 
-static void intel_fbc_update_state_cache(struct intel_crtc *crtc)
+static void intel_fbc_update_state_cache(struct intel_crtc *crtc,
+                                         struct intel_crtc_state *crtc_state,
+                                         struct intel_plane_state *plane_state)
 {
-        struct drm_i915_private *dev_priv = crtc->base.dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
         struct intel_fbc *fbc = &dev_priv->fbc;
         struct intel_fbc_state_cache *cache = &fbc->state_cache;
-        struct intel_crtc_state *crtc_state =
-                to_intel_crtc_state(crtc->base.state);
-        struct intel_plane_state *plane_state =
-                to_intel_plane_state(crtc->base.primary->state);
         struct drm_framebuffer *fb = plane_state->base.fb;
         struct drm_i915_gem_object *obj;
 
-        WARN_ON(!drm_modeset_is_locked(&crtc->base.mutex));
-        WARN_ON(!drm_modeset_is_locked(&crtc->base.primary->mutex));
-
         cache->crtc.mode_flags = crtc_state->base.adjusted_mode.flags;
         if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
                 cache->crtc.hsw_bdw_pixel_rate =
@@ -740,7 +736,7 @@ static void intel_fbc_update_state_cache(struct intel_crtc *crtc)
 
         /* FIXME: We lack the proper locking here, so only run this on the
          * platforms that need. */
-        if (INTEL_INFO(dev_priv)->gen >= 5 && INTEL_INFO(dev_priv)->gen < 7)
+        if (IS_GEN(dev_priv, 5, 6))
                 cache->fb.ilk_ggtt_offset = i915_gem_obj_ggtt_offset(obj);
         cache->fb.pixel_format = fb->pixel_format;
         cache->fb.stride = fb->pitches[0];
@@ -750,7 +746,7 @@ static void intel_fbc_update_state_cache(struct intel_crtc *crtc)
 
 static bool intel_fbc_can_activate(struct intel_crtc *crtc)
 {
-        struct drm_i915_private *dev_priv = crtc->base.dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
         struct intel_fbc *fbc = &dev_priv->fbc;
         struct intel_fbc_state_cache *cache = &fbc->state_cache;
 
@@ -822,22 +818,16 @@ static bool intel_fbc_can_activate(struct intel_crtc *crtc)
 
 static bool intel_fbc_can_choose(struct intel_crtc *crtc)
 {
-        struct drm_i915_private *dev_priv = crtc->base.dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
         struct intel_fbc *fbc = &dev_priv->fbc;
-        bool enable_by_default = IS_BROADWELL(dev_priv);
 
-        if (intel_vgpu_active(dev_priv->dev)) {
+        if (intel_vgpu_active(dev_priv)) {
                 fbc->no_fbc_reason = "VGPU is active";
                 return false;
         }
 
-        if (i915.enable_fbc < 0 && !enable_by_default) {
-                fbc->no_fbc_reason = "disabled per chip default";
-                return false;
-        }
-
         if (!i915.enable_fbc) {
-                fbc->no_fbc_reason = "disabled per module param";
+                fbc->no_fbc_reason = "disabled per module param or by default";
                 return false;
         }
 
@@ -857,7 +847,7 @@ static bool intel_fbc_can_choose(struct intel_crtc *crtc)
 static void intel_fbc_get_reg_params(struct intel_crtc *crtc,
                                      struct intel_fbc_reg_params *params)
 {
-        struct drm_i915_private *dev_priv = crtc->base.dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
         struct intel_fbc *fbc = &dev_priv->fbc;
         struct intel_fbc_state_cache *cache = &fbc->state_cache;
 
@@ -886,9 +876,11 @@ static bool intel_fbc_reg_params_equal(struct intel_fbc_reg_params *params1,
         return memcmp(params1, params2, sizeof(*params1)) == 0;
 }
 
-void intel_fbc_pre_update(struct intel_crtc *crtc)
+void intel_fbc_pre_update(struct intel_crtc *crtc,
+                          struct intel_crtc_state *crtc_state,
+                          struct intel_plane_state *plane_state)
 {
-        struct drm_i915_private *dev_priv = crtc->base.dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
         struct intel_fbc *fbc = &dev_priv->fbc;
 
         if (!fbc_supported(dev_priv))
@@ -896,7 +888,7 @@ void intel_fbc_pre_update(struct intel_crtc *crtc)
 
         mutex_lock(&fbc->lock);
 
-        if (!multiple_pipes_ok(crtc)) {
+        if (!multiple_pipes_ok(crtc, plane_state)) {
                 fbc->no_fbc_reason = "more than one pipe active";
                 goto deactivate;
         }
@@ -904,7 +896,7 @@ void intel_fbc_pre_update(struct intel_crtc *crtc)
         if (!fbc->enabled || fbc->crtc != crtc)
                 goto unlock;
 
-        intel_fbc_update_state_cache(crtc);
+        intel_fbc_update_state_cache(crtc, crtc_state, plane_state);
 
 deactivate:
         intel_fbc_deactivate(dev_priv);
@@ -914,7 +906,7 @@ unlock:
 
 static void __intel_fbc_post_update(struct intel_crtc *crtc)
 {
-        struct drm_i915_private *dev_priv = crtc->base.dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
         struct intel_fbc *fbc = &dev_priv->fbc;
         struct intel_fbc_reg_params old_params;
 
@@ -947,7 +939,7 @@ static void __intel_fbc_post_update(struct intel_crtc *crtc)
 
 void intel_fbc_post_update(struct intel_crtc *crtc)
 {
-        struct drm_i915_private *dev_priv = crtc->base.dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
         struct intel_fbc *fbc = &dev_priv->fbc;
 
         if (!fbc_supported(dev_priv))
@@ -996,13 +988,13 @@ void intel_fbc_flush(struct drm_i915_private *dev_priv,
         if (!fbc_supported(dev_priv))
                 return;
 
-        if (origin == ORIGIN_GTT || origin == ORIGIN_FLIP)
-                return;
-
         mutex_lock(&fbc->lock);
 
         fbc->busy_bits &= ~frontbuffer_bits;
 
+        if (origin == ORIGIN_GTT || origin == ORIGIN_FLIP)
+                goto out;
+
         if (!fbc->busy_bits && fbc->enabled &&
             (frontbuffer_bits & intel_fbc_get_frontbuffer_bit(fbc))) {
                 if (fbc->active)
@@ -1011,6 +1003,7 @@ void intel_fbc_flush(struct drm_i915_private *dev_priv,
                         __intel_fbc_post_update(fbc->crtc);
         }
 
+out:
         mutex_unlock(&fbc->lock);
 }
 
@@ -1088,9 +1081,11 @@ out:
  * intel_fbc_enable multiple times for the same pipe without an
  * intel_fbc_disable in the middle, as long as it is deactivated.
  */
-void intel_fbc_enable(struct intel_crtc *crtc)
+void intel_fbc_enable(struct intel_crtc *crtc,
+                      struct intel_crtc_state *crtc_state,
+                      struct intel_plane_state *plane_state)
 {
-        struct drm_i915_private *dev_priv = crtc->base.dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
         struct intel_fbc *fbc = &dev_priv->fbc;
 
         if (!fbc_supported(dev_priv))
@@ -1101,19 +1096,19 @@ void intel_fbc_enable(struct intel_crtc *crtc)
         if (fbc->enabled) {
                 WARN_ON(fbc->crtc == NULL);
                 if (fbc->crtc == crtc) {
-                        WARN_ON(!crtc->config->enable_fbc);
+                        WARN_ON(!crtc_state->enable_fbc);
                         WARN_ON(fbc->active);
                 }
                 goto out;
         }
 
-        if (!crtc->config->enable_fbc)
+        if (!crtc_state->enable_fbc)
                 goto out;
 
         WARN_ON(fbc->active);
         WARN_ON(fbc->crtc != NULL);
 
-        intel_fbc_update_state_cache(crtc);
+        intel_fbc_update_state_cache(crtc, crtc_state, plane_state);
         if (intel_fbc_alloc_cfb(crtc)) {
                 fbc->no_fbc_reason = "not enough stolen memory";
                 goto out;
@@ -1161,7 +1156,7 @@ static void __intel_fbc_disable(struct drm_i915_private *dev_priv)
  */
 void intel_fbc_disable(struct intel_crtc *crtc)
 {
-        struct drm_i915_private *dev_priv = crtc->base.dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
         struct intel_fbc *fbc = &dev_priv->fbc;
 
         if (!fbc_supported(dev_priv))
@@ -1215,12 +1210,32 @@ void intel_fbc_init_pipe_state(struct drm_i915_private *dev_priv)
         if (!no_fbc_on_multiple_pipes(dev_priv))
                 return;
 
-        for_each_intel_crtc(dev_priv->dev, crtc)
+        for_each_intel_crtc(&dev_priv->drm, crtc)
                 if (intel_crtc_active(&crtc->base) &&
                     to_intel_plane_state(crtc->base.primary->state)->visible)
                         dev_priv->fbc.visible_pipes_mask |= (1 << crtc->pipe);
 }
 
+/*
+ * The DDX driver changes its behavior depending on the value it reads from
+ * i915.enable_fbc, so sanitize it by translating the default value into either
+ * 0 or 1 in order to allow it to know what's going on.
+ *
+ * Notice that this is done at driver initialization and we still allow user
+ * space to change the value during runtime without sanitizing it again. IGT
+ * relies on being able to change i915.enable_fbc at runtime.
+ */
+static int intel_sanitize_fbc_option(struct drm_i915_private *dev_priv)
+{
+        if (i915.enable_fbc >= 0)
+                return !!i915.enable_fbc;
+
+        if (IS_BROADWELL(dev_priv))
+                return 1;
+
+        return 0;
+}
+
 /**
  * intel_fbc_init - Initialize FBC
  * @dev_priv: the i915 device
@@ -1238,6 +1253,9 @@ void intel_fbc_init(struct drm_i915_private *dev_priv)
         fbc->active = false;
         fbc->work.scheduled = false;
 
+        i915.enable_fbc = intel_sanitize_fbc_option(dev_priv);
+        DRM_DEBUG_KMS("Sanitized enable_fbc value: %d\n", i915.enable_fbc);
+
         if (!HAS_FBC(dev_priv)) {
                 fbc->no_fbc_reason = "unsupported by this chipset";
                 return;
diff --git a/drivers/gpu/drm/i915/intel_fbdev.c b/drivers/gpu/drm/i915/intel_fbdev.c
index ab8d09a81f14..86b00c6db1a6 100644
--- a/drivers/gpu/drm/i915/intel_fbdev.c
+++ b/drivers/gpu/drm/i915/intel_fbdev.c
@@ -150,10 +150,10 @@ static int intelfb_alloc(struct drm_fb_helper *helper,
         if (size * 2 < ggtt->stolen_usable_size)
                 obj = i915_gem_object_create_stolen(dev, size);
         if (obj == NULL)
-                obj = i915_gem_alloc_object(dev, size);
-        if (!obj) {
+                obj = i915_gem_object_create(dev, size);
+        if (IS_ERR(obj)) {
                 DRM_ERROR("failed to allocate framebuffer\n");
-                ret = -ENOMEM;
+                ret = PTR_ERR(obj);
                 goto out;
         }
 
@@ -186,9 +186,11 @@ static int intelfb_create(struct drm_fb_helper *helper,
         struct i915_ggtt *ggtt = &dev_priv->ggtt;
         struct fb_info *info;
         struct drm_framebuffer *fb;
+        struct i915_vma *vma;
         struct drm_i915_gem_object *obj;
-        int size, ret;
         bool prealloc = false;
+        void *vaddr;
+        int ret;
 
         if (intel_fb &&
             (sizes->fb_width > intel_fb->base.width ||
@@ -214,7 +216,6 @@ static int intelfb_create(struct drm_fb_helper *helper,
         }
 
         obj = intel_fb->obj;
-        size = obj->base.size;
 
         mutex_lock(&dev->struct_mutex);
 
@@ -244,22 +245,23 @@ static int intelfb_create(struct drm_fb_helper *helper,
         info->flags = FBINFO_DEFAULT | FBINFO_CAN_FORCE_OUTPUT;
         info->fbops = &intelfb_ops;
 
+        vma = i915_gem_obj_to_ggtt(obj);
+
         /* setup aperture base/size for vesafb takeover */
         info->apertures->ranges[0].base = dev->mode_config.fb_base;
         info->apertures->ranges[0].size = ggtt->mappable_end;
 
-        info->fix.smem_start = dev->mode_config.fb_base + i915_gem_obj_ggtt_offset(obj);
-        info->fix.smem_len = size;
+        info->fix.smem_start = dev->mode_config.fb_base + vma->node.start;
+        info->fix.smem_len = vma->node.size;
 
-        info->screen_base =
-                ioremap_wc(ggtt->mappable_base + i915_gem_obj_ggtt_offset(obj),
-                           size);
-        if (!info->screen_base) {
+        vaddr = i915_vma_pin_iomap(vma);
+        if (IS_ERR(vaddr)) {
                 DRM_ERROR("Failed to remap framebuffer into virtual memory\n");
-                ret = -ENOSPC;
+                ret = PTR_ERR(vaddr);
                 goto out_destroy_fbi;
         }
-        info->screen_size = size;
+        info->screen_base = vaddr;
+        info->screen_size = vma->node.size;
 
         /* This driver doesn't need a VT switch to restore the mode on resume */
         info->skip_vt_switch = true;
@@ -287,7 +289,7 @@ static int intelfb_create(struct drm_fb_helper *helper,
 out_destroy_fbi:
         drm_fb_helper_release_fbi(helper);
 out_unpin:
-        i915_gem_object_ggtt_unpin(obj);
+        intel_unpin_fb_obj(&ifbdev->fb->base, BIT(DRM_ROTATE_0));
 out_unlock:
         mutex_unlock(&dev->struct_mutex);
         return ret;
@@ -360,23 +362,24 @@ static bool intel_fb_initial_config(struct drm_fb_helper *fb_helper,
                                     bool *enabled, int width, int height)
 {
         struct drm_device *dev = fb_helper->dev;
+        unsigned long conn_configured, mask;
+        unsigned int count = min(fb_helper->connector_count, BITS_PER_LONG);
         int i, j;
         bool *save_enabled;
         bool fallback = true;
         int num_connectors_enabled = 0;
         int num_connectors_detected = 0;
-        uint64_t conn_configured = 0, mask;
         int pass = 0;
 
-        save_enabled = kcalloc(fb_helper->connector_count, sizeof(bool),
-                               GFP_KERNEL);
+        save_enabled = kcalloc(count, sizeof(bool), GFP_KERNEL);
         if (!save_enabled)
                 return false;
 
-        memcpy(save_enabled, enabled, fb_helper->connector_count);
-        mask = (1 << fb_helper->connector_count) - 1;
+        memcpy(save_enabled, enabled, count);
+        mask = BIT(count) - 1;
+        conn_configured = 0;
 retry:
-        for (i = 0; i < fb_helper->connector_count; i++) {
+        for (i = 0; i < count; i++) {
                 struct drm_fb_helper_connector *fb_conn;
                 struct drm_connector *connector;
                 struct drm_encoder *encoder;
@@ -386,7 +389,7 @@ retry:
                 fb_conn = fb_helper->connector_info[i];
                 connector = fb_conn->connector;
 
-                if (conn_configured & (1 << i))
+                if (conn_configured & BIT(i))
                         continue;
 
                 if (pass == 0 && !connector->has_tile)
@@ -398,7 +401,7 @@ retry:
                 if (!enabled[i]) {
                         DRM_DEBUG_KMS("connector %s not enabled, skipping\n",
                                       connector->name);
-                        conn_configured |= (1 << i);
+                        conn_configured |= BIT(i);
                         continue;
                 }
 
@@ -417,7 +420,7 @@ retry:
                         DRM_DEBUG_KMS("connector %s has no encoder or crtc, skipping\n",
                                       connector->name);
                         enabled[i] = false;
-                        conn_configured |= (1 << i);
+                        conn_configured |= BIT(i);
                         continue;
                 }
 
@@ -430,14 +433,15 @@ retry:
                         intel_crtc->lut_b[j] = j;
                 }
 
-                new_crtc = intel_fb_helper_crtc(fb_helper, connector->state->crtc);
+                new_crtc = intel_fb_helper_crtc(fb_helper,
+                                                connector->state->crtc);
 
                 /*
                  * Make sure we're not trying to drive multiple connectors
                  * with a single CRTC, since our cloning support may not
                  * match the BIOS.
                  */
-                for (j = 0; j < fb_helper->connector_count; j++) {
+                for (j = 0; j < count; j++) {
                         if (crtcs[j] == new_crtc) {
                                 DRM_DEBUG_KMS("fallback: cloned configuration\n");
                                 goto bail;
@@ -488,15 +492,15 @@ retry:
                 }
                 crtcs[i] = new_crtc;
 
-                DRM_DEBUG_KMS("connector %s on pipe %c [CRTC:%d]: %dx%d%s\n",
+                DRM_DEBUG_KMS("connector %s on [CRTC:%d:%s]: %dx%d%s\n",
                               connector->name,
-                              pipe_name(to_intel_crtc(connector->state->crtc)->pipe),
                               connector->state->crtc->base.id,
+                              connector->state->crtc->name,
                               modes[i]->hdisplay, modes[i]->vdisplay,
                               modes[i]->flags & DRM_MODE_FLAG_INTERLACE ? "i" :"");
 
                 fallback = false;
-                conn_configured |= (1 << i);
+                conn_configured |= BIT(i);
         }
 
         if ((conn_configured & mask) != mask) {
@@ -520,7 +524,7 @@ retry:
         if (fallback) {
 bail:
                 DRM_DEBUG_KMS("Not using firmware configuration\n");
-                memcpy(enabled, save_enabled, fb_helper->connector_count);
+                memcpy(enabled, save_enabled, count);
                 kfree(save_enabled);
                 return false;
         }
@@ -536,8 +540,7 @@ static const struct drm_fb_helper_funcs intel_fb_helper_funcs = {
         .fb_probe = intelfb_create,
 };
 
-static void intel_fbdev_destroy(struct drm_device *dev,
-                                struct intel_fbdev *ifbdev)
+static void intel_fbdev_destroy(struct intel_fbdev *ifbdev)
 {
         /* We rely on the object-free to release the VMA pinning for
          * the info->screen_base mmaping. Leaking the VMA is simpler than
@@ -550,9 +553,14 @@ static void intel_fbdev_destroy(struct drm_device *dev,
         drm_fb_helper_fini(&ifbdev->helper);
 
         if (ifbdev->fb) {
-                drm_framebuffer_unregister_private(&ifbdev->fb->base);
+                mutex_lock(&ifbdev->helper.dev->struct_mutex);
+                intel_unpin_fb_obj(&ifbdev->fb->base, BIT(DRM_ROTATE_0));
+                mutex_unlock(&ifbdev->helper.dev->struct_mutex);
+
                 drm_framebuffer_remove(&ifbdev->fb->base);
         }
+
+        kfree(ifbdev);
 }
 
 /*
@@ -685,9 +693,9 @@ out:
 
 static void intel_fbdev_suspend_worker(struct work_struct *work)
 {
-        intel_fbdev_set_suspend(container_of(work,
-                                             struct drm_i915_private,
-                                             fbdev_suspend_work)->dev,
+        intel_fbdev_set_suspend(&container_of(work,
+                                              struct drm_i915_private,
+                                              fbdev_suspend_work)->drm,
                                 FBINFO_STATE_RUNNING,
                                 true);
 }
@@ -695,7 +703,7 @@ static void intel_fbdev_suspend_worker(struct work_struct *work)
 int intel_fbdev_init(struct drm_device *dev)
 {
         struct intel_fbdev *ifbdev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         int ret;
 
         if (WARN_ON(INTEL_INFO(dev)->num_pipes == 0))
@@ -717,8 +725,6 @@ int intel_fbdev_init(struct drm_device *dev)
                 return ret;
         }
 
-        ifbdev->helper.atomic = true;
-
         dev_priv->fbdev = ifbdev;
         INIT_WORK(&dev_priv->fbdev_suspend_work, intel_fbdev_suspend_worker);
 
@@ -729,38 +735,50 @@ int intel_fbdev_init(struct drm_device *dev)
 
 static void intel_fbdev_initial_config(void *data, async_cookie_t cookie)
 {
-        struct drm_i915_private *dev_priv = data;
-        struct intel_fbdev *ifbdev = dev_priv->fbdev;
+        struct intel_fbdev *ifbdev = data;
 
         /* Due to peculiar init order wrt to hpd handling this is separate. */
         if (drm_fb_helper_initial_config(&ifbdev->helper,
                                          ifbdev->preferred_bpp))
-                intel_fbdev_fini(dev_priv->dev);
+                intel_fbdev_fini(ifbdev->helper.dev);
 }
 
 void intel_fbdev_initial_config_async(struct drm_device *dev)
 {
-        async_schedule(intel_fbdev_initial_config, to_i915(dev));
+        struct intel_fbdev *ifbdev = to_i915(dev)->fbdev;
+
+        ifbdev->cookie = async_schedule(intel_fbdev_initial_config, ifbdev);
+}
+
+static void intel_fbdev_sync(struct intel_fbdev *ifbdev)
+{
+        if (!ifbdev->cookie)
+                return;
+
+        /* Only serialises with all preceding async calls, hence +1 */
+        async_synchronize_cookie(ifbdev->cookie + 1);
+        ifbdev->cookie = 0;
 }
 
 void intel_fbdev_fini(struct drm_device *dev)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
-        if (!dev_priv->fbdev)
+        struct drm_i915_private *dev_priv = to_i915(dev);
+        struct intel_fbdev *ifbdev = dev_priv->fbdev;
+
+        if (!ifbdev)
                 return;
 
         flush_work(&dev_priv->fbdev_suspend_work);
-
         if (!current_is_async())
-                async_synchronize_full();
-        intel_fbdev_destroy(dev, dev_priv->fbdev);
-        kfree(dev_priv->fbdev);
+                intel_fbdev_sync(ifbdev);
+
+        intel_fbdev_destroy(ifbdev);
         dev_priv->fbdev = NULL;
 }
 
 void intel_fbdev_set_suspend(struct drm_device *dev, int state, bool synchronous)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         struct intel_fbdev *ifbdev = dev_priv->fbdev;
         struct fb_info *info;
 
@@ -809,7 +827,7 @@ void intel_fbdev_set_suspend(struct drm_device *dev, int state, bool synchronous
 
 void intel_fbdev_output_poll_changed(struct drm_device *dev)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         if (dev_priv->fbdev)
                 drm_fb_helper_hotplug_event(&dev_priv->fbdev->helper);
 }
@@ -817,13 +835,15 @@ void intel_fbdev_output_poll_changed(struct drm_device *dev)
 void intel_fbdev_restore_mode(struct drm_device *dev)
 {
         int ret;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         struct intel_fbdev *ifbdev = dev_priv->fbdev;
         struct drm_fb_helper *fb_helper;
 
         if (!ifbdev)
                 return;
 
+        intel_fbdev_sync(ifbdev);
+
         fb_helper = &ifbdev->helper;
 
         ret = drm_fb_helper_restore_fbdev_mode_unlocked(fb_helper);
diff --git a/drivers/gpu/drm/i915/intel_fifo_underrun.c b/drivers/gpu/drm/i915/intel_fifo_underrun.c
index 9be839a242f9..2aa744081f09 100644
--- a/drivers/gpu/drm/i915/intel_fifo_underrun.c
+++ b/drivers/gpu/drm/i915/intel_fifo_underrun.c
@@ -50,7 +50,7 @@
 
 static bool ivb_can_enable_err_int(struct drm_device *dev)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         struct intel_crtc *crtc;
         enum pipe pipe;
 
@@ -68,7 +68,7 @@ static bool ivb_can_enable_err_int(struct drm_device *dev)
 
 static bool cpt_can_enable_serr_int(struct drm_device *dev)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         enum pipe pipe;
         struct intel_crtc *crtc;
 
@@ -105,7 +105,7 @@ static void i9xx_set_fifo_underrun_reporting(struct drm_device *dev,
                                              enum pipe pipe,
                                              bool enable, bool old)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         i915_reg_t reg = PIPESTAT(pipe);
         u32 pipestat = I915_READ(reg) & 0xffff0000;
 
@@ -123,7 +123,7 @@ static void i9xx_set_fifo_underrun_reporting(struct drm_device *dev,
 static void ironlake_set_fifo_underrun_reporting(struct drm_device *dev,
                                                  enum pipe pipe, bool enable)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         uint32_t bit = (pipe == PIPE_A) ? DE_PIPEA_FIFO_UNDERRUN :
                                           DE_PIPEB_FIFO_UNDERRUN;
 
@@ -154,7 +154,7 @@ static void ivybridge_set_fifo_underrun_reporting(struct drm_device *dev,
                                                   enum pipe pipe,
                                                   bool enable, bool old)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         if (enable) {
                 I915_WRITE(GEN7_ERR_INT, ERR_INT_FIFO_UNDERRUN(pipe));
 
@@ -176,7 +176,7 @@ static void ivybridge_set_fifo_underrun_reporting(struct drm_device *dev,
 static void broadwell_set_fifo_underrun_reporting(struct drm_device *dev,
                                                   enum pipe pipe, bool enable)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
 
         if (enable)
                 bdw_enable_pipe_irq(dev_priv, pipe, GEN8_PIPE_FIFO_UNDERRUN);
@@ -188,7 +188,7 @@ static void ibx_set_fifo_underrun_reporting(struct drm_device *dev,
                                             enum transcoder pch_transcoder,
                                             bool enable)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         uint32_t bit = (pch_transcoder == TRANSCODER_A) ?
                        SDE_TRANSA_FIFO_UNDER : SDE_TRANSB_FIFO_UNDER;
 
@@ -220,7 +220,7 @@ static void cpt_set_fifo_underrun_reporting(struct drm_device *dev,
                                             enum transcoder pch_transcoder,
                                             bool enable, bool old)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
 
         if (enable) {
                 I915_WRITE(SERR_INT,
@@ -244,7 +244,7 @@ static void cpt_set_fifo_underrun_reporting(struct drm_device *dev,
 static bool __intel_set_cpu_fifo_underrun_reporting(struct drm_device *dev,
                                                     enum pipe pipe, bool enable)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         struct drm_crtc *crtc = dev_priv->pipe_to_crtc_mapping[pipe];
         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
         bool old;
@@ -289,7 +289,7 @@ bool intel_set_cpu_fifo_underrun_reporting(struct drm_i915_private *dev_priv,
         bool ret;
 
         spin_lock_irqsave(&dev_priv->irq_lock, flags);
-        ret = __intel_set_cpu_fifo_underrun_reporting(dev_priv->dev, pipe,
+        ret = __intel_set_cpu_fifo_underrun_reporting(&dev_priv->drm, pipe,
                                                       enable);
         spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
 
@@ -334,10 +334,12 @@ bool intel_set_pch_fifo_underrun_reporting(struct drm_i915_private *dev_priv,
         intel_crtc->pch_fifo_underrun_disabled = !enable;
 
         if (HAS_PCH_IBX(dev_priv))
-                ibx_set_fifo_underrun_reporting(dev_priv->dev, pch_transcoder,
+                ibx_set_fifo_underrun_reporting(&dev_priv->drm,
+                                                pch_transcoder,
                                                 enable);
         else
-                cpt_set_fifo_underrun_reporting(dev_priv->dev, pch_transcoder,
+                cpt_set_fifo_underrun_reporting(&dev_priv->drm,
+                                                pch_transcoder,
                                                 enable, old);
 
         spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
@@ -405,7 +407,7 @@ void intel_check_cpu_fifo_underruns(struct drm_i915_private *dev_priv)
 
         spin_lock_irq(&dev_priv->irq_lock);
 
-        for_each_intel_crtc(dev_priv->dev, crtc) {
+        for_each_intel_crtc(&dev_priv->drm, crtc) {
                 if (crtc->cpu_fifo_underrun_disabled)
                         continue;
 
@@ -432,7 +434,7 @@ void intel_check_pch_fifo_underruns(struct drm_i915_private *dev_priv)
 
         spin_lock_irq(&dev_priv->irq_lock);
 
-        for_each_intel_crtc(dev_priv->dev, crtc) {
+        for_each_intel_crtc(&dev_priv->drm, crtc) {
                 if (crtc->pch_fifo_underrun_disabled)
                         continue;
 
diff --git a/drivers/gpu/drm/i915/intel_guc.h b/drivers/gpu/drm/i915/intel_guc.h
index 9d79c4c3e256..3e3e743740c0 100644
--- a/drivers/gpu/drm/i915/intel_guc.h
+++ b/drivers/gpu/drm/i915/intel_guc.h
@@ -26,6 +26,7 @@
 
 #include "intel_guc_fwif.h"
 #include "i915_guc_reg.h"
+#include "intel_ringbuffer.h"
 
 struct drm_i915_gem_request;
 
@@ -48,14 +49,23 @@ struct drm_i915_gem_request;
  * queue (a circular array of work items), again described in the process
  * descriptor. Work queue pages are mapped momentarily as required.
  *
- * Finally, we also keep a few statistics here, including the number of
- * submissions to each engine, and a record of the last submission failure
- * (if any).
+ * We also keep a few statistics on failures. Ideally, these should all
+ * be zero!
+ *   no_wq_space: times that the submission pre-check found no space was
+ *                available in the work queue (note, the queue is shared,
+ *                not per-engine). It is OK for this to be nonzero, but
+ *                it should not be huge!
+ *   q_fail: failed to enqueue a work item. This should never happen,
+ *           because we check for space beforehand.
+ *   b_fail: failed to ring the doorbell. This should never happen, unless
+ *           somehow the hardware misbehaves, or maybe if the GuC firmware
+ *           crashes? We probably need to reset the GPU to recover.
+ *   retcode: errno from last guc_submit()
  */
 struct i915_guc_client {
         struct drm_i915_gem_object *client_obj;
         void *client_base;              /* first page (only) of above   */
-        struct intel_context *owner;
+        struct i915_gem_context *owner;
         struct intel_guc *guc;
         uint32_t priority;
         uint32_t ctx_index;
@@ -71,12 +81,13 @@ struct i915_guc_client {
         uint32_t wq_tail;
         uint32_t unused;                /* Was 'wq_head'                */
 
-        /* GuC submission statistics & status */
-        uint64_t submissions[GUC_MAX_ENGINES_NUM];
-        uint32_t q_fail;
+        uint32_t no_wq_space;
+        uint32_t q_fail;                /* No longer used               */
         uint32_t b_fail;
         int retcode;
-        int spare;                      /* pad to 32 DWords             */
+
+        /* Per-engine counts of GuC submissions */
+        uint64_t submissions[I915_NUM_ENGINES];
 };
 
 enum intel_guc_fw_status {
@@ -133,25 +144,24 @@ struct intel_guc {
         uint32_t action_fail;           /* Total number of failures     */
         int32_t action_err;             /* Last error code              */
 
-        uint64_t submissions[GUC_MAX_ENGINES_NUM];
-        uint32_t last_seqno[GUC_MAX_ENGINES_NUM];
+        uint64_t submissions[I915_NUM_ENGINES];
+        uint32_t last_seqno[I915_NUM_ENGINES];
 };
 
 /* intel_guc_loader.c */
-extern void intel_guc_ucode_init(struct drm_device *dev);
-extern int intel_guc_ucode_load(struct drm_device *dev);
-extern void intel_guc_ucode_fini(struct drm_device *dev);
+extern void intel_guc_init(struct drm_device *dev);
+extern int intel_guc_setup(struct drm_device *dev);
+extern void intel_guc_fini(struct drm_device *dev);
 extern const char *intel_guc_fw_status_repr(enum intel_guc_fw_status status);
 extern int intel_guc_suspend(struct drm_device *dev);
 extern int intel_guc_resume(struct drm_device *dev);
 
 /* i915_guc_submission.c */
-int i915_guc_submission_init(struct drm_device *dev);
-int i915_guc_submission_enable(struct drm_device *dev);
-int i915_guc_submit(struct i915_guc_client *client,
-                    struct drm_i915_gem_request *rq);
-void i915_guc_submission_disable(struct drm_device *dev);
-void i915_guc_submission_fini(struct drm_device *dev);
-int i915_guc_wq_check_space(struct i915_guc_client *client);
+int i915_guc_submission_init(struct drm_i915_private *dev_priv);
+int i915_guc_submission_enable(struct drm_i915_private *dev_priv);
+int i915_guc_wq_check_space(struct drm_i915_gem_request *rq);
+int i915_guc_submit(struct drm_i915_gem_request *rq);
+void i915_guc_submission_disable(struct drm_i915_private *dev_priv);
+void i915_guc_submission_fini(struct drm_i915_private *dev_priv);
 
 #endif
diff --git a/drivers/gpu/drm/i915/intel_guc_fwif.h b/drivers/gpu/drm/i915/intel_guc_fwif.h
index 2de57ffe5e18..944786d7075b 100644
--- a/drivers/gpu/drm/i915/intel_guc_fwif.h
+++ b/drivers/gpu/drm/i915/intel_guc_fwif.h
@@ -71,7 +71,8 @@
 #define   WQ_WORKLOAD_TOUCH             (2 << WQ_WORKLOAD_SHIFT)
 
 #define WQ_RING_TAIL_SHIFT              20
-#define WQ_RING_TAIL_MASK               (0x7FF << WQ_RING_TAIL_SHIFT)
+#define WQ_RING_TAIL_MAX                0x7FF   /* 2^11 QWords */
+#define WQ_RING_TAIL_MASK               (WQ_RING_TAIL_MAX << WQ_RING_TAIL_SHIFT)
 
 #define GUC_DOORBELL_ENABLED            1
 #define GUC_DOORBELL_DISABLED           0
diff --git a/drivers/gpu/drm/i915/intel_guc_loader.c b/drivers/gpu/drm/i915/intel_guc_loader.c
index 876e5da44c4e..605c69658d2c 100644
--- a/drivers/gpu/drm/i915/intel_guc_loader.c
+++ b/drivers/gpu/drm/i915/intel_guc_loader.c
@@ -59,9 +59,15 @@
  *
  */
 
-#define I915_SKL_GUC_UCODE "i915/skl_guc_ver6.bin"
+#define I915_SKL_GUC_UCODE "i915/skl_guc_ver6_1.bin"
 MODULE_FIRMWARE(I915_SKL_GUC_UCODE);
 
+#define I915_BXT_GUC_UCODE "i915/bxt_guc_ver8_7.bin"
+MODULE_FIRMWARE(I915_BXT_GUC_UCODE);
+
+#define I915_KBL_GUC_UCODE "i915/kbl_guc_ver9_14.bin"
+MODULE_FIRMWARE(I915_KBL_GUC_UCODE);
+
 /* User-friendly representation of an enum */
 const char *intel_guc_fw_status_repr(enum intel_guc_fw_status status)
 {
@@ -84,7 +90,7 @@ static void direct_interrupts_to_host(struct drm_i915_private *dev_priv)
         struct intel_engine_cs *engine;
         int irqs;
 
-        /* tell all command streamers NOT to forward interrupts and vblank to GuC */
+        /* tell all command streamers NOT to forward interrupts or vblank to GuC */
         irqs = _MASKED_FIELD(GFX_FORWARD_VBLANK_MASK, GFX_FORWARD_VBLANK_NEVER);
         irqs |= _MASKED_BIT_DISABLE(GFX_INTERRUPT_STEERING);
         for_each_engine(engine, dev_priv)
@@ -100,10 +106,10 @@ static void direct_interrupts_to_guc(struct drm_i915_private *dev_priv)
 {
         struct intel_engine_cs *engine;
         int irqs;
+        u32 tmp;
 
-        /* tell all command streamers to forward interrupts and vblank to GuC */
-        irqs = _MASKED_FIELD(GFX_FORWARD_VBLANK_MASK, GFX_FORWARD_VBLANK_ALWAYS);
-        irqs |= _MASKED_BIT_ENABLE(GFX_INTERRUPT_STEERING);
+        /* tell all command streamers to forward interrupts (but not vblank) to GuC */
+        irqs = _MASKED_BIT_ENABLE(GFX_INTERRUPT_STEERING);
         for_each_engine(engine, dev_priv)
                 I915_WRITE(RING_MODE_GEN7(engine), irqs);
 
@@ -114,6 +120,16 @@ static void direct_interrupts_to_guc(struct drm_i915_private *dev_priv)
         I915_WRITE(GUC_BCS_RCS_IER, ~irqs);
         I915_WRITE(GUC_VCS2_VCS1_IER, ~irqs);
         I915_WRITE(GUC_WD_VECS_IER, ~irqs);
+
+        /*
+         * If GuC has routed PM interrupts to itself, don't keep it.
+         * and keep other interrupts those are unmasked by GuC.
+        */
+        tmp = I915_READ(GEN6_PMINTRMSK);
+        if (tmp & GEN8_PMINTR_REDIRECT_TO_NON_DISP) {
+                dev_priv->rps.pm_intr_keep |= ~(tmp & ~GEN8_PMINTR_REDIRECT_TO_NON_DISP);
+                dev_priv->rps.pm_intr_keep &= ~GEN8_PMINTR_REDIRECT_TO_NON_DISP;
+        }
 }
 
 static u32 get_gttype(struct drm_i915_private *dev_priv)
@@ -281,13 +297,24 @@ static int guc_ucode_xfer_dma(struct drm_i915_private *dev_priv)
         return ret;
 }
 
+static u32 guc_wopcm_size(struct drm_i915_private *dev_priv)
+{
+        u32 wopcm_size = GUC_WOPCM_TOP;
+
+        /* On BXT, the top of WOPCM is reserved for RC6 context */
+        if (IS_BROXTON(dev_priv))
+                wopcm_size -= BXT_GUC_WOPCM_RC6_RESERVED;
+
+        return wopcm_size;
+}
+
 /*
  * Load the GuC firmware blob into the MinuteIA.
  */
 static int guc_ucode_xfer(struct drm_i915_private *dev_priv)
 {
         struct intel_guc_fw *guc_fw = &dev_priv->guc.guc_fw;
-        struct drm_device *dev = dev_priv->dev;
+        struct drm_device *dev = &dev_priv->drm;
         int ret;
 
         ret = i915_gem_object_set_to_gtt_domain(guc_fw->guc_fw_obj, false);
@@ -308,7 +335,7 @@ static int guc_ucode_xfer(struct drm_i915_private *dev_priv)
         intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
 
         /* init WOPCM */
-        I915_WRITE(GUC_WOPCM_SIZE, GUC_WOPCM_SIZE_VALUE);
+        I915_WRITE(GUC_WOPCM_SIZE, guc_wopcm_size(dev_priv));
         I915_WRITE(DMA_GUC_WOPCM_OFFSET, GUC_WOPCM_OFFSET_VALUE);
 
         /* Enable MIA caching. GuC clock gating is disabled. */
@@ -372,66 +399,63 @@ static int i915_reset_guc(struct drm_i915_private *dev_priv)
 }
 
 /**
- * intel_guc_ucode_load() - load GuC uCode into the device
+ * intel_guc_setup() - finish preparing the GuC for activity
  * @dev:        drm device
  *
  * Called from gem_init_hw() during driver loading and also after a GPU reset.
  *
+ * The main action required here it to load the GuC uCode into the device.
  * The firmware image should have already been fetched into memory by the
- * earlier call to intel_guc_ucode_init(), so here we need only check that
- * is succeeded, and then transfer the image to the h/w.
+ * earlier call to intel_guc_init(), so here we need only check that worked,
+ * and then transfer the image to the h/w.
  *
  * Return:      non-zero code on error
  */
-int intel_guc_ucode_load(struct drm_device *dev)
+int intel_guc_setup(struct drm_device *dev)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         struct intel_guc_fw *guc_fw = &dev_priv->guc.guc_fw;
-        int retries, err = 0;
+        const char *fw_path = guc_fw->guc_fw_path;
+        int retries, ret, err;
 
-        if (!i915.enable_guc_submission)
-                return 0;
-
-        DRM_DEBUG_DRIVER("GuC fw status: fetch %s, load %s\n",
+        DRM_DEBUG_DRIVER("GuC fw status: path %s, fetch %s, load %s\n",
+                fw_path,
                 intel_guc_fw_status_repr(guc_fw->guc_fw_fetch_status),
                 intel_guc_fw_status_repr(guc_fw->guc_fw_load_status));
 
-        direct_interrupts_to_host(dev_priv);
-
-        if (guc_fw->guc_fw_fetch_status == GUC_FIRMWARE_NONE)
-                return 0;
-
-        if (guc_fw->guc_fw_fetch_status == GUC_FIRMWARE_SUCCESS &&
-            guc_fw->guc_fw_load_status == GUC_FIRMWARE_FAIL)
-                return -ENOEXEC;
-
-        guc_fw->guc_fw_load_status = GUC_FIRMWARE_PENDING;
-
-        DRM_DEBUG_DRIVER("GuC fw fetch status %s\n",
-                intel_guc_fw_status_repr(guc_fw->guc_fw_fetch_status));
-
-        switch (guc_fw->guc_fw_fetch_status) {
-        case GUC_FIRMWARE_FAIL:
-                /* something went wrong :( */
-                err = -EIO;
+        /* Loading forbidden, or no firmware to load? */
+        if (!i915.enable_guc_loading) {
+                err = 0;
                 goto fail;
-
-        case GUC_FIRMWARE_NONE:
-        case GUC_FIRMWARE_PENDING:
-        default:
-                /* "can't happen" */
-                WARN_ONCE(1, "GuC fw %s invalid guc_fw_fetch_status %s [%d]\n",
-                        guc_fw->guc_fw_path,
-                        intel_guc_fw_status_repr(guc_fw->guc_fw_fetch_status),
-                        guc_fw->guc_fw_fetch_status);
+        } else if (fw_path == NULL) {
+                /* Device is known to have no uCode (e.g. no GuC) */
                 err = -ENXIO;
                 goto fail;
+        } else if (*fw_path == '\0') {
+                /* Device has a GuC but we don't know what f/w to load? */
+                DRM_INFO("No GuC firmware known for this platform\n");
+                err = -ENODEV;
+                goto fail;
+        }
 
-        case GUC_FIRMWARE_SUCCESS:
-                break;
+        /* Fetch failed, or already fetched but failed to load? */
+        if (guc_fw->guc_fw_fetch_status != GUC_FIRMWARE_SUCCESS) {
+                err = -EIO;
+                goto fail;
+        } else if (guc_fw->guc_fw_load_status == GUC_FIRMWARE_FAIL) {
+                err = -ENOEXEC;
+                goto fail;
         }
 
-        err = i915_guc_submission_init(dev);
+        direct_interrupts_to_host(dev_priv);
+
+        guc_fw->guc_fw_load_status = GUC_FIRMWARE_PENDING;
+
+        DRM_DEBUG_DRIVER("GuC fw status: fetch %s, load %s\n",
+                intel_guc_fw_status_repr(guc_fw->guc_fw_fetch_status),
+                intel_guc_fw_status_repr(guc_fw->guc_fw_load_status));
+
+        err = i915_guc_submission_init(dev_priv);
         if (err)
                 goto fail;
 
@@ -448,7 +472,7 @@ int intel_guc_ucode_load(struct drm_device *dev)
                  */
                 err = i915_reset_guc(dev_priv);
                 if (err) {
-                        DRM_ERROR("GuC reset failed, err %d\n", err);
+                        DRM_ERROR("GuC reset failed: %d\n", err);
                         goto fail;
                 }
 
@@ -459,8 +483,8 @@ int intel_guc_ucode_load(struct drm_device *dev)
                 if (--retries == 0)
                         goto fail;
 
-                DRM_INFO("GuC fw load failed, err %d; will reset and "
-                        "retry %d more time(s)\n", err, retries);
+                DRM_INFO("GuC fw load failed: %d; will reset and "
+                         "retry %d more time(s)\n", err, retries);
         }
 
         guc_fw->guc_fw_load_status = GUC_FIRMWARE_SUCCESS;
@@ -470,10 +494,7 @@ int intel_guc_ucode_load(struct drm_device *dev)
                 intel_guc_fw_status_repr(guc_fw->guc_fw_load_status));
 
         if (i915.enable_guc_submission) {
-                /* The execbuf_client will be recreated. Release it first. */
-                i915_guc_submission_disable(dev);
-
-                err = i915_guc_submission_enable(dev);
+                err = i915_guc_submission_enable(dev_priv);
                 if (err)
                         goto fail;
                 direct_interrupts_to_guc(dev_priv);
@@ -482,15 +503,50 @@ int intel_guc_ucode_load(struct drm_device *dev)
         return 0;
 
 fail:
-        DRM_ERROR("GuC firmware load failed, err %d\n", err);
         if (guc_fw->guc_fw_load_status == GUC_FIRMWARE_PENDING)
                 guc_fw->guc_fw_load_status = GUC_FIRMWARE_FAIL;
 
         direct_interrupts_to_host(dev_priv);
-        i915_guc_submission_disable(dev);
-        i915_guc_submission_fini(dev);
+        i915_guc_submission_disable(dev_priv);
+        i915_guc_submission_fini(dev_priv);
 
-        return err;
+        /*
+         * We've failed to load the firmware :(
+         *
+         * Decide whether to disable GuC submission and fall back to
+         * execlist mode, and whether to hide the error by returning
+         * zero or to return -EIO, which the caller will treat as a
+         * nonfatal error (i.e. it doesn't prevent driver load, but
+         * marks the GPU as wedged until reset).
+         */
+        if (i915.enable_guc_loading > 1) {
+                ret = -EIO;
+        } else if (i915.enable_guc_submission > 1) {
+                ret = -EIO;
+        } else {
+                ret = 0;
+        }
+
+        if (err == 0 && !HAS_GUC_UCODE(dev))
+                ;       /* Don't mention the GuC! */
+        else if (err == 0)
+                DRM_INFO("GuC firmware load skipped\n");
+        else if (ret != -EIO)
+                DRM_INFO("GuC firmware load failed: %d\n", err);
+        else
+                DRM_ERROR("GuC firmware load failed: %d\n", err);
+
+        if (i915.enable_guc_submission) {
+                if (fw_path == NULL)
+                        DRM_INFO("GuC submission without firmware not supported\n");
+                if (ret == 0)
+                        DRM_INFO("Falling back from GuC submission to execlist mode\n");
+                else
+                        DRM_ERROR("GuC init failed: %d\n", ret);
+        }
+        i915.enable_guc_submission = 0;
+
+        return ret;
 }
 
 static void guc_fw_fetch(struct drm_device *dev, struct intel_guc_fw *guc_fw)
@@ -552,9 +608,7 @@ static void guc_fw_fetch(struct drm_device *dev, struct intel_guc_fw *guc_fw)
 
         /* Header and uCode will be loaded to WOPCM. Size of the two. */
         size = guc_fw->header_size + guc_fw->ucode_size;
-
-        /* Top 32k of WOPCM is reserved (8K stack + 24k RC6 context). */
-        if (size > GUC_WOPCM_SIZE_VALUE - 0x8000) {
+        if (size > guc_wopcm_size(to_i915(dev))) {
                 DRM_ERROR("Firmware is too large to fit in WOPCM\n");
                 goto fail;
         }
@@ -617,22 +671,25 @@ fail:
 }
 
 /**
- * intel_guc_ucode_init() - define parameters and fetch firmware
+ * intel_guc_init() - define parameters and fetch firmware
  * @dev:        drm device
  *
  * Called early during driver load, but after GEM is initialised.
  *
  * The firmware will be transferred to the GuC's memory later,
- * when intel_guc_ucode_load() is called.
+ * when intel_guc_setup() is called.
  */
-void intel_guc_ucode_init(struct drm_device *dev)
+void intel_guc_init(struct drm_device *dev)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         struct intel_guc_fw *guc_fw = &dev_priv->guc.guc_fw;
         const char *fw_path;
 
-        if (!HAS_GUC_SCHED(dev))
-                i915.enable_guc_submission = false;
+        /* A negative value means "use platform default" */
+        if (i915.enable_guc_loading < 0)
+                i915.enable_guc_loading = HAS_GUC_UCODE(dev);
+        if (i915.enable_guc_submission < 0)
+                i915.enable_guc_submission = HAS_GUC_SCHED(dev);
 
         if (!HAS_GUC_UCODE(dev)) {
                 fw_path = NULL;
@@ -640,27 +697,30 @@ void intel_guc_ucode_init(struct drm_device *dev)
                 fw_path = I915_SKL_GUC_UCODE;
                 guc_fw->guc_fw_major_wanted = 6;
                 guc_fw->guc_fw_minor_wanted = 1;
+        } else if (IS_BROXTON(dev)) {
+                fw_path = I915_BXT_GUC_UCODE;
+                guc_fw->guc_fw_major_wanted = 8;
+                guc_fw->guc_fw_minor_wanted = 7;
+        } else if (IS_KABYLAKE(dev)) {
+                fw_path = I915_KBL_GUC_UCODE;
+                guc_fw->guc_fw_major_wanted = 9;
+                guc_fw->guc_fw_minor_wanted = 14;
         } else {
-                i915.enable_guc_submission = false;
                 fw_path = "";   /* unknown device */
         }
 
-        if (!i915.enable_guc_submission)
-                return;
-
         guc_fw->guc_dev = dev;
         guc_fw->guc_fw_path = fw_path;
         guc_fw->guc_fw_fetch_status = GUC_FIRMWARE_NONE;
         guc_fw->guc_fw_load_status = GUC_FIRMWARE_NONE;
 
+        /* Early (and silent) return if GuC loading is disabled */
+        if (!i915.enable_guc_loading)
+                return;
         if (fw_path == NULL)
                 return;
-
-        if (*fw_path == '\0') {
-                DRM_ERROR("No GuC firmware known for this platform\n");
-                guc_fw->guc_fw_fetch_status = GUC_FIRMWARE_FAIL;
+        if (*fw_path == '\0')
                 return;
-        }
 
         guc_fw->guc_fw_fetch_status = GUC_FIRMWARE_PENDING;
         DRM_DEBUG_DRIVER("GuC firmware pending, path %s\n", fw_path);
@@ -669,18 +729,18 @@ void intel_guc_ucode_init(struct drm_device *dev)
 }
 
 /**
- * intel_guc_ucode_fini() - clean up all allocated resources
+ * intel_guc_fini() - clean up all allocated resources
  * @dev:        drm device
  */
-void intel_guc_ucode_fini(struct drm_device *dev)
+void intel_guc_fini(struct drm_device *dev)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         struct intel_guc_fw *guc_fw = &dev_priv->guc.guc_fw;
 
         mutex_lock(&dev->struct_mutex);
         direct_interrupts_to_host(dev_priv);
-        i915_guc_submission_disable(dev);
-        i915_guc_submission_fini(dev);
+        i915_guc_submission_disable(dev_priv);
+        i915_guc_submission_fini(dev_priv);
 
         if (guc_fw->guc_fw_obj)
                 drm_gem_object_unreference(&guc_fw->guc_fw_obj->base);
diff --git a/drivers/gpu/drm/i915/intel_gvt.c b/drivers/gpu/drm/i915/intel_gvt.c
new file mode 100644
index 000000000000..434f4d5c553d
--- /dev/null
+++ b/drivers/gpu/drm/i915/intel_gvt.c
@@ -0,0 +1,104 @@
+/*
+ * Copyright(c) 2011-2016 Intel Corporation. All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#include "i915_drv.h"
+#include "intel_gvt.h"
+
+/**
+ * DOC: Intel GVT-g host support
+ *
+ * Intel GVT-g is a graphics virtualization technology which shares the
+ * GPU among multiple virtual machines on a time-sharing basis. Each
+ * virtual machine is presented a virtual GPU (vGPU), which has equivalent
+ * features as the underlying physical GPU (pGPU), so i915 driver can run
+ * seamlessly in a virtual machine. This file provides the englightments
+ * of GVT and the necessary components used by GVT in i915 driver.
+ */
+
+static bool is_supported_device(struct drm_i915_private *dev_priv)
+{
+        if (IS_BROADWELL(dev_priv))
+                return true;
+        return false;
+}
+
+/**
+ * intel_gvt_init - initialize GVT components
+ * @dev_priv: drm i915 private data
+ *
+ * This function is called at the initialization stage to create a GVT device.
+ *
+ * Returns:
+ * Zero on success, negative error code if failed.
+ *
+ */
+int intel_gvt_init(struct drm_i915_private *dev_priv)
+{
+        int ret;
+
+        if (!i915.enable_gvt) {
+                DRM_DEBUG_DRIVER("GVT-g is disabled by kernel params\n");
+                return 0;
+        }
+
+        if (!is_supported_device(dev_priv)) {
+                DRM_DEBUG_DRIVER("Unsupported device. GVT-g is disabled\n");
+                goto bail;
+        }
+
+        /*
+         * We're not in host or fail to find a MPT module, disable GVT-g
+         */
+        ret = intel_gvt_init_host();
+        if (ret) {
+                DRM_DEBUG_DRIVER("Not in host or MPT modules not found\n");
+                goto bail;
+        }
+
+        ret = intel_gvt_init_device(dev_priv);
+        if (ret) {
+                DRM_DEBUG_DRIVER("Fail to init GVT device\n");
+                goto bail;
+        }
+
+        return 0;
+
+bail:
+        i915.enable_gvt = 0;
+        return 0;
+}
+
+/**
+ * intel_gvt_cleanup - cleanup GVT components when i915 driver is unloading
+ * @dev_priv: drm i915 private *
+ *
+ * This function is called at the i915 driver unloading stage, to shutdown
+ * GVT components and release the related resources.
+ */
+void intel_gvt_cleanup(struct drm_i915_private *dev_priv)
+{
+        if (!intel_gvt_active(dev_priv))
+                return;
+
+        intel_gvt_clean_device(dev_priv);
+}
diff --git a/drivers/gpu/drm/i915/intel_gvt.h b/drivers/gpu/drm/i915/intel_gvt.h
new file mode 100644
index 000000000000..960211df74db
--- /dev/null
+++ b/drivers/gpu/drm/i915/intel_gvt.h
@@ -0,0 +1,45 @@
+/*
+ * Copyright(c) 2011-2016 Intel Corporation. All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#ifndef _INTEL_GVT_H_
+#define _INTEL_GVT_H_
+
+#include "gvt/gvt.h"
+
+#ifdef CONFIG_DRM_I915_GVT
+int intel_gvt_init(struct drm_i915_private *dev_priv);
+void intel_gvt_cleanup(struct drm_i915_private *dev_priv);
+int intel_gvt_init_device(struct drm_i915_private *dev_priv);
+void intel_gvt_clean_device(struct drm_i915_private *dev_priv);
+int intel_gvt_init_host(void);
+#else
+static inline int intel_gvt_init(struct drm_i915_private *dev_priv)
+{
+        return 0;
+}
+static inline void intel_gvt_cleanup(struct drm_i915_private *dev_priv)
+{
+}
+#endif
+
+#endif /* _INTEL_GVT_H_ */
diff --git a/drivers/gpu/drm/i915/intel_hdmi.c b/drivers/gpu/drm/i915/intel_hdmi.c
index a8844702d11b..4df9f384910c 100644
--- a/drivers/gpu/drm/i915/intel_hdmi.c
+++ b/drivers/gpu/drm/i915/intel_hdmi.c
@@ -47,7 +47,7 @@ static void
 assert_hdmi_port_disabled(struct intel_hdmi *intel_hdmi)
 {
         struct drm_device *dev = intel_hdmi_to_dev(intel_hdmi);
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         uint32_t enabled_bits;
 
         enabled_bits = HAS_DDI(dev) ? DDI_BUF_CTL_ENABLE : SDVO_ENABLE;
@@ -138,7 +138,7 @@ static void g4x_write_infoframe(struct drm_encoder *encoder,
 {
         const uint32_t *data = frame;
         struct drm_device *dev = encoder->dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         u32 val = I915_READ(VIDEO_DIP_CTL);
         int i;
 
@@ -192,7 +192,7 @@ static void ibx_write_infoframe(struct drm_encoder *encoder,
 {
         const uint32_t *data = frame;
         struct drm_device *dev = encoder->dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         struct intel_crtc *intel_crtc = to_intel_crtc(encoder->crtc);
         i915_reg_t reg = TVIDEO_DIP_CTL(intel_crtc->pipe);
         u32 val = I915_READ(reg);
@@ -251,7 +251,7 @@ static void cpt_write_infoframe(struct drm_encoder *encoder,
 {
         const uint32_t *data = frame;
         struct drm_device *dev = encoder->dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         struct intel_crtc *intel_crtc = to_intel_crtc(encoder->crtc);
         i915_reg_t reg = TVIDEO_DIP_CTL(intel_crtc->pipe);
         u32 val = I915_READ(reg);
@@ -308,7 +308,7 @@ static void vlv_write_infoframe(struct drm_encoder *encoder,
 {
         const uint32_t *data = frame;
         struct drm_device *dev = encoder->dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         struct intel_crtc *intel_crtc = to_intel_crtc(encoder->crtc);
         i915_reg_t reg = VLV_TVIDEO_DIP_CTL(intel_crtc->pipe);
         u32 val = I915_READ(reg);
@@ -366,7 +366,7 @@ static void hsw_write_infoframe(struct drm_encoder *encoder,
 {
         const uint32_t *data = frame;
         struct drm_device *dev = encoder->dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         struct intel_crtc *intel_crtc = to_intel_crtc(encoder->crtc);
         enum transcoder cpu_transcoder = intel_crtc->config->cpu_transcoder;
         i915_reg_t ctl_reg = HSW_TVIDEO_DIP_CTL(cpu_transcoder);
@@ -508,7 +508,7 @@ static void g4x_set_infoframes(struct drm_encoder *encoder,
                                bool enable,
                                const struct drm_display_mode *adjusted_mode)
 {
-        struct drm_i915_private *dev_priv = encoder->dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(encoder->dev);
         struct intel_digital_port *intel_dig_port = enc_to_dig_port(encoder);
         struct intel_hdmi *intel_hdmi = &intel_dig_port->hdmi;
         i915_reg_t reg = VIDEO_DIP_CTL;
@@ -629,7 +629,7 @@ static bool gcp_default_phase_possible(int pipe_bpp,
 
 static bool intel_hdmi_set_gcp_infoframe(struct drm_encoder *encoder)
 {
-        struct drm_i915_private *dev_priv = encoder->dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(encoder->dev);
         struct intel_crtc *crtc = to_intel_crtc(encoder->crtc);
         i915_reg_t reg;
         u32 val = 0;
@@ -661,7 +661,7 @@ static void ibx_set_infoframes(struct drm_encoder *encoder,
                                bool enable,
                                const struct drm_display_mode *adjusted_mode)
 {
-        struct drm_i915_private *dev_priv = encoder->dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(encoder->dev);
         struct intel_crtc *intel_crtc = to_intel_crtc(encoder->crtc);
         struct intel_digital_port *intel_dig_port = enc_to_dig_port(encoder);
         struct intel_hdmi *intel_hdmi = &intel_dig_port->hdmi;
@@ -713,7 +713,7 @@ static void cpt_set_infoframes(struct drm_encoder *encoder,
                                bool enable,
                                const struct drm_display_mode *adjusted_mode)
 {
-        struct drm_i915_private *dev_priv = encoder->dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(encoder->dev);
         struct intel_crtc *intel_crtc = to_intel_crtc(encoder->crtc);
         struct intel_hdmi *intel_hdmi = enc_to_intel_hdmi(encoder);
         i915_reg_t reg = TVIDEO_DIP_CTL(intel_crtc->pipe);
@@ -755,7 +755,7 @@ static void vlv_set_infoframes(struct drm_encoder *encoder,
                                bool enable,
                                const struct drm_display_mode *adjusted_mode)
 {
-        struct drm_i915_private *dev_priv = encoder->dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(encoder->dev);
         struct intel_digital_port *intel_dig_port = enc_to_dig_port(encoder);
         struct intel_crtc *intel_crtc = to_intel_crtc(encoder->crtc);
         struct intel_hdmi *intel_hdmi = enc_to_intel_hdmi(encoder);
@@ -807,7 +807,7 @@ static void hsw_set_infoframes(struct drm_encoder *encoder,
                                bool enable,
                                const struct drm_display_mode *adjusted_mode)
 {
-        struct drm_i915_private *dev_priv = encoder->dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(encoder->dev);
         struct intel_crtc *intel_crtc = to_intel_crtc(encoder->crtc);
         struct intel_hdmi *intel_hdmi = enc_to_intel_hdmi(encoder);
         i915_reg_t reg = HSW_TVIDEO_DIP_CTL(intel_crtc->config->cpu_transcoder);
@@ -855,7 +855,7 @@ void intel_dp_dual_mode_set_tmds_output(struct intel_hdmi *hdmi, bool enable)
 static void intel_hdmi_prepare(struct intel_encoder *encoder)
 {
         struct drm_device *dev = encoder->base.dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         struct intel_crtc *crtc = to_intel_crtc(encoder->base.crtc);
         struct intel_hdmi *intel_hdmi = enc_to_intel_hdmi(&encoder->base);
         const struct drm_display_mode *adjusted_mode = &crtc->config->base.adjusted_mode;
@@ -894,7 +894,7 @@ static bool intel_hdmi_get_hw_state(struct intel_encoder *encoder,
                                     enum pipe *pipe)
 {
         struct drm_device *dev = encoder->base.dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         struct intel_hdmi *intel_hdmi = enc_to_intel_hdmi(&encoder->base);
         enum intel_display_power_domain power_domain;
         u32 tmp;
@@ -931,7 +931,7 @@ static void intel_hdmi_get_config(struct intel_encoder *encoder,
 {
         struct intel_hdmi *intel_hdmi = enc_to_intel_hdmi(&encoder->base);
         struct drm_device *dev = encoder->base.dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         u32 tmp, flags = 0;
         int dotclock;
 
@@ -988,7 +988,7 @@ static void intel_enable_hdmi_audio(struct intel_encoder *encoder)
 static void g4x_enable_hdmi(struct intel_encoder *encoder)
 {
         struct drm_device *dev = encoder->base.dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         struct intel_crtc *crtc = to_intel_crtc(encoder->base.crtc);
         struct intel_hdmi *intel_hdmi = enc_to_intel_hdmi(&encoder->base);
         u32 temp;
@@ -1009,7 +1009,7 @@ static void g4x_enable_hdmi(struct intel_encoder *encoder)
 static void ibx_enable_hdmi(struct intel_encoder *encoder)
 {
         struct drm_device *dev = encoder->base.dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         struct intel_crtc *crtc = to_intel_crtc(encoder->base.crtc);
         struct intel_hdmi *intel_hdmi = enc_to_intel_hdmi(&encoder->base);
         u32 temp;
@@ -1058,7 +1058,7 @@ static void ibx_enable_hdmi(struct intel_encoder *encoder)
 static void cpt_enable_hdmi(struct intel_encoder *encoder)
 {
         struct drm_device *dev = encoder->base.dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         struct intel_crtc *crtc = to_intel_crtc(encoder->base.crtc);
         struct intel_hdmi *intel_hdmi = enc_to_intel_hdmi(&encoder->base);
         enum pipe pipe = crtc->pipe;
@@ -1115,7 +1115,7 @@ static void vlv_enable_hdmi(struct intel_encoder *encoder)
 static void intel_disable_hdmi(struct intel_encoder *encoder)
 {
         struct drm_device *dev = encoder->base.dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         struct intel_hdmi *intel_hdmi = enc_to_intel_hdmi(&encoder->base);
         struct intel_crtc *crtc = to_intel_crtc(encoder->base.crtc);
         u32 temp;
@@ -1154,7 +1154,7 @@ static void intel_disable_hdmi(struct intel_encoder *encoder)
                 I915_WRITE(intel_hdmi->hdmi_reg, temp);
                 POSTING_READ(intel_hdmi->hdmi_reg);
 
-                intel_wait_for_vblank_if_active(dev_priv->dev, PIPE_A);
+                intel_wait_for_vblank_if_active(&dev_priv->drm, PIPE_A);
                 intel_set_cpu_fifo_underrun_reporting(dev_priv, PIPE_A, true);
                 intel_set_pch_fifo_underrun_reporting(dev_priv, PIPE_A, true);
         }
@@ -1273,33 +1273,15 @@ intel_hdmi_mode_valid(struct drm_connector *connector,
 static bool hdmi_12bpc_possible(struct intel_crtc_state *crtc_state)
 {
         struct drm_device *dev = crtc_state->base.crtc->dev;
-        struct drm_atomic_state *state;
-        struct intel_encoder *encoder;
-        struct drm_connector *connector;
-        struct drm_connector_state *connector_state;
-        int count = 0, count_hdmi = 0;
-        int i;
 
         if (HAS_GMCH_DISPLAY(dev))
                 return false;
 
-        state = crtc_state->base.state;
-
-        for_each_connector_in_state(state, connector, connector_state, i) {
-                if (connector_state->crtc != crtc_state->base.crtc)
-                        continue;
-
-                encoder = to_intel_encoder(connector_state->best_encoder);
-
-                count_hdmi += encoder->type == INTEL_OUTPUT_HDMI;
-                count++;
-        }
-
         /*
          * HDMI 12bpc affects the clocks, so it's only possible
          * when not cloning with other encoder types.
          */
-        return count_hdmi > 0 && count_hdmi == count;
+        return crtc_state->output_types == 1 << INTEL_OUTPUT_HDMI;
 }
 
 bool intel_hdmi_compute_config(struct intel_encoder *encoder,
@@ -1575,7 +1557,7 @@ intel_hdmi_set_property(struct drm_connector *connector,
         struct intel_hdmi *intel_hdmi = intel_attached_hdmi(connector);
         struct intel_digital_port *intel_dig_port =
                 hdmi_to_dig_port(intel_hdmi);
-        struct drm_i915_private *dev_priv = connector->dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(connector->dev);
         int ret;
 
         ret = drm_object_property_set_value(&connector->base, property, val);
@@ -1674,39 +1656,16 @@ static void vlv_hdmi_pre_enable(struct intel_encoder *encoder)
         struct intel_digital_port *dport = enc_to_dig_port(&encoder->base);
         struct intel_hdmi *intel_hdmi = &dport->hdmi;
         struct drm_device *dev = encoder->base.dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         struct intel_crtc *intel_crtc =
                 to_intel_crtc(encoder->base.crtc);
         const struct drm_display_mode *adjusted_mode = &intel_crtc->config->base.adjusted_mode;
-        enum dpio_channel port = vlv_dport_to_channel(dport);
-        int pipe = intel_crtc->pipe;
-        u32 val;
 
-        /* Enable clock channels for this port */
-        mutex_lock(&dev_priv->sb_lock);
-        val = vlv_dpio_read(dev_priv, pipe, VLV_PCS01_DW8(port));
-        val = 0;
-        if (pipe)
-                val |= (1<<21);
-        else
-                val &= ~(1<<21);
-        val |= 0x001000c4;
-        vlv_dpio_write(dev_priv, pipe, VLV_PCS_DW8(port), val);
+        vlv_phy_pre_encoder_enable(encoder);
 
         /* HDMI 1.0V-2dB */
-        vlv_dpio_write(dev_priv, pipe, VLV_TX_DW5(port), 0);
-        vlv_dpio_write(dev_priv, pipe, VLV_TX_DW4(port), 0x2b245f5f);
-        vlv_dpio_write(dev_priv, pipe, VLV_TX_DW2(port), 0x5578b83a);
-        vlv_dpio_write(dev_priv, pipe, VLV_TX_DW3(port), 0x0c782040);
-        vlv_dpio_write(dev_priv, pipe, VLV_TX3_DW4(port), 0x2b247878);
-        vlv_dpio_write(dev_priv, pipe, VLV_PCS_DW11(port), 0x00030000);
-        vlv_dpio_write(dev_priv, pipe, VLV_PCS_DW9(port), 0x00002000);
-        vlv_dpio_write(dev_priv, pipe, VLV_TX_DW5(port), DPIO_TX_OCALINIT_EN);
-
-        /* Program lane clock */
-        vlv_dpio_write(dev_priv, pipe, VLV_PCS_DW14(port), 0x00760018);
-        vlv_dpio_write(dev_priv, pipe, VLV_PCS_DW23(port), 0x00400888);
-        mutex_unlock(&dev_priv->sb_lock);
+        vlv_set_phy_signal_level(encoder, 0x2b245f5f, 0x00002000, 0x5578b83a,
+                                 0x2b247878);
 
         intel_hdmi->set_infoframes(&encoder->base,
                                    intel_crtc->config->has_hdmi_sink,
@@ -1719,213 +1678,33 @@ static void vlv_hdmi_pre_enable(struct intel_encoder *encoder)
 
 static void vlv_hdmi_pre_pll_enable(struct intel_encoder *encoder)
 {
-        struct intel_digital_port *dport = enc_to_dig_port(&encoder->base);
-        struct drm_device *dev = encoder->base.dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
-        struct intel_crtc *intel_crtc =
-                to_intel_crtc(encoder->base.crtc);
-        enum dpio_channel port = vlv_dport_to_channel(dport);
-        int pipe = intel_crtc->pipe;
-
         intel_hdmi_prepare(encoder);
 
-        /* Program Tx lane resets to default */
-        mutex_lock(&dev_priv->sb_lock);
-        vlv_dpio_write(dev_priv, pipe, VLV_PCS_DW0(port),
-                         DPIO_PCS_TX_LANE2_RESET |
-                         DPIO_PCS_TX_LANE1_RESET);
-        vlv_dpio_write(dev_priv, pipe, VLV_PCS_DW1(port),
-                         DPIO_PCS_CLK_CRI_RXEB_EIOS_EN |
-                         DPIO_PCS_CLK_CRI_RXDIGFILTSG_EN |
-                         (1<<DPIO_PCS_CLK_DATAWIDTH_SHIFT) |
-                         DPIO_PCS_CLK_SOFT_RESET);
-
-        /* Fix up inter-pair skew failure */
-        vlv_dpio_write(dev_priv, pipe, VLV_PCS_DW12(port), 0x00750f00);
-        vlv_dpio_write(dev_priv, pipe, VLV_TX_DW11(port), 0x00001500);
-        vlv_dpio_write(dev_priv, pipe, VLV_TX_DW14(port), 0x40400000);
-
-        vlv_dpio_write(dev_priv, pipe, VLV_PCS_DW9(port), 0x00002000);
-        vlv_dpio_write(dev_priv, pipe, VLV_TX_DW5(port), DPIO_TX_OCALINIT_EN);
-        mutex_unlock(&dev_priv->sb_lock);
-}
-
-static void chv_data_lane_soft_reset(struct intel_encoder *encoder,
-                                     bool reset)
-{
-        struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
-        enum dpio_channel ch = vlv_dport_to_channel(enc_to_dig_port(&encoder->base));
-        struct intel_crtc *crtc = to_intel_crtc(encoder->base.crtc);
-        enum pipe pipe = crtc->pipe;
-        uint32_t val;
-
-        val = vlv_dpio_read(dev_priv, pipe, VLV_PCS01_DW0(ch));
-        if (reset)
-                val &= ~(DPIO_PCS_TX_LANE2_RESET | DPIO_PCS_TX_LANE1_RESET);
-        else
-                val |= DPIO_PCS_TX_LANE2_RESET | DPIO_PCS_TX_LANE1_RESET;
-        vlv_dpio_write(dev_priv, pipe, VLV_PCS01_DW0(ch), val);
-
-        if (crtc->config->lane_count > 2) {
-                val = vlv_dpio_read(dev_priv, pipe, VLV_PCS23_DW0(ch));
-                if (reset)
-                        val &= ~(DPIO_PCS_TX_LANE2_RESET | DPIO_PCS_TX_LANE1_RESET);
-                else
-                        val |= DPIO_PCS_TX_LANE2_RESET | DPIO_PCS_TX_LANE1_RESET;
-                vlv_dpio_write(dev_priv, pipe, VLV_PCS23_DW0(ch), val);
-        }
-
-        val = vlv_dpio_read(dev_priv, pipe, VLV_PCS01_DW1(ch));
-        val |= CHV_PCS_REQ_SOFTRESET_EN;
-        if (reset)
-                val &= ~DPIO_PCS_CLK_SOFT_RESET;
-        else
-                val |= DPIO_PCS_CLK_SOFT_RESET;
-        vlv_dpio_write(dev_priv, pipe, VLV_PCS01_DW1(ch), val);
-
-        if (crtc->config->lane_count > 2) {
-                val = vlv_dpio_read(dev_priv, pipe, VLV_PCS23_DW1(ch));
-                val |= CHV_PCS_REQ_SOFTRESET_EN;
-                if (reset)
-                        val &= ~DPIO_PCS_CLK_SOFT_RESET;
-                else
-                        val |= DPIO_PCS_CLK_SOFT_RESET;
-                vlv_dpio_write(dev_priv, pipe, VLV_PCS23_DW1(ch), val);
-        }
+        vlv_phy_pre_pll_enable(encoder);
 }
 
 static void chv_hdmi_pre_pll_enable(struct intel_encoder *encoder)
 {
-        struct intel_digital_port *dport = enc_to_dig_port(&encoder->base);
-        struct drm_device *dev = encoder->base.dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
-        struct intel_crtc *intel_crtc =
-                to_intel_crtc(encoder->base.crtc);
-        enum dpio_channel ch = vlv_dport_to_channel(dport);
-        enum pipe pipe = intel_crtc->pipe;
-        u32 val;
-
         intel_hdmi_prepare(encoder);
 
-        /*
-         * Must trick the second common lane into life.
-         * Otherwise we can't even access the PLL.
-         */
-        if (ch == DPIO_CH0 && pipe == PIPE_B)
-                dport->release_cl2_override =
-                        !chv_phy_powergate_ch(dev_priv, DPIO_PHY0, DPIO_CH1, true);
-
-        chv_phy_powergate_lanes(encoder, true, 0x0);
-
-        mutex_lock(&dev_priv->sb_lock);
-
-        /* Assert data lane reset */
-        chv_data_lane_soft_reset(encoder, true);
-
-        /* program left/right clock distribution */
-        if (pipe != PIPE_B) {
-                val = vlv_dpio_read(dev_priv, pipe, _CHV_CMN_DW5_CH0);
-                val &= ~(CHV_BUFLEFTENA1_MASK | CHV_BUFRIGHTENA1_MASK);
-                if (ch == DPIO_CH0)
-                        val |= CHV_BUFLEFTENA1_FORCE;
-                if (ch == DPIO_CH1)
-                        val |= CHV_BUFRIGHTENA1_FORCE;
-                vlv_dpio_write(dev_priv, pipe, _CHV_CMN_DW5_CH0, val);
-        } else {
-                val = vlv_dpio_read(dev_priv, pipe, _CHV_CMN_DW1_CH1);
-                val &= ~(CHV_BUFLEFTENA2_MASK | CHV_BUFRIGHTENA2_MASK);
-                if (ch == DPIO_CH0)
-                        val |= CHV_BUFLEFTENA2_FORCE;
-                if (ch == DPIO_CH1)
-                        val |= CHV_BUFRIGHTENA2_FORCE;
-                vlv_dpio_write(dev_priv, pipe, _CHV_CMN_DW1_CH1, val);
-        }
-
-        /* program clock channel usage */
-        val = vlv_dpio_read(dev_priv, pipe, VLV_PCS01_DW8(ch));
-        val |= CHV_PCS_USEDCLKCHANNEL_OVRRIDE;
-        if (pipe != PIPE_B)
-                val &= ~CHV_PCS_USEDCLKCHANNEL;
-        else
-                val |= CHV_PCS_USEDCLKCHANNEL;
-        vlv_dpio_write(dev_priv, pipe, VLV_PCS01_DW8(ch), val);
-
-        val = vlv_dpio_read(dev_priv, pipe, VLV_PCS23_DW8(ch));
-        val |= CHV_PCS_USEDCLKCHANNEL_OVRRIDE;
-        if (pipe != PIPE_B)
-                val &= ~CHV_PCS_USEDCLKCHANNEL;
-        else
-                val |= CHV_PCS_USEDCLKCHANNEL;
-        vlv_dpio_write(dev_priv, pipe, VLV_PCS23_DW8(ch), val);
-
-        /*
-         * This a a bit weird since generally CL
-         * matches the pipe, but here we need to
-         * pick the CL based on the port.
-         */
-        val = vlv_dpio_read(dev_priv, pipe, CHV_CMN_DW19(ch));
-        if (pipe != PIPE_B)
-                val &= ~CHV_CMN_USEDCLKCHANNEL;
-        else
-                val |= CHV_CMN_USEDCLKCHANNEL;
-        vlv_dpio_write(dev_priv, pipe, CHV_CMN_DW19(ch), val);
-
-        mutex_unlock(&dev_priv->sb_lock);
+        chv_phy_pre_pll_enable(encoder);
 }
 
 static void chv_hdmi_post_pll_disable(struct intel_encoder *encoder)
 {
-        struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
-        enum pipe pipe = to_intel_crtc(encoder->base.crtc)->pipe;
-        u32 val;
-
-        mutex_lock(&dev_priv->sb_lock);
-
-        /* disable left/right clock distribution */
-        if (pipe != PIPE_B) {
-                val = vlv_dpio_read(dev_priv, pipe, _CHV_CMN_DW5_CH0);
-                val &= ~(CHV_BUFLEFTENA1_MASK | CHV_BUFRIGHTENA1_MASK);
-                vlv_dpio_write(dev_priv, pipe, _CHV_CMN_DW5_CH0, val);
-        } else {
-                val = vlv_dpio_read(dev_priv, pipe, _CHV_CMN_DW1_CH1);
-                val &= ~(CHV_BUFLEFTENA2_MASK | CHV_BUFRIGHTENA2_MASK);
-                vlv_dpio_write(dev_priv, pipe, _CHV_CMN_DW1_CH1, val);
-        }
-
-        mutex_unlock(&dev_priv->sb_lock);
-
-        /*
-         * Leave the power down bit cleared for at least one
-         * lane so that chv_powergate_phy_ch() will power
-         * on something when the channel is otherwise unused.
-         * When the port is off and the override is removed
-         * the lanes power down anyway, so otherwise it doesn't
-         * really matter what the state of power down bits is
-         * after this.
-         */
-        chv_phy_powergate_lanes(encoder, false, 0x0);
+        chv_phy_post_pll_disable(encoder);
 }
 
 static void vlv_hdmi_post_disable(struct intel_encoder *encoder)
 {
-        struct intel_digital_port *dport = enc_to_dig_port(&encoder->base);
-        struct drm_i915_private *dev_priv = encoder->base.dev->dev_private;
-        struct intel_crtc *intel_crtc =
-                to_intel_crtc(encoder->base.crtc);
-        enum dpio_channel port = vlv_dport_to_channel(dport);
-        int pipe = intel_crtc->pipe;
-
         /* Reset lanes to avoid HDMI flicker (VLV w/a) */
-        mutex_lock(&dev_priv->sb_lock);
-        vlv_dpio_write(dev_priv, pipe, VLV_PCS_DW0(port), 0x00000000);
-        vlv_dpio_write(dev_priv, pipe, VLV_PCS_DW1(port), 0x00e00060);
-        mutex_unlock(&dev_priv->sb_lock);
+        vlv_phy_reset_lanes(encoder);
 }
 
 static void chv_hdmi_post_disable(struct intel_encoder *encoder)
 {
         struct drm_device *dev = encoder->base.dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
 
         mutex_lock(&dev_priv->sb_lock);
 
@@ -1940,142 +1719,16 @@ static void chv_hdmi_pre_enable(struct intel_encoder *encoder)
         struct intel_digital_port *dport = enc_to_dig_port(&encoder->base);
         struct intel_hdmi *intel_hdmi = &dport->hdmi;
         struct drm_device *dev = encoder->base.dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         struct intel_crtc *intel_crtc =
                 to_intel_crtc(encoder->base.crtc);
         const struct drm_display_mode *adjusted_mode = &intel_crtc->config->base.adjusted_mode;
-        enum dpio_channel ch = vlv_dport_to_channel(dport);
-        int pipe = intel_crtc->pipe;
-        int data, i, stagger;
-        u32 val;
 
-        mutex_lock(&dev_priv->sb_lock);
-
-        /* allow hardware to manage TX FIFO reset source */
-        val = vlv_dpio_read(dev_priv, pipe, VLV_PCS01_DW11(ch));
-        val &= ~DPIO_LANEDESKEW_STRAP_OVRD;
-        vlv_dpio_write(dev_priv, pipe, VLV_PCS01_DW11(ch), val);
-
-        val = vlv_dpio_read(dev_priv, pipe, VLV_PCS23_DW11(ch));
-        val &= ~DPIO_LANEDESKEW_STRAP_OVRD;
-        vlv_dpio_write(dev_priv, pipe, VLV_PCS23_DW11(ch), val);
-
-        /* Program Tx latency optimal setting */
-        for (i = 0; i < 4; i++) {
-                /* Set the upar bit */
-                data = (i == 1) ? 0x0 : 0x1;
-                vlv_dpio_write(dev_priv, pipe, CHV_TX_DW14(ch, i),
-                                data << DPIO_UPAR_SHIFT);
-        }
-
-        /* Data lane stagger programming */
-        if (intel_crtc->config->port_clock > 270000)
-                stagger = 0x18;
-        else if (intel_crtc->config->port_clock > 135000)
-                stagger = 0xd;
-        else if (intel_crtc->config->port_clock > 67500)
-                stagger = 0x7;
-        else if (intel_crtc->config->port_clock > 33750)
-                stagger = 0x4;
-        else
-                stagger = 0x2;
-
-        val = vlv_dpio_read(dev_priv, pipe, VLV_PCS01_DW11(ch));
-        val |= DPIO_TX2_STAGGER_MASK(0x1f);
-        vlv_dpio_write(dev_priv, pipe, VLV_PCS01_DW11(ch), val);
-
-        val = vlv_dpio_read(dev_priv, pipe, VLV_PCS23_DW11(ch));
-        val |= DPIO_TX2_STAGGER_MASK(0x1f);
-        vlv_dpio_write(dev_priv, pipe, VLV_PCS23_DW11(ch), val);
-
-        vlv_dpio_write(dev_priv, pipe, VLV_PCS01_DW12(ch),
-                       DPIO_LANESTAGGER_STRAP(stagger) |
-                       DPIO_LANESTAGGER_STRAP_OVRD |
-                       DPIO_TX1_STAGGER_MASK(0x1f) |
-                       DPIO_TX1_STAGGER_MULT(6) |
-                       DPIO_TX2_STAGGER_MULT(0));
-
-        vlv_dpio_write(dev_priv, pipe, VLV_PCS23_DW12(ch),
-                       DPIO_LANESTAGGER_STRAP(stagger) |
-                       DPIO_LANESTAGGER_STRAP_OVRD |
-                       DPIO_TX1_STAGGER_MASK(0x1f) |
-                       DPIO_TX1_STAGGER_MULT(7) |
-                       DPIO_TX2_STAGGER_MULT(5));
-
-        /* Deassert data lane reset */
-        chv_data_lane_soft_reset(encoder, false);
-
-        /* Clear calc init */
-        val = vlv_dpio_read(dev_priv, pipe, VLV_PCS01_DW10(ch));
-        val &= ~(DPIO_PCS_SWING_CALC_TX0_TX2 | DPIO_PCS_SWING_CALC_TX1_TX3);
-        val &= ~(DPIO_PCS_TX1DEEMP_MASK | DPIO_PCS_TX2DEEMP_MASK);
-        val |= DPIO_PCS_TX1DEEMP_9P5 | DPIO_PCS_TX2DEEMP_9P5;
-        vlv_dpio_write(dev_priv, pipe, VLV_PCS01_DW10(ch), val);
-
-        val = vlv_dpio_read(dev_priv, pipe, VLV_PCS23_DW10(ch));
-        val &= ~(DPIO_PCS_SWING_CALC_TX0_TX2 | DPIO_PCS_SWING_CALC_TX1_TX3);
-        val &= ~(DPIO_PCS_TX1DEEMP_MASK | DPIO_PCS_TX2DEEMP_MASK);
-        val |= DPIO_PCS_TX1DEEMP_9P5 | DPIO_PCS_TX2DEEMP_9P5;
-        vlv_dpio_write(dev_priv, pipe, VLV_PCS23_DW10(ch), val);
-
-        val = vlv_dpio_read(dev_priv, pipe, VLV_PCS01_DW9(ch));
-        val &= ~(DPIO_PCS_TX1MARGIN_MASK | DPIO_PCS_TX2MARGIN_MASK);
-        val |= DPIO_PCS_TX1MARGIN_000 | DPIO_PCS_TX2MARGIN_000;
-        vlv_dpio_write(dev_priv, pipe, VLV_PCS01_DW9(ch), val);
-
-        val = vlv_dpio_read(dev_priv, pipe, VLV_PCS23_DW9(ch));
-        val &= ~(DPIO_PCS_TX1MARGIN_MASK | DPIO_PCS_TX2MARGIN_MASK);
-        val |= DPIO_PCS_TX1MARGIN_000 | DPIO_PCS_TX2MARGIN_000;
-        vlv_dpio_write(dev_priv, pipe, VLV_PCS23_DW9(ch), val);
+        chv_phy_pre_encoder_enable(encoder);
 
         /* FIXME: Program the support xxx V-dB */
         /* Use 800mV-0dB */
-        for (i = 0; i < 4; i++) {
-                val = vlv_dpio_read(dev_priv, pipe, CHV_TX_DW4(ch, i));
-                val &= ~DPIO_SWING_DEEMPH9P5_MASK;
-                val |= 128 << DPIO_SWING_DEEMPH9P5_SHIFT;
-                vlv_dpio_write(dev_priv, pipe, CHV_TX_DW4(ch, i), val);
-        }
-
-        for (i = 0; i < 4; i++) {
-                val = vlv_dpio_read(dev_priv, pipe, CHV_TX_DW2(ch, i));
-
-                val &= ~DPIO_SWING_MARGIN000_MASK;
-                val |= 102 << DPIO_SWING_MARGIN000_SHIFT;
-
-                /*
-                 * Supposedly this value shouldn't matter when unique transition
-                 * scale is disabled, but in fact it does matter. Let's just
-                 * always program the same value and hope it's OK.
-                 */
-                val &= ~(0xff << DPIO_UNIQ_TRANS_SCALE_SHIFT);
-                val |= 0x9a << DPIO_UNIQ_TRANS_SCALE_SHIFT;
-
-                vlv_dpio_write(dev_priv, pipe, CHV_TX_DW2(ch, i), val);
-        }
-
-        /*
-         * The document said it needs to set bit 27 for ch0 and bit 26
-         * for ch1. Might be a typo in the doc.
-         * For now, for this unique transition scale selection, set bit
-         * 27 for ch0 and ch1.
-         */
-        for (i = 0; i < 4; i++) {
-                val = vlv_dpio_read(dev_priv, pipe, CHV_TX_DW3(ch, i));
-                val &= ~DPIO_TX_UNIQ_TRANS_SCALE_EN;
-                vlv_dpio_write(dev_priv, pipe, CHV_TX_DW3(ch, i), val);
-        }
-
-        /* Start swing calculation */
-        val = vlv_dpio_read(dev_priv, pipe, VLV_PCS01_DW10(ch));
-        val |= DPIO_PCS_SWING_CALC_TX0_TX2 | DPIO_PCS_SWING_CALC_TX1_TX3;
-        vlv_dpio_write(dev_priv, pipe, VLV_PCS01_DW10(ch), val);
-
-        val = vlv_dpio_read(dev_priv, pipe, VLV_PCS23_DW10(ch));
-        val |= DPIO_PCS_SWING_CALC_TX0_TX2 | DPIO_PCS_SWING_CALC_TX1_TX3;
-        vlv_dpio_write(dev_priv, pipe, VLV_PCS23_DW10(ch), val);
-
-        mutex_unlock(&dev_priv->sb_lock);
+        chv_set_phy_signal_level(encoder, 128, 102, false);
 
         intel_hdmi->set_infoframes(&encoder->base,
                                    intel_crtc->config->has_hdmi_sink,
@@ -2086,10 +1739,7 @@ static void chv_hdmi_pre_enable(struct intel_encoder *encoder)
         vlv_wait_port_ready(dev_priv, dport, 0x0);
 
         /* Second common lane will stay alive on its own now */
-        if (dport->release_cl2_override) {
-                chv_phy_powergate_ch(dev_priv, DPIO_PHY0, DPIO_CH1, false);
-                dport->release_cl2_override = false;
-        }
+        chv_phy_release_cl2_override(encoder);
 }
 
 static void intel_hdmi_destroy(struct drm_connector *connector)
@@ -2106,6 +1756,8 @@ static const struct drm_connector_funcs intel_hdmi_connector_funcs = {
         .fill_modes = drm_helper_probe_single_connector_modes,
         .set_property = intel_hdmi_set_property,
         .atomic_get_property = intel_connector_atomic_get_property,
+        .late_register = intel_connector_register,
+        .early_unregister = intel_connector_unregister,
         .destroy = intel_hdmi_destroy,
         .atomic_destroy_state = drm_atomic_helper_connector_destroy_state,
         .atomic_duplicate_state = drm_atomic_helper_connector_duplicate_state,
@@ -2114,7 +1766,6 @@ static const struct drm_connector_funcs intel_hdmi_connector_funcs = {
 static const struct drm_connector_helper_funcs intel_hdmi_connector_helper_funcs = {
         .get_modes = intel_hdmi_get_modes,
         .mode_valid = intel_hdmi_mode_valid,
-        .best_encoder = intel_best_encoder,
 };
 
 static const struct drm_encoder_funcs intel_hdmi_enc_funcs = {
@@ -2138,7 +1789,7 @@ void intel_hdmi_init_connector(struct intel_digital_port *intel_dig_port,
         struct intel_hdmi *intel_hdmi = &intel_dig_port->hdmi;
         struct intel_encoder *intel_encoder = &intel_dig_port->base;
         struct drm_device *dev = intel_encoder->base.dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         enum port port = intel_dig_port->port;
         uint8_t alternate_ddc_pin;
 
@@ -2242,12 +1893,10 @@ void intel_hdmi_init_connector(struct intel_digital_port *intel_dig_port,
                 intel_connector->get_hw_state = intel_ddi_connector_get_hw_state;
         else
                 intel_connector->get_hw_state = intel_connector_get_hw_state;
-        intel_connector->unregister = intel_connector_unregister;
 
         intel_hdmi_add_properties(intel_hdmi, connector);
 
         intel_connector_attach_encoder(intel_connector, intel_encoder);
-        drm_connector_register(connector);
         intel_hdmi->attached_connector = intel_connector;
 
         /* For G4X desktop chip, PEG_BAND_GAP_DATA 3:0 must first be written
@@ -2280,7 +1929,7 @@ void intel_hdmi_init(struct drm_device *dev,
         intel_encoder = &intel_dig_port->base;
 
         drm_encoder_init(dev, &intel_encoder->base, &intel_hdmi_enc_funcs,
-                         DRM_MODE_ENCODER_TMDS, NULL);
+                         DRM_MODE_ENCODER_TMDS, "HDMI %c", port_name(port));
 
         intel_encoder->compute_config = intel_hdmi_compute_config;
         if (HAS_PCH_SPLIT(dev)) {
diff --git a/drivers/gpu/drm/i915/intel_hotplug.c b/drivers/gpu/drm/i915/intel_hotplug.c
index bee673005d48..f48957ea100d 100644
--- a/drivers/gpu/drm/i915/intel_hotplug.c
+++ b/drivers/gpu/drm/i915/intel_hotplug.c
@@ -144,7 +144,7 @@ static bool intel_hpd_irq_storm_detect(struct drm_i915_private *dev_priv,
 
 static void intel_hpd_irq_storm_disable(struct drm_i915_private *dev_priv)
 {
-        struct drm_device *dev = dev_priv->dev;
+        struct drm_device *dev = &dev_priv->drm;
         struct drm_mode_config *mode_config = &dev->mode_config;
         struct intel_connector *intel_connector;
         struct intel_encoder *intel_encoder;
@@ -191,7 +191,7 @@ static void intel_hpd_irq_storm_reenable_work(struct work_struct *work)
         struct drm_i915_private *dev_priv =
                 container_of(work, typeof(*dev_priv),
                              hotplug.reenable_work.work);
-        struct drm_device *dev = dev_priv->dev;
+        struct drm_device *dev = &dev_priv->drm;
         struct drm_mode_config *mode_config = &dev->mode_config;
         int i;
 
@@ -220,7 +220,7 @@ static void intel_hpd_irq_storm_reenable_work(struct work_struct *work)
                 }
         }
         if (dev_priv->display.hpd_irq_setup)
-                dev_priv->display.hpd_irq_setup(dev);
+                dev_priv->display.hpd_irq_setup(dev_priv);
         spin_unlock_irq(&dev_priv->irq_lock);
 
         intel_runtime_pm_put(dev_priv);
@@ -302,7 +302,7 @@ static void i915_hotplug_work_func(struct work_struct *work)
 {
         struct drm_i915_private *dev_priv =
                 container_of(work, struct drm_i915_private, hotplug.hotplug_work);
-        struct drm_device *dev = dev_priv->dev;
+        struct drm_device *dev = &dev_priv->drm;
         struct drm_mode_config *mode_config = &dev->mode_config;
         struct intel_connector *intel_connector;
         struct intel_encoder *intel_encoder;
@@ -346,7 +346,7 @@ static void i915_hotplug_work_func(struct work_struct *work)
 
 /**
  * intel_hpd_irq_handler - main hotplug irq handler
- * @dev: drm device
+ * @dev_priv: drm_i915_private
  * @pin_mask: a mask of hpd pins that have triggered the irq
  * @long_mask: a mask of hpd pins that may be long hpd pulses
  *
@@ -360,10 +360,9 @@ static void i915_hotplug_work_func(struct work_struct *work)
  * Here, we do hotplug irq storm detection and mitigation, and pass further
  * processing to appropriate bottom halves.
  */
-void intel_hpd_irq_handler(struct drm_device *dev,
+void intel_hpd_irq_handler(struct drm_i915_private *dev_priv,
                            u32 pin_mask, u32 long_mask)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
         int i;
         enum port port;
         bool storm_detected = false;
@@ -407,7 +406,7 @@ void intel_hpd_irq_handler(struct drm_device *dev,
                          * hotplug bits itself. So only WARN about unexpected
                          * interrupts on saner platforms.
                          */
-                        WARN_ONCE(!HAS_GMCH_DISPLAY(dev),
+                        WARN_ONCE(!HAS_GMCH_DISPLAY(dev_priv),
                                   "Received HPD interrupt on pin %d although disabled\n", i);
                         continue;
                 }
@@ -427,7 +426,7 @@ void intel_hpd_irq_handler(struct drm_device *dev,
         }
 
         if (storm_detected)
-                dev_priv->display.hpd_irq_setup(dev);
+                dev_priv->display.hpd_irq_setup(dev_priv);
         spin_unlock(&dev_priv->irq_lock);
 
         /*
@@ -453,20 +452,47 @@ void intel_hpd_irq_handler(struct drm_device *dev,
  *
  * This is a separate step from interrupt enabling to simplify the locking rules
  * in the driver load and resume code.
+ *
+ * Also see: intel_hpd_poll_init(), which enables connector polling
  */
 void intel_hpd_init(struct drm_i915_private *dev_priv)
 {
-        struct drm_device *dev = dev_priv->dev;
-        struct drm_mode_config *mode_config = &dev->mode_config;
-        struct drm_connector *connector;
         int i;
 
         for_each_hpd_pin(i) {
                 dev_priv->hotplug.stats[i].count = 0;
                 dev_priv->hotplug.stats[i].state = HPD_ENABLED;
         }
+
+        WRITE_ONCE(dev_priv->hotplug.poll_enabled, false);
+        schedule_work(&dev_priv->hotplug.poll_init_work);
+
+        /*
+         * Interrupt setup is already guaranteed to be single-threaded, this is
+         * just to make the assert_spin_locked checks happy.
+         */
+        spin_lock_irq(&dev_priv->irq_lock);
+        if (dev_priv->display.hpd_irq_setup)
+                dev_priv->display.hpd_irq_setup(dev_priv);
+        spin_unlock_irq(&dev_priv->irq_lock);
+}
+
+void i915_hpd_poll_init_work(struct work_struct *work) {
+        struct drm_i915_private *dev_priv =
+                container_of(work, struct drm_i915_private,
+                             hotplug.poll_init_work);
+        struct drm_device *dev = &dev_priv->drm;
+        struct drm_mode_config *mode_config = &dev->mode_config;
+        struct drm_connector *connector;
+        bool enabled;
+
+        mutex_lock(&dev->mode_config.mutex);
+
+        enabled = READ_ONCE(dev_priv->hotplug.poll_enabled);
+
         list_for_each_entry(connector, &mode_config->connector_list, head) {
-                struct intel_connector *intel_connector = to_intel_connector(connector);
+                struct intel_connector *intel_connector =
+                        to_intel_connector(connector);
                 connector->polled = intel_connector->polled;
 
                 /* MST has a dynamic intel_connector->encoder and it's reprobing
@@ -475,24 +501,62 @@ void intel_hpd_init(struct drm_i915_private *dev_priv)
                         continue;
 
                 if (!connector->polled && I915_HAS_HOTPLUG(dev) &&
-                    intel_connector->encoder->hpd_pin > HPD_NONE)
-                        connector->polled = DRM_CONNECTOR_POLL_HPD;
+                    intel_connector->encoder->hpd_pin > HPD_NONE) {
+                        connector->polled = enabled ?
+                                DRM_CONNECTOR_POLL_CONNECT |
+                                DRM_CONNECTOR_POLL_DISCONNECT :
+                                DRM_CONNECTOR_POLL_HPD;
+                }
         }
 
+        if (enabled)
+                drm_kms_helper_poll_enable_locked(dev);
+
+        mutex_unlock(&dev->mode_config.mutex);
+
         /*
-         * Interrupt setup is already guaranteed to be single-threaded, this is
-         * just to make the assert_spin_locked checks happy.
+         * We might have missed any hotplugs that happened while we were
+         * in the middle of disabling polling
          */
-        spin_lock_irq(&dev_priv->irq_lock);
-        if (dev_priv->display.hpd_irq_setup)
-                dev_priv->display.hpd_irq_setup(dev);
-        spin_unlock_irq(&dev_priv->irq_lock);
+        if (!enabled)
+                drm_helper_hpd_irq_event(dev);
+}
+
+/**
+ * intel_hpd_poll_init - enables/disables polling for connectors with hpd
+ * @dev_priv: i915 device instance
+ * @enabled: Whether to enable or disable polling
+ *
+ * This function enables polling for all connectors, regardless of whether or
+ * not they support hotplug detection. Under certain conditions HPD may not be
+ * functional. On most Intel GPUs, this happens when we enter runtime suspend.
+ * On Valleyview and Cherryview systems, this also happens when we shut off all
+ * of the powerwells.
+ *
+ * Since this function can get called in contexts where we're already holding
+ * dev->mode_config.mutex, we do the actual hotplug enabling in a seperate
+ * worker.
+ *
+ * Also see: intel_hpd_init(), which restores hpd handling.
+ */
+void intel_hpd_poll_init(struct drm_i915_private *dev_priv)
+{
+        WRITE_ONCE(dev_priv->hotplug.poll_enabled, true);
+
+        /*
+         * We might already be holding dev->mode_config.mutex, so do this in a
+         * seperate worker
+         * As well, there's no issue if we race here since we always reschedule
+         * this worker anyway
+         */
+        schedule_work(&dev_priv->hotplug.poll_init_work);
 }
 
 void intel_hpd_init_work(struct drm_i915_private *dev_priv)
 {
         INIT_WORK(&dev_priv->hotplug.hotplug_work, i915_hotplug_work_func);
         INIT_WORK(&dev_priv->hotplug.dig_port_work, i915_digport_work_func);
+        INIT_WORK(&dev_priv->hotplug.poll_init_work, i915_hpd_poll_init_work);
         INIT_DELAYED_WORK(&dev_priv->hotplug.reenable_work,
                           intel_hpd_irq_storm_reenable_work);
 }
@@ -509,5 +573,33 @@ void intel_hpd_cancel_work(struct drm_i915_private *dev_priv)
 
         cancel_work_sync(&dev_priv->hotplug.dig_port_work);
         cancel_work_sync(&dev_priv->hotplug.hotplug_work);
+        cancel_work_sync(&dev_priv->hotplug.poll_init_work);
         cancel_delayed_work_sync(&dev_priv->hotplug.reenable_work);
 }
+
+bool intel_hpd_disable(struct drm_i915_private *dev_priv, enum hpd_pin pin)
+{
+        bool ret = false;
+
+        if (pin == HPD_NONE)
+                return false;
+
+        spin_lock_irq(&dev_priv->irq_lock);
+        if (dev_priv->hotplug.stats[pin].state == HPD_ENABLED) {
+                dev_priv->hotplug.stats[pin].state = HPD_DISABLED;
+                ret = true;
+        }
+        spin_unlock_irq(&dev_priv->irq_lock);
+
+        return ret;
+}
+
+void intel_hpd_enable(struct drm_i915_private *dev_priv, enum hpd_pin pin)
+{
+        if (pin == HPD_NONE)
+                return;
+
+        spin_lock_irq(&dev_priv->irq_lock);
+        dev_priv->hotplug.stats[pin].state = HPD_ENABLED;
+        spin_unlock_irq(&dev_priv->irq_lock);
+}
diff --git a/drivers/gpu/drm/i915/intel_i2c.c b/drivers/gpu/drm/i915/intel_i2c.c
index 81de23098be7..1f266d7df2ec 100644
--- a/drivers/gpu/drm/i915/intel_i2c.c
+++ b/drivers/gpu/drm/i915/intel_i2c.c
@@ -113,7 +113,7 @@ to_intel_gmbus(struct i2c_adapter *i2c)
 void
 intel_i2c_reset(struct drm_device *dev)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
 
         I915_WRITE(GMBUS0, 0);
         I915_WRITE(GMBUS4, 0);
@@ -138,7 +138,7 @@ static void intel_i2c_quirk_set(struct drm_i915_private *dev_priv, bool enable)
 static u32 get_reserved(struct intel_gmbus *bus)
 {
         struct drm_i915_private *dev_priv = bus->dev_priv;
-        struct drm_device *dev = dev_priv->dev;
+        struct drm_device *dev = &dev_priv->drm;
         u32 reserved = 0;
 
         /* On most chips, these bits must be preserved in software. */
@@ -212,7 +212,7 @@ intel_gpio_pre_xfer(struct i2c_adapter *adapter)
                                                adapter);
         struct drm_i915_private *dev_priv = bus->dev_priv;
 
-        intel_i2c_reset(dev_priv->dev);
+        intel_i2c_reset(&dev_priv->drm);
         intel_i2c_quirk_set(dev_priv, true);
         set_data(bus, 1);
         set_clock(bus, 1);
@@ -298,15 +298,16 @@ gmbus_wait_idle(struct drm_i915_private *dev_priv)
 {
         int ret;
 
-#define C ((I915_READ_NOTRACE(GMBUS2) & GMBUS_ACTIVE) == 0)
-
         if (!HAS_GMBUS_IRQ(dev_priv))
-                return wait_for(C, 10);
+                return intel_wait_for_register(dev_priv,
+                                               GMBUS2, GMBUS_ACTIVE, 0,
+                                               10);
 
         /* Important: The hw handles only the first bit, so set only one! */
         I915_WRITE(GMBUS4, GMBUS_IDLE_EN);
 
-        ret = wait_event_timeout(dev_priv->gmbus_wait_queue, C,
+        ret = wait_event_timeout(dev_priv->gmbus_wait_queue,
+                                 (I915_READ_NOTRACE(GMBUS2) & GMBUS_ACTIVE) == 0,
                                  msecs_to_jiffies_timeout(10));
 
         I915_WRITE(GMBUS4, 0);
@@ -315,7 +316,6 @@ gmbus_wait_idle(struct drm_i915_private *dev_priv)
                 return 0;
         else
                 return -ETIMEDOUT;
-#undef C
 }
 
 static int
@@ -632,7 +632,7 @@ static const struct i2c_algorithm gmbus_algorithm = {
  */
 int intel_setup_gmbus(struct drm_device *dev)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         struct intel_gmbus *bus;
         unsigned int pin;
         int ret;
@@ -688,7 +688,7 @@ int intel_setup_gmbus(struct drm_device *dev)
                         goto err;
         }
 
-        intel_i2c_reset(dev_priv->dev);
+        intel_i2c_reset(&dev_priv->drm);
 
         return 0;
 
@@ -736,7 +736,7 @@ void intel_gmbus_force_bit(struct i2c_adapter *adapter, bool force_bit)
 
 void intel_teardown_gmbus(struct drm_device *dev)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         struct intel_gmbus *bus;
         unsigned int pin;
 
diff --git a/drivers/gpu/drm/i915/intel_lrc.c b/drivers/gpu/drm/i915/intel_lrc.c
index 7f2d8415ed8b..414ddda43922 100644
--- a/drivers/gpu/drm/i915/intel_lrc.c
+++ b/drivers/gpu/drm/i915/intel_lrc.c
@@ -208,31 +208,27 @@
 } while (0)
 
 enum {
-        ADVANCED_CONTEXT = 0,
-        LEGACY_32B_CONTEXT,
-        ADVANCED_AD_CONTEXT,
-        LEGACY_64B_CONTEXT
-};
-#define GEN8_CTX_ADDRESSING_MODE_SHIFT 3
-#define GEN8_CTX_ADDRESSING_MODE(dev)  (USES_FULL_48BIT_PPGTT(dev) ?\
-                LEGACY_64B_CONTEXT :\
-                LEGACY_32B_CONTEXT)
-enum {
         FAULT_AND_HANG = 0,
         FAULT_AND_HALT, /* Debug only */
         FAULT_AND_STREAM,
         FAULT_AND_CONTINUE /* Unsupported */
 };
 #define GEN8_CTX_ID_SHIFT 32
+#define GEN8_CTX_ID_WIDTH 21
 #define GEN8_CTX_RCS_INDIRECT_CTX_OFFSET_DEFAULT        0x17
 #define GEN9_CTX_RCS_INDIRECT_CTX_OFFSET_DEFAULT        0x26
 
-static int intel_lr_context_pin(struct intel_context *ctx,
+/* Typical size of the average request (2 pipecontrols and a MI_BB) */
+#define EXECLISTS_REQUEST_SIZE 64 /* bytes */
+
+static int execlists_context_deferred_alloc(struct i915_gem_context *ctx,
+                                            struct intel_engine_cs *engine);
+static int intel_lr_context_pin(struct i915_gem_context *ctx,
                                 struct intel_engine_cs *engine);
 
 /**
  * intel_sanitize_enable_execlists() - sanitize i915.enable_execlists
- * @dev: DRM device.
+ * @dev_priv: i915 device private
  * @enable_execlists: value of i915.enable_execlists module parameter.
  *
  * Only certain platforms support Execlists (the prerequisites being
@@ -240,23 +236,22 @@ static int intel_lr_context_pin(struct intel_context *ctx,
  *
  * Return: 1 if Execlists is supported and has to be enabled.
  */
-int intel_sanitize_enable_execlists(struct drm_device *dev, int enable_execlists)
+int intel_sanitize_enable_execlists(struct drm_i915_private *dev_priv, int enable_execlists)
 {
-        WARN_ON(i915.enable_ppgtt == -1);
-
         /* On platforms with execlist available, vGPU will only
          * support execlist mode, no ring buffer mode.
          */
-        if (HAS_LOGICAL_RING_CONTEXTS(dev) && intel_vgpu_active(dev))
+        if (HAS_LOGICAL_RING_CONTEXTS(dev_priv) && intel_vgpu_active(dev_priv))
                 return 1;
 
-        if (INTEL_INFO(dev)->gen >= 9)
+        if (INTEL_GEN(dev_priv) >= 9)
                 return 1;
 
         if (enable_execlists == 0)
                 return 0;
 
-        if (HAS_LOGICAL_RING_CONTEXTS(dev) && USES_PPGTT(dev) &&
+        if (HAS_LOGICAL_RING_CONTEXTS(dev_priv) &&
+            USES_PPGTT(dev_priv) &&
             i915.use_mmio_flip >= 0)
                 return 1;
 
@@ -266,19 +261,17 @@ int intel_sanitize_enable_execlists(struct drm_device *dev, int enable_execlists
 static void
 logical_ring_init_platform_invariants(struct intel_engine_cs *engine)
 {
-        struct drm_device *dev = engine->dev;
+        struct drm_i915_private *dev_priv = engine->i915;
 
-        if (IS_GEN8(dev) || IS_GEN9(dev))
+        if (IS_GEN8(dev_priv) || IS_GEN9(dev_priv))
                 engine->idle_lite_restore_wa = ~0;
 
-        engine->disable_lite_restore_wa = (IS_SKL_REVID(dev, 0, SKL_REVID_B0) ||
-                                        IS_BXT_REVID(dev, 0, BXT_REVID_A1)) &&
+        engine->disable_lite_restore_wa = (IS_SKL_REVID(dev_priv, 0, SKL_REVID_B0) ||
+                                        IS_BXT_REVID(dev_priv, 0, BXT_REVID_A1)) &&
                                         (engine->id == VCS || engine->id == VCS2);
 
         engine->ctx_desc_template = GEN8_CTX_VALID;
-        engine->ctx_desc_template |= GEN8_CTX_ADDRESSING_MODE(dev) <<
-                                   GEN8_CTX_ADDRESSING_MODE_SHIFT;
-        if (IS_GEN8(dev))
+        if (IS_GEN8(dev_priv))
                 engine->ctx_desc_template |= GEN8_CTX_L3LLC_COHERENT;
         engine->ctx_desc_template |= GEN8_CTX_PRIVILEGE;
 
@@ -297,7 +290,7 @@ logical_ring_init_platform_invariants(struct intel_engine_cs *engine)
  *                                        descriptor for a pinned context
  *
  * @ctx: Context to work on
- * @ring: Engine the descriptor will be used with
+ * @engine: Engine the descriptor will be used with
  *
  * The context descriptor encodes various attributes of a context,
  * including its GTT address and some flags. Because it's fairly
@@ -305,62 +298,42 @@ logical_ring_init_platform_invariants(struct intel_engine_cs *engine)
  * which remains valid until the context is unpinned.
  *
  * This is what a descriptor looks like, from LSB to MSB:
- *    bits 0-11:    flags, GEN8_CTX_* (cached in ctx_desc_template)
+ *    bits  0-11:    flags, GEN8_CTX_* (cached in ctx_desc_template)
  *    bits 12-31:    LRCA, GTT address of (the HWSP of) this context
- *    bits 32-51:    ctx ID, a globally unique tag (the LRCA again!)
- *    bits 52-63:    reserved, may encode the engine ID (for GuC)
+ *    bits 32-52:    ctx ID, a globally unique tag
+ *    bits 53-54:    mbz, reserved for use by hardware
+ *    bits 55-63:    group ID, currently unused and set to 0
  */
 static void
-intel_lr_context_descriptor_update(struct intel_context *ctx,
+intel_lr_context_descriptor_update(struct i915_gem_context *ctx,
                                    struct intel_engine_cs *engine)
 {
-        uint64_t lrca, desc;
+        struct intel_context *ce = &ctx->engine[engine->id];
+        u64 desc;
 
-        lrca = ctx->engine[engine->id].lrc_vma->node.start +
-               LRC_PPHWSP_PN * PAGE_SIZE;
+        BUILD_BUG_ON(MAX_CONTEXT_HW_ID > (1<<GEN8_CTX_ID_WIDTH));
 
-        desc = engine->ctx_desc_template;                          /* bits  0-11 */
-        desc |= lrca;                                      /* bits 12-31 */
-        desc |= (lrca >> PAGE_SHIFT) << GEN8_CTX_ID_SHIFT; /* bits 32-51 */
+        desc = ctx->desc_template;                              /* bits  3-4  */
+        desc |= engine->ctx_desc_template;                      /* bits  0-11 */
+        desc |= ce->lrc_vma->node.start + LRC_PPHWSP_PN * PAGE_SIZE;
+                                                                /* bits 12-31 */
+        desc |= (u64)ctx->hw_id << GEN8_CTX_ID_SHIFT;           /* bits 32-52 */
 
-        ctx->engine[engine->id].lrc_desc = desc;
+        ce->lrc_desc = desc;
 }
 
-uint64_t intel_lr_context_descriptor(struct intel_context *ctx,
+uint64_t intel_lr_context_descriptor(struct i915_gem_context *ctx,
                                      struct intel_engine_cs *engine)
 {
         return ctx->engine[engine->id].lrc_desc;
 }
 
-/**
- * intel_execlists_ctx_id() - get the Execlists Context ID
- * @ctx: Context to get the ID for
- * @ring: Engine to get the ID for
- *
- * Do not confuse with ctx->id! Unfortunately we have a name overload
- * here: the old context ID we pass to userspace as a handler so that
- * they can refer to a context, and the new context ID we pass to the
- * ELSP so that the GPU can inform us of the context status via
- * interrupts.
- *
- * The context ID is a portion of the context descriptor, so we can
- * just extract the required part from the cached descriptor.
- *
- * Return: 20-bits globally unique context ID.
- */
-u32 intel_execlists_ctx_id(struct intel_context *ctx,
-                           struct intel_engine_cs *engine)
-{
-        return intel_lr_context_descriptor(ctx, engine) >> GEN8_CTX_ID_SHIFT;
-}
-
 static void execlists_elsp_write(struct drm_i915_gem_request *rq0,
                                  struct drm_i915_gem_request *rq1)
 {
 
         struct intel_engine_cs *engine = rq0->engine;
-        struct drm_device *dev = engine->dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = rq0->i915;
         uint64_t desc[2];
 
         if (rq1) {
@@ -431,6 +404,20 @@ static void execlists_submit_requests(struct drm_i915_gem_request *rq0,
         spin_unlock_irq(&dev_priv->uncore.lock);
 }
 
+static inline void execlists_context_status_change(
+                struct drm_i915_gem_request *rq,
+                unsigned long status)
+{
+        /*
+         * Only used when GVT-g is enabled now. When GVT-g is disabled,
+         * The compiler should eliminate this function as dead-code.
+         */
+        if (!IS_ENABLED(CONFIG_DRM_I915_GVT))
+                return;
+
+        atomic_notifier_call_chain(&rq->ctx->status_notifier, status, rq);
+}
+
 static void execlists_context_unqueue(struct intel_engine_cs *engine)
 {
         struct drm_i915_gem_request *req0 = NULL, *req1 = NULL;
@@ -442,7 +429,7 @@ static void execlists_context_unqueue(struct intel_engine_cs *engine)
          * If irqs are not active generate a warning as batches that finish
          * without the irqs may get lost and a GPU Hang may occur.
          */
-        WARN_ON(!intel_irqs_enabled(engine->dev->dev_private));
+        WARN_ON(!intel_irqs_enabled(engine->i915));
 
         /* Try to read in pairs */
         list_for_each_entry_safe(cursor, tmp, &engine->execlist_queue,
@@ -453,10 +440,24 @@ static void execlists_context_unqueue(struct intel_engine_cs *engine)
                         /* Same ctx: ignore first request, as second request
                          * will update tail past first request's workload */
                         cursor->elsp_submitted = req0->elsp_submitted;
-                        list_move_tail(&req0->execlist_link,
-                                       &engine->execlist_retired_req_list);
+                        list_del(&req0->execlist_link);
+                        i915_gem_request_unreference(req0);
                         req0 = cursor;
                 } else {
+                        if (IS_ENABLED(CONFIG_DRM_I915_GVT)) {
+                                /*
+                                 * req0 (after merged) ctx requires single
+                                 * submission, stop picking
+                                 */
+                                if (req0->ctx->execlists_force_single_submission)
+                                        break;
+                                /*
+                                 * req0 ctx doesn't require single submission,
+                                 * but next req ctx requires, stop picking
+                                 */
+                                if (cursor->ctx->execlists_force_single_submission)
+                                        break;
+                        }
                         req1 = cursor;
                         WARN_ON(req1->elsp_submitted);
                         break;
@@ -466,6 +467,12 @@ static void execlists_context_unqueue(struct intel_engine_cs *engine)
         if (unlikely(!req0))
                 return;
 
+        execlists_context_status_change(req0, INTEL_CONTEXT_SCHEDULE_IN);
+
+        if (req1)
+                execlists_context_status_change(req1,
+                                                INTEL_CONTEXT_SCHEDULE_IN);
+
         if (req0->elsp_submitted & engine->idle_lite_restore_wa) {
                 /*
                  * WaIdleLiteRestore: make sure we never cause a lite restore
@@ -486,7 +493,7 @@ static void execlists_context_unqueue(struct intel_engine_cs *engine)
 }
 
 static unsigned int
-execlists_check_remove_request(struct intel_engine_cs *engine, u32 request_id)
+execlists_check_remove_request(struct intel_engine_cs *engine, u32 ctx_id)
 {
         struct drm_i915_gem_request *head_req;
 
@@ -496,19 +503,18 @@ execlists_check_remove_request(struct intel_engine_cs *engine, u32 request_id)
                                             struct drm_i915_gem_request,
                                             execlist_link);
 
-        if (!head_req)
-                return 0;
-
-        if (unlikely(intel_execlists_ctx_id(head_req->ctx, engine) != request_id))
-                return 0;
+        if (WARN_ON(!head_req || (head_req->ctx_hw_id != ctx_id)))
+               return 0;
 
         WARN(head_req->elsp_submitted == 0, "Never submitted head request\n");
 
         if (--head_req->elsp_submitted > 0)
                 return 0;
 
-        list_move_tail(&head_req->execlist_link,
-                       &engine->execlist_retired_req_list);
+        execlists_context_status_change(head_req, INTEL_CONTEXT_SCHEDULE_OUT);
+
+        list_del(&head_req->execlist_link);
+        i915_gem_request_unreference(head_req);
 
         return 1;
 }
@@ -517,7 +523,7 @@ static u32
 get_context_status(struct intel_engine_cs *engine, unsigned int read_pointer,
                    u32 *context_id)
 {
-        struct drm_i915_private *dev_priv = engine->dev->dev_private;
+        struct drm_i915_private *dev_priv = engine->i915;
         u32 status;
 
         read_pointer %= GEN8_CSB_ENTRIES;
@@ -535,7 +541,7 @@ get_context_status(struct intel_engine_cs *engine, unsigned int read_pointer,
 
 /**
  * intel_lrc_irq_handler() - handle Context Switch interrupts
- * @engine: Engine Command Streamer to handle.
+ * @data: tasklet handler passed in unsigned long
  *
  * Check the unread Context Status Buffers and manage the submission of new
  * contexts to the ELSP accordingly.
@@ -543,7 +549,7 @@ get_context_status(struct intel_engine_cs *engine, unsigned int read_pointer,
 static void intel_lrc_irq_handler(unsigned long data)
 {
         struct intel_engine_cs *engine = (struct intel_engine_cs *)data;
-        struct drm_i915_private *dev_priv = engine->dev->dev_private;
+        struct drm_i915_private *dev_priv = engine->i915;
         u32 status_pointer;
         unsigned int read_pointer, write_pointer;
         u32 csb[GEN8_CSB_ENTRIES][2];
@@ -612,11 +618,6 @@ static void execlists_context_queue(struct drm_i915_gem_request *request)
         struct drm_i915_gem_request *cursor;
         int num_elements = 0;
 
-        if (request->ctx != request->i915->kernel_context)
-                intel_lr_context_pin(request->ctx, engine);
-
-        i915_gem_request_reference(request);
-
         spin_lock_bh(&engine->execlist_lock);
 
         list_for_each_entry(cursor, &engine->execlist_queue, execlist_link)
@@ -633,12 +634,14 @@ static void execlists_context_queue(struct drm_i915_gem_request *request)
                 if (request->ctx == tail_req->ctx) {
                         WARN(tail_req->elsp_submitted != 0,
                                 "More than 2 already-submitted reqs queued\n");
-                        list_move_tail(&tail_req->execlist_link,
-                                       &engine->execlist_retired_req_list);
+                        list_del(&tail_req->execlist_link);
+                        i915_gem_request_unreference(tail_req);
                 }
         }
 
+        i915_gem_request_reference(request);
         list_add_tail(&request->execlist_link, &engine->execlist_queue);
+        request->ctx_hw_id = request->ctx->hw_id;
         if (num_elements == 0)
                 execlists_context_unqueue(engine);
 
@@ -698,9 +701,23 @@ static int execlists_move_to_gpu(struct drm_i915_gem_request *req,
 
 int intel_logical_ring_alloc_request_extras(struct drm_i915_gem_request *request)
 {
-        int ret = 0;
+        struct intel_engine_cs *engine = request->engine;
+        struct intel_context *ce = &request->ctx->engine[engine->id];
+        int ret;
 
-        request->ringbuf = request->ctx->engine[request->engine->id].ringbuf;
+        /* Flush enough space to reduce the likelihood of waiting after
+         * we start building the request - in which case we will just
+         * have to repeat work.
+         */
+        request->reserved_space += EXECLISTS_REQUEST_SIZE;
+
+        if (!ce->state) {
+                ret = execlists_context_deferred_alloc(request->ctx, engine);
+                if (ret)
+                        return ret;
+        }
+
+        request->ringbuf = ce->ringbuf;
 
         if (i915.enable_guc_submission) {
                 /*
@@ -708,16 +725,39 @@ int intel_logical_ring_alloc_request_extras(struct drm_i915_gem_request *request
                  * going any further, as the i915_add_request() call
                  * later on mustn't fail ...
                  */
-                struct intel_guc *guc = &request->i915->guc;
-
-                ret = i915_guc_wq_check_space(guc->execbuf_client);
+                ret = i915_guc_wq_check_space(request);
                 if (ret)
                         return ret;
         }
 
-        if (request->ctx != request->i915->kernel_context)
-                ret = intel_lr_context_pin(request->ctx, request->engine);
+        ret = intel_lr_context_pin(request->ctx, engine);
+        if (ret)
+                return ret;
+
+        ret = intel_ring_begin(request, 0);
+        if (ret)
+                goto err_unpin;
+
+        if (!ce->initialised) {
+                ret = engine->init_context(request);
+                if (ret)
+                        goto err_unpin;
+
+                ce->initialised = true;
+        }
+
+        /* Note that after this point, we have committed to using
+         * this request as it is being used to both track the
+         * state of engine initialisation and liveness of the
+         * golden renderstate above. Think twice before you try
+         * to cancel/unwind this request now.
+         */
 
+        request->reserved_space -= EXECLISTS_REQUEST_SIZE;
+        return 0;
+
+err_unpin:
+        intel_lr_context_unpin(request->ctx, engine);
         return ret;
 }
 
@@ -734,7 +774,6 @@ static int
 intel_logical_ring_advance_and_submit(struct drm_i915_gem_request *request)
 {
         struct intel_ringbuffer *ringbuf = request->ringbuf;
-        struct drm_i915_private *dev_priv = request->i915;
         struct intel_engine_cs *engine = request->engine;
 
         intel_logical_ring_advance(ringbuf);
@@ -750,54 +789,28 @@ intel_logical_ring_advance_and_submit(struct drm_i915_gem_request *request)
         intel_logical_ring_emit(ringbuf, MI_NOOP);
         intel_logical_ring_advance(ringbuf);
 
-        if (intel_engine_stopped(engine))
-                return 0;
-
-        if (engine->last_context != request->ctx) {
-                if (engine->last_context)
-                        intel_lr_context_unpin(engine->last_context, engine);
-                if (request->ctx != request->i915->kernel_context) {
-                        intel_lr_context_pin(request->ctx, engine);
-                        engine->last_context = request->ctx;
-                } else {
-                        engine->last_context = NULL;
-                }
-        }
+        /* We keep the previous context alive until we retire the following
+         * request. This ensures that any the context object is still pinned
+         * for any residual writes the HW makes into it on the context switch
+         * into the next object following the breadcrumb. Otherwise, we may
+         * retire the context too early.
+         */
+        request->previous_context = engine->last_context;
+        engine->last_context = request->ctx;
 
-        if (dev_priv->guc.execbuf_client)
-                i915_guc_submit(dev_priv->guc.execbuf_client, request);
+        if (i915.enable_guc_submission)
+                i915_guc_submit(request);
         else
                 execlists_context_queue(request);
 
         return 0;
 }
 
-int intel_logical_ring_reserve_space(struct drm_i915_gem_request *request)
-{
-        /*
-         * The first call merely notes the reserve request and is common for
-         * all back ends. The subsequent localised _begin() call actually
-         * ensures that the reservation is available. Without the begin, if
-         * the request creator immediately submitted the request without
-         * adding any commands to it then there might not actually be
-         * sufficient room for the submission commands.
-         */
-        intel_ring_reserved_space_reserve(request->ringbuf, MIN_SPACE_FOR_ADD_REQUEST);
-
-        return intel_ring_begin(request, 0);
-}
-
 /**
  * execlists_submission() - submit a batchbuffer for execution, Execlists style
- * @dev: DRM device.
- * @file: DRM file.
- * @ring: Engine Command Streamer to submit to.
- * @ctx: Context to employ for this submission.
+ * @params: execbuffer call parameters.
  * @args: execbuffer call arguments.
  * @vmas: list of vmas.
- * @batch_obj: the batchbuffer to submit.
- * @exec_start: batchbuffer start virtual address pointer.
- * @dispatch_flags: translated execbuffer call flags.
  *
  * This is the evil twin version of i915_gem_ringbuffer_submission. It abstracts
  * away the submission details of the execbuffer ioctl call.
@@ -810,7 +823,7 @@ int intel_execlists_submission(struct i915_execbuffer_params *params,
 {
         struct drm_device       *dev = params->dev;
         struct intel_engine_cs *engine = params->engine;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         struct intel_ringbuffer *ringbuf = params->ctx->engine[engine->id].ringbuf;
         u64 exec_start;
         int instp_mode;
@@ -881,28 +894,18 @@ int intel_execlists_submission(struct i915_execbuffer_params *params,
         return 0;
 }
 
-void intel_execlists_retire_requests(struct intel_engine_cs *engine)
+void intel_execlists_cancel_requests(struct intel_engine_cs *engine)
 {
         struct drm_i915_gem_request *req, *tmp;
-        struct list_head retired_list;
+        LIST_HEAD(cancel_list);
 
-        WARN_ON(!mutex_is_locked(&engine->dev->struct_mutex));
-        if (list_empty(&engine->execlist_retired_req_list))
-                return;
+        WARN_ON(!mutex_is_locked(&engine->i915->drm.struct_mutex));
 
-        INIT_LIST_HEAD(&retired_list);
         spin_lock_bh(&engine->execlist_lock);
-        list_replace_init(&engine->execlist_retired_req_list, &retired_list);
+        list_replace_init(&engine->execlist_queue, &cancel_list);
         spin_unlock_bh(&engine->execlist_lock);
 
-        list_for_each_entry_safe(req, tmp, &retired_list, execlist_link) {
-                struct intel_context *ctx = req->ctx;
-                struct drm_i915_gem_object *ctx_obj =
-                                ctx->engine[engine->id].state;
-
-                if (ctx_obj && (ctx != req->i915->kernel_context))
-                        intel_lr_context_unpin(ctx, engine);
-
+        list_for_each_entry_safe(req, tmp, &cancel_list, execlist_link) {
                 list_del(&req->execlist_link);
                 i915_gem_request_unreference(req);
         }
@@ -910,7 +913,7 @@ void intel_execlists_retire_requests(struct intel_engine_cs *engine)
 
 void intel_logical_ring_stop(struct intel_engine_cs *engine)
 {
-        struct drm_i915_private *dev_priv = engine->dev->dev_private;
+        struct drm_i915_private *dev_priv = engine->i915;
         int ret;
 
         if (!intel_engine_initialized(engine))
@@ -923,7 +926,10 @@ void intel_logical_ring_stop(struct intel_engine_cs *engine)
 
         /* TODO: Is this correct with Execlists enabled? */
         I915_WRITE_MODE(engine, _MASKED_BIT_ENABLE(STOP_RING));
-        if (wait_for((I915_READ_MODE(engine) & MODE_IDLE) != 0, 1000)) {
+        if (intel_wait_for_register(dev_priv,
+                                    RING_MI_MODE(engine->mmio_base),
+                                    MODE_IDLE, MODE_IDLE,
+                                    1000)) {
                 DRM_ERROR("%s :timed out trying to stop ring\n", engine->name);
                 return;
         }
@@ -946,25 +952,26 @@ int logical_ring_flush_all_caches(struct drm_i915_gem_request *req)
         return 0;
 }
 
-static int intel_lr_context_do_pin(struct intel_context *ctx,
-                                   struct intel_engine_cs *engine)
+static int intel_lr_context_pin(struct i915_gem_context *ctx,
+                                struct intel_engine_cs *engine)
 {
-        struct drm_device *dev = engine->dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
-        struct drm_i915_gem_object *ctx_obj = ctx->engine[engine->id].state;
-        struct intel_ringbuffer *ringbuf = ctx->engine[engine->id].ringbuf;
+        struct drm_i915_private *dev_priv = ctx->i915;
+        struct intel_context *ce = &ctx->engine[engine->id];
         void *vaddr;
         u32 *lrc_reg_state;
         int ret;
 
-        WARN_ON(!mutex_is_locked(&engine->dev->struct_mutex));
+        lockdep_assert_held(&ctx->i915->drm.struct_mutex);
 
-        ret = i915_gem_obj_ggtt_pin(ctx_obj, GEN8_LR_CONTEXT_ALIGN,
-                        PIN_OFFSET_BIAS | GUC_WOPCM_TOP);
+        if (ce->pin_count++)
+                return 0;
+
+        ret = i915_gem_obj_ggtt_pin(ce->state, GEN8_LR_CONTEXT_ALIGN,
+                                    PIN_OFFSET_BIAS | GUC_WOPCM_TOP);
         if (ret)
-                return ret;
+                goto err;
 
-        vaddr = i915_gem_object_pin_map(ctx_obj);
+        vaddr = i915_gem_object_pin_map(ce->state);
         if (IS_ERR(vaddr)) {
                 ret = PTR_ERR(vaddr);
                 goto unpin_ctx_obj;
@@ -972,65 +979,54 @@ static int intel_lr_context_do_pin(struct intel_context *ctx,
 
         lrc_reg_state = vaddr + LRC_STATE_PN * PAGE_SIZE;
 
-        ret = intel_pin_and_map_ringbuffer_obj(engine->dev, ringbuf);
+        ret = intel_pin_and_map_ringbuffer_obj(dev_priv, ce->ringbuf);
         if (ret)
                 goto unpin_map;
 
-        ctx->engine[engine->id].lrc_vma = i915_gem_obj_to_ggtt(ctx_obj);
+        i915_gem_context_reference(ctx);
+        ce->lrc_vma = i915_gem_obj_to_ggtt(ce->state);
         intel_lr_context_descriptor_update(ctx, engine);
-        lrc_reg_state[CTX_RING_BUFFER_START+1] = ringbuf->vma->node.start;
-        ctx->engine[engine->id].lrc_reg_state = lrc_reg_state;
-        ctx_obj->dirty = true;
+
+        lrc_reg_state[CTX_RING_BUFFER_START+1] = ce->ringbuf->vma->node.start;
+        ce->lrc_reg_state = lrc_reg_state;
+        ce->state->dirty = true;
 
         /* Invalidate GuC TLB. */
         if (i915.enable_guc_submission)
                 I915_WRITE(GEN8_GTCR, GEN8_GTCR_INVALIDATE);
 
-        return ret;
+        return 0;
 
 unpin_map:
-        i915_gem_object_unpin_map(ctx_obj);
+        i915_gem_object_unpin_map(ce->state);
 unpin_ctx_obj:
-        i915_gem_object_ggtt_unpin(ctx_obj);
-
+        i915_gem_object_ggtt_unpin(ce->state);
+err:
+        ce->pin_count = 0;
         return ret;
 }
 
-static int intel_lr_context_pin(struct intel_context *ctx,
-                                struct intel_engine_cs *engine)
+void intel_lr_context_unpin(struct i915_gem_context *ctx,
+                            struct intel_engine_cs *engine)
 {
-        int ret = 0;
+        struct intel_context *ce = &ctx->engine[engine->id];
 
-        if (ctx->engine[engine->id].pin_count++ == 0) {
-                ret = intel_lr_context_do_pin(ctx, engine);
-                if (ret)
-                        goto reset_pin_count;
+        lockdep_assert_held(&ctx->i915->drm.struct_mutex);
+        GEM_BUG_ON(ce->pin_count == 0);
 
-                i915_gem_context_reference(ctx);
-        }
-        return ret;
+        if (--ce->pin_count)
+                return;
 
-reset_pin_count:
-        ctx->engine[engine->id].pin_count = 0;
-        return ret;
-}
+        intel_unpin_ringbuffer_obj(ce->ringbuf);
 
-void intel_lr_context_unpin(struct intel_context *ctx,
-                            struct intel_engine_cs *engine)
-{
-        struct drm_i915_gem_object *ctx_obj = ctx->engine[engine->id].state;
+        i915_gem_object_unpin_map(ce->state);
+        i915_gem_object_ggtt_unpin(ce->state);
 
-        WARN_ON(!mutex_is_locked(&ctx->i915->dev->struct_mutex));
-        if (--ctx->engine[engine->id].pin_count == 0) {
-                i915_gem_object_unpin_map(ctx_obj);
-                intel_unpin_ringbuffer_obj(ctx->engine[engine->id].ringbuf);
-                i915_gem_object_ggtt_unpin(ctx_obj);
-                ctx->engine[engine->id].lrc_vma = NULL;
-                ctx->engine[engine->id].lrc_desc = 0;
-                ctx->engine[engine->id].lrc_reg_state = NULL;
+        ce->lrc_vma = NULL;
+        ce->lrc_desc = 0;
+        ce->lrc_reg_state = NULL;
 
-                i915_gem_context_unreference(ctx);
-        }
+        i915_gem_context_unreference(ctx);
 }
 
 static int intel_logical_ring_workarounds_emit(struct drm_i915_gem_request *req)
@@ -1038,9 +1034,7 @@ static int intel_logical_ring_workarounds_emit(struct drm_i915_gem_request *req)
         int ret, i;
         struct intel_engine_cs *engine = req->engine;
         struct intel_ringbuffer *ringbuf = req->ringbuf;
-        struct drm_device *dev = engine->dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
-        struct i915_workarounds *w = &dev_priv->workarounds;
+        struct i915_workarounds *w = &req->i915->workarounds;
 
         if (w->count == 0)
                 return 0;
@@ -1103,7 +1097,7 @@ static inline int gen8_emit_flush_coherentl3_wa(struct intel_engine_cs *engine,
                                                 uint32_t *const batch,
                                                 uint32_t index)
 {
-        struct drm_i915_private *dev_priv = engine->dev->dev_private;
+        struct drm_i915_private *dev_priv = engine->i915;
         uint32_t l3sqc4_flush = (0x40400000 | GEN8_LQSC_FLUSH_COHERENT_LINES);
 
         /*
@@ -1165,7 +1159,7 @@ static inline int wa_ctx_end(struct i915_wa_ctx_bb *wa_ctx,
 /**
  * gen8_init_indirectctx_bb() - initialize indirect ctx batch with WA
  *
- * @ring: only applicable for RCS
+ * @engine: only applicable for RCS
  * @wa_ctx: structure representing wa_ctx
  *  offset: specifies start of the batch, should be cache-aligned. This is updated
  *    with the offset value received as input.
@@ -1202,7 +1196,7 @@ static int gen8_init_indirectctx_bb(struct intel_engine_cs *engine,
         wa_ctx_emit(batch, index, MI_ARB_ON_OFF | MI_ARB_DISABLE);
 
         /* WaFlushCoherentL3CacheLinesAtContextSwitch:bdw */
-        if (IS_BROADWELL(engine->dev)) {
+        if (IS_BROADWELL(engine->i915)) {
                 int rc = gen8_emit_flush_coherentl3_wa(engine, batch, index);
                 if (rc < 0)
                         return rc;
@@ -1239,7 +1233,7 @@ static int gen8_init_indirectctx_bb(struct intel_engine_cs *engine,
 /**
  * gen8_init_perctx_bb() - initialize per ctx batch with WA
  *
- * @ring: only applicable for RCS
+ * @engine: only applicable for RCS
  * @wa_ctx: structure representing wa_ctx
  *  offset: specifies start of the batch, should be cache-aligned.
  *  size: size of the batch in DWORDS but HW expects in terms of cachelines
@@ -1274,13 +1268,12 @@ static int gen9_init_indirectctx_bb(struct intel_engine_cs *engine,
                                     uint32_t *offset)
 {
         int ret;
-        struct drm_device *dev = engine->dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = engine->i915;
         uint32_t index = wa_ctx_start(wa_ctx, *offset, CACHELINE_DWORDS);
 
         /* WaDisableCtxRestoreArbitration:skl,bxt */
-        if (IS_SKL_REVID(dev, 0, SKL_REVID_D0) ||
-            IS_BXT_REVID(dev, 0, BXT_REVID_A1))
+        if (IS_SKL_REVID(dev_priv, 0, SKL_REVID_D0) ||
+            IS_BXT_REVID(dev_priv, 0, BXT_REVID_A1))
                 wa_ctx_emit(batch, index, MI_ARB_ON_OFF | MI_ARB_DISABLE);
 
         /* WaFlushCoherentL3CacheLinesAtContextSwitch:skl,bxt */
@@ -1305,6 +1298,31 @@ static int gen9_init_indirectctx_bb(struct intel_engine_cs *engine,
                 wa_ctx_emit(batch, index, 0);
                 wa_ctx_emit(batch, index, 0);
         }
+
+        /* WaMediaPoolStateCmdInWABB:bxt */
+        if (HAS_POOLED_EU(engine->i915)) {
+                /*
+                 * EU pool configuration is setup along with golden context
+                 * during context initialization. This value depends on
+                 * device type (2x6 or 3x6) and needs to be updated based
+                 * on which subslice is disabled especially for 2x6
+                 * devices, however it is safe to load default
+                 * configuration of 3x6 device instead of masking off
+                 * corresponding bits because HW ignores bits of a disabled
+                 * subslice and drops down to appropriate config. Please
+                 * see render_state_setup() in i915_gem_render_state.c for
+                 * possible configurations, to avoid duplication they are
+                 * not shown here again.
+                 */
+                u32 eu_pool_config = 0x00777000;
+                wa_ctx_emit(batch, index, GEN9_MEDIA_POOL_STATE);
+                wa_ctx_emit(batch, index, GEN9_MEDIA_POOL_ENABLE);
+                wa_ctx_emit(batch, index, eu_pool_config);
+                wa_ctx_emit(batch, index, 0);
+                wa_ctx_emit(batch, index, 0);
+                wa_ctx_emit(batch, index, 0);
+        }
+
         /* Pad to end of cacheline */
         while (index % CACHELINE_DWORDS)
                 wa_ctx_emit(batch, index, MI_NOOP);
@@ -1317,12 +1335,11 @@ static int gen9_init_perctx_bb(struct intel_engine_cs *engine,
                                uint32_t *const batch,
                                uint32_t *offset)
 {
-        struct drm_device *dev = engine->dev;
         uint32_t index = wa_ctx_start(wa_ctx, *offset, CACHELINE_DWORDS);
 
         /* WaSetDisablePixMaskCammingAndRhwoInCommonSliceChicken:skl,bxt */
-        if (IS_SKL_REVID(dev, 0, SKL_REVID_B0) ||
-            IS_BXT_REVID(dev, 0, BXT_REVID_A1)) {
+        if (IS_SKL_REVID(engine->i915, 0, SKL_REVID_B0) ||
+            IS_BXT_REVID(engine->i915, 0, BXT_REVID_A1)) {
                 wa_ctx_emit(batch, index, MI_LOAD_REGISTER_IMM(1));
                 wa_ctx_emit_reg(batch, index, GEN9_SLICE_COMMON_ECO_CHICKEN0);
                 wa_ctx_emit(batch, index,
@@ -1331,7 +1348,7 @@ static int gen9_init_perctx_bb(struct intel_engine_cs *engine,
         }
 
         /* WaClearTdlStateAckDirtyBits:bxt */
-        if (IS_BXT_REVID(dev, 0, BXT_REVID_B0)) {
+        if (IS_BXT_REVID(engine->i915, 0, BXT_REVID_B0)) {
                 wa_ctx_emit(batch, index, MI_LOAD_REGISTER_IMM(4));
 
                 wa_ctx_emit_reg(batch, index, GEN8_STATE_ACK);
@@ -1350,8 +1367,8 @@ static int gen9_init_perctx_bb(struct intel_engine_cs *engine,
         }
 
         /* WaDisableCtxRestoreArbitration:skl,bxt */
-        if (IS_SKL_REVID(dev, 0, SKL_REVID_D0) ||
-            IS_BXT_REVID(dev, 0, BXT_REVID_A1))
+        if (IS_SKL_REVID(engine->i915, 0, SKL_REVID_D0) ||
+            IS_BXT_REVID(engine->i915, 0, BXT_REVID_A1))
                 wa_ctx_emit(batch, index, MI_ARB_ON_OFF | MI_ARB_ENABLE);
 
         wa_ctx_emit(batch, index, MI_BATCH_BUFFER_END);
@@ -1363,11 +1380,13 @@ static int lrc_setup_wa_ctx_obj(struct intel_engine_cs *engine, u32 size)
 {
         int ret;
 
-        engine->wa_ctx.obj = i915_gem_alloc_object(engine->dev,
-                                                   PAGE_ALIGN(size));
-        if (!engine->wa_ctx.obj) {
+        engine->wa_ctx.obj = i915_gem_object_create(&engine->i915->drm,
+                                                    PAGE_ALIGN(size));
+        if (IS_ERR(engine->wa_ctx.obj)) {
                 DRM_DEBUG_DRIVER("alloc LRC WA ctx backing obj failed.\n");
-                return -ENOMEM;
+                ret = PTR_ERR(engine->wa_ctx.obj);
+                engine->wa_ctx.obj = NULL;
+                return ret;
         }
 
         ret = i915_gem_obj_ggtt_pin(engine->wa_ctx.obj, PAGE_SIZE, 0);
@@ -1401,9 +1420,9 @@ static int intel_init_workaround_bb(struct intel_engine_cs *engine)
         WARN_ON(engine->id != RCS);
 
         /* update this when WA for higher Gen are added */
-        if (INTEL_INFO(engine->dev)->gen > 9) {
+        if (INTEL_GEN(engine->i915) > 9) {
                 DRM_ERROR("WA batch buffer is not initialized for Gen%d\n",
-                          INTEL_INFO(engine->dev)->gen);
+                          INTEL_GEN(engine->i915));
                 return 0;
         }
 
@@ -1423,7 +1442,7 @@ static int intel_init_workaround_bb(struct intel_engine_cs *engine)
         batch = kmap_atomic(page);
         offset = 0;
 
-        if (INTEL_INFO(engine->dev)->gen == 8) {
+        if (IS_GEN8(engine->i915)) {
                 ret = gen8_init_indirectctx_bb(engine,
                                                &wa_ctx->indirect_ctx,
                                                batch,
@@ -1437,7 +1456,7 @@ static int intel_init_workaround_bb(struct intel_engine_cs *engine)
                                           &offset);
                 if (ret)
                         goto out;
-        } else if (INTEL_INFO(engine->dev)->gen == 9) {
+        } else if (IS_GEN9(engine->i915)) {
                 ret = gen9_init_indirectctx_bb(engine,
                                                &wa_ctx->indirect_ctx,
                                                batch,
@@ -1463,7 +1482,7 @@ out:
 
 static void lrc_init_hws(struct intel_engine_cs *engine)
 {
-        struct drm_i915_private *dev_priv = engine->dev->dev_private;
+        struct drm_i915_private *dev_priv = engine->i915;
 
         I915_WRITE(RING_HWS_PGA(engine->mmio_base),
                    (u32)engine->status_page.gfx_addr);
@@ -1472,8 +1491,7 @@ static void lrc_init_hws(struct intel_engine_cs *engine)
 
 static int gen8_init_common_ring(struct intel_engine_cs *engine)
 {
-        struct drm_device *dev = engine->dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = engine->i915;
         unsigned int next_context_status_buffer_hw;
 
         lrc_init_hws(engine);
@@ -1520,8 +1538,7 @@ static int gen8_init_common_ring(struct intel_engine_cs *engine)
 
 static int gen8_init_render_ring(struct intel_engine_cs *engine)
 {
-        struct drm_device *dev = engine->dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = engine->i915;
         int ret;
 
         ret = gen8_init_common_ring(engine);
@@ -1598,7 +1615,7 @@ static int gen8_emit_bb_start(struct drm_i915_gem_request *req,
         if (req->ctx->ppgtt &&
             (intel_engine_flag(req->engine) & req->ctx->ppgtt->pd_dirty_rings)) {
                 if (!USES_FULL_48BIT_PPGTT(req->i915) &&
-                    !intel_vgpu_active(req->i915->dev)) {
+                    !intel_vgpu_active(req->i915)) {
                         ret = intel_logical_ring_emit_pdps(req);
                         if (ret)
                                 return ret;
@@ -1624,38 +1641,18 @@ static int gen8_emit_bb_start(struct drm_i915_gem_request *req,
         return 0;
 }
 
-static bool gen8_logical_ring_get_irq(struct intel_engine_cs *engine)
+static void gen8_logical_ring_enable_irq(struct intel_engine_cs *engine)
 {
-        struct drm_device *dev = engine->dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
-        unsigned long flags;
-
-        if (WARN_ON(!intel_irqs_enabled(dev_priv)))
-                return false;
-
-        spin_lock_irqsave(&dev_priv->irq_lock, flags);
-        if (engine->irq_refcount++ == 0) {
-                I915_WRITE_IMR(engine,
-                               ~(engine->irq_enable_mask | engine->irq_keep_mask));
-                POSTING_READ(RING_IMR(engine->mmio_base));
-        }
-        spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
-
-        return true;
+        struct drm_i915_private *dev_priv = engine->i915;
+        I915_WRITE_IMR(engine,
+                       ~(engine->irq_enable_mask | engine->irq_keep_mask));
+        POSTING_READ_FW(RING_IMR(engine->mmio_base));
 }
 
-static void gen8_logical_ring_put_irq(struct intel_engine_cs *engine)
+static void gen8_logical_ring_disable_irq(struct intel_engine_cs *engine)
 {
-        struct drm_device *dev = engine->dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
-        unsigned long flags;
-
-        spin_lock_irqsave(&dev_priv->irq_lock, flags);
-        if (--engine->irq_refcount == 0) {
-                I915_WRITE_IMR(engine, ~engine->irq_keep_mask);
-                POSTING_READ(RING_IMR(engine->mmio_base));
-        }
-        spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
+        struct drm_i915_private *dev_priv = engine->i915;
+        I915_WRITE_IMR(engine, ~engine->irq_keep_mask);
 }
 
 static int gen8_emit_flush(struct drm_i915_gem_request *request,
@@ -1664,8 +1661,7 @@ static int gen8_emit_flush(struct drm_i915_gem_request *request,
 {
         struct intel_ringbuffer *ringbuf = request->ringbuf;
         struct intel_engine_cs *engine = ringbuf->engine;
-        struct drm_device *dev = engine->dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = request->i915;
         uint32_t cmd;
         int ret;
 
@@ -1734,7 +1730,7 @@ static int gen8_emit_flush_render(struct drm_i915_gem_request *request,
                  * On GEN9: before VF_CACHE_INVALIDATE we need to emit a NULL
                  * pipe control.
                  */
-                if (IS_GEN9(engine->dev))
+                if (IS_GEN9(request->i915))
                         vf_flush_wa = true;
 
                 /* WaForGAMHang:kbl */
@@ -1793,16 +1789,6 @@ static int gen8_emit_flush_render(struct drm_i915_gem_request *request,
         return 0;
 }
 
-static u32 gen8_get_seqno(struct intel_engine_cs *engine)
-{
-        return intel_read_status_page(engine, I915_GEM_HWS_INDEX);
-}
-
-static void gen8_set_seqno(struct intel_engine_cs *engine, u32 seqno)
-{
-        intel_write_status_page(engine, I915_GEM_HWS_INDEX, seqno);
-}
-
 static void bxt_a_seqno_barrier(struct intel_engine_cs *engine)
 {
         /*
@@ -1818,14 +1804,6 @@ static void bxt_a_seqno_barrier(struct intel_engine_cs *engine)
         intel_flush_status_page(engine, I915_GEM_HWS_INDEX);
 }
 
-static void bxt_a_set_seqno(struct intel_engine_cs *engine, u32 seqno)
-{
-        intel_write_status_page(engine, I915_GEM_HWS_INDEX, seqno);
-
-        /* See bxt_a_get_seqno() explaining the reason for the clflush. */
-        intel_flush_status_page(engine, I915_GEM_HWS_INDEX);
-}
-
 /*
  * Reserve space for 2 NOOPs at the end of each request to be
  * used as a workaround for not being allowed to do lite
@@ -1833,11 +1811,6 @@ static void bxt_a_set_seqno(struct intel_engine_cs *engine, u32 seqno)
  */
 #define WA_TAIL_DWORDS 2
 
-static inline u32 hws_seqno_address(struct intel_engine_cs *engine)
-{
-        return engine->status_page.gfx_addr + I915_GEM_HWS_INDEX_ADDR;
-}
-
 static int gen8_emit_request(struct drm_i915_gem_request *request)
 {
         struct intel_ringbuffer *ringbuf = request->ringbuf;
@@ -1853,10 +1826,10 @@ static int gen8_emit_request(struct drm_i915_gem_request *request)
         intel_logical_ring_emit(ringbuf,
                                 (MI_FLUSH_DW + 1) | MI_FLUSH_DW_OP_STOREDW);
         intel_logical_ring_emit(ringbuf,
-                                hws_seqno_address(request->engine) |
+                                intel_hws_seqno_address(request->engine) |
                                 MI_FLUSH_DW_USE_GTT);
         intel_logical_ring_emit(ringbuf, 0);
-        intel_logical_ring_emit(ringbuf, i915_gem_request_get_seqno(request));
+        intel_logical_ring_emit(ringbuf, request->seqno);
         intel_logical_ring_emit(ringbuf, MI_USER_INTERRUPT);
         intel_logical_ring_emit(ringbuf, MI_NOOP);
         return intel_logical_ring_advance_and_submit(request);
@@ -1883,7 +1856,8 @@ static int gen8_emit_request_render(struct drm_i915_gem_request *request)
                                 (PIPE_CONTROL_GLOBAL_GTT_IVB |
                                  PIPE_CONTROL_CS_STALL |
                                  PIPE_CONTROL_QW_WRITE));
-        intel_logical_ring_emit(ringbuf, hws_seqno_address(request->engine));
+        intel_logical_ring_emit(ringbuf,
+                                intel_hws_seqno_address(request->engine));
         intel_logical_ring_emit(ringbuf, 0);
         intel_logical_ring_emit(ringbuf, i915_gem_request_get_seqno(request));
         /* We're thrashing one dword of HWS. */
@@ -1945,7 +1919,7 @@ static int gen8_init_rcs_context(struct drm_i915_gem_request *req)
 /**
  * intel_logical_ring_cleanup() - deallocate the Engine Command Streamer
  *
- * @ring: Engine Command Streamer.
+ * @engine: Engine Command Streamer.
  *
  */
 void intel_logical_ring_cleanup(struct intel_engine_cs *engine)
@@ -1962,7 +1936,7 @@ void intel_logical_ring_cleanup(struct intel_engine_cs *engine)
         if (WARN_ON(test_bit(TASKLET_STATE_SCHED, &engine->irq_tasklet.state)))
                 tasklet_kill(&engine->irq_tasklet);
 
-        dev_priv = engine->dev->dev_private;
+        dev_priv = engine->i915;
 
         if (engine->buffer) {
                 intel_logical_ring_stop(engine);
@@ -1975,36 +1949,34 @@ void intel_logical_ring_cleanup(struct intel_engine_cs *engine)
         i915_cmd_parser_fini_ring(engine);
         i915_gem_batch_pool_fini(&engine->batch_pool);
 
+        intel_engine_fini_breadcrumbs(engine);
+
         if (engine->status_page.obj) {
                 i915_gem_object_unpin_map(engine->status_page.obj);
                 engine->status_page.obj = NULL;
         }
+        intel_lr_context_unpin(dev_priv->kernel_context, engine);
 
         engine->idle_lite_restore_wa = 0;
         engine->disable_lite_restore_wa = false;
         engine->ctx_desc_template = 0;
 
         lrc_destroy_wa_ctx_obj(engine);
-        engine->dev = NULL;
+        engine->i915 = NULL;
 }
 
 static void
-logical_ring_default_vfuncs(struct drm_device *dev,
-                            struct intel_engine_cs *engine)
+logical_ring_default_vfuncs(struct intel_engine_cs *engine)
 {
         /* Default vfuncs which can be overriden by each engine. */
         engine->init_hw = gen8_init_common_ring;
         engine->emit_request = gen8_emit_request;
         engine->emit_flush = gen8_emit_flush;
-        engine->irq_get = gen8_logical_ring_get_irq;
-        engine->irq_put = gen8_logical_ring_put_irq;
+        engine->irq_enable = gen8_logical_ring_enable_irq;
+        engine->irq_disable = gen8_logical_ring_disable_irq;
         engine->emit_bb_start = gen8_emit_bb_start;
-        engine->get_seqno = gen8_get_seqno;
-        engine->set_seqno = gen8_set_seqno;
-        if (IS_BXT_REVID(dev, 0, BXT_REVID_A1)) {
+        if (IS_BXT_REVID(engine->i915, 0, BXT_REVID_A1))
                 engine->irq_seqno_barrier = bxt_a_seqno_barrier;
-                engine->set_seqno = bxt_a_set_seqno;
-        }
 }
 
 static inline void
@@ -2033,60 +2005,28 @@ lrc_setup_hws(struct intel_engine_cs *engine,
 }
 
 static int
-logical_ring_init(struct drm_device *dev, struct intel_engine_cs *engine)
+logical_ring_init(struct intel_engine_cs *engine)
 {
-        struct drm_i915_private *dev_priv = to_i915(dev);
-        struct intel_context *dctx = dev_priv->kernel_context;
-        enum forcewake_domains fw_domains;
+        struct i915_gem_context *dctx = engine->i915->kernel_context;
         int ret;
 
-        /* Intentionally left blank. */
-        engine->buffer = NULL;
-
-        engine->dev = dev;
-        INIT_LIST_HEAD(&engine->active_list);
-        INIT_LIST_HEAD(&engine->request_list);
-        i915_gem_batch_pool_init(dev, &engine->batch_pool);
-        init_waitqueue_head(&engine->irq_queue);
-
-        INIT_LIST_HEAD(&engine->buffers);
-        INIT_LIST_HEAD(&engine->execlist_queue);
-        INIT_LIST_HEAD(&engine->execlist_retired_req_list);
-        spin_lock_init(&engine->execlist_lock);
-
-        tasklet_init(&engine->irq_tasklet,
-                     intel_lrc_irq_handler, (unsigned long)engine);
-
-        logical_ring_init_platform_invariants(engine);
-
-        fw_domains = intel_uncore_forcewake_for_reg(dev_priv,
-                                                    RING_ELSP(engine),
-                                                    FW_REG_WRITE);
-
-        fw_domains |= intel_uncore_forcewake_for_reg(dev_priv,
-                                                     RING_CONTEXT_STATUS_PTR(engine),
-                                                     FW_REG_READ | FW_REG_WRITE);
-
-        fw_domains |= intel_uncore_forcewake_for_reg(dev_priv,
-                                                     RING_CONTEXT_STATUS_BUF_BASE(engine),
-                                                     FW_REG_READ);
-
-        engine->fw_domains = fw_domains;
+        ret = intel_engine_init_breadcrumbs(engine);
+        if (ret)
+                goto error;
 
         ret = i915_cmd_parser_init_ring(engine);
         if (ret)
                 goto error;
 
-        ret = intel_lr_context_deferred_alloc(dctx, engine);
+        ret = execlists_context_deferred_alloc(dctx, engine);
         if (ret)
                 goto error;
 
         /* As this is the default context, always pin it */
-        ret = intel_lr_context_do_pin(dctx, engine);
+        ret = intel_lr_context_pin(dctx, engine);
         if (ret) {
-                DRM_ERROR(
-                        "Failed to pin and map ringbuffer %s: %d\n",
-                        engine->name, ret);
+                DRM_ERROR("Failed to pin context for %s: %d\n",
+                          engine->name, ret);
                 goto error;
         }
 
@@ -2104,26 +2044,16 @@ error:
         return ret;
 }
 
-static int logical_render_ring_init(struct drm_device *dev)
+static int logical_render_ring_init(struct intel_engine_cs *engine)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
-        struct intel_engine_cs *engine = &dev_priv->engine[RCS];
+        struct drm_i915_private *dev_priv = engine->i915;
         int ret;
 
-        engine->name = "render ring";
-        engine->id = RCS;
-        engine->exec_id = I915_EXEC_RENDER;
-        engine->guc_id = GUC_RENDER_ENGINE;
-        engine->mmio_base = RENDER_RING_BASE;
-
-        logical_ring_default_irqs(engine, GEN8_RCS_IRQ_SHIFT);
-        if (HAS_L3_DPF(dev))
+        if (HAS_L3_DPF(dev_priv))
                 engine->irq_keep_mask |= GT_RENDER_L3_PARITY_ERROR_INTERRUPT;
 
-        logical_ring_default_vfuncs(dev, engine);
-
         /* Override some for render ring. */
-        if (INTEL_INFO(dev)->gen >= 9)
+        if (INTEL_GEN(dev_priv) >= 9)
                 engine->init_hw = gen9_init_render_ring;
         else
                 engine->init_hw = gen8_init_render_ring;
@@ -2132,9 +2062,7 @@ static int logical_render_ring_init(struct drm_device *dev)
         engine->emit_flush = gen8_emit_flush_render;
         engine->emit_request = gen8_emit_request_render;
 
-        engine->dev = dev;
-
-        ret = intel_init_pipe_control(engine);
+        ret = intel_init_pipe_control(engine, 4096);
         if (ret)
                 return ret;
 
@@ -2149,7 +2077,7 @@ static int logical_render_ring_init(struct drm_device *dev)
                           ret);
         }
 
-        ret = logical_ring_init(dev, engine);
+        ret = logical_ring_init(engine);
         if (ret) {
                 lrc_destroy_wa_ctx_obj(engine);
         }
@@ -2157,133 +2085,164 @@ static int logical_render_ring_init(struct drm_device *dev)
         return ret;
 }
 
-static int logical_bsd_ring_init(struct drm_device *dev)
-{
-        struct drm_i915_private *dev_priv = dev->dev_private;
-        struct intel_engine_cs *engine = &dev_priv->engine[VCS];
-
-        engine->name = "bsd ring";
-        engine->id = VCS;
-        engine->exec_id = I915_EXEC_BSD;
-        engine->guc_id = GUC_VIDEO_ENGINE;
-        engine->mmio_base = GEN6_BSD_RING_BASE;
-
-        logical_ring_default_irqs(engine, GEN8_VCS1_IRQ_SHIFT);
-        logical_ring_default_vfuncs(dev, engine);
-
-        return logical_ring_init(dev, engine);
-}
+static const struct logical_ring_info {
+        const char *name;
+        unsigned exec_id;
+        unsigned guc_id;
+        u32 mmio_base;
+        unsigned irq_shift;
+        int (*init)(struct intel_engine_cs *engine);
+} logical_rings[] = {
+        [RCS] = {
+                .name = "render ring",
+                .exec_id = I915_EXEC_RENDER,
+                .guc_id = GUC_RENDER_ENGINE,
+                .mmio_base = RENDER_RING_BASE,
+                .irq_shift = GEN8_RCS_IRQ_SHIFT,
+                .init = logical_render_ring_init,
+        },
+        [BCS] = {
+                .name = "blitter ring",
+                .exec_id = I915_EXEC_BLT,
+                .guc_id = GUC_BLITTER_ENGINE,
+                .mmio_base = BLT_RING_BASE,
+                .irq_shift = GEN8_BCS_IRQ_SHIFT,
+                .init = logical_ring_init,
+        },
+        [VCS] = {
+                .name = "bsd ring",
+                .exec_id = I915_EXEC_BSD,
+                .guc_id = GUC_VIDEO_ENGINE,
+                .mmio_base = GEN6_BSD_RING_BASE,
+                .irq_shift = GEN8_VCS1_IRQ_SHIFT,
+                .init = logical_ring_init,
+        },
+        [VCS2] = {
+                .name = "bsd2 ring",
+                .exec_id = I915_EXEC_BSD,
+                .guc_id = GUC_VIDEO_ENGINE2,
+                .mmio_base = GEN8_BSD2_RING_BASE,
+                .irq_shift = GEN8_VCS2_IRQ_SHIFT,
+                .init = logical_ring_init,
+        },
+        [VECS] = {
+                .name = "video enhancement ring",
+                .exec_id = I915_EXEC_VEBOX,
+                .guc_id = GUC_VIDEOENHANCE_ENGINE,
+                .mmio_base = VEBOX_RING_BASE,
+                .irq_shift = GEN8_VECS_IRQ_SHIFT,
+                .init = logical_ring_init,
+        },
+};
 
-static int logical_bsd2_ring_init(struct drm_device *dev)
+static struct intel_engine_cs *
+logical_ring_setup(struct drm_i915_private *dev_priv, enum intel_engine_id id)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
-        struct intel_engine_cs *engine = &dev_priv->engine[VCS2];
+        const struct logical_ring_info *info = &logical_rings[id];
+        struct intel_engine_cs *engine = &dev_priv->engine[id];
+        enum forcewake_domains fw_domains;
 
-        engine->name = "bsd2 ring";
-        engine->id = VCS2;
-        engine->exec_id = I915_EXEC_BSD;
-        engine->guc_id = GUC_VIDEO_ENGINE2;
-        engine->mmio_base = GEN8_BSD2_RING_BASE;
+        engine->id = id;
+        engine->name = info->name;
+        engine->exec_id = info->exec_id;
+        engine->guc_id = info->guc_id;
+        engine->mmio_base = info->mmio_base;
 
-        logical_ring_default_irqs(engine, GEN8_VCS2_IRQ_SHIFT);
-        logical_ring_default_vfuncs(dev, engine);
+        engine->i915 = dev_priv;
 
-        return logical_ring_init(dev, engine);
-}
+        /* Intentionally left blank. */
+        engine->buffer = NULL;
 
-static int logical_blt_ring_init(struct drm_device *dev)
-{
-        struct drm_i915_private *dev_priv = dev->dev_private;
-        struct intel_engine_cs *engine = &dev_priv->engine[BCS];
+        fw_domains = intel_uncore_forcewake_for_reg(dev_priv,
+                                                    RING_ELSP(engine),
+                                                    FW_REG_WRITE);
 
-        engine->name = "blitter ring";
-        engine->id = BCS;
-        engine->exec_id = I915_EXEC_BLT;
-        engine->guc_id = GUC_BLITTER_ENGINE;
-        engine->mmio_base = BLT_RING_BASE;
+        fw_domains |= intel_uncore_forcewake_for_reg(dev_priv,
+                                                     RING_CONTEXT_STATUS_PTR(engine),
+                                                     FW_REG_READ | FW_REG_WRITE);
 
-        logical_ring_default_irqs(engine, GEN8_BCS_IRQ_SHIFT);
-        logical_ring_default_vfuncs(dev, engine);
+        fw_domains |= intel_uncore_forcewake_for_reg(dev_priv,
+                                                     RING_CONTEXT_STATUS_BUF_BASE(engine),
+                                                     FW_REG_READ);
 
-        return logical_ring_init(dev, engine);
-}
+        engine->fw_domains = fw_domains;
 
-static int logical_vebox_ring_init(struct drm_device *dev)
-{
-        struct drm_i915_private *dev_priv = dev->dev_private;
-        struct intel_engine_cs *engine = &dev_priv->engine[VECS];
+        INIT_LIST_HEAD(&engine->active_list);
+        INIT_LIST_HEAD(&engine->request_list);
+        INIT_LIST_HEAD(&engine->buffers);
+        INIT_LIST_HEAD(&engine->execlist_queue);
+        spin_lock_init(&engine->execlist_lock);
 
-        engine->name = "video enhancement ring";
-        engine->id = VECS;
-        engine->exec_id = I915_EXEC_VEBOX;
-        engine->guc_id = GUC_VIDEOENHANCE_ENGINE;
-        engine->mmio_base = VEBOX_RING_BASE;
+        tasklet_init(&engine->irq_tasklet,
+                     intel_lrc_irq_handler, (unsigned long)engine);
 
-        logical_ring_default_irqs(engine, GEN8_VECS_IRQ_SHIFT);
-        logical_ring_default_vfuncs(dev, engine);
+        logical_ring_init_platform_invariants(engine);
+        logical_ring_default_vfuncs(engine);
+        logical_ring_default_irqs(engine, info->irq_shift);
+
+        intel_engine_init_hangcheck(engine);
+        i915_gem_batch_pool_init(&dev_priv->drm, &engine->batch_pool);
 
-        return logical_ring_init(dev, engine);
+        return engine;
 }
 
 /**
  * intel_logical_rings_init() - allocate, populate and init the Engine Command Streamers
  * @dev: DRM device.
  *
- * This function inits the engines for an Execlists submission style (the equivalent in the
- * legacy ringbuffer submission world would be i915_gem_init_engines). It does it only for
- * those engines that are present in the hardware.
+ * This function inits the engines for an Execlists submission style (the
+ * equivalent in the legacy ringbuffer submission world would be
+ * i915_gem_init_engines). It does it only for those engines that are present in
+ * the hardware.
  *
  * Return: non-zero if the initialization failed.
  */
 int intel_logical_rings_init(struct drm_device *dev)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
+        unsigned int mask = 0;
+        unsigned int i;
         int ret;
 
-        ret = logical_render_ring_init(dev);
-        if (ret)
-                return ret;
+        WARN_ON(INTEL_INFO(dev_priv)->ring_mask &
+                GENMASK(sizeof(mask) * BITS_PER_BYTE - 1, I915_NUM_ENGINES));
 
-        if (HAS_BSD(dev)) {
-                ret = logical_bsd_ring_init(dev);
-                if (ret)
-                        goto cleanup_render_ring;
-        }
+        for (i = 0; i < ARRAY_SIZE(logical_rings); i++) {
+                if (!HAS_ENGINE(dev_priv, i))
+                        continue;
 
-        if (HAS_BLT(dev)) {
-                ret = logical_blt_ring_init(dev);
-                if (ret)
-                        goto cleanup_bsd_ring;
-        }
+                if (!logical_rings[i].init)
+                        continue;
 
-        if (HAS_VEBOX(dev)) {
-                ret = logical_vebox_ring_init(dev);
+                ret = logical_rings[i].init(logical_ring_setup(dev_priv, i));
                 if (ret)
-                        goto cleanup_blt_ring;
+                        goto cleanup;
+
+                mask |= ENGINE_MASK(i);
         }
 
-        if (HAS_BSD2(dev)) {
-                ret = logical_bsd2_ring_init(dev);
-                if (ret)
-                        goto cleanup_vebox_ring;
+        /*
+         * Catch failures to update logical_rings table when the new engines
+         * are added to the driver by a warning and disabling the forgotten
+         * engines.
+         */
+        if (WARN_ON(mask != INTEL_INFO(dev_priv)->ring_mask)) {
+                struct intel_device_info *info =
+                        (struct intel_device_info *)&dev_priv->info;
+                info->ring_mask = mask;
         }
 
         return 0;
 
-cleanup_vebox_ring:
-        intel_logical_ring_cleanup(&dev_priv->engine[VECS]);
-cleanup_blt_ring:
-        intel_logical_ring_cleanup(&dev_priv->engine[BCS]);
-cleanup_bsd_ring:
-        intel_logical_ring_cleanup(&dev_priv->engine[VCS]);
-cleanup_render_ring:
-        intel_logical_ring_cleanup(&dev_priv->engine[RCS]);
+cleanup:
+        for (i = 0; i < I915_NUM_ENGINES; i++)
+                intel_logical_ring_cleanup(&dev_priv->engine[i]);
 
         return ret;
 }
 
 static u32
-make_rpcs(struct drm_device *dev)
+make_rpcs(struct drm_i915_private *dev_priv)
 {
         u32 rpcs = 0;
 
@@ -2291,7 +2250,7 @@ make_rpcs(struct drm_device *dev)
          * No explicit RPCS request is needed to ensure full
          * slice/subslice/EU enablement prior to Gen9.
         */
-        if (INTEL_INFO(dev)->gen < 9)
+        if (INTEL_GEN(dev_priv) < 9)
                 return 0;
 
         /*
@@ -2300,24 +2259,24 @@ make_rpcs(struct drm_device *dev)
          * must make an explicit request through RPCS for full
          * enablement.
         */
-        if (INTEL_INFO(dev)->has_slice_pg) {
+        if (INTEL_INFO(dev_priv)->has_slice_pg) {
                 rpcs |= GEN8_RPCS_S_CNT_ENABLE;
-                rpcs |= INTEL_INFO(dev)->slice_total <<
+                rpcs |= INTEL_INFO(dev_priv)->slice_total <<
                         GEN8_RPCS_S_CNT_SHIFT;
                 rpcs |= GEN8_RPCS_ENABLE;
         }
 
-        if (INTEL_INFO(dev)->has_subslice_pg) {
+        if (INTEL_INFO(dev_priv)->has_subslice_pg) {
                 rpcs |= GEN8_RPCS_SS_CNT_ENABLE;
-                rpcs |= INTEL_INFO(dev)->subslice_per_slice <<
+                rpcs |= INTEL_INFO(dev_priv)->subslice_per_slice <<
                         GEN8_RPCS_SS_CNT_SHIFT;
                 rpcs |= GEN8_RPCS_ENABLE;
         }
 
-        if (INTEL_INFO(dev)->has_eu_pg) {
-                rpcs |= INTEL_INFO(dev)->eu_per_subslice <<
+        if (INTEL_INFO(dev_priv)->has_eu_pg) {
+                rpcs |= INTEL_INFO(dev_priv)->eu_per_subslice <<
                         GEN8_RPCS_EU_MIN_SHIFT;
-                rpcs |= INTEL_INFO(dev)->eu_per_subslice <<
+                rpcs |= INTEL_INFO(dev_priv)->eu_per_subslice <<
                         GEN8_RPCS_EU_MAX_SHIFT;
                 rpcs |= GEN8_RPCS_ENABLE;
         }
@@ -2329,9 +2288,9 @@ static u32 intel_lr_indirect_ctx_offset(struct intel_engine_cs *engine)
 {
         u32 indirect_ctx_offset;
 
-        switch (INTEL_INFO(engine->dev)->gen) {
+        switch (INTEL_GEN(engine->i915)) {
         default:
-                MISSING_CASE(INTEL_INFO(engine->dev)->gen);
+                MISSING_CASE(INTEL_GEN(engine->i915));
                 /* fall through */
         case 9:
                 indirect_ctx_offset =
@@ -2347,13 +2306,12 @@ static u32 intel_lr_indirect_ctx_offset(struct intel_engine_cs *engine)
 }
 
 static int
-populate_lr_context(struct intel_context *ctx,
+populate_lr_context(struct i915_gem_context *ctx,
                     struct drm_i915_gem_object *ctx_obj,
                     struct intel_engine_cs *engine,
                     struct intel_ringbuffer *ringbuf)
 {
-        struct drm_device *dev = engine->dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = ctx->i915;
         struct i915_hw_ppgtt *ppgtt = ctx->ppgtt;
         void *vaddr;
         u32 *reg_state;
@@ -2391,7 +2349,7 @@ populate_lr_context(struct intel_context *ctx,
                        RING_CONTEXT_CONTROL(engine),
                        _MASKED_BIT_ENABLE(CTX_CTRL_INHIBIT_SYN_CTX_SWITCH |
                                           CTX_CTRL_ENGINE_CTX_RESTORE_INHIBIT |
-                                          (HAS_RESOURCE_STREAMER(dev) ?
+                                          (HAS_RESOURCE_STREAMER(dev_priv) ?
                                             CTX_CTRL_RS_CTX_ENABLE : 0)));
         ASSIGN_CTX_REG(reg_state, CTX_RING_HEAD, RING_HEAD(engine->mmio_base),
                        0);
@@ -2480,7 +2438,7 @@ populate_lr_context(struct intel_context *ctx,
         if (engine->id == RCS) {
                 reg_state[CTX_LRI_HEADER_2] = MI_LOAD_REGISTER_IMM(1);
                 ASSIGN_CTX_REG(reg_state, CTX_R_PWR_CLK_STATE, GEN8_R_PWR_CLK_STATE,
-                               make_rpcs(dev));
+                               make_rpcs(dev_priv));
         }
 
         i915_gem_object_unpin_map(ctx_obj);
@@ -2489,39 +2447,8 @@ populate_lr_context(struct intel_context *ctx,
 }
 
 /**
- * intel_lr_context_free() - free the LRC specific bits of a context
- * @ctx: the LR context to free.
- *
- * The real context freeing is done in i915_gem_context_free: this only
- * takes care of the bits that are LRC related: the per-engine backing
- * objects and the logical ringbuffer.
- */
-void intel_lr_context_free(struct intel_context *ctx)
-{
-        int i;
-
-        for (i = I915_NUM_ENGINES; --i >= 0; ) {
-                struct intel_ringbuffer *ringbuf = ctx->engine[i].ringbuf;
-                struct drm_i915_gem_object *ctx_obj = ctx->engine[i].state;
-
-                if (!ctx_obj)
-                        continue;
-
-                if (ctx == ctx->i915->kernel_context) {
-                        intel_unpin_ringbuffer_obj(ringbuf);
-                        i915_gem_object_ggtt_unpin(ctx_obj);
-                        i915_gem_object_unpin_map(ctx_obj);
-                }
-
-                WARN_ON(ctx->engine[i].pin_count);
-                intel_ringbuffer_free(ringbuf);
-                drm_gem_object_unreference(&ctx_obj->base);
-        }
-}
-
-/**
  * intel_lr_context_size() - return the size of the context for an engine
- * @ring: which engine to find the context size for
+ * @engine: which engine to find the context size for
  *
  * Each engine may require a different amount of space for a context image,
  * so when allocating (or copying) an image, this function can be used to
@@ -2537,11 +2464,11 @@ uint32_t intel_lr_context_size(struct intel_engine_cs *engine)
 {
         int ret = 0;
 
-        WARN_ON(INTEL_INFO(engine->dev)->gen < 8);
+        WARN_ON(INTEL_GEN(engine->i915) < 8);
 
         switch (engine->id) {
         case RCS:
-                if (INTEL_INFO(engine->dev)->gen >= 9)
+                if (INTEL_GEN(engine->i915) >= 9)
                         ret = GEN9_LR_CONTEXT_RENDER_SIZE;
                 else
                         ret = GEN8_LR_CONTEXT_RENDER_SIZE;
@@ -2558,9 +2485,9 @@ uint32_t intel_lr_context_size(struct intel_engine_cs *engine)
 }
 
 /**
- * intel_lr_context_deferred_alloc() - create the LRC specific bits of a context
+ * execlists_context_deferred_alloc() - create the LRC specific bits of a context
  * @ctx: LR context to create.
- * @ring: engine to be used with the context.
+ * @engine: engine to be used with the context.
  *
  * This function can be called more than once, with different engines, if we plan
  * to use the context with them. The context backing objects and the ringbuffers
@@ -2570,31 +2497,29 @@ uint32_t intel_lr_context_size(struct intel_engine_cs *engine)
  *
  * Return: non-zero on error.
  */
-
-int intel_lr_context_deferred_alloc(struct intel_context *ctx,
-                                    struct intel_engine_cs *engine)
+static int execlists_context_deferred_alloc(struct i915_gem_context *ctx,
+                                            struct intel_engine_cs *engine)
 {
-        struct drm_device *dev = engine->dev;
         struct drm_i915_gem_object *ctx_obj;
+        struct intel_context *ce = &ctx->engine[engine->id];
         uint32_t context_size;
         struct intel_ringbuffer *ringbuf;
         int ret;
 
-        WARN_ON(ctx->legacy_hw_ctx.rcs_state != NULL);
-        WARN_ON(ctx->engine[engine->id].state);
+        WARN_ON(ce->state);
 
         context_size = round_up(intel_lr_context_size(engine), 4096);
 
         /* One extra page as the sharing data between driver and GuC */
         context_size += PAGE_SIZE * LRC_PPHWSP_PN;
 
-        ctx_obj = i915_gem_alloc_object(dev, context_size);
-        if (!ctx_obj) {
+        ctx_obj = i915_gem_object_create(&ctx->i915->drm, context_size);
+        if (IS_ERR(ctx_obj)) {
                 DRM_DEBUG_DRIVER("Alloc LRC backing obj failed.\n");
-                return -ENOMEM;
+                return PTR_ERR(ctx_obj);
         }
 
-        ringbuf = intel_engine_create_ringbuffer(engine, 4 * PAGE_SIZE);
+        ringbuf = intel_engine_create_ringbuffer(engine, ctx->ring_size);
         if (IS_ERR(ringbuf)) {
                 ret = PTR_ERR(ringbuf);
                 goto error_deref_obj;
@@ -2606,48 +2531,29 @@ int intel_lr_context_deferred_alloc(struct intel_context *ctx,
                 goto error_ringbuf;
         }
 
-        ctx->engine[engine->id].ringbuf = ringbuf;
-        ctx->engine[engine->id].state = ctx_obj;
+        ce->ringbuf = ringbuf;
+        ce->state = ctx_obj;
+        ce->initialised = engine->init_context == NULL;
 
-        if (ctx != ctx->i915->kernel_context && engine->init_context) {
-                struct drm_i915_gem_request *req;
-
-                req = i915_gem_request_alloc(engine, ctx);
-                if (IS_ERR(req)) {
-                        ret = PTR_ERR(req);
-                        DRM_ERROR("ring create req: %d\n", ret);
-                        goto error_ringbuf;
-                }
-
-                ret = engine->init_context(req);
-                i915_add_request_no_flush(req);
-                if (ret) {
-                        DRM_ERROR("ring init context: %d\n",
-                                ret);
-                        goto error_ringbuf;
-                }
-        }
         return 0;
 
 error_ringbuf:
         intel_ringbuffer_free(ringbuf);
 error_deref_obj:
         drm_gem_object_unreference(&ctx_obj->base);
-        ctx->engine[engine->id].ringbuf = NULL;
-        ctx->engine[engine->id].state = NULL;
+        ce->ringbuf = NULL;
+        ce->state = NULL;
         return ret;
 }
 
 void intel_lr_context_reset(struct drm_i915_private *dev_priv,
-                            struct intel_context *ctx)
+                            struct i915_gem_context *ctx)
 {
         struct intel_engine_cs *engine;
 
         for_each_engine(engine, dev_priv) {
-                struct drm_i915_gem_object *ctx_obj =
-                                ctx->engine[engine->id].state;
-                struct intel_ringbuffer *ringbuf =
-                                ctx->engine[engine->id].ringbuf;
+                struct intel_context *ce = &ctx->engine[engine->id];
+                struct drm_i915_gem_object *ctx_obj = ce->state;
                 void *vaddr;
                 uint32_t *reg_state;
 
@@ -2666,7 +2572,7 @@ void intel_lr_context_reset(struct drm_i915_private *dev_priv,
 
                 i915_gem_object_unpin_map(ctx_obj);
 
-                ringbuf->head = 0;
-                ringbuf->tail = 0;
+                ce->ringbuf->head = 0;
+                ce->ringbuf->tail = 0;
         }
 }
diff --git a/drivers/gpu/drm/i915/intel_lrc.h b/drivers/gpu/drm/i915/intel_lrc.h
index 60a7385bc531..2b8255c19dcc 100644
--- a/drivers/gpu/drm/i915/intel_lrc.h
+++ b/drivers/gpu/drm/i915/intel_lrc.h
@@ -57,6 +57,11 @@
 #define GEN8_CSB_READ_PTR(csb_status) \
         (((csb_status) & GEN8_CSB_READ_PTR_MASK) >> 8)
 
+enum {
+        INTEL_CONTEXT_SCHEDULE_IN = 0,
+        INTEL_CONTEXT_SCHEDULE_OUT,
+};
+
 /* Logical Rings */
 int intel_logical_ring_alloc_request_extras(struct drm_i915_gem_request *request);
 int intel_logical_ring_reserve_space(struct drm_i915_gem_request *request);
@@ -99,30 +104,27 @@ static inline void intel_logical_ring_emit_reg(struct intel_ringbuffer *ringbuf,
 #define LRC_PPHWSP_PN   (LRC_GUCSHR_PN + 1)
 #define LRC_STATE_PN    (LRC_PPHWSP_PN + 1)
 
-void intel_lr_context_free(struct intel_context *ctx);
+struct i915_gem_context;
+
 uint32_t intel_lr_context_size(struct intel_engine_cs *engine);
-int intel_lr_context_deferred_alloc(struct intel_context *ctx,
-                                    struct intel_engine_cs *engine);
-void intel_lr_context_unpin(struct intel_context *ctx,
+void intel_lr_context_unpin(struct i915_gem_context *ctx,
                             struct intel_engine_cs *engine);
 
 struct drm_i915_private;
 
 void intel_lr_context_reset(struct drm_i915_private *dev_priv,
-                            struct intel_context *ctx);
-uint64_t intel_lr_context_descriptor(struct intel_context *ctx,
+                            struct i915_gem_context *ctx);
+uint64_t intel_lr_context_descriptor(struct i915_gem_context *ctx,
                                      struct intel_engine_cs *engine);
 
-u32 intel_execlists_ctx_id(struct intel_context *ctx,
-                           struct intel_engine_cs *engine);
-
 /* Execlists */
-int intel_sanitize_enable_execlists(struct drm_device *dev, int enable_execlists);
+int intel_sanitize_enable_execlists(struct drm_i915_private *dev_priv,
+                                    int enable_execlists);
 struct i915_execbuffer_params;
 int intel_execlists_submission(struct i915_execbuffer_params *params,
                                struct drm_i915_gem_execbuffer2 *args,
                                struct list_head *vmas);
 
-void intel_execlists_retire_requests(struct intel_engine_cs *engine);
+void intel_execlists_cancel_requests(struct intel_engine_cs *engine);
 
 #endif /* _INTEL_LRC_H_ */
diff --git a/drivers/gpu/drm/i915/intel_lvds.c b/drivers/gpu/drm/i915/intel_lvds.c
index 96281e628d2a..49550470483e 100644
--- a/drivers/gpu/drm/i915/intel_lvds.c
+++ b/drivers/gpu/drm/i915/intel_lvds.c
@@ -72,7 +72,7 @@ static bool intel_lvds_get_hw_state(struct intel_encoder *encoder,
                                     enum pipe *pipe)
 {
         struct drm_device *dev = encoder->base.dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         struct intel_lvds_encoder *lvds_encoder = to_lvds_encoder(&encoder->base);
         enum intel_display_power_domain power_domain;
         u32 tmp;
@@ -106,7 +106,7 @@ static void intel_lvds_get_config(struct intel_encoder *encoder,
                                   struct intel_crtc_state *pipe_config)
 {
         struct drm_device *dev = encoder->base.dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         struct intel_lvds_encoder *lvds_encoder = to_lvds_encoder(&encoder->base);
         u32 tmp, flags = 0;
 
@@ -140,7 +140,7 @@ static void intel_pre_enable_lvds(struct intel_encoder *encoder)
 {
         struct intel_lvds_encoder *lvds_encoder = to_lvds_encoder(&encoder->base);
         struct drm_device *dev = encoder->base.dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         struct intel_crtc *crtc = to_intel_crtc(encoder->base.crtc);
         const struct drm_display_mode *adjusted_mode = &crtc->config->base.adjusted_mode;
         int pipe = crtc->pipe;
@@ -184,13 +184,13 @@ static void intel_pre_enable_lvds(struct intel_encoder *encoder)
          * panels behave in the two modes. For now, let's just maintain the
          * value we got from the BIOS.
          */
-         temp &= ~LVDS_A3_POWER_MASK;
-         temp |= lvds_encoder->a3_power;
+        temp &= ~LVDS_A3_POWER_MASK;
+        temp |= lvds_encoder->a3_power;
 
         /* Set the dithering flag on LVDS as needed, note that there is no
          * special lvds dither control bit on pch-split platforms, dithering is
          * only controlled through the PIPECONF reg. */
-        if (INTEL_INFO(dev)->gen == 4) {
+        if (IS_GEN4(dev_priv)) {
                 /* Bspec wording suggests that LVDS port dithering only exists
                  * for 18bpp panels. */
                 if (crtc->config->dither && crtc->config->pipe_bpp == 18)
@@ -216,7 +216,7 @@ static void intel_enable_lvds(struct intel_encoder *encoder)
         struct intel_lvds_encoder *lvds_encoder = to_lvds_encoder(&encoder->base);
         struct intel_connector *intel_connector =
                 &lvds_encoder->attached_connector->base;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         i915_reg_t ctl_reg, stat_reg;
 
         if (HAS_PCH_SPLIT(dev)) {
@@ -231,7 +231,7 @@ static void intel_enable_lvds(struct intel_encoder *encoder)
 
         I915_WRITE(ctl_reg, I915_READ(ctl_reg) | POWER_TARGET_ON);
         POSTING_READ(lvds_encoder->reg);
-        if (wait_for((I915_READ(stat_reg) & PP_ON) != 0, 1000))
+        if (intel_wait_for_register(dev_priv, stat_reg, PP_ON, PP_ON, 1000))
                 DRM_ERROR("timed out waiting for panel to power on\n");
 
         intel_panel_enable_backlight(intel_connector);
@@ -241,7 +241,7 @@ static void intel_disable_lvds(struct intel_encoder *encoder)
 {
         struct drm_device *dev = encoder->base.dev;
         struct intel_lvds_encoder *lvds_encoder = to_lvds_encoder(&encoder->base);
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         i915_reg_t ctl_reg, stat_reg;
 
         if (HAS_PCH_SPLIT(dev)) {
@@ -253,7 +253,7 @@ static void intel_disable_lvds(struct intel_encoder *encoder)
         }
 
         I915_WRITE(ctl_reg, I915_READ(ctl_reg) & ~POWER_TARGET_ON);
-        if (wait_for((I915_READ(stat_reg) & PP_ON) == 0, 1000))
+        if (intel_wait_for_register(dev_priv, stat_reg, PP_ON, 0, 1000))
                 DRM_ERROR("timed out waiting for panel to power off\n");
 
         I915_WRITE(lvds_encoder->reg, I915_READ(lvds_encoder->reg) & ~LVDS_PORT_EN);
@@ -442,7 +442,7 @@ static int intel_lid_notify(struct notifier_block *nb, unsigned long val,
                 container_of(nb, struct intel_lvds_connector, lid_notifier);
         struct drm_connector *connector = &lvds_connector->base.base;
         struct drm_device *dev = connector->dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
 
         if (dev->switch_power_state != DRM_SWITCH_POWER_ON)
                 return NOTIFY_OK;
@@ -547,7 +547,6 @@ static int intel_lvds_set_property(struct drm_connector *connector,
 static const struct drm_connector_helper_funcs intel_lvds_connector_helper_funcs = {
         .get_modes = intel_lvds_get_modes,
         .mode_valid = intel_lvds_mode_valid,
-        .best_encoder = intel_best_encoder,
 };
 
 static const struct drm_connector_funcs intel_lvds_connector_funcs = {
@@ -556,6 +555,8 @@ static const struct drm_connector_funcs intel_lvds_connector_funcs = {
         .fill_modes = drm_helper_probe_single_connector_modes,
         .set_property = intel_lvds_set_property,
         .atomic_get_property = intel_connector_atomic_get_property,
+        .late_register = intel_connector_register,
+        .early_unregister = intel_connector_unregister,
         .destroy = intel_lvds_destroy,
         .atomic_destroy_state = drm_atomic_helper_connector_destroy_state,
         .atomic_duplicate_state = drm_atomic_helper_connector_duplicate_state,
@@ -810,27 +811,29 @@ static const struct dmi_system_id intel_dual_link_lvds[] = {
         { }     /* terminating entry */
 };
 
-bool intel_is_dual_link_lvds(struct drm_device *dev)
+struct intel_encoder *intel_get_lvds_encoder(struct drm_device *dev)
 {
-        struct intel_encoder *encoder;
-        struct intel_lvds_encoder *lvds_encoder;
+        struct intel_encoder *intel_encoder;
 
-        for_each_intel_encoder(dev, encoder) {
-                if (encoder->type == INTEL_OUTPUT_LVDS) {
-                        lvds_encoder = to_lvds_encoder(&encoder->base);
+        for_each_intel_encoder(dev, intel_encoder)
+                if (intel_encoder->type == INTEL_OUTPUT_LVDS)
+                        return intel_encoder;
 
-                        return lvds_encoder->is_dual_link;
-                }
-        }
+        return NULL;
+}
 
-        return false;
+bool intel_is_dual_link_lvds(struct drm_device *dev)
+{
+        struct intel_encoder *encoder = intel_get_lvds_encoder(dev);
+
+        return encoder && to_lvds_encoder(&encoder->base)->is_dual_link;
 }
 
 static bool compute_is_dual_link_lvds(struct intel_lvds_encoder *lvds_encoder)
 {
         struct drm_device *dev = lvds_encoder->base.base.dev;
         unsigned int val;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
 
         /* use the module option value if specified */
         if (i915.lvds_channel_mode > 0)
@@ -880,7 +883,7 @@ static bool intel_lvds_supported(struct drm_device *dev)
  */
 void intel_lvds_init(struct drm_device *dev)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         struct intel_lvds_encoder *lvds_encoder;
         struct intel_encoder *intel_encoder;
         struct intel_lvds_connector *lvds_connector;
@@ -978,7 +981,7 @@ void intel_lvds_init(struct drm_device *dev)
                            DRM_MODE_CONNECTOR_LVDS);
 
         drm_encoder_init(dev, &intel_encoder->base, &intel_lvds_enc_funcs,
-                         DRM_MODE_ENCODER_LVDS, NULL);
+                         DRM_MODE_ENCODER_LVDS, "LVDS");
 
         intel_encoder->enable = intel_enable_lvds;
         intel_encoder->pre_enable = intel_pre_enable_lvds;
@@ -992,7 +995,6 @@ void intel_lvds_init(struct drm_device *dev)
         intel_encoder->get_hw_state = intel_lvds_get_hw_state;
         intel_encoder->get_config = intel_lvds_get_config;
         intel_connector->get_hw_state = intel_connector_get_hw_state;
-        intel_connector->unregister = intel_connector_unregister;
 
         intel_connector_attach_encoder(intel_connector, intel_encoder);
         intel_encoder->type = INTEL_OUTPUT_LVDS;
@@ -1119,6 +1121,7 @@ out:
         mutex_unlock(&dev->mode_config.mutex);
 
         intel_panel_init(&intel_connector->panel, fixed_mode, downclock_mode);
+        intel_panel_setup_backlight(connector, INVALID_PIPE);
 
         lvds_encoder->is_dual_link = compute_is_dual_link_lvds(lvds_encoder);
         DRM_DEBUG_KMS("detected %s-link lvds configuration\n",
@@ -1131,9 +1134,6 @@ out:
                 DRM_DEBUG_KMS("lid notifier registration failed\n");
                 lvds_connector->lid_notifier.notifier_call = NULL;
         }
-        drm_connector_register(connector);
-
-        intel_panel_setup_backlight(connector, INVALID_PIPE);
 
         return;
 
diff --git a/drivers/gpu/drm/i915/intel_mocs.c b/drivers/gpu/drm/i915/intel_mocs.c
index 6ba4bf7f2a89..927825f5b284 100644
--- a/drivers/gpu/drm/i915/intel_mocs.c
+++ b/drivers/gpu/drm/i915/intel_mocs.c
@@ -66,9 +66,10 @@ struct drm_i915_mocs_table {
 #define L3_WB                   3
 
 /* Target cache */
-#define ELLC                    0
-#define LLC                     1
-#define LLC_ELLC                2
+#define LE_TC_PAGETABLE         0
+#define LE_TC_LLC               1
+#define LE_TC_LLC_ELLC          2
+#define LE_TC_LLC_ELLC_ALT      3
 
 /*
  * MOCS tables
@@ -96,34 +97,67 @@ struct drm_i915_mocs_table {
  *       end.
  */
 static const struct drm_i915_mocs_entry skylake_mocs_table[] = {
-        /* { 0x00000009, 0x0010 } */
-        { (LE_CACHEABILITY(LE_UC) | LE_TGT_CACHE(LLC_ELLC) | LE_LRUM(0) |
-           LE_AOM(0) | LE_RSC(0) | LE_SCC(0) | LE_PFM(0) | LE_SCF(0)),
-          (L3_ESC(0) | L3_SCC(0) | L3_CACHEABILITY(L3_UC)) },
-        /* { 0x00000038, 0x0030 } */
-        { (LE_CACHEABILITY(LE_PAGETABLE) | LE_TGT_CACHE(LLC_ELLC) | LE_LRUM(3) |
-           LE_AOM(0) | LE_RSC(0) | LE_SCC(0) | LE_PFM(0) | LE_SCF(0)),
-          (L3_ESC(0) | L3_SCC(0) | L3_CACHEABILITY(L3_WB)) },
-        /* { 0x0000003b, 0x0030 } */
-        { (LE_CACHEABILITY(LE_WB) | LE_TGT_CACHE(LLC_ELLC) | LE_LRUM(3) |
-           LE_AOM(0) | LE_RSC(0) | LE_SCC(0) | LE_PFM(0) | LE_SCF(0)),
-          (L3_ESC(0) | L3_SCC(0) | L3_CACHEABILITY(L3_WB)) }
+        { /* 0x00000009 */
+          .control_value = LE_CACHEABILITY(LE_UC) |
+                           LE_TGT_CACHE(LE_TC_LLC_ELLC) |
+                           LE_LRUM(0) | LE_AOM(0) | LE_RSC(0) | LE_SCC(0) |
+                           LE_PFM(0) | LE_SCF(0),
+
+          /* 0x0010 */
+          .l3cc_value =    L3_ESC(0) | L3_SCC(0) | L3_CACHEABILITY(L3_UC),
+        },
+        {
+          /* 0x00000038 */
+          .control_value = LE_CACHEABILITY(LE_PAGETABLE) |
+                           LE_TGT_CACHE(LE_TC_LLC_ELLC) |
+                           LE_LRUM(3) | LE_AOM(0) | LE_RSC(0) | LE_SCC(0) |
+                           LE_PFM(0) | LE_SCF(0),
+          /* 0x0030 */
+          .l3cc_value =    L3_ESC(0) | L3_SCC(0) | L3_CACHEABILITY(L3_WB),
+        },
+        {
+          /* 0x0000003b */
+          .control_value = LE_CACHEABILITY(LE_WB) |
+                           LE_TGT_CACHE(LE_TC_LLC_ELLC) |
+                           LE_LRUM(3) | LE_AOM(0) | LE_RSC(0) | LE_SCC(0) |
+                           LE_PFM(0) | LE_SCF(0),
+          /* 0x0030 */
+          .l3cc_value =   L3_ESC(0) | L3_SCC(0) | L3_CACHEABILITY(L3_WB),
+        },
 };
 
 /* NOTE: the LE_TGT_CACHE is not used on Broxton */
 static const struct drm_i915_mocs_entry broxton_mocs_table[] = {
-        /* { 0x00000009, 0x0010 } */
-        { (LE_CACHEABILITY(LE_UC) | LE_TGT_CACHE(LLC_ELLC) | LE_LRUM(0) |
-           LE_AOM(0) | LE_RSC(0) | LE_SCC(0) | LE_PFM(0) | LE_SCF(0)),
-          (L3_ESC(0) | L3_SCC(0) | L3_CACHEABILITY(L3_UC)) },
-        /* { 0x00000038, 0x0030 } */
-        { (LE_CACHEABILITY(LE_PAGETABLE) | LE_TGT_CACHE(LLC_ELLC) | LE_LRUM(3) |
-           LE_AOM(0) | LE_RSC(0) | LE_SCC(0) | LE_PFM(0) | LE_SCF(0)),
-          (L3_ESC(0) | L3_SCC(0) | L3_CACHEABILITY(L3_WB)) },
-        /* { 0x0000003b, 0x0030 } */
-        { (LE_CACHEABILITY(LE_WB) | LE_TGT_CACHE(LLC_ELLC) | LE_LRUM(3) |
-           LE_AOM(0) | LE_RSC(0) | LE_SCC(0) | LE_PFM(0) | LE_SCF(0)),
-          (L3_ESC(0) | L3_SCC(0) | L3_CACHEABILITY(L3_WB)) }
+        {
+          /* 0x00000009 */
+          .control_value = LE_CACHEABILITY(LE_UC) |
+                           LE_TGT_CACHE(LE_TC_LLC_ELLC) |
+                           LE_LRUM(0) | LE_AOM(0) | LE_RSC(0) | LE_SCC(0) |
+                           LE_PFM(0) | LE_SCF(0),
+
+          /* 0x0010 */
+          .l3cc_value =    L3_ESC(0) | L3_SCC(0) | L3_CACHEABILITY(L3_UC),
+        },
+        {
+          /* 0x00000038 */
+          .control_value = LE_CACHEABILITY(LE_PAGETABLE) |
+                           LE_TGT_CACHE(LE_TC_LLC_ELLC) |
+                           LE_LRUM(3) | LE_AOM(0) | LE_RSC(0) | LE_SCC(0) |
+                           LE_PFM(0) | LE_SCF(0),
+
+          /* 0x0030 */
+          .l3cc_value =    L3_ESC(0) | L3_SCC(0) | L3_CACHEABILITY(L3_WB),
+        },
+        {
+          /* 0x00000039 */
+          .control_value = LE_CACHEABILITY(LE_UC) |
+                           LE_TGT_CACHE(LE_TC_LLC_ELLC) |
+                           LE_LRUM(3) | LE_AOM(0) | LE_RSC(0) | LE_SCC(0) |
+                           LE_PFM(0) | LE_SCF(0),
+
+          /* 0x0030 */
+          .l3cc_value =    L3_ESC(0) | L3_SCC(0) | L3_CACHEABILITY(L3_WB),
+        },
 };
 
 /**
@@ -156,6 +190,16 @@ static bool get_mocs_settings(struct drm_i915_private *dev_priv,
                           "Platform that should have a MOCS table does not.\n");
         }
 
+        /* WaDisableSkipCaching:skl,bxt,kbl */
+        if (IS_GEN9(dev_priv)) {
+                int i;
+
+                for (i = 0; i < table->size; i++)
+                        if (WARN_ON(table->table[i].l3cc_value &
+                                    (L3_ESC(1) | L3_SCC(0x7))))
+                                return false;
+        }
+
         return result;
 }
 
@@ -189,7 +233,7 @@ static i915_reg_t mocs_register(enum intel_engine_id ring, int index)
  */
 int intel_mocs_init_engine(struct intel_engine_cs *engine)
 {
-        struct drm_i915_private *dev_priv = to_i915(engine->dev);
+        struct drm_i915_private *dev_priv = engine->i915;
         struct drm_i915_mocs_table table;
         unsigned int index;
 
diff --git a/drivers/gpu/drm/i915/intel_modes.c b/drivers/gpu/drm/i915/intel_modes.c
index 38a4c8ce7e63..f2584d0a01ab 100644
--- a/drivers/gpu/drm/i915/intel_modes.c
+++ b/drivers/gpu/drm/i915/intel_modes.c
@@ -82,7 +82,7 @@ void
 intel_attach_force_audio_property(struct drm_connector *connector)
 {
         struct drm_device *dev = connector->dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         struct drm_property *prop;
 
         prop = dev_priv->force_audio_property;
@@ -109,7 +109,7 @@ void
 intel_attach_broadcast_rgb_property(struct drm_connector *connector)
 {
         struct drm_device *dev = connector->dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         struct drm_property *prop;
 
         prop = dev_priv->broadcast_rgb_property;
diff --git a/drivers/gpu/drm/i915/intel_opregion.c b/drivers/gpu/drm/i915/intel_opregion.c
index 16e209d326b6..adca262d591a 100644
--- a/drivers/gpu/drm/i915/intel_opregion.c
+++ b/drivers/gpu/drm/i915/intel_opregion.c
@@ -232,18 +232,36 @@ struct opregion_asle_ext {
 #define SWSCI_SBCB_POST_VBE_PM          SWSCI_FUNCTION_CODE(SWSCI_SBCB, 19)
 #define SWSCI_SBCB_ENABLE_DISABLE_AUDIO SWSCI_FUNCTION_CODE(SWSCI_SBCB, 21)
 
-#define ACPI_OTHER_OUTPUT (0<<8)
-#define ACPI_VGA_OUTPUT (1<<8)
-#define ACPI_TV_OUTPUT (2<<8)
-#define ACPI_DIGITAL_OUTPUT (3<<8)
-#define ACPI_LVDS_OUTPUT (4<<8)
+/*
+ * ACPI Specification, Revision 5.0, Appendix B.3.2 _DOD (Enumerate All Devices
+ * Attached to the Display Adapter).
+ */
+#define ACPI_DISPLAY_INDEX_SHIFT                0
+#define ACPI_DISPLAY_INDEX_MASK                 (0xf << 0)
+#define ACPI_DISPLAY_PORT_ATTACHMENT_SHIFT      4
+#define ACPI_DISPLAY_PORT_ATTACHMENT_MASK       (0xf << 4)
+#define ACPI_DISPLAY_TYPE_SHIFT                 8
+#define ACPI_DISPLAY_TYPE_MASK                  (0xf << 8)
+#define ACPI_DISPLAY_TYPE_OTHER                 (0 << 8)
+#define ACPI_DISPLAY_TYPE_VGA                   (1 << 8)
+#define ACPI_DISPLAY_TYPE_TV                    (2 << 8)
+#define ACPI_DISPLAY_TYPE_EXTERNAL_DIGITAL      (3 << 8)
+#define ACPI_DISPLAY_TYPE_INTERNAL_DIGITAL      (4 << 8)
+#define ACPI_VENDOR_SPECIFIC_SHIFT              12
+#define ACPI_VENDOR_SPECIFIC_MASK               (0xf << 12)
+#define ACPI_BIOS_CAN_DETECT                    (1 << 16)
+#define ACPI_DEPENDS_ON_VGA                     (1 << 17)
+#define ACPI_PIPE_ID_SHIFT                      18
+#define ACPI_PIPE_ID_MASK                       (7 << 18)
+#define ACPI_DEVICE_ID_SCHEME                   (1 << 31)
 
 #define MAX_DSLP        1500
 
-static int swsci(struct drm_device *dev, u32 function, u32 parm, u32 *parm_out)
+static int swsci(struct drm_i915_private *dev_priv,
+                 u32 function, u32 parm, u32 *parm_out)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
         struct opregion_swsci *swsci = dev_priv->opregion.swsci;
+        struct pci_dev *pdev = dev_priv->drm.pdev;
         u32 main_function, sub_function, scic;
         u16 swsci_val;
         u32 dslp;
@@ -293,16 +311,16 @@ static int swsci(struct drm_device *dev, u32 function, u32 parm, u32 *parm_out)
         swsci->scic = scic;
 
         /* Ensure SCI event is selected and event trigger is cleared. */
-        pci_read_config_word(dev->pdev, SWSCI, &swsci_val);
+        pci_read_config_word(pdev, SWSCI, &swsci_val);
         if (!(swsci_val & SWSCI_SCISEL) || (swsci_val & SWSCI_GSSCIE)) {
                 swsci_val |= SWSCI_SCISEL;
                 swsci_val &= ~SWSCI_GSSCIE;
-                pci_write_config_word(dev->pdev, SWSCI, swsci_val);
+                pci_write_config_word(pdev, SWSCI, swsci_val);
         }
 
         /* Use event trigger to tell bios to check the mail. */
         swsci_val |= SWSCI_GSSCIE;
-        pci_write_config_word(dev->pdev, SWSCI, swsci_val);
+        pci_write_config_word(pdev, SWSCI, swsci_val);
 
         /* Poll for the result. */
 #define C (((scic = swsci->scic) & SWSCI_SCIC_INDICATOR) == 0)
@@ -336,13 +354,13 @@ static int swsci(struct drm_device *dev, u32 function, u32 parm, u32 *parm_out)
 int intel_opregion_notify_encoder(struct intel_encoder *intel_encoder,
                                   bool enable)
 {
-        struct drm_device *dev = intel_encoder->base.dev;
+        struct drm_i915_private *dev_priv = to_i915(intel_encoder->base.dev);
         u32 parm = 0;
         u32 type = 0;
         u32 port;
 
         /* don't care about old stuff for now */
-        if (!HAS_DDI(dev))
+        if (!HAS_DDI(dev_priv))
                 return 0;
 
         if (intel_encoder->type == INTEL_OUTPUT_DSI)
@@ -365,7 +383,7 @@ int intel_opregion_notify_encoder(struct intel_encoder *intel_encoder,
                 type = DISPLAY_TYPE_CRT;
                 break;
         case INTEL_OUTPUT_UNKNOWN:
-        case INTEL_OUTPUT_DISPLAYPORT:
+        case INTEL_OUTPUT_DP:
         case INTEL_OUTPUT_HDMI:
         case INTEL_OUTPUT_DP_MST:
                 type = DISPLAY_TYPE_EXTERNAL_FLAT_PANEL;
@@ -382,7 +400,7 @@ int intel_opregion_notify_encoder(struct intel_encoder *intel_encoder,
 
         parm |= type << (16 + port * 3);
 
-        return swsci(dev, SWSCI_SBCB_DISPLAY_POWER_STATE, parm, NULL);
+        return swsci(dev_priv, SWSCI_SBCB_DISPLAY_POWER_STATE, parm, NULL);
 }
 
 static const struct {
@@ -396,27 +414,28 @@ static const struct {
         { PCI_D3cold,   0x04 },
 };
 
-int intel_opregion_notify_adapter(struct drm_device *dev, pci_power_t state)
+int intel_opregion_notify_adapter(struct drm_i915_private *dev_priv,
+                                  pci_power_t state)
 {
         int i;
 
-        if (!HAS_DDI(dev))
+        if (!HAS_DDI(dev_priv))
                 return 0;
 
         for (i = 0; i < ARRAY_SIZE(power_state_map); i++) {
                 if (state == power_state_map[i].pci_power_state)
-                        return swsci(dev, SWSCI_SBCB_ADAPTER_POWER_STATE,
+                        return swsci(dev_priv, SWSCI_SBCB_ADAPTER_POWER_STATE,
                                      power_state_map[i].parm, NULL);
         }
 
         return -EINVAL;
 }
 
-static u32 asle_set_backlight(struct drm_device *dev, u32 bclp)
+static u32 asle_set_backlight(struct drm_i915_private *dev_priv, u32 bclp)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
         struct intel_connector *connector;
         struct opregion_asle *asle = dev_priv->opregion.asle;
+        struct drm_device *dev = &dev_priv->drm;
 
         DRM_DEBUG_DRIVER("bclp = 0x%08x\n", bclp);
 
@@ -449,7 +468,7 @@ static u32 asle_set_backlight(struct drm_device *dev, u32 bclp)
         return 0;
 }
 
-static u32 asle_set_als_illum(struct drm_device *dev, u32 alsi)
+static u32 asle_set_als_illum(struct drm_i915_private *dev_priv, u32 alsi)
 {
         /* alsi is the current ALS reading in lux. 0 indicates below sensor
            range, 0xffff indicates above sensor range. 1-0xfffe are valid */
@@ -457,13 +476,13 @@ static u32 asle_set_als_illum(struct drm_device *dev, u32 alsi)
         return ASLC_ALS_ILLUM_FAILED;
 }
 
-static u32 asle_set_pwm_freq(struct drm_device *dev, u32 pfmb)
+static u32 asle_set_pwm_freq(struct drm_i915_private *dev_priv, u32 pfmb)
 {
         DRM_DEBUG_DRIVER("PWM freq is not supported\n");
         return ASLC_PWM_FREQ_FAILED;
 }
 
-static u32 asle_set_pfit(struct drm_device *dev, u32 pfit)
+static u32 asle_set_pfit(struct drm_i915_private *dev_priv, u32 pfit)
 {
         /* Panel fitting is currently controlled by the X code, so this is a
            noop until modesetting support works fully */
@@ -471,13 +490,13 @@ static u32 asle_set_pfit(struct drm_device *dev, u32 pfit)
         return ASLC_PFIT_FAILED;
 }
 
-static u32 asle_set_supported_rotation_angles(struct drm_device *dev, u32 srot)
+static u32 asle_set_supported_rotation_angles(struct drm_i915_private *dev_priv, u32 srot)
 {
         DRM_DEBUG_DRIVER("SROT is not supported\n");
         return ASLC_ROTATION_ANGLES_FAILED;
 }
 
-static u32 asle_set_button_array(struct drm_device *dev, u32 iuer)
+static u32 asle_set_button_array(struct drm_i915_private *dev_priv, u32 iuer)
 {
         if (!iuer)
                 DRM_DEBUG_DRIVER("Button array event is not supported (nothing)\n");
@@ -495,7 +514,7 @@ static u32 asle_set_button_array(struct drm_device *dev, u32 iuer)
         return ASLC_BUTTON_ARRAY_FAILED;
 }
 
-static u32 asle_set_convertible(struct drm_device *dev, u32 iuer)
+static u32 asle_set_convertible(struct drm_i915_private *dev_priv, u32 iuer)
 {
         if (iuer & ASLE_IUER_CONVERTIBLE)
                 DRM_DEBUG_DRIVER("Convertible is not supported (clamshell)\n");
@@ -505,7 +524,7 @@ static u32 asle_set_convertible(struct drm_device *dev, u32 iuer)
         return ASLC_CONVERTIBLE_FAILED;
 }
 
-static u32 asle_set_docking(struct drm_device *dev, u32 iuer)
+static u32 asle_set_docking(struct drm_i915_private *dev_priv, u32 iuer)
 {
         if (iuer & ASLE_IUER_DOCKING)
                 DRM_DEBUG_DRIVER("Docking is not supported (docked)\n");
@@ -515,7 +534,7 @@ static u32 asle_set_docking(struct drm_device *dev, u32 iuer)
         return ASLC_DOCKING_FAILED;
 }
 
-static u32 asle_isct_state(struct drm_device *dev)
+static u32 asle_isct_state(struct drm_i915_private *dev_priv)
 {
         DRM_DEBUG_DRIVER("ISCT is not supported\n");
         return ASLC_ISCT_STATE_FAILED;
@@ -527,7 +546,6 @@ static void asle_work(struct work_struct *work)
                 container_of(work, struct intel_opregion, asle_work);
         struct drm_i915_private *dev_priv =
                 container_of(opregion, struct drm_i915_private, opregion);
-        struct drm_device *dev = dev_priv->dev;
         struct opregion_asle *asle = dev_priv->opregion.asle;
         u32 aslc_stat = 0;
         u32 aslc_req;
@@ -544,40 +562,38 @@ static void asle_work(struct work_struct *work)
         }
 
         if (aslc_req & ASLC_SET_ALS_ILLUM)
-                aslc_stat |= asle_set_als_illum(dev, asle->alsi);
+                aslc_stat |= asle_set_als_illum(dev_priv, asle->alsi);
 
         if (aslc_req & ASLC_SET_BACKLIGHT)
-                aslc_stat |= asle_set_backlight(dev, asle->bclp);
+                aslc_stat |= asle_set_backlight(dev_priv, asle->bclp);
 
         if (aslc_req & ASLC_SET_PFIT)
-                aslc_stat |= asle_set_pfit(dev, asle->pfit);
+                aslc_stat |= asle_set_pfit(dev_priv, asle->pfit);
 
         if (aslc_req & ASLC_SET_PWM_FREQ)
-                aslc_stat |= asle_set_pwm_freq(dev, asle->pfmb);
+                aslc_stat |= asle_set_pwm_freq(dev_priv, asle->pfmb);
 
         if (aslc_req & ASLC_SUPPORTED_ROTATION_ANGLES)
-                aslc_stat |= asle_set_supported_rotation_angles(dev,
+                aslc_stat |= asle_set_supported_rotation_angles(dev_priv,
                                                         asle->srot);
 
         if (aslc_req & ASLC_BUTTON_ARRAY)
-                aslc_stat |= asle_set_button_array(dev, asle->iuer);
+                aslc_stat |= asle_set_button_array(dev_priv, asle->iuer);
 
         if (aslc_req & ASLC_CONVERTIBLE_INDICATOR)
-                aslc_stat |= asle_set_convertible(dev, asle->iuer);
+                aslc_stat |= asle_set_convertible(dev_priv, asle->iuer);
 
         if (aslc_req & ASLC_DOCKING_INDICATOR)
-                aslc_stat |= asle_set_docking(dev, asle->iuer);
+                aslc_stat |= asle_set_docking(dev_priv, asle->iuer);
 
         if (aslc_req & ASLC_ISCT_STATE_CHANGE)
-                aslc_stat |= asle_isct_state(dev);
+                aslc_stat |= asle_isct_state(dev_priv);
 
         asle->aslc = aslc_stat;
 }
 
-void intel_opregion_asle_intr(struct drm_device *dev)
+void intel_opregion_asle_intr(struct drm_i915_private *dev_priv)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
-
         if (dev_priv->opregion.asle)
                 schedule_work(&dev_priv->opregion.asle_work);
 }
@@ -658,10 +674,51 @@ static void set_did(struct intel_opregion *opregion, int i, u32 val)
         }
 }
 
-static void intel_didl_outputs(struct drm_device *dev)
+static u32 acpi_display_type(struct drm_connector *connector)
+{
+        u32 display_type;
+
+        switch (connector->connector_type) {
+        case DRM_MODE_CONNECTOR_VGA:
+        case DRM_MODE_CONNECTOR_DVIA:
+                display_type = ACPI_DISPLAY_TYPE_VGA;
+                break;
+        case DRM_MODE_CONNECTOR_Composite:
+        case DRM_MODE_CONNECTOR_SVIDEO:
+        case DRM_MODE_CONNECTOR_Component:
+        case DRM_MODE_CONNECTOR_9PinDIN:
+        case DRM_MODE_CONNECTOR_TV:
+                display_type = ACPI_DISPLAY_TYPE_TV;
+                break;
+        case DRM_MODE_CONNECTOR_DVII:
+        case DRM_MODE_CONNECTOR_DVID:
+        case DRM_MODE_CONNECTOR_DisplayPort:
+        case DRM_MODE_CONNECTOR_HDMIA:
+        case DRM_MODE_CONNECTOR_HDMIB:
+                display_type = ACPI_DISPLAY_TYPE_EXTERNAL_DIGITAL;
+                break;
+        case DRM_MODE_CONNECTOR_LVDS:
+        case DRM_MODE_CONNECTOR_eDP:
+        case DRM_MODE_CONNECTOR_DSI:
+                display_type = ACPI_DISPLAY_TYPE_INTERNAL_DIGITAL;
+                break;
+        case DRM_MODE_CONNECTOR_Unknown:
+        case DRM_MODE_CONNECTOR_VIRTUAL:
+                display_type = ACPI_DISPLAY_TYPE_OTHER;
+                break;
+        default:
+                MISSING_CASE(connector->connector_type);
+                display_type = ACPI_DISPLAY_TYPE_OTHER;
+                break;
+        }
+
+        return display_type;
+}
+
+static void intel_didl_outputs(struct drm_i915_private *dev_priv)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
         struct intel_opregion *opregion = &dev_priv->opregion;
+        struct pci_dev *pdev = dev_priv->drm.pdev;
         struct drm_connector *connector;
         acpi_handle handle;
         struct acpi_device *acpi_dev, *acpi_cdev, *acpi_video_bus = NULL;
@@ -670,7 +727,7 @@ static void intel_didl_outputs(struct drm_device *dev)
         u32 temp, max_outputs;
         int i = 0;
 
-        handle = ACPI_HANDLE(&dev->pdev->dev);
+        handle = ACPI_HANDLE(&pdev->dev);
         if (!handle || acpi_bus_get_device(handle, &acpi_dev))
                 return;
 
@@ -725,45 +782,25 @@ end:
 
 blind_set:
         i = 0;
-        list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
-                int output_type = ACPI_OTHER_OUTPUT;
+        list_for_each_entry(connector,
+                            &dev_priv->drm.mode_config.connector_list, head) {
+                int display_type = acpi_display_type(connector);
+
                 if (i >= max_outputs) {
                         DRM_DEBUG_KMS("More than %u outputs in connector list\n",
                                       max_outputs);
                         return;
                 }
-                switch (connector->connector_type) {
-                case DRM_MODE_CONNECTOR_VGA:
-                case DRM_MODE_CONNECTOR_DVIA:
-                        output_type = ACPI_VGA_OUTPUT;
-                        break;
-                case DRM_MODE_CONNECTOR_Composite:
-                case DRM_MODE_CONNECTOR_SVIDEO:
-                case DRM_MODE_CONNECTOR_Component:
-                case DRM_MODE_CONNECTOR_9PinDIN:
-                        output_type = ACPI_TV_OUTPUT;
-                        break;
-                case DRM_MODE_CONNECTOR_DVII:
-                case DRM_MODE_CONNECTOR_DVID:
-                case DRM_MODE_CONNECTOR_DisplayPort:
-                case DRM_MODE_CONNECTOR_HDMIA:
-                case DRM_MODE_CONNECTOR_HDMIB:
-                        output_type = ACPI_DIGITAL_OUTPUT;
-                        break;
-                case DRM_MODE_CONNECTOR_LVDS:
-                        output_type = ACPI_LVDS_OUTPUT;
-                        break;
-                }
+
                 temp = get_did(opregion, i);
-                set_did(opregion, i, temp | (1 << 31) | output_type | i);
+                set_did(opregion, i, temp | (1 << 31) | display_type | i);
                 i++;
         }
         goto end;
 }
 
-static void intel_setup_cadls(struct drm_device *dev)
+static void intel_setup_cadls(struct drm_i915_private *dev_priv)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
         struct intel_opregion *opregion = &dev_priv->opregion;
         int i = 0;
         u32 disp_id;
@@ -780,17 +817,16 @@ static void intel_setup_cadls(struct drm_device *dev)
         } while (++i < 8 && disp_id != 0);
 }
 
-void intel_opregion_init(struct drm_device *dev)
+void intel_opregion_register(struct drm_i915_private *dev_priv)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
         struct intel_opregion *opregion = &dev_priv->opregion;
 
         if (!opregion->header)
                 return;
 
         if (opregion->acpi) {
-                intel_didl_outputs(dev);
-                intel_setup_cadls(dev);
+                intel_didl_outputs(dev_priv);
+                intel_setup_cadls(dev_priv);
 
                 /* Notify BIOS we are ready to handle ACPI video ext notifs.
                  * Right now, all the events are handled by the ACPI video module.
@@ -808,9 +844,8 @@ void intel_opregion_init(struct drm_device *dev)
         }
 }
 
-void intel_opregion_fini(struct drm_device *dev)
+void intel_opregion_unregister(struct drm_i915_private *dev_priv)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
         struct intel_opregion *opregion = &dev_priv->opregion;
 
         if (!opregion->header)
@@ -842,9 +877,8 @@ void intel_opregion_fini(struct drm_device *dev)
         opregion->lid_state = NULL;
 }
 
-static void swsci_setup(struct drm_device *dev)
+static void swsci_setup(struct drm_i915_private *dev_priv)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
         struct intel_opregion *opregion = &dev_priv->opregion;
         bool requested_callbacks = false;
         u32 tmp;
@@ -854,7 +888,7 @@ static void swsci_setup(struct drm_device *dev)
         opregion->swsci_sbcb_sub_functions = 1;
 
         /* We use GBDA to ask for supported GBDA calls. */
-        if (swsci(dev, SWSCI_GBDA_SUPPORTED_CALLS, 0, &tmp) == 0) {
+        if (swsci(dev_priv, SWSCI_GBDA_SUPPORTED_CALLS, 0, &tmp) == 0) {
                 /* make the bits match the sub-function codes */
                 tmp <<= 1;
                 opregion->swsci_gbda_sub_functions |= tmp;
@@ -865,7 +899,7 @@ static void swsci_setup(struct drm_device *dev)
          * must not call interfaces that are not specifically requested by the
          * bios.
          */
-        if (swsci(dev, SWSCI_GBDA_REQUESTED_CALLBACKS, 0, &tmp) == 0) {
+        if (swsci(dev_priv, SWSCI_GBDA_REQUESTED_CALLBACKS, 0, &tmp) == 0) {
                 /* here, the bits already match sub-function codes */
                 opregion->swsci_sbcb_sub_functions |= tmp;
                 requested_callbacks = true;
@@ -876,7 +910,7 @@ static void swsci_setup(struct drm_device *dev)
          * the callback is _requested_. But we still can't call interfaces that
          * are not requested.
          */
-        if (swsci(dev, SWSCI_SBCB_SUPPORTED_CALLBACKS, 0, &tmp) == 0) {
+        if (swsci(dev_priv, SWSCI_SBCB_SUPPORTED_CALLBACKS, 0, &tmp) == 0) {
                 /* make the bits match the sub-function codes */
                 u32 low = tmp & 0x7ff;
                 u32 high = tmp & ~0xfff; /* bit 11 is reserved */
@@ -918,10 +952,10 @@ static const struct dmi_system_id intel_no_opregion_vbt[] = {
         { }
 };
 
-int intel_opregion_setup(struct drm_device *dev)
+int intel_opregion_setup(struct drm_i915_private *dev_priv)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
         struct intel_opregion *opregion = &dev_priv->opregion;
+        struct pci_dev *pdev = dev_priv->drm.pdev;
         u32 asls, mboxes;
         char buf[sizeof(OPREGION_SIGNATURE)];
         int err = 0;
@@ -933,7 +967,7 @@ int intel_opregion_setup(struct drm_device *dev)
         BUILD_BUG_ON(sizeof(struct opregion_asle) != 0x100);
         BUILD_BUG_ON(sizeof(struct opregion_asle_ext) != 0x400);
 
-        pci_read_config_dword(dev->pdev, ASLS, &asls);
+        pci_read_config_dword(pdev, ASLS, &asls);
         DRM_DEBUG_DRIVER("graphic opregion physical addr: 0x%x\n", asls);
         if (asls == 0) {
                 DRM_DEBUG_DRIVER("ACPI OpRegion not supported!\n");
@@ -965,7 +999,7 @@ int intel_opregion_setup(struct drm_device *dev)
         if (mboxes & MBOX_SWSCI) {
                 DRM_DEBUG_DRIVER("SWSCI supported\n");
                 opregion->swsci = base + OPREGION_SWSCI_OFFSET;
-                swsci_setup(dev);
+                swsci_setup(dev_priv);
         }
 
         if (mboxes & MBOX_ASLE) {
@@ -1014,12 +1048,12 @@ err_out:
 }
 
 int
-intel_opregion_get_panel_type(struct drm_device *dev)
+intel_opregion_get_panel_type(struct drm_i915_private *dev_priv)
 {
         u32 panel_details;
         int ret;
 
-        ret = swsci(dev, SWSCI_GBDA_PANEL_DETAILS, 0x0, &panel_details);
+        ret = swsci(dev_priv, SWSCI_GBDA_PANEL_DETAILS, 0x0, &panel_details);
         if (ret) {
                 DRM_DEBUG_KMS("Failed to get panel details from OpRegion (%d)\n",
                               ret);
@@ -1044,7 +1078,7 @@ intel_opregion_get_panel_type(struct drm_device *dev)
          * vswing instead. Low vswing results in some display flickers, so
          * let's simply ignore the OpRegion panel type on SKL for now.
          */
-        if (IS_SKYLAKE(dev)) {
+        if (IS_SKYLAKE(dev_priv)) {
                 DRM_DEBUG_KMS("Ignoring OpRegion panel type (%d)\n", ret - 1);
                 return -ENODEV;
         }
diff --git a/drivers/gpu/drm/i915/intel_overlay.c b/drivers/gpu/drm/i915/intel_overlay.c
index bd38e49f7334..3212d8806b5a 100644
--- a/drivers/gpu/drm/i915/intel_overlay.c
+++ b/drivers/gpu/drm/i915/intel_overlay.c
@@ -168,7 +168,7 @@ struct overlay_registers {
 };
 
 struct intel_overlay {
-        struct drm_device *dev;
+        struct drm_i915_private *i915;
         struct intel_crtc *crtc;
         struct drm_i915_gem_object *vid_bo;
         struct drm_i915_gem_object *old_vid_bo;
@@ -190,15 +190,15 @@ struct intel_overlay {
 static struct overlay_registers __iomem *
 intel_overlay_map_regs(struct intel_overlay *overlay)
 {
-        struct drm_i915_private *dev_priv = to_i915(overlay->dev);
-        struct i915_ggtt *ggtt = &dev_priv->ggtt;
+        struct drm_i915_private *dev_priv = overlay->i915;
         struct overlay_registers __iomem *regs;
 
-        if (OVERLAY_NEEDS_PHYSICAL(overlay->dev))
+        if (OVERLAY_NEEDS_PHYSICAL(dev_priv))
                 regs = (struct overlay_registers __iomem *)overlay->reg_bo->phys_handle->vaddr;
         else
-                regs = io_mapping_map_wc(ggtt->mappable,
-                                         i915_gem_obj_ggtt_offset(overlay->reg_bo));
+                regs = io_mapping_map_wc(dev_priv->ggtt.mappable,
+                                         overlay->flip_addr,
+                                         PAGE_SIZE);
 
         return regs;
 }
@@ -206,7 +206,7 @@ intel_overlay_map_regs(struct intel_overlay *overlay)
 static void intel_overlay_unmap_regs(struct intel_overlay *overlay,
                                      struct overlay_registers __iomem *regs)
 {
-        if (!OVERLAY_NEEDS_PHYSICAL(overlay->dev))
+        if (!OVERLAY_NEEDS_PHYSICAL(overlay->i915))
                 io_mapping_unmap(regs);
 }
 
@@ -232,14 +232,13 @@ static int intel_overlay_do_wait_request(struct intel_overlay *overlay,
 /* overlay needs to be disable in OCMD reg */
 static int intel_overlay_on(struct intel_overlay *overlay)
 {
-        struct drm_device *dev = overlay->dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = overlay->i915;
         struct intel_engine_cs *engine = &dev_priv->engine[RCS];
         struct drm_i915_gem_request *req;
         int ret;
 
         WARN_ON(overlay->active);
-        WARN_ON(IS_I830(dev) && !(dev_priv->quirks & QUIRK_PIPEA_FORCE));
+        WARN_ON(IS_I830(dev_priv) && !(dev_priv->quirks & QUIRK_PIPEA_FORCE));
 
         req = i915_gem_request_alloc(engine, NULL);
         if (IS_ERR(req))
@@ -266,8 +265,7 @@ static int intel_overlay_on(struct intel_overlay *overlay)
 static int intel_overlay_continue(struct intel_overlay *overlay,
                                   bool load_polyphase_filter)
 {
-        struct drm_device *dev = overlay->dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = overlay->i915;
         struct intel_engine_cs *engine = &dev_priv->engine[RCS];
         struct drm_i915_gem_request *req;
         u32 flip_addr = overlay->flip_addr;
@@ -335,8 +333,7 @@ static void intel_overlay_off_tail(struct intel_overlay *overlay)
 /* overlay needs to be disabled in OCMD reg */
 static int intel_overlay_off(struct intel_overlay *overlay)
 {
-        struct drm_device *dev = overlay->dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = overlay->i915;
         struct intel_engine_cs *engine = &dev_priv->engine[RCS];
         struct drm_i915_gem_request *req;
         u32 flip_addr = overlay->flip_addr;
@@ -365,7 +362,7 @@ static int intel_overlay_off(struct intel_overlay *overlay)
         intel_ring_emit(engine, flip_addr);
         intel_ring_emit(engine, MI_WAIT_FOR_EVENT | MI_WAIT_FOR_OVERLAY_FLIP);
         /* turn overlay off */
-        if (IS_I830(dev)) {
+        if (IS_I830(dev_priv)) {
                 /* Workaround: Don't disable the overlay fully, since otherwise
                  * it dies on the next OVERLAY_ON cmd. */
                 intel_ring_emit(engine, MI_NOOP);
@@ -408,12 +405,11 @@ static int intel_overlay_recover_from_interrupt(struct intel_overlay *overlay)
  */
 static int intel_overlay_release_old_vid(struct intel_overlay *overlay)
 {
-        struct drm_device *dev = overlay->dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = overlay->i915;
         struct intel_engine_cs *engine = &dev_priv->engine[RCS];
         int ret;
 
-        WARN_ON(!mutex_is_locked(&dev->struct_mutex));
+        lockdep_assert_held(&dev_priv->drm.struct_mutex);
 
         /* Only wait if there is actually an old frame to release to
          * guarantee forward progress.
@@ -537,10 +533,10 @@ static int uv_vsubsampling(u32 format)
         }
 }
 
-static u32 calc_swidthsw(struct drm_device *dev, u32 offset, u32 width)
+static u32 calc_swidthsw(struct drm_i915_private *dev_priv, u32 offset, u32 width)
 {
         u32 mask, shift, ret;
-        if (IS_GEN2(dev)) {
+        if (IS_GEN2(dev_priv)) {
                 mask = 0x1f;
                 shift = 5;
         } else {
@@ -548,7 +544,7 @@ static u32 calc_swidthsw(struct drm_device *dev, u32 offset, u32 width)
                 shift = 6;
         }
         ret = ((offset + width + mask) >> shift) - (offset >> shift);
-        if (!IS_GEN2(dev))
+        if (!IS_GEN2(dev_priv))
                 ret <<= 1;
         ret -= 1;
         return ret << 2;
@@ -741,12 +737,12 @@ static int intel_overlay_do_put_image(struct intel_overlay *overlay,
         int ret, tmp_width;
         struct overlay_registers __iomem *regs;
         bool scale_changed = false;
-        struct drm_device *dev = overlay->dev;
+        struct drm_i915_private *dev_priv = overlay->i915;
         u32 swidth, swidthsw, sheight, ostride;
         enum pipe pipe = overlay->crtc->pipe;
 
-        WARN_ON(!mutex_is_locked(&dev->struct_mutex));
-        WARN_ON(!drm_modeset_is_locked(&dev->mode_config.connection_mutex));
+        lockdep_assert_held(&dev_priv->drm.struct_mutex);
+        WARN_ON(!drm_modeset_is_locked(&dev_priv->drm.mode_config.connection_mutex));
 
         ret = intel_overlay_release_old_vid(overlay);
         if (ret != 0)
@@ -769,7 +765,7 @@ static int intel_overlay_do_put_image(struct intel_overlay *overlay,
                         goto out_unpin;
                 }
                 oconfig = OCONF_CC_OUT_8BIT;
-                if (IS_GEN4(overlay->dev))
+                if (IS_GEN4(dev_priv))
                         oconfig |= OCONF_CSC_MODE_BT709;
                 oconfig |= pipe == 0 ?
                         OCONF_PIPE_A : OCONF_PIPE_B;
@@ -796,7 +792,7 @@ static int intel_overlay_do_put_image(struct intel_overlay *overlay,
                 tmp_width = params->src_w;
 
         swidth = params->src_w;
-        swidthsw = calc_swidthsw(overlay->dev, params->offset_Y, tmp_width);
+        swidthsw = calc_swidthsw(dev_priv, params->offset_Y, tmp_width);
         sheight = params->src_h;
         iowrite32(i915_gem_obj_ggtt_offset(new_bo) + params->offset_Y, &regs->OBUF_0Y);
         ostride = params->stride_Y;
@@ -806,9 +802,9 @@ static int intel_overlay_do_put_image(struct intel_overlay *overlay,
                 int uv_vscale = uv_vsubsampling(params->format);
                 u32 tmp_U, tmp_V;
                 swidth |= (params->src_w/uv_hscale) << 16;
-                tmp_U = calc_swidthsw(overlay->dev, params->offset_U,
+                tmp_U = calc_swidthsw(dev_priv, params->offset_U,
                                       params->src_w/uv_hscale);
-                tmp_V = calc_swidthsw(overlay->dev, params->offset_V,
+                tmp_V = calc_swidthsw(dev_priv, params->offset_V,
                                       params->src_w/uv_hscale);
                 swidthsw |= max_t(u32, tmp_U, tmp_V) << 16;
                 sheight |= (params->src_h/uv_vscale) << 16;
@@ -840,7 +836,7 @@ static int intel_overlay_do_put_image(struct intel_overlay *overlay,
         overlay->old_vid_bo = overlay->vid_bo;
         overlay->vid_bo = new_bo;
 
-        intel_frontbuffer_flip(dev,
+        intel_frontbuffer_flip(&dev_priv->drm,
                                INTEL_FRONTBUFFER_OVERLAY(pipe));
 
         return 0;
@@ -852,12 +848,12 @@ out_unpin:
 
 int intel_overlay_switch_off(struct intel_overlay *overlay)
 {
+        struct drm_i915_private *dev_priv = overlay->i915;
         struct overlay_registers __iomem *regs;
-        struct drm_device *dev = overlay->dev;
         int ret;
 
-        WARN_ON(!mutex_is_locked(&dev->struct_mutex));
-        WARN_ON(!drm_modeset_is_locked(&dev->mode_config.connection_mutex));
+        lockdep_assert_held(&dev_priv->drm.struct_mutex);
+        WARN_ON(!drm_modeset_is_locked(&dev_priv->drm.mode_config.connection_mutex));
 
         ret = intel_overlay_recover_from_interrupt(overlay);
         if (ret != 0)
@@ -897,15 +893,14 @@ static int check_overlay_possible_on_crtc(struct intel_overlay *overlay,
 
 static void update_pfit_vscale_ratio(struct intel_overlay *overlay)
 {
-        struct drm_device *dev = overlay->dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = overlay->i915;
         u32 pfit_control = I915_READ(PFIT_CONTROL);
         u32 ratio;
 
         /* XXX: This is not the same logic as in the xorg driver, but more in
          * line with the intel documentation for the i965
          */
-        if (INTEL_INFO(dev)->gen >= 4) {
+        if (INTEL_GEN(dev_priv) >= 4) {
                 /* on i965 use the PGM reg to read out the autoscaler values */
                 ratio = I915_READ(PFIT_PGM_RATIOS) >> PFIT_VERT_SCALE_SHIFT_965;
         } else {
@@ -948,7 +943,7 @@ static int check_overlay_scaling(struct put_image_params *rec)
         return 0;
 }
 
-static int check_overlay_src(struct drm_device *dev,
+static int check_overlay_src(struct drm_i915_private *dev_priv,
                              struct drm_intel_overlay_put_image *rec,
                              struct drm_i915_gem_object *new_bo)
 {
@@ -959,7 +954,7 @@ static int check_overlay_src(struct drm_device *dev,
         u32 tmp;
 
         /* check src dimensions */
-        if (IS_845G(dev) || IS_I830(dev)) {
+        if (IS_845G(dev_priv) || IS_I830(dev_priv)) {
                 if (rec->src_height > IMAGE_MAX_HEIGHT_LEGACY ||
                     rec->src_width  > IMAGE_MAX_WIDTH_LEGACY)
                         return -EINVAL;
@@ -1011,14 +1006,14 @@ static int check_overlay_src(struct drm_device *dev,
                 return -EINVAL;
 
         /* stride checking */
-        if (IS_I830(dev) || IS_845G(dev))
+        if (IS_I830(dev_priv) || IS_845G(dev_priv))
                 stride_mask = 255;
         else
                 stride_mask = 63;
 
         if (rec->stride_Y & stride_mask || rec->stride_UV & stride_mask)
                 return -EINVAL;
-        if (IS_GEN4(dev) && rec->stride_Y < 512)
+        if (IS_GEN4(dev_priv) && rec->stride_Y < 512)
                 return -EINVAL;
 
         tmp = (rec->flags & I915_OVERLAY_TYPE_MASK) == I915_OVERLAY_YUV_PLANAR ?
@@ -1063,13 +1058,13 @@ static int check_overlay_src(struct drm_device *dev,
  * Return the pipe currently connected to the panel fitter,
  * or -1 if the panel fitter is not present or not in use
  */
-static int intel_panel_fitter_pipe(struct drm_device *dev)
+static int intel_panel_fitter_pipe(struct drm_i915_private *dev_priv)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
         u32  pfit_control;
 
         /* i830 doesn't have a panel fitter */
-        if (INTEL_INFO(dev)->gen <= 3 && (IS_I830(dev) || !IS_MOBILE(dev)))
+        if (INTEL_GEN(dev_priv) <= 3 &&
+            (IS_I830(dev_priv) || !IS_MOBILE(dev_priv)))
                 return -1;
 
         pfit_control = I915_READ(PFIT_CONTROL);
@@ -1079,18 +1074,18 @@ static int intel_panel_fitter_pipe(struct drm_device *dev)
                 return -1;
 
         /* 965 can place panel fitter on either pipe */
-        if (IS_GEN4(dev))
+        if (IS_GEN4(dev_priv))
                 return (pfit_control >> 29) & 0x3;
 
         /* older chips can only use pipe 1 */
         return 1;
 }
 
-int intel_overlay_put_image(struct drm_device *dev, void *data,
-                            struct drm_file *file_priv)
+int intel_overlay_put_image_ioctl(struct drm_device *dev, void *data,
+                                  struct drm_file *file_priv)
 {
         struct drm_intel_overlay_put_image *put_image_rec = data;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         struct intel_overlay *overlay;
         struct drm_crtc *drmmode_crtc;
         struct intel_crtc *crtc;
@@ -1162,7 +1157,7 @@ int intel_overlay_put_image(struct drm_device *dev, void *data,
 
                 /* line too wide, i.e. one-line-mode */
                 if (mode->hdisplay > 1024 &&
-                    intel_panel_fitter_pipe(dev) == crtc->pipe) {
+                    intel_panel_fitter_pipe(dev_priv) == crtc->pipe) {
                         overlay->pfit_active = true;
                         update_pfit_vscale_ratio(overlay);
                 } else
@@ -1196,7 +1191,7 @@ int intel_overlay_put_image(struct drm_device *dev, void *data,
                 goto out_unlock;
         }
 
-        ret = check_overlay_src(dev, put_image_rec, new_bo);
+        ret = check_overlay_src(dev_priv, put_image_rec, new_bo);
         if (ret != 0)
                 goto out_unlock;
         params->format = put_image_rec->flags & ~I915_OVERLAY_FLAGS_MASK;
@@ -1284,11 +1279,11 @@ static int check_gamma(struct drm_intel_overlay_attrs *attrs)
         return 0;
 }
 
-int intel_overlay_attrs(struct drm_device *dev, void *data,
-                        struct drm_file *file_priv)
+int intel_overlay_attrs_ioctl(struct drm_device *dev, void *data,
+                              struct drm_file *file_priv)
 {
         struct drm_intel_overlay_attrs *attrs = data;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         struct intel_overlay *overlay;
         struct overlay_registers __iomem *regs;
         int ret;
@@ -1309,7 +1304,7 @@ int intel_overlay_attrs(struct drm_device *dev, void *data,
                 attrs->contrast   = overlay->contrast;
                 attrs->saturation = overlay->saturation;
 
-                if (!IS_GEN2(dev)) {
+                if (!IS_GEN2(dev_priv)) {
                         attrs->gamma0 = I915_READ(OGAMC0);
                         attrs->gamma1 = I915_READ(OGAMC1);
                         attrs->gamma2 = I915_READ(OGAMC2);
@@ -1341,7 +1336,7 @@ int intel_overlay_attrs(struct drm_device *dev, void *data,
                 intel_overlay_unmap_regs(overlay, regs);
 
                 if (attrs->flags & I915_OVERLAY_UPDATE_GAMMA) {
-                        if (IS_GEN2(dev))
+                        if (IS_GEN2(dev_priv))
                                 goto out_unlock;
 
                         if (overlay->active) {
@@ -1371,37 +1366,37 @@ out_unlock:
         return ret;
 }
 
-void intel_setup_overlay(struct drm_device *dev)
+void intel_setup_overlay(struct drm_i915_private *dev_priv)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
         struct intel_overlay *overlay;
         struct drm_i915_gem_object *reg_bo;
         struct overlay_registers __iomem *regs;
         int ret;
 
-        if (!HAS_OVERLAY(dev))
+        if (!HAS_OVERLAY(dev_priv))
                 return;
 
         overlay = kzalloc(sizeof(*overlay), GFP_KERNEL);
         if (!overlay)
                 return;
 
-        mutex_lock(&dev->struct_mutex);
+        mutex_lock(&dev_priv->drm.struct_mutex);
         if (WARN_ON(dev_priv->overlay))
                 goto out_free;
 
-        overlay->dev = dev;
+        overlay->i915 = dev_priv;
 
         reg_bo = NULL;
-        if (!OVERLAY_NEEDS_PHYSICAL(dev))
-                reg_bo = i915_gem_object_create_stolen(dev, PAGE_SIZE);
-        if (reg_bo == NULL)
-                reg_bo = i915_gem_alloc_object(dev, PAGE_SIZE);
+        if (!OVERLAY_NEEDS_PHYSICAL(dev_priv))
+                reg_bo = i915_gem_object_create_stolen(&dev_priv->drm,
+                                                       PAGE_SIZE);
         if (reg_bo == NULL)
+                reg_bo = i915_gem_object_create(&dev_priv->drm, PAGE_SIZE);
+        if (IS_ERR(reg_bo))
                 goto out_free;
         overlay->reg_bo = reg_bo;
 
-        if (OVERLAY_NEEDS_PHYSICAL(dev)) {
+        if (OVERLAY_NEEDS_PHYSICAL(dev_priv)) {
                 ret = i915_gem_object_attach_phys(reg_bo, PAGE_SIZE);
                 if (ret) {
                         DRM_ERROR("failed to attach phys overlay regs\n");
@@ -1441,25 +1436,23 @@ void intel_setup_overlay(struct drm_device *dev)
         intel_overlay_unmap_regs(overlay, regs);
 
         dev_priv->overlay = overlay;
-        mutex_unlock(&dev->struct_mutex);
+        mutex_unlock(&dev_priv->drm.struct_mutex);
         DRM_INFO("initialized overlay support\n");
         return;
 
 out_unpin_bo:
-        if (!OVERLAY_NEEDS_PHYSICAL(dev))
+        if (!OVERLAY_NEEDS_PHYSICAL(dev_priv))
                 i915_gem_object_ggtt_unpin(reg_bo);
 out_free_bo:
         drm_gem_object_unreference(&reg_bo->base);
 out_free:
-        mutex_unlock(&dev->struct_mutex);
+        mutex_unlock(&dev_priv->drm.struct_mutex);
         kfree(overlay);
         return;
 }
 
-void intel_cleanup_overlay(struct drm_device *dev)
+void intel_cleanup_overlay(struct drm_i915_private *dev_priv)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
-
         if (!dev_priv->overlay)
                 return;
 
@@ -1482,18 +1475,17 @@ struct intel_overlay_error_state {
 static struct overlay_registers __iomem *
 intel_overlay_map_regs_atomic(struct intel_overlay *overlay)
 {
-        struct drm_i915_private *dev_priv = to_i915(overlay->dev);
-        struct i915_ggtt *ggtt = &dev_priv->ggtt;
+        struct drm_i915_private *dev_priv = overlay->i915;
         struct overlay_registers __iomem *regs;
 
-        if (OVERLAY_NEEDS_PHYSICAL(overlay->dev))
+        if (OVERLAY_NEEDS_PHYSICAL(dev_priv))
                 /* Cast to make sparse happy, but it's wc memory anyway, so
                  * equivalent to the wc io mapping on X86. */
                 regs = (struct overlay_registers __iomem *)
                         overlay->reg_bo->phys_handle->vaddr;
         else
-                regs = io_mapping_map_atomic_wc(ggtt->mappable,
-                                                i915_gem_obj_ggtt_offset(overlay->reg_bo));
+                regs = io_mapping_map_atomic_wc(dev_priv->ggtt.mappable,
+                                                overlay->flip_addr);
 
         return regs;
 }
@@ -1501,15 +1493,13 @@ intel_overlay_map_regs_atomic(struct intel_overlay *overlay)
 static void intel_overlay_unmap_regs_atomic(struct intel_overlay *overlay,
                                         struct overlay_registers __iomem *regs)
 {
-        if (!OVERLAY_NEEDS_PHYSICAL(overlay->dev))
+        if (!OVERLAY_NEEDS_PHYSICAL(overlay->i915))
                 io_mapping_unmap_atomic(regs);
 }
 
-
 struct intel_overlay_error_state *
-intel_overlay_capture_error_state(struct drm_device *dev)
+intel_overlay_capture_error_state(struct drm_i915_private *dev_priv)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
         struct intel_overlay *overlay = dev_priv->overlay;
         struct intel_overlay_error_state *error;
         struct overlay_registers __iomem *regs;
@@ -1523,10 +1513,7 @@ intel_overlay_capture_error_state(struct drm_device *dev)
 
         error->dovsta = I915_READ(DOVSTA);
         error->isr = I915_READ(ISR);
-        if (OVERLAY_NEEDS_PHYSICAL(overlay->dev))
-                error->base = (__force long)overlay->reg_bo->phys_handle->vaddr;
-        else
-                error->base = i915_gem_obj_ggtt_offset(overlay->reg_bo);
+        error->base = overlay->flip_addr;
 
         regs = intel_overlay_map_regs_atomic(overlay);
         if (!regs)
diff --git a/drivers/gpu/drm/i915/intel_panel.c b/drivers/gpu/drm/i915/intel_panel.c
index aba94099886b..96c65d77e886 100644
--- a/drivers/gpu/drm/i915/intel_panel.c
+++ b/drivers/gpu/drm/i915/intel_panel.c
@@ -377,7 +377,7 @@ out:
 enum drm_connector_status
 intel_panel_detect(struct drm_device *dev)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
 
         /* Assume that the BIOS does not lie through the OpRegion... */
         if (!i915.panel_ignore_lid && dev_priv->opregion.lid_state) {
@@ -504,7 +504,7 @@ static u32 i9xx_get_backlight(struct intel_connector *connector)
         if (panel->backlight.combination_mode) {
                 u8 lbpc;
 
-                pci_read_config_byte(dev_priv->dev->pdev, LBPC, &lbpc);
+                pci_read_config_byte(dev_priv->drm.pdev, LBPC, &lbpc);
                 val *= lbpc;
         }
 
@@ -592,7 +592,7 @@ static void i9xx_set_backlight(struct intel_connector *connector, u32 level)
 
                 lbpc = level * 0xfe / panel->backlight.max + 1;
                 level /= lbpc;
-                pci_write_config_byte(dev_priv->dev->pdev, LBPC, lbpc);
+                pci_write_config_byte(dev_priv->drm.pdev, LBPC, lbpc);
         }
 
         if (IS_GEN4(dev_priv)) {
@@ -822,7 +822,7 @@ void intel_panel_disable_backlight(struct intel_connector *connector)
          * backlight. This will leave the backlight on unnecessarily when
          * another client is not activated.
          */
-        if (dev_priv->dev->switch_power_state == DRM_SWITCH_POWER_CHANGING) {
+        if (dev_priv->drm.switch_power_state == DRM_SWITCH_POWER_CHANGING) {
                 DRM_DEBUG_DRIVER("Skipping backlight disable on vga switch\n");
                 return;
         }
@@ -1142,7 +1142,7 @@ static int intel_backlight_device_get_brightness(struct backlight_device *bd)
 {
         struct intel_connector *connector = bl_get_data(bd);
         struct drm_device *dev = connector->base.dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         u32 hw_level;
         int ret;
 
@@ -1163,7 +1163,7 @@ static const struct backlight_ops intel_backlight_device_ops = {
         .get_brightness = intel_backlight_device_get_brightness,
 };
 
-static int intel_backlight_device_register(struct intel_connector *connector)
+int intel_backlight_device_register(struct intel_connector *connector)
 {
         struct intel_panel *panel = &connector->panel;
         struct backlight_properties props;
@@ -1216,7 +1216,7 @@ static int intel_backlight_device_register(struct intel_connector *connector)
         return 0;
 }
 
-static void intel_backlight_device_unregister(struct intel_connector *connector)
+void intel_backlight_device_unregister(struct intel_connector *connector)
 {
         struct intel_panel *panel = &connector->panel;
 
@@ -1225,14 +1225,6 @@ static void intel_backlight_device_unregister(struct intel_connector *connector)
                 panel->backlight.device = NULL;
         }
 }
-#else /* CONFIG_BACKLIGHT_CLASS_DEVICE */
-static int intel_backlight_device_register(struct intel_connector *connector)
-{
-        return 0;
-}
-static void intel_backlight_device_unregister(struct intel_connector *connector)
-{
-}
 #endif /* CONFIG_BACKLIGHT_CLASS_DEVICE */
 
 /*
@@ -1324,7 +1316,7 @@ static u32 i9xx_hz_to_pwm(struct intel_connector *connector, u32 pwm_freq_hz)
 static u32 i965_hz_to_pwm(struct intel_connector *connector, u32 pwm_freq_hz)
 {
         struct drm_device *dev = connector->base.dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         int clock;
 
         if (IS_G4X(dev_priv))
@@ -1724,6 +1716,14 @@ intel_panel_init_backlight_funcs(struct intel_panel *panel)
                 container_of(panel, struct intel_connector, panel);
         struct drm_i915_private *dev_priv = to_i915(connector->base.dev);
 
+        if (connector->base.connector_type == DRM_MODE_CONNECTOR_eDP &&
+            intel_dp_aux_init_backlight_funcs(connector) == 0)
+                return;
+
+        if (connector->base.connector_type == DRM_MODE_CONNECTOR_DSI &&
+            intel_dsi_dcs_init_backlight_funcs(connector) == 0)
+                return;
+
         if (IS_BROXTON(dev_priv)) {
                 panel->backlight.setup = bxt_setup_backlight;
                 panel->backlight.enable = bxt_enable_backlight;
@@ -1805,19 +1805,3 @@ void intel_panel_fini(struct intel_panel *panel)
                 drm_mode_destroy(intel_connector->base.dev,
                                 panel->downclock_mode);
 }
-
-void intel_backlight_register(struct drm_device *dev)
-{
-        struct intel_connector *connector;
-
-        for_each_intel_connector(dev, connector)
-                intel_backlight_device_register(connector);
-}
-
-void intel_backlight_unregister(struct drm_device *dev)
-{
-        struct intel_connector *connector;
-
-        for_each_intel_connector(dev, connector)
-                intel_backlight_device_unregister(connector);
-}
diff --git a/drivers/gpu/drm/i915/intel_pm.c b/drivers/gpu/drm/i915/intel_pm.c
index 2863b92c9da6..f4f3fcc8b3be 100644
--- a/drivers/gpu/drm/i915/intel_pm.c
+++ b/drivers/gpu/drm/i915/intel_pm.c
@@ -26,6 +26,7 @@
  */
 
 #include <linux/cpufreq.h>
+#include <drm/drm_plane_helper.h>
 #include "i915_drv.h"
 #include "intel_drv.h"
 #include "../../../platform/x86/intel_ips.h"
@@ -82,7 +83,7 @@ static void gen9_init_clock_gating(struct drm_device *dev)
 
 static void bxt_init_clock_gating(struct drm_device *dev)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
 
         gen9_init_clock_gating(dev);
 
@@ -108,7 +109,7 @@ static void bxt_init_clock_gating(struct drm_device *dev)
 
 static void i915_pineview_get_mem_freq(struct drm_device *dev)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         u32 tmp;
 
         tmp = I915_READ(CLKCFG);
@@ -147,7 +148,7 @@ static void i915_pineview_get_mem_freq(struct drm_device *dev)
 
 static void i915_ironlake_get_mem_freq(struct drm_device *dev)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         u16 ddrpll, csipll;
 
         ddrpll = I915_READ16(DDRMPLL1);
@@ -318,7 +319,7 @@ static void chv_set_memory_pm5(struct drm_i915_private *dev_priv, bool enable)
 
 void intel_set_memory_cxsr(struct drm_i915_private *dev_priv, bool enable)
 {
-        struct drm_device *dev = dev_priv->dev;
+        struct drm_device *dev = &dev_priv->drm;
         u32 val;
 
         if (IS_VALLEYVIEW(dev) || IS_CHERRYVIEW(dev)) {
@@ -374,7 +375,7 @@ static const int pessimal_latency_ns = 5000;
 static int vlv_get_fifo_size(struct drm_device *dev,
                               enum pipe pipe, int plane)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         int sprite0_start, sprite1_start, size;
 
         switch (pipe) {
@@ -425,7 +426,7 @@ static int vlv_get_fifo_size(struct drm_device *dev,
 
 static int i9xx_get_fifo_size(struct drm_device *dev, int plane)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         uint32_t dsparb = I915_READ(DSPARB);
         int size;
 
@@ -441,7 +442,7 @@ static int i9xx_get_fifo_size(struct drm_device *dev, int plane)
 
 static int i830_get_fifo_size(struct drm_device *dev, int plane)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         uint32_t dsparb = I915_READ(DSPARB);
         int size;
 
@@ -458,7 +459,7 @@ static int i830_get_fifo_size(struct drm_device *dev, int plane)
 
 static int i845_get_fifo_size(struct drm_device *dev, int plane)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         uint32_t dsparb = I915_READ(DSPARB);
         int size;
 
@@ -636,7 +637,7 @@ static struct drm_crtc *single_enabled_crtc(struct drm_device *dev)
 static void pineview_update_wm(struct drm_crtc *unused_crtc)
 {
         struct drm_device *dev = unused_crtc->dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         struct drm_crtc *crtc;
         const struct cxsr_latency *latency;
         u32 reg;
@@ -933,7 +934,7 @@ static unsigned int vlv_wm_method2(unsigned int pixel_rate,
 
 static void vlv_setup_wm_latency(struct drm_device *dev)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
 
         /* all latencies in usec */
         dev_priv->wm.pri_latency[VLV_WM_LEVEL_PM2] = 3;
@@ -1324,7 +1325,7 @@ static void vlv_merge_wm(struct drm_device *dev,
 static void vlv_update_wm(struct drm_crtc *crtc)
 {
         struct drm_device *dev = crtc->dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
         enum pipe pipe = intel_crtc->pipe;
         struct vlv_wm_values wm = {};
@@ -1380,7 +1381,7 @@ static void g4x_update_wm(struct drm_crtc *crtc)
 {
         struct drm_device *dev = crtc->dev;
         static const int sr_latency_ns = 12000;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         int planea_wm, planeb_wm, cursora_wm, cursorb_wm;
         int plane_sr, cursor_sr;
         unsigned int enabled = 0;
@@ -1437,7 +1438,7 @@ static void g4x_update_wm(struct drm_crtc *crtc)
 static void i965_update_wm(struct drm_crtc *unused_crtc)
 {
         struct drm_device *dev = unused_crtc->dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         struct drm_crtc *crtc;
         int srwm = 1;
         int cursor_sr = 16;
@@ -1511,7 +1512,7 @@ static void i965_update_wm(struct drm_crtc *unused_crtc)
 static void i9xx_update_wm(struct drm_crtc *unused_crtc)
 {
         struct drm_device *dev = unused_crtc->dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         const struct intel_watermark_params *wm_info;
         uint32_t fwater_lo;
         uint32_t fwater_hi;
@@ -1641,7 +1642,7 @@ static void i9xx_update_wm(struct drm_crtc *unused_crtc)
 static void i845_update_wm(struct drm_crtc *unused_crtc)
 {
         struct drm_device *dev = unused_crtc->dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         struct drm_crtc *crtc;
         const struct drm_display_mode *adjusted_mode;
         uint32_t fwater_lo;
@@ -2040,10 +2041,10 @@ static void ilk_compute_wm_level(const struct drm_i915_private *dev_priv,
 }
 
 static uint32_t
-hsw_compute_linetime_wm(struct drm_device *dev,
-                        struct intel_crtc_state *cstate)
+hsw_compute_linetime_wm(const struct intel_crtc_state *cstate)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        const struct intel_atomic_state *intel_state =
+                to_intel_atomic_state(cstate->base.state);
         const struct drm_display_mode *adjusted_mode =
                 &cstate->base.adjusted_mode;
         u32 linetime, ips_linetime;
@@ -2052,7 +2053,7 @@ hsw_compute_linetime_wm(struct drm_device *dev,
                 return 0;
         if (WARN_ON(adjusted_mode->crtc_clock == 0))
                 return 0;
-        if (WARN_ON(dev_priv->cdclk_freq == 0))
+        if (WARN_ON(intel_state->cdclk == 0))
                 return 0;
 
         /* The WM are computed with base on how long it takes to fill a single
@@ -2061,7 +2062,7 @@ hsw_compute_linetime_wm(struct drm_device *dev,
         linetime = DIV_ROUND_CLOSEST(adjusted_mode->crtc_htotal * 1000 * 8,
                                      adjusted_mode->crtc_clock);
         ips_linetime = DIV_ROUND_CLOSEST(adjusted_mode->crtc_htotal * 1000 * 8,
-                                         dev_priv->cdclk_freq);
+                                         intel_state->cdclk);
 
         return PIPE_WM_LINETIME_IPS_LINETIME(ips_linetime) |
                PIPE_WM_LINETIME_TIME(linetime);
@@ -2069,7 +2070,7 @@ hsw_compute_linetime_wm(struct drm_device *dev,
 
 static void intel_read_wm_latency(struct drm_device *dev, uint16_t wm[8])
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
 
         if (IS_GEN9(dev)) {
                 uint32_t val;
@@ -2174,14 +2175,14 @@ static void intel_read_wm_latency(struct drm_device *dev, uint16_t wm[8])
 static void intel_fixup_spr_wm_latency(struct drm_device *dev, uint16_t wm[5])
 {
         /* ILK sprite LP0 latency is 1300 ns */
-        if (INTEL_INFO(dev)->gen == 5)
+        if (IS_GEN5(dev))
                 wm[0] = 13;
 }
 
 static void intel_fixup_cur_wm_latency(struct drm_device *dev, uint16_t wm[5])
 {
         /* ILK cursor LP0 latency is 1300 ns */
-        if (INTEL_INFO(dev)->gen == 5)
+        if (IS_GEN5(dev))
                 wm[0] = 13;
 
         /* WaDoubleCursorLP3Latency:ivb */
@@ -2235,7 +2236,7 @@ static void intel_print_wm_latency(struct drm_device *dev,
 static bool ilk_increase_wm_latency(struct drm_i915_private *dev_priv,
                                     uint16_t wm[5], uint16_t min)
 {
-        int level, max_level = ilk_wm_max_level(dev_priv->dev);
+        int level, max_level = ilk_wm_max_level(&dev_priv->drm);
 
         if (wm[0] >= min)
                 return false;
@@ -2249,7 +2250,7 @@ static bool ilk_increase_wm_latency(struct drm_i915_private *dev_priv,
 
 static void snb_wm_latency_quirk(struct drm_device *dev)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         bool changed;
 
         /*
@@ -2271,7 +2272,7 @@ static void snb_wm_latency_quirk(struct drm_device *dev)
 
 static void ilk_setup_wm_latency(struct drm_device *dev)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
 
         intel_read_wm_latency(dev, dev_priv->wm.pri_latency);
 
@@ -2293,7 +2294,7 @@ static void ilk_setup_wm_latency(struct drm_device *dev)
 
 static void skl_setup_wm_latency(struct drm_device *dev)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
 
         intel_read_wm_latency(dev, dev_priv->wm.skl_latency);
         intel_print_wm_latency(dev, "Gen9 Plane", dev_priv->wm.skl_latency);
@@ -2329,7 +2330,7 @@ static int ilk_compute_pipe_wm(struct intel_crtc_state *cstate)
         struct intel_crtc *intel_crtc = to_intel_crtc(cstate->base.crtc);
         struct intel_pipe_wm *pipe_wm;
         struct drm_device *dev = state->dev;
-        const struct drm_i915_private *dev_priv = dev->dev_private;
+        const struct drm_i915_private *dev_priv = to_i915(dev);
         struct intel_plane *intel_plane;
         struct intel_plane_state *pristate = NULL;
         struct intel_plane_state *sprstate = NULL;
@@ -2337,7 +2338,7 @@ static int ilk_compute_pipe_wm(struct intel_crtc_state *cstate)
         int level, max_level = ilk_wm_max_level(dev), usable_level;
         struct ilk_wm_maximums max;
 
-        pipe_wm = &cstate->wm.optimal.ilk;
+        pipe_wm = &cstate->wm.ilk.optimal;
 
         for_each_intel_plane_on_crtc(dev, intel_crtc, intel_plane) {
                 struct intel_plane_state *ps;
@@ -2380,7 +2381,7 @@ static int ilk_compute_pipe_wm(struct intel_crtc_state *cstate)
         pipe_wm->wm[0] = pipe_wm->raw_wm[0];
 
         if (IS_HASWELL(dev) || IS_BROADWELL(dev))
-                pipe_wm->linetime = hsw_compute_linetime_wm(dev, cstate);
+                pipe_wm->linetime = hsw_compute_linetime_wm(cstate);
 
         if (!ilk_validate_pipe_wm(dev, pipe_wm))
                 return -EINVAL;
@@ -2419,7 +2420,7 @@ static int ilk_compute_intermediate_wm(struct drm_device *dev,
                                        struct intel_crtc *intel_crtc,
                                        struct intel_crtc_state *newstate)
 {
-        struct intel_pipe_wm *a = &newstate->wm.intermediate;
+        struct intel_pipe_wm *a = &newstate->wm.ilk.intermediate;
         struct intel_pipe_wm *b = &intel_crtc->wm.active.ilk;
         int level, max_level = ilk_wm_max_level(dev);
 
@@ -2428,7 +2429,7 @@ static int ilk_compute_intermediate_wm(struct drm_device *dev,
          * currently active watermarks to get values that are safe both before
          * and after the vblank.
          */
-        *a = newstate->wm.optimal.ilk;
+        *a = newstate->wm.ilk.optimal;
         a->pipe_enabled |= b->pipe_enabled;
         a->sprites_enabled |= b->sprites_enabled;
         a->sprites_scaled |= b->sprites_scaled;
@@ -2457,7 +2458,7 @@ static int ilk_compute_intermediate_wm(struct drm_device *dev,
          * If our intermediate WM are identical to the final WM, then we can
          * omit the post-vblank programming; only update if it's different.
          */
-        if (memcmp(a, &newstate->wm.optimal.ilk, sizeof(*a)) == 0)
+        if (memcmp(a, &newstate->wm.ilk.optimal, sizeof(*a)) == 0)
                 newstate->wm.need_postvbl_update = false;
 
         return 0;
@@ -2504,7 +2505,7 @@ static void ilk_wm_merge(struct drm_device *dev,
                          const struct ilk_wm_maximums *max,
                          struct intel_pipe_wm *merged)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         int level, max_level = ilk_wm_max_level(dev);
         int last_enabled_level = max_level;
 
@@ -2564,7 +2565,7 @@ static int ilk_wm_lp_to_level(int wm_lp, const struct intel_pipe_wm *pipe_wm)
 /* The value we need to program into the WM_LPx latency field */
 static unsigned int ilk_wm_lp_latency(struct drm_device *dev, int level)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
 
         if (IS_HASWELL(dev) || IS_BROADWELL(dev))
                 return 2 * level;
@@ -2764,7 +2765,7 @@ static bool _ilk_disable_lp_wm(struct drm_i915_private *dev_priv,
 static void ilk_write_wm_values(struct drm_i915_private *dev_priv,
                                 struct ilk_wm_values *results)
 {
-        struct drm_device *dev = dev_priv->dev;
+        struct drm_device *dev = &dev_priv->drm;
         struct ilk_wm_values *previous = &dev_priv->wm.hw;
         unsigned int dirty;
         uint32_t val;
@@ -2839,7 +2840,7 @@ static void ilk_write_wm_values(struct drm_i915_private *dev_priv,
 
 bool ilk_disable_lp_wm(struct drm_device *dev)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
 
         return _ilk_disable_lp_wm(dev_priv, WM_DIRTY_LP_ALL);
 }
@@ -2877,20 +2878,29 @@ skl_wm_plane_id(const struct intel_plane *plane)
 static void
 skl_ddb_get_pipe_allocation_limits(struct drm_device *dev,
                                    const struct intel_crtc_state *cstate,
-                                   const struct intel_wm_config *config,
-                                   struct skl_ddb_entry *alloc /* out */)
+                                   struct skl_ddb_entry *alloc, /* out */
+                                   int *num_active /* out */)
 {
+        struct drm_atomic_state *state = cstate->base.state;
+        struct intel_atomic_state *intel_state = to_intel_atomic_state(state);
+        struct drm_i915_private *dev_priv = to_i915(dev);
         struct drm_crtc *for_crtc = cstate->base.crtc;
-        struct drm_crtc *crtc;
         unsigned int pipe_size, ddb_size;
         int nth_active_pipe;
+        int pipe = to_intel_crtc(for_crtc)->pipe;
 
-        if (!cstate->base.active) {
+        if (WARN_ON(!state) || !cstate->base.active) {
                 alloc->start = 0;
                 alloc->end = 0;
+                *num_active = hweight32(dev_priv->active_crtcs);
                 return;
         }
 
+        if (intel_state->active_pipe_changes)
+                *num_active = hweight32(intel_state->active_crtcs);
+        else
+                *num_active = hweight32(dev_priv->active_crtcs);
+
         if (IS_BROXTON(dev))
                 ddb_size = BXT_DDB_SIZE;
         else
@@ -2898,25 +2908,29 @@ skl_ddb_get_pipe_allocation_limits(struct drm_device *dev,
 
         ddb_size -= 4; /* 4 blocks for bypass path allocation */
 
-        nth_active_pipe = 0;
-        for_each_crtc(dev, crtc) {
-                if (!to_intel_crtc(crtc)->active)
-                        continue;
-
-                if (crtc == for_crtc)
-                        break;
-
-                nth_active_pipe++;
+        /*
+         * If the state doesn't change the active CRTC's, then there's
+         * no need to recalculate; the existing pipe allocation limits
+         * should remain unchanged.  Note that we're safe from racing
+         * commits since any racing commit that changes the active CRTC
+         * list would need to grab _all_ crtc locks, including the one
+         * we currently hold.
+         */
+        if (!intel_state->active_pipe_changes) {
+                *alloc = dev_priv->wm.skl_hw.ddb.pipe[pipe];
+                return;
         }
 
-        pipe_size = ddb_size / config->num_pipes_active;
-        alloc->start = nth_active_pipe * ddb_size / config->num_pipes_active;
+        nth_active_pipe = hweight32(intel_state->active_crtcs &
+                                    (drm_crtc_mask(for_crtc) - 1));
+        pipe_size = ddb_size / hweight32(intel_state->active_crtcs);
+        alloc->start = nth_active_pipe * ddb_size / *num_active;
         alloc->end = alloc->start + pipe_size;
 }
 
-static unsigned int skl_cursor_allocation(const struct intel_wm_config *config)
+static unsigned int skl_cursor_allocation(int num_active)
 {
-        if (config->num_pipes_active == 1)
+        if (num_active == 1)
                 return 32;
 
         return 8;
@@ -2960,6 +2974,46 @@ void skl_ddb_get_hw_state(struct drm_i915_private *dev_priv,
         }
 }
 
+/*
+ * Determines the downscale amount of a plane for the purposes of watermark calculations.
+ * The bspec defines downscale amount as:
+ *
+ * """
+ * Horizontal down scale amount = maximum[1, Horizontal source size /
+ *                                           Horizontal destination size]
+ * Vertical down scale amount = maximum[1, Vertical source size /
+ *                                         Vertical destination size]
+ * Total down scale amount = Horizontal down scale amount *
+ *                           Vertical down scale amount
+ * """
+ *
+ * Return value is provided in 16.16 fixed point form to retain fractional part.
+ * Caller should take care of dividing & rounding off the value.
+ */
+static uint32_t
+skl_plane_downscale_amount(const struct intel_plane_state *pstate)
+{
+        uint32_t downscale_h, downscale_w;
+        uint32_t src_w, src_h, dst_w, dst_h;
+
+        if (WARN_ON(!pstate->visible))
+                return DRM_PLANE_HELPER_NO_SCALING;
+
+        /* n.b., src is 16.16 fixed point, dst is whole integer */
+        src_w = drm_rect_width(&pstate->src);
+        src_h = drm_rect_height(&pstate->src);
+        dst_w = drm_rect_width(&pstate->dst);
+        dst_h = drm_rect_height(&pstate->dst);
+        if (intel_rotation_90_or_270(pstate->base.rotation))
+                swap(dst_w, dst_h);
+
+        downscale_h = max(src_h / dst_h, (uint32_t)DRM_PLANE_HELPER_NO_SCALING);
+        downscale_w = max(src_w / dst_w, (uint32_t)DRM_PLANE_HELPER_NO_SCALING);
+
+        /* Provide result in 16.16 fixed point */
+        return (uint64_t)downscale_w * downscale_h >> 16;
+}
+
 static unsigned int
 skl_plane_relative_data_rate(const struct intel_crtc_state *cstate,
                              const struct drm_plane_state *pstate,
@@ -2967,7 +3021,16 @@ skl_plane_relative_data_rate(const struct intel_crtc_state *cstate,
 {
         struct intel_plane_state *intel_pstate = to_intel_plane_state(pstate);
         struct drm_framebuffer *fb = pstate->fb;
+        uint32_t down_scale_amount, data_rate;
         uint32_t width = 0, height = 0;
+        unsigned format = fb ? fb->pixel_format : DRM_FORMAT_XRGB8888;
+
+        if (!intel_pstate->visible)
+                return 0;
+        if (pstate->plane->type == DRM_PLANE_TYPE_CURSOR)
+                return 0;
+        if (y && format != DRM_FORMAT_NV12)
+                return 0;
 
         width = drm_rect_width(&intel_pstate->src) >> 16;
         height = drm_rect_height(&intel_pstate->src) >> 16;
@@ -2976,17 +3039,21 @@ skl_plane_relative_data_rate(const struct intel_crtc_state *cstate,
                 swap(width, height);
 
         /* for planar format */
-        if (fb->pixel_format == DRM_FORMAT_NV12) {
+        if (format == DRM_FORMAT_NV12) {
                 if (y)  /* y-plane data rate */
-                        return width * height *
-                                drm_format_plane_cpp(fb->pixel_format, 0);
+                        data_rate = width * height *
+                                drm_format_plane_cpp(format, 0);
                 else    /* uv-plane data rate */
-                        return (width / 2) * (height / 2) *
-                                drm_format_plane_cpp(fb->pixel_format, 1);
+                        data_rate = (width / 2) * (height / 2) *
+                                drm_format_plane_cpp(format, 1);
+        } else {
+                /* for packed formats */
+                data_rate = width * height * drm_format_plane_cpp(format, 0);
         }
 
-        /* for packed formats */
-        return width * height * drm_format_plane_cpp(fb->pixel_format, 0);
+        down_scale_amount = skl_plane_downscale_amount(intel_pstate);
+
+        return (uint64_t)data_rate * down_scale_amount >> 16;
 }
 
 /*
@@ -2995,86 +3062,188 @@ skl_plane_relative_data_rate(const struct intel_crtc_state *cstate,
  *   3 * 4096 * 8192  * 4 < 2^32
  */
 static unsigned int
-skl_get_total_relative_data_rate(const struct intel_crtc_state *cstate)
+skl_get_total_relative_data_rate(struct intel_crtc_state *intel_cstate)
 {
-        struct intel_crtc *intel_crtc = to_intel_crtc(cstate->base.crtc);
-        struct drm_device *dev = intel_crtc->base.dev;
+        struct drm_crtc_state *cstate = &intel_cstate->base;
+        struct drm_atomic_state *state = cstate->state;
+        struct drm_crtc *crtc = cstate->crtc;
+        struct drm_device *dev = crtc->dev;
+        struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
+        const struct drm_plane *plane;
         const struct intel_plane *intel_plane;
-        unsigned int total_data_rate = 0;
+        struct drm_plane_state *pstate;
+        unsigned int rate, total_data_rate = 0;
+        int id;
+        int i;
 
-        for_each_intel_plane_on_crtc(dev, intel_crtc, intel_plane) {
-                const struct drm_plane_state *pstate = intel_plane->base.state;
+        if (WARN_ON(!state))
+                return 0;
 
-                if (pstate->fb == NULL)
-                        continue;
+        /* Calculate and cache data rate for each plane */
+        for_each_plane_in_state(state, plane, pstate, i) {
+                id = skl_wm_plane_id(to_intel_plane(plane));
+                intel_plane = to_intel_plane(plane);
 
-                if (intel_plane->base.type == DRM_PLANE_TYPE_CURSOR)
+                if (intel_plane->pipe != intel_crtc->pipe)
                         continue;
 
                 /* packed/uv */
-                total_data_rate += skl_plane_relative_data_rate(cstate,
-                                                                pstate,
-                                                                0);
+                rate = skl_plane_relative_data_rate(intel_cstate,
+                                                    pstate, 0);
+                intel_cstate->wm.skl.plane_data_rate[id] = rate;
+
+                /* y-plane */
+                rate = skl_plane_relative_data_rate(intel_cstate,
+                                                    pstate, 1);
+                intel_cstate->wm.skl.plane_y_data_rate[id] = rate;
+        }
+
+        /* Calculate CRTC's total data rate from cached values */
+        for_each_intel_plane_on_crtc(dev, intel_crtc, intel_plane) {
+                int id = skl_wm_plane_id(intel_plane);
 
-                if (pstate->fb->pixel_format == DRM_FORMAT_NV12)
-                        /* y-plane */
-                        total_data_rate += skl_plane_relative_data_rate(cstate,
-                                                                        pstate,
-                                                                        1);
+                /* packed/uv */
+                total_data_rate += intel_cstate->wm.skl.plane_data_rate[id];
+                total_data_rate += intel_cstate->wm.skl.plane_y_data_rate[id];
         }
 
+        WARN_ON(cstate->plane_mask && total_data_rate == 0);
+
         return total_data_rate;
 }
 
-static void
+static uint16_t
+skl_ddb_min_alloc(const struct drm_plane_state *pstate,
+                  const int y)
+{
+        struct drm_framebuffer *fb = pstate->fb;
+        struct intel_plane_state *intel_pstate = to_intel_plane_state(pstate);
+        uint32_t src_w, src_h;
+        uint32_t min_scanlines = 8;
+        uint8_t plane_bpp;
+
+        if (WARN_ON(!fb))
+                return 0;
+
+        /* For packed formats, no y-plane, return 0 */
+        if (y && fb->pixel_format != DRM_FORMAT_NV12)
+                return 0;
+
+        /* For Non Y-tile return 8-blocks */
+        if (fb->modifier[0] != I915_FORMAT_MOD_Y_TILED &&
+            fb->modifier[0] != I915_FORMAT_MOD_Yf_TILED)
+                return 8;
+
+        src_w = drm_rect_width(&intel_pstate->src) >> 16;
+        src_h = drm_rect_height(&intel_pstate->src) >> 16;
+
+        if (intel_rotation_90_or_270(pstate->rotation))
+                swap(src_w, src_h);
+
+        /* Halve UV plane width and height for NV12 */
+        if (fb->pixel_format == DRM_FORMAT_NV12 && !y) {
+                src_w /= 2;
+                src_h /= 2;
+        }
+
+        if (fb->pixel_format == DRM_FORMAT_NV12 && !y)
+                plane_bpp = drm_format_plane_cpp(fb->pixel_format, 1);
+        else
+                plane_bpp = drm_format_plane_cpp(fb->pixel_format, 0);
+
+        if (intel_rotation_90_or_270(pstate->rotation)) {
+                switch (plane_bpp) {
+                case 1:
+                        min_scanlines = 32;
+                        break;
+                case 2:
+                        min_scanlines = 16;
+                        break;
+                case 4:
+                        min_scanlines = 8;
+                        break;
+                case 8:
+                        min_scanlines = 4;
+                        break;
+                default:
+                        WARN(1, "Unsupported pixel depth %u for rotation",
+                             plane_bpp);
+                        min_scanlines = 32;
+                }
+        }
+
+        return DIV_ROUND_UP((4 * src_w * plane_bpp), 512) * min_scanlines/4 + 3;
+}
+
+static int
 skl_allocate_pipe_ddb(struct intel_crtc_state *cstate,
                       struct skl_ddb_allocation *ddb /* out */)
 {
+        struct drm_atomic_state *state = cstate->base.state;
         struct drm_crtc *crtc = cstate->base.crtc;
         struct drm_device *dev = crtc->dev;
-        struct drm_i915_private *dev_priv = to_i915(dev);
-        struct intel_wm_config *config = &dev_priv->wm.config;
         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
         struct intel_plane *intel_plane;
+        struct drm_plane *plane;
+        struct drm_plane_state *pstate;
         enum pipe pipe = intel_crtc->pipe;
         struct skl_ddb_entry *alloc = &ddb->pipe[pipe];
         uint16_t alloc_size, start, cursor_blocks;
-        uint16_t minimum[I915_MAX_PLANES];
-        uint16_t y_minimum[I915_MAX_PLANES];
+        uint16_t *minimum = cstate->wm.skl.minimum_blocks;
+        uint16_t *y_minimum = cstate->wm.skl.minimum_y_blocks;
         unsigned int total_data_rate;
+        int num_active;
+        int id, i;
 
-        skl_ddb_get_pipe_allocation_limits(dev, cstate, config, alloc);
+        if (WARN_ON(!state))
+                return 0;
+
+        if (!cstate->base.active) {
+                ddb->pipe[pipe].start = ddb->pipe[pipe].end = 0;
+                memset(ddb->plane[pipe], 0, sizeof(ddb->plane[pipe]));
+                memset(ddb->y_plane[pipe], 0, sizeof(ddb->y_plane[pipe]));
+                return 0;
+        }
+
+        skl_ddb_get_pipe_allocation_limits(dev, cstate, alloc, &num_active);
         alloc_size = skl_ddb_entry_size(alloc);
         if (alloc_size == 0) {
                 memset(ddb->plane[pipe], 0, sizeof(ddb->plane[pipe]));
-                memset(&ddb->plane[pipe][PLANE_CURSOR], 0,
-                       sizeof(ddb->plane[pipe][PLANE_CURSOR]));
-                return;
+                return 0;
         }
 
-        cursor_blocks = skl_cursor_allocation(config);
+        cursor_blocks = skl_cursor_allocation(num_active);
         ddb->plane[pipe][PLANE_CURSOR].start = alloc->end - cursor_blocks;
         ddb->plane[pipe][PLANE_CURSOR].end = alloc->end;
 
         alloc_size -= cursor_blocks;
-        alloc->end -= cursor_blocks;
 
         /* 1. Allocate the mininum required blocks for each active plane */
-        for_each_intel_plane_on_crtc(dev, intel_crtc, intel_plane) {
-                struct drm_plane *plane = &intel_plane->base;
-                struct drm_framebuffer *fb = plane->state->fb;
-                int id = skl_wm_plane_id(intel_plane);
+        for_each_plane_in_state(state, plane, pstate, i) {
+                intel_plane = to_intel_plane(plane);
+                id = skl_wm_plane_id(intel_plane);
 
-                if (!to_intel_plane_state(plane->state)->visible)
+                if (intel_plane->pipe != pipe)
                         continue;
 
-                if (plane->type == DRM_PLANE_TYPE_CURSOR)
+                if (!to_intel_plane_state(pstate)->visible) {
+                        minimum[id] = 0;
+                        y_minimum[id] = 0;
+                        continue;
+                }
+                if (plane->type == DRM_PLANE_TYPE_CURSOR) {
+                        minimum[id] = 0;
+                        y_minimum[id] = 0;
                         continue;
+                }
+
+                minimum[id] = skl_ddb_min_alloc(pstate, 0);
+                y_minimum[id] = skl_ddb_min_alloc(pstate, 1);
+        }
 
-                minimum[id] = 8;
-                alloc_size -= minimum[id];
-                y_minimum[id] = (fb->pixel_format == DRM_FORMAT_NV12) ? 8 : 0;
-                alloc_size -= y_minimum[id];
+        for (i = 0; i < PLANE_CURSOR; i++) {
+                alloc_size -= minimum[i];
+                alloc_size -= y_minimum[i];
         }
 
         /*
@@ -3084,21 +3253,16 @@ skl_allocate_pipe_ddb(struct intel_crtc_state *cstate,
          * FIXME: we may not allocate every single block here.
          */
         total_data_rate = skl_get_total_relative_data_rate(cstate);
+        if (total_data_rate == 0)
+                return 0;
 
         start = alloc->start;
         for_each_intel_plane_on_crtc(dev, intel_crtc, intel_plane) {
-                struct drm_plane *plane = &intel_plane->base;
-                struct drm_plane_state *pstate = intel_plane->base.state;
                 unsigned int data_rate, y_data_rate;
                 uint16_t plane_blocks, y_plane_blocks = 0;
                 int id = skl_wm_plane_id(intel_plane);
 
-                if (!to_intel_plane_state(pstate)->visible)
-                        continue;
-                if (plane->type == DRM_PLANE_TYPE_CURSOR)
-                        continue;
-
-                data_rate = skl_plane_relative_data_rate(cstate, pstate, 0);
+                data_rate = cstate->wm.skl.plane_data_rate[id];
 
                 /*
                  * allocation for (packed formats) or (uv-plane part of planar format):
@@ -3109,30 +3273,32 @@ skl_allocate_pipe_ddb(struct intel_crtc_state *cstate,
                 plane_blocks += div_u64((uint64_t)alloc_size * data_rate,
                                         total_data_rate);
 
-                ddb->plane[pipe][id].start = start;
-                ddb->plane[pipe][id].end = start + plane_blocks;
+                /* Leave disabled planes at (0,0) */
+                if (data_rate) {
+                        ddb->plane[pipe][id].start = start;
+                        ddb->plane[pipe][id].end = start + plane_blocks;
+                }
 
                 start += plane_blocks;
 
                 /*
                  * allocation for y_plane part of planar format:
                  */
-                if (pstate->fb->pixel_format == DRM_FORMAT_NV12) {
-                        y_data_rate = skl_plane_relative_data_rate(cstate,
-                                                                   pstate,
-                                                                   1);
-                        y_plane_blocks = y_minimum[id];
-                        y_plane_blocks += div_u64((uint64_t)alloc_size * y_data_rate,
-                                                total_data_rate);
+                y_data_rate = cstate->wm.skl.plane_y_data_rate[id];
+
+                y_plane_blocks = y_minimum[id];
+                y_plane_blocks += div_u64((uint64_t)alloc_size * y_data_rate,
+                                        total_data_rate);
 
+                if (y_data_rate) {
                         ddb->y_plane[pipe][id].start = start;
                         ddb->y_plane[pipe][id].end = start + y_plane_blocks;
-
-                        start += y_plane_blocks;
                 }
 
+                start += y_plane_blocks;
         }
 
+        return 0;
 }
 
 static uint32_t skl_pipe_pixel_rate(const struct intel_crtc_state *config)
@@ -3189,35 +3355,41 @@ static uint32_t skl_wm_method2(uint32_t pixel_rate, uint32_t pipe_htotal,
         return ret;
 }
 
-static bool skl_ddb_allocation_changed(const struct skl_ddb_allocation *new_ddb,
-                                       const struct intel_crtc *intel_crtc)
+static uint32_t skl_adjusted_plane_pixel_rate(const struct intel_crtc_state *cstate,
+                                              struct intel_plane_state *pstate)
 {
-        struct drm_device *dev = intel_crtc->base.dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
-        const struct skl_ddb_allocation *cur_ddb = &dev_priv->wm.skl_hw.ddb;
+        uint64_t adjusted_pixel_rate;
+        uint64_t downscale_amount;
+        uint64_t pixel_rate;
+
+        /* Shouldn't reach here on disabled planes... */
+        if (WARN_ON(!pstate->visible))
+                return 0;
 
         /*
-         * If ddb allocation of pipes changed, it may require recalculation of
-         * watermarks
+         * Adjusted plane pixel rate is just the pipe's adjusted pixel rate
+         * with additional adjustments for plane-specific scaling.
          */
-        if (memcmp(new_ddb->pipe, cur_ddb->pipe, sizeof(new_ddb->pipe)))
-                return true;
+        adjusted_pixel_rate = skl_pipe_pixel_rate(cstate);
+        downscale_amount = skl_plane_downscale_amount(pstate);
+
+        pixel_rate = adjusted_pixel_rate * downscale_amount >> 16;
+        WARN_ON(pixel_rate != clamp_t(uint32_t, pixel_rate, 0, ~0));
 
-        return false;
+        return pixel_rate;
 }
 
-static bool skl_compute_plane_wm(const struct drm_i915_private *dev_priv,
-                                 struct intel_crtc_state *cstate,
-                                 struct intel_plane *intel_plane,
-                                 uint16_t ddb_allocation,
-                                 int level,
-                                 uint16_t *out_blocks, /* out */
-                                 uint8_t *out_lines /* out */)
+static int skl_compute_plane_wm(const struct drm_i915_private *dev_priv,
+                                struct intel_crtc_state *cstate,
+                                struct intel_plane_state *intel_pstate,
+                                uint16_t ddb_allocation,
+                                int level,
+                                uint16_t *out_blocks, /* out */
+                                uint8_t *out_lines, /* out */
+                                bool *enabled /* out */)
 {
-        struct drm_plane *plane = &intel_plane->base;
-        struct drm_framebuffer *fb = plane->state->fb;
-        struct intel_plane_state *intel_pstate =
-                                        to_intel_plane_state(plane->state);
+        struct drm_plane_state *pstate = &intel_pstate->base;
+        struct drm_framebuffer *fb = pstate->fb;
         uint32_t latency = dev_priv->wm.skl_latency[level];
         uint32_t method1, method2;
         uint32_t plane_bytes_per_line, plane_blocks_per_line;
@@ -3225,20 +3397,24 @@ static bool skl_compute_plane_wm(const struct drm_i915_private *dev_priv,
         uint32_t selected_result;
         uint8_t cpp;
         uint32_t width = 0, height = 0;
+        uint32_t plane_pixel_rate;
 
-        if (latency == 0 || !cstate->base.active || !intel_pstate->visible)
-                return false;
+        if (latency == 0 || !cstate->base.active || !intel_pstate->visible) {
+                *enabled = false;
+                return 0;
+        }
 
         width = drm_rect_width(&intel_pstate->src) >> 16;
         height = drm_rect_height(&intel_pstate->src) >> 16;
 
-        if (intel_rotation_90_or_270(plane->state->rotation))
+        if (intel_rotation_90_or_270(pstate->rotation))
                 swap(width, height);
 
         cpp = drm_format_plane_cpp(fb->pixel_format, 0);
-        method1 = skl_wm_method1(skl_pipe_pixel_rate(cstate),
-                                 cpp, latency);
-        method2 = skl_wm_method2(skl_pipe_pixel_rate(cstate),
+        plane_pixel_rate = skl_adjusted_plane_pixel_rate(cstate, intel_pstate);
+
+        method1 = skl_wm_method1(plane_pixel_rate, cpp, latency);
+        method2 = skl_wm_method2(plane_pixel_rate,
                                  cstate->base.adjusted_mode.crtc_htotal,
                                  width,
                                  cpp,
@@ -3252,7 +3428,7 @@ static bool skl_compute_plane_wm(const struct drm_i915_private *dev_priv,
             fb->modifier[0] == I915_FORMAT_MOD_Yf_TILED) {
                 uint32_t min_scanlines = 4;
                 uint32_t y_tile_minimum;
-                if (intel_rotation_90_or_270(plane->state->rotation)) {
+                if (intel_rotation_90_or_270(pstate->rotation)) {
                         int cpp = (fb->pixel_format == DRM_FORMAT_NV12) ?
                                 drm_format_plane_cpp(fb->pixel_format, 1) :
                                 drm_format_plane_cpp(fb->pixel_format, 0);
@@ -3288,40 +3464,100 @@ static bool skl_compute_plane_wm(const struct drm_i915_private *dev_priv,
                         res_blocks++;
         }
 
-        if (res_blocks >= ddb_allocation || res_lines > 31)
-                return false;
+        if (res_blocks >= ddb_allocation || res_lines > 31) {
+                *enabled = false;
+
+                /*
+                 * If there are no valid level 0 watermarks, then we can't
+                 * support this display configuration.
+                 */
+                if (level) {
+                        return 0;
+                } else {
+                        DRM_DEBUG_KMS("Requested display configuration exceeds system watermark limitations\n");
+                        DRM_DEBUG_KMS("Plane %d.%d: blocks required = %u/%u, lines required = %u/31\n",
+                                      to_intel_crtc(cstate->base.crtc)->pipe,
+                                      skl_wm_plane_id(to_intel_plane(pstate->plane)),
+                                      res_blocks, ddb_allocation, res_lines);
+
+                        return -EINVAL;
+                }
+        }
 
         *out_blocks = res_blocks;
         *out_lines = res_lines;
+        *enabled = true;
 
-        return true;
+        return 0;
 }
 
-static void skl_compute_wm_level(const struct drm_i915_private *dev_priv,
-                                 struct skl_ddb_allocation *ddb,
-                                 struct intel_crtc_state *cstate,
-                                 int level,
-                                 struct skl_wm_level *result)
+static int
+skl_compute_wm_level(const struct drm_i915_private *dev_priv,
+                     struct skl_ddb_allocation *ddb,
+                     struct intel_crtc_state *cstate,
+                     int level,
+                     struct skl_wm_level *result)
 {
-        struct drm_device *dev = dev_priv->dev;
+        struct drm_atomic_state *state = cstate->base.state;
         struct intel_crtc *intel_crtc = to_intel_crtc(cstate->base.crtc);
+        struct drm_plane *plane;
         struct intel_plane *intel_plane;
+        struct intel_plane_state *intel_pstate;
         uint16_t ddb_blocks;
         enum pipe pipe = intel_crtc->pipe;
+        int ret;
 
-        for_each_intel_plane_on_crtc(dev, intel_crtc, intel_plane) {
+        /*
+         * We'll only calculate watermarks for planes that are actually
+         * enabled, so make sure all other planes are set as disabled.
+         */
+        memset(result, 0, sizeof(*result));
+
+        for_each_intel_plane_mask(&dev_priv->drm,
+                                  intel_plane,
+                                  cstate->base.plane_mask) {
                 int i = skl_wm_plane_id(intel_plane);
 
+                plane = &intel_plane->base;
+                intel_pstate = NULL;
+                if (state)
+                        intel_pstate =
+                                intel_atomic_get_existing_plane_state(state,
+                                                                      intel_plane);
+
+                /*
+                 * Note: If we start supporting multiple pending atomic commits
+                 * against the same planes/CRTC's in the future, plane->state
+                 * will no longer be the correct pre-state to use for the
+                 * calculations here and we'll need to change where we get the
+                 * 'unchanged' plane data from.
+                 *
+                 * For now this is fine because we only allow one queued commit
+                 * against a CRTC.  Even if the plane isn't modified by this
+                 * transaction and we don't have a plane lock, we still have
+                 * the CRTC's lock, so we know that no other transactions are
+                 * racing with us to update it.
+                 */
+                if (!intel_pstate)
+                        intel_pstate = to_intel_plane_state(plane->state);
+
+                WARN_ON(!intel_pstate->base.fb);
+
                 ddb_blocks = skl_ddb_entry_size(&ddb->plane[pipe][i]);
 
-                result->plane_en[i] = skl_compute_plane_wm(dev_priv,
-                                                cstate,
-                                                intel_plane,
-                                                ddb_blocks,
-                                                level,
-                                                &result->plane_res_b[i],
-                                                &result->plane_res_l[i]);
+                ret = skl_compute_plane_wm(dev_priv,
+                                           cstate,
+                                           intel_pstate,
+                                           ddb_blocks,
+                                           level,
+                                           &result->plane_res_b[i],
+                                           &result->plane_res_l[i],
+                                           &result->plane_en[i]);
+                if (ret)
+                        return ret;
         }
+
+        return 0;
 }
 
 static uint32_t
@@ -3355,21 +3591,26 @@ static void skl_compute_transition_wm(struct intel_crtc_state *cstate,
         }
 }
 
-static void skl_compute_pipe_wm(struct intel_crtc_state *cstate,
-                                struct skl_ddb_allocation *ddb,
-                                struct skl_pipe_wm *pipe_wm)
+static int skl_build_pipe_wm(struct intel_crtc_state *cstate,
+                             struct skl_ddb_allocation *ddb,
+                             struct skl_pipe_wm *pipe_wm)
 {
         struct drm_device *dev = cstate->base.crtc->dev;
-        const struct drm_i915_private *dev_priv = dev->dev_private;
+        const struct drm_i915_private *dev_priv = to_i915(dev);
         int level, max_level = ilk_wm_max_level(dev);
+        int ret;
 
         for (level = 0; level <= max_level; level++) {
-                skl_compute_wm_level(dev_priv, ddb, cstate,
-                                     level, &pipe_wm->wm[level]);
+                ret = skl_compute_wm_level(dev_priv, ddb, cstate,
+                                           level, &pipe_wm->wm[level]);
+                if (ret)
+                        return ret;
         }
         pipe_wm->linetime = skl_compute_linetime_wm(cstate);
 
         skl_compute_transition_wm(cstate, &pipe_wm->trans_wm);
+
+        return 0;
 }
 
 static void skl_compute_wm_results(struct drm_device *dev,
@@ -3442,14 +3683,16 @@ static void skl_ddb_entry_write(struct drm_i915_private *dev_priv,
 static void skl_write_wm_values(struct drm_i915_private *dev_priv,
                                 const struct skl_wm_values *new)
 {
-        struct drm_device *dev = dev_priv->dev;
+        struct drm_device *dev = &dev_priv->drm;
         struct intel_crtc *crtc;
 
         for_each_intel_crtc(dev, crtc) {
                 int i, level, max_level = ilk_wm_max_level(dev);
                 enum pipe pipe = crtc->pipe;
 
-                if (!new->dirty[pipe])
+                if ((new->dirty_pipes & drm_crtc_mask(&crtc->base)) == 0)
+                        continue;
+                if (!crtc->active)
                         continue;
 
                 I915_WRITE(PIPE_WM_LINETIME(pipe), new->wm_linetime[pipe]);
@@ -3537,7 +3780,7 @@ skl_ddb_allocation_included(const struct skl_ddb_allocation *old,
 static void skl_flush_wm_values(struct drm_i915_private *dev_priv,
                                 struct skl_wm_values *new_values)
 {
-        struct drm_device *dev = dev_priv->dev;
+        struct drm_device *dev = &dev_priv->drm;
         struct skl_ddb_allocation *cur_ddb, *new_ddb;
         bool reallocated[I915_MAX_PIPES] = {};
         struct intel_crtc *crtc;
@@ -3616,116 +3859,182 @@ static void skl_flush_wm_values(struct drm_i915_private *dev_priv,
         }
 }
 
-static bool skl_update_pipe_wm(struct drm_crtc *crtc,
-                               struct skl_ddb_allocation *ddb, /* out */
-                               struct skl_pipe_wm *pipe_wm /* out */)
+static int skl_update_pipe_wm(struct drm_crtc_state *cstate,
+                              struct skl_ddb_allocation *ddb, /* out */
+                              struct skl_pipe_wm *pipe_wm, /* out */
+                              bool *changed /* out */)
 {
-        struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
-        struct intel_crtc_state *cstate = to_intel_crtc_state(crtc->state);
+        struct intel_crtc *intel_crtc = to_intel_crtc(cstate->crtc);
+        struct intel_crtc_state *intel_cstate = to_intel_crtc_state(cstate);
+        int ret;
 
-        skl_allocate_pipe_ddb(cstate, ddb);
-        skl_compute_pipe_wm(cstate, ddb, pipe_wm);
+        ret = skl_build_pipe_wm(intel_cstate, ddb, pipe_wm);
+        if (ret)
+                return ret;
 
         if (!memcmp(&intel_crtc->wm.active.skl, pipe_wm, sizeof(*pipe_wm)))
-                return false;
+                *changed = false;
+        else
+                *changed = true;
 
-        intel_crtc->wm.active.skl = *pipe_wm;
+        return 0;
+}
 
-        return true;
+static uint32_t
+pipes_modified(struct drm_atomic_state *state)
+{
+        struct drm_crtc *crtc;
+        struct drm_crtc_state *cstate;
+        uint32_t i, ret = 0;
+
+        for_each_crtc_in_state(state, crtc, cstate, i)
+                ret |= drm_crtc_mask(crtc);
+
+        return ret;
 }
 
-static void skl_update_other_pipe_wm(struct drm_device *dev,
-                                     struct drm_crtc *crtc,
-                                     struct skl_wm_values *r)
+static int
+skl_compute_ddb(struct drm_atomic_state *state)
 {
+        struct drm_device *dev = state->dev;
+        struct drm_i915_private *dev_priv = to_i915(dev);
+        struct intel_atomic_state *intel_state = to_intel_atomic_state(state);
         struct intel_crtc *intel_crtc;
-        struct intel_crtc *this_crtc = to_intel_crtc(crtc);
+        struct skl_ddb_allocation *ddb = &intel_state->wm_results.ddb;
+        uint32_t realloc_pipes = pipes_modified(state);
+        int ret;
 
         /*
-         * If the WM update hasn't changed the allocation for this_crtc (the
-         * crtc we are currently computing the new WM values for), other
-         * enabled crtcs will keep the same allocation and we don't need to
-         * recompute anything for them.
+         * If this is our first atomic update following hardware readout,
+         * we can't trust the DDB that the BIOS programmed for us.  Let's
+         * pretend that all pipes switched active status so that we'll
+         * ensure a full DDB recompute.
          */
-        if (!skl_ddb_allocation_changed(&r->ddb, this_crtc))
-                return;
+        if (dev_priv->wm.distrust_bios_wm)
+                intel_state->active_pipe_changes = ~0;
 
         /*
-         * Otherwise, because of this_crtc being freshly enabled/disabled, the
-         * other active pipes need new DDB allocation and WM values.
+         * If the modeset changes which CRTC's are active, we need to
+         * recompute the DDB allocation for *all* active pipes, even
+         * those that weren't otherwise being modified in any way by this
+         * atomic commit.  Due to the shrinking of the per-pipe allocations
+         * when new active CRTC's are added, it's possible for a pipe that
+         * we were already using and aren't changing at all here to suddenly
+         * become invalid if its DDB needs exceeds its new allocation.
+         *
+         * Note that if we wind up doing a full DDB recompute, we can't let
+         * any other display updates race with this transaction, so we need
+         * to grab the lock on *all* CRTC's.
          */
-        for_each_intel_crtc(dev, intel_crtc) {
-                struct skl_pipe_wm pipe_wm = {};
-                bool wm_changed;
-
-                if (this_crtc->pipe == intel_crtc->pipe)
-                        continue;
+        if (intel_state->active_pipe_changes) {
+                realloc_pipes = ~0;
+                intel_state->wm_results.dirty_pipes = ~0;
+        }
 
-                if (!intel_crtc->active)
-                        continue;
+        for_each_intel_crtc_mask(dev, intel_crtc, realloc_pipes) {
+                struct intel_crtc_state *cstate;
 
-                wm_changed = skl_update_pipe_wm(&intel_crtc->base,
-                                                &r->ddb, &pipe_wm);
+                cstate = intel_atomic_get_crtc_state(state, intel_crtc);
+                if (IS_ERR(cstate))
+                        return PTR_ERR(cstate);
 
-                /*
-                 * If we end up re-computing the other pipe WM values, it's
-                 * because it was really needed, so we expect the WM values to
-                 * be different.
-                 */
-                WARN_ON(!wm_changed);
-
-                skl_compute_wm_results(dev, &pipe_wm, r, intel_crtc);
-                r->dirty[intel_crtc->pipe] = true;
+                ret = skl_allocate_pipe_ddb(cstate, ddb);
+                if (ret)
+                        return ret;
         }
+
+        return 0;
 }
 
-static void skl_clear_wm(struct skl_wm_values *watermarks, enum pipe pipe)
+static int
+skl_compute_wm(struct drm_atomic_state *state)
 {
-        watermarks->wm_linetime[pipe] = 0;
-        memset(watermarks->plane[pipe], 0,
-               sizeof(uint32_t) * 8 * I915_MAX_PLANES);
-        memset(watermarks->plane_trans[pipe],
-               0, sizeof(uint32_t) * I915_MAX_PLANES);
-        watermarks->plane_trans[pipe][PLANE_CURSOR] = 0;
+        struct drm_crtc *crtc;
+        struct drm_crtc_state *cstate;
+        struct intel_atomic_state *intel_state = to_intel_atomic_state(state);
+        struct skl_wm_values *results = &intel_state->wm_results;
+        struct skl_pipe_wm *pipe_wm;
+        bool changed = false;
+        int ret, i;
+
+        /*
+         * If this transaction isn't actually touching any CRTC's, don't
+         * bother with watermark calculation.  Note that if we pass this
+         * test, we're guaranteed to hold at least one CRTC state mutex,
+         * which means we can safely use values like dev_priv->active_crtcs
+         * since any racing commits that want to update them would need to
+         * hold _all_ CRTC state mutexes.
+         */
+        for_each_crtc_in_state(state, crtc, cstate, i)
+                changed = true;
+        if (!changed)
+                return 0;
+
+        /* Clear all dirty flags */
+        results->dirty_pipes = 0;
+
+        ret = skl_compute_ddb(state);
+        if (ret)
+                return ret;
+
+        /*
+         * Calculate WM's for all pipes that are part of this transaction.
+         * Note that the DDB allocation above may have added more CRTC's that
+         * weren't otherwise being modified (and set bits in dirty_pipes) if
+         * pipe allocations had to change.
+         *
+         * FIXME:  Now that we're doing this in the atomic check phase, we
+         * should allow skl_update_pipe_wm() to return failure in cases where
+         * no suitable watermark values can be found.
+         */
+        for_each_crtc_in_state(state, crtc, cstate, i) {
+                struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
+                struct intel_crtc_state *intel_cstate =
+                        to_intel_crtc_state(cstate);
+
+                pipe_wm = &intel_cstate->wm.skl.optimal;
+                ret = skl_update_pipe_wm(cstate, &results->ddb, pipe_wm,
+                                         &changed);
+                if (ret)
+                        return ret;
 
-        /* Clear ddb entries for pipe */
-        memset(&watermarks->ddb.pipe[pipe], 0, sizeof(struct skl_ddb_entry));
-        memset(&watermarks->ddb.plane[pipe], 0,
-               sizeof(struct skl_ddb_entry) * I915_MAX_PLANES);
-        memset(&watermarks->ddb.y_plane[pipe], 0,
-               sizeof(struct skl_ddb_entry) * I915_MAX_PLANES);
-        memset(&watermarks->ddb.plane[pipe][PLANE_CURSOR], 0,
-               sizeof(struct skl_ddb_entry));
+                if (changed)
+                        results->dirty_pipes |= drm_crtc_mask(crtc);
 
+                if ((results->dirty_pipes & drm_crtc_mask(crtc)) == 0)
+                        /* This pipe's WM's did not change */
+                        continue;
+
+                intel_cstate->update_wm_pre = true;
+                skl_compute_wm_results(crtc->dev, pipe_wm, results, intel_crtc);
+        }
+
+        return 0;
 }
 
 static void skl_update_wm(struct drm_crtc *crtc)
 {
         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
         struct drm_device *dev = crtc->dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         struct skl_wm_values *results = &dev_priv->wm.skl_results;
         struct intel_crtc_state *cstate = to_intel_crtc_state(crtc->state);
-        struct skl_pipe_wm *pipe_wm = &cstate->wm.optimal.skl;
+        struct skl_pipe_wm *pipe_wm = &cstate->wm.skl.optimal;
 
-
-        /* Clear all dirty flags */
-        memset(results->dirty, 0, sizeof(bool) * I915_MAX_PIPES);
-
-        skl_clear_wm(results, intel_crtc->pipe);
-
-        if (!skl_update_pipe_wm(crtc, &results->ddb, pipe_wm))
+        if ((results->dirty_pipes & drm_crtc_mask(crtc)) == 0)
                 return;
 
-        skl_compute_wm_results(dev, pipe_wm, results, intel_crtc);
-        results->dirty[intel_crtc->pipe] = true;
+        intel_crtc->wm.active.skl = *pipe_wm;
+
+        mutex_lock(&dev_priv->wm.wm_mutex);
 
-        skl_update_other_pipe_wm(dev, crtc, results);
         skl_write_wm_values(dev_priv, results);
         skl_flush_wm_values(dev_priv, results);
 
         /* store the new configuration */
         dev_priv->wm.skl_hw = *results;
+
+        mutex_unlock(&dev_priv->wm.wm_mutex);
 }
 
 static void ilk_compute_wm_config(struct drm_device *dev,
@@ -3748,7 +4057,7 @@ static void ilk_compute_wm_config(struct drm_device *dev,
 
 static void ilk_program_watermarks(struct drm_i915_private *dev_priv)
 {
-        struct drm_device *dev = dev_priv->dev;
+        struct drm_device *dev = &dev_priv->drm;
         struct intel_pipe_wm lp_wm_1_2 = {}, lp_wm_5_6 = {}, *best_lp_wm;
         struct ilk_wm_maximums max;
         struct intel_wm_config config = {};
@@ -3785,7 +4094,7 @@ static void ilk_initial_watermarks(struct intel_crtc_state *cstate)
         struct intel_crtc *intel_crtc = to_intel_crtc(cstate->base.crtc);
 
         mutex_lock(&dev_priv->wm.wm_mutex);
-        intel_crtc->wm.active.ilk = cstate->wm.intermediate;
+        intel_crtc->wm.active.ilk = cstate->wm.ilk.intermediate;
         ilk_program_watermarks(dev_priv);
         mutex_unlock(&dev_priv->wm.wm_mutex);
 }
@@ -3797,7 +4106,7 @@ static void ilk_optimize_watermarks(struct intel_crtc_state *cstate)
 
         mutex_lock(&dev_priv->wm.wm_mutex);
         if (cstate->wm.need_postvbl_update) {
-                intel_crtc->wm.active.ilk = cstate->wm.optimal.ilk;
+                intel_crtc->wm.active.ilk = cstate->wm.ilk.optimal;
                 ilk_program_watermarks(dev_priv);
         }
         mutex_unlock(&dev_priv->wm.wm_mutex);
@@ -3850,11 +4159,11 @@ static void skl_pipe_wm_active_state(uint32_t val,
 static void skl_pipe_wm_get_hw_state(struct drm_crtc *crtc)
 {
         struct drm_device *dev = crtc->dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         struct skl_wm_values *hw = &dev_priv->wm.skl_hw;
         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
         struct intel_crtc_state *cstate = to_intel_crtc_state(crtc->state);
-        struct skl_pipe_wm *active = &cstate->wm.optimal.skl;
+        struct skl_pipe_wm *active = &cstate->wm.skl.optimal;
         enum pipe pipe = intel_crtc->pipe;
         int level, i, max_level;
         uint32_t temp;
@@ -3877,7 +4186,7 @@ static void skl_pipe_wm_get_hw_state(struct drm_crtc *crtc)
         if (!intel_crtc->active)
                 return;
 
-        hw->dirty[pipe] = true;
+        hw->dirty_pipes |= drm_crtc_mask(crtc);
 
         active->linetime = hw->wm_linetime[pipe];
 
@@ -3904,23 +4213,31 @@ static void skl_pipe_wm_get_hw_state(struct drm_crtc *crtc)
 
 void skl_wm_get_hw_state(struct drm_device *dev)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         struct skl_ddb_allocation *ddb = &dev_priv->wm.skl_hw.ddb;
         struct drm_crtc *crtc;
 
         skl_ddb_get_hw_state(dev_priv, ddb);
         list_for_each_entry(crtc, &dev->mode_config.crtc_list, head)
                 skl_pipe_wm_get_hw_state(crtc);
+
+        if (dev_priv->active_crtcs) {
+                /* Fully recompute DDB on first atomic commit */
+                dev_priv->wm.distrust_bios_wm = true;
+        } else {
+                /* Easy/common case; just sanitize DDB now if everything off */
+                memset(ddb, 0, sizeof(*ddb));
+        }
 }
 
 static void ilk_pipe_wm_get_hw_state(struct drm_crtc *crtc)
 {
         struct drm_device *dev = crtc->dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         struct ilk_wm_values *hw = &dev_priv->wm.hw;
         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
         struct intel_crtc_state *cstate = to_intel_crtc_state(crtc->state);
-        struct intel_pipe_wm *active = &cstate->wm.optimal.ilk;
+        struct intel_pipe_wm *active = &cstate->wm.ilk.optimal;
         enum pipe pipe = intel_crtc->pipe;
         static const i915_reg_t wm0_pipe_reg[] = {
                 [PIPE_A] = WM0_PIPEA_ILK,
@@ -4120,7 +4437,7 @@ void vlv_wm_get_hw_state(struct drm_device *dev)
 
 void ilk_wm_get_hw_state(struct drm_device *dev)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         struct ilk_wm_values *hw = &dev_priv->wm.hw;
         struct drm_crtc *crtc;
 
@@ -4182,7 +4499,7 @@ void ilk_wm_get_hw_state(struct drm_device *dev)
  */
 void intel_update_watermarks(struct drm_crtc *crtc)
 {
-        struct drm_i915_private *dev_priv = crtc->dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(crtc->dev);
 
         if (dev_priv->display.update_wm)
                 dev_priv->display.update_wm(crtc);
@@ -4197,9 +4514,8 @@ DEFINE_SPINLOCK(mchdev_lock);
  * mchdev_lock. */
 static struct drm_i915_private *i915_mch_dev;
 
-bool ironlake_set_drps(struct drm_device *dev, u8 val)
+bool ironlake_set_drps(struct drm_i915_private *dev_priv, u8 val)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
         u16 rgvswctl;
 
         assert_spin_locked(&mchdev_lock);
@@ -4221,9 +4537,8 @@ bool ironlake_set_drps(struct drm_device *dev, u8 val)
         return true;
 }
 
-static void ironlake_enable_drps(struct drm_device *dev)
+static void ironlake_enable_drps(struct drm_i915_private *dev_priv)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
         u32 rgvmodectl;
         u8 fmax, fmin, fstart, vstart;
 
@@ -4280,7 +4595,7 @@ static void ironlake_enable_drps(struct drm_device *dev)
                 DRM_ERROR("stuck trying to change perf mode\n");
         mdelay(1);
 
-        ironlake_set_drps(dev, fstart);
+        ironlake_set_drps(dev_priv, fstart);
 
         dev_priv->ips.last_count1 = I915_READ(DMIEC) +
                 I915_READ(DDREC) + I915_READ(CSIEC);
@@ -4291,9 +4606,8 @@ static void ironlake_enable_drps(struct drm_device *dev)
         spin_unlock_irq(&mchdev_lock);
 }
 
-static void ironlake_disable_drps(struct drm_device *dev)
+static void ironlake_disable_drps(struct drm_i915_private *dev_priv)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
         u16 rgvswctl;
 
         spin_lock_irq(&mchdev_lock);
@@ -4308,7 +4622,7 @@ static void ironlake_disable_drps(struct drm_device *dev)
         I915_WRITE(DEIMR, I915_READ(DEIMR) | DE_PCU_EVENT);
 
         /* Go back to the starting frequency */
-        ironlake_set_drps(dev, dev_priv->ips.fstart);
+        ironlake_set_drps(dev_priv, dev_priv->ips.fstart);
         mdelay(1);
         rgvswctl |= MEMCTL_CMD_STS;
         I915_WRITE(MEMSWCTL, rgvswctl);
@@ -4354,19 +4668,23 @@ static void gen6_set_rps_thresholds(struct drm_i915_private *dev_priv, u8 val)
         new_power = dev_priv->rps.power;
         switch (dev_priv->rps.power) {
         case LOW_POWER:
-                if (val > dev_priv->rps.efficient_freq + 1 && val > dev_priv->rps.cur_freq)
+                if (val > dev_priv->rps.efficient_freq + 1 &&
+                    val > dev_priv->rps.cur_freq)
                         new_power = BETWEEN;
                 break;
 
         case BETWEEN:
-                if (val <= dev_priv->rps.efficient_freq && val < dev_priv->rps.cur_freq)
+                if (val <= dev_priv->rps.efficient_freq &&
+                    val < dev_priv->rps.cur_freq)
                         new_power = LOW_POWER;
-                else if (val >= dev_priv->rps.rp0_freq && val > dev_priv->rps.cur_freq)
+                else if (val >= dev_priv->rps.rp0_freq &&
+                         val > dev_priv->rps.cur_freq)
                         new_power = HIGH_POWER;
                 break;
 
         case HIGH_POWER:
-                if (val < (dev_priv->rps.rp1_freq + dev_priv->rps.rp0_freq) >> 1 && val < dev_priv->rps.cur_freq)
+                if (val < (dev_priv->rps.rp1_freq + dev_priv->rps.rp0_freq) >> 1 &&
+                    val < dev_priv->rps.cur_freq)
                         new_power = BETWEEN;
                 break;
         }
@@ -4412,22 +4730,24 @@ static void gen6_set_rps_thresholds(struct drm_i915_private *dev_priv, u8 val)
         }
 
         I915_WRITE(GEN6_RP_UP_EI,
-                GT_INTERVAL_FROM_US(dev_priv, ei_up));
+                   GT_INTERVAL_FROM_US(dev_priv, ei_up));
         I915_WRITE(GEN6_RP_UP_THRESHOLD,
-                GT_INTERVAL_FROM_US(dev_priv, (ei_up * threshold_up / 100)));
+                   GT_INTERVAL_FROM_US(dev_priv,
+                                       ei_up * threshold_up / 100));
 
         I915_WRITE(GEN6_RP_DOWN_EI,
-                GT_INTERVAL_FROM_US(dev_priv, ei_down));
+                   GT_INTERVAL_FROM_US(dev_priv, ei_down));
         I915_WRITE(GEN6_RP_DOWN_THRESHOLD,
-                GT_INTERVAL_FROM_US(dev_priv, (ei_down * threshold_down / 100)));
+                   GT_INTERVAL_FROM_US(dev_priv,
+                                       ei_down * threshold_down / 100));
 
-         I915_WRITE(GEN6_RP_CONTROL,
-                    GEN6_RP_MEDIA_TURBO |
-                    GEN6_RP_MEDIA_HW_NORMAL_MODE |
-                    GEN6_RP_MEDIA_IS_GFX |
-                    GEN6_RP_ENABLE |
-                    GEN6_RP_UP_BUSY_AVG |
-                    GEN6_RP_DOWN_IDLE_AVG);
+        I915_WRITE(GEN6_RP_CONTROL,
+                   GEN6_RP_MEDIA_TURBO |
+                   GEN6_RP_MEDIA_HW_NORMAL_MODE |
+                   GEN6_RP_MEDIA_IS_GFX |
+                   GEN6_RP_ENABLE |
+                   GEN6_RP_UP_BUSY_AVG |
+                   GEN6_RP_DOWN_IDLE_AVG);
 
         dev_priv->rps.power = new_power;
         dev_priv->rps.up_threshold = threshold_up;
@@ -4452,12 +4772,10 @@ static u32 gen6_rps_pm_mask(struct drm_i915_private *dev_priv, u8 val)
 /* gen6_set_rps is called to update the frequency request, but should also be
  * called when the range (min_delay and max_delay) is modified so that we can
  * update the GEN6_RP_INTERRUPT_LIMITS register accordingly. */
-static void gen6_set_rps(struct drm_device *dev, u8 val)
+static void gen6_set_rps(struct drm_i915_private *dev_priv, u8 val)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
-
         /* WaGsvDisableTurbo: Workaround to disable turbo on BXT A* */
-        if (IS_BXT_REVID(dev, 0, BXT_REVID_A1))
+        if (IS_BXT_REVID(dev_priv, 0, BXT_REVID_A1))
                 return;
 
         WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock));
@@ -4470,10 +4788,10 @@ static void gen6_set_rps(struct drm_device *dev, u8 val)
         if (val != dev_priv->rps.cur_freq) {
                 gen6_set_rps_thresholds(dev_priv, val);
 
-                if (IS_GEN9(dev))
+                if (IS_GEN9(dev_priv))
                         I915_WRITE(GEN6_RPNSWREQ,
                                    GEN9_FREQUENCY(val));
-                else if (IS_HASWELL(dev) || IS_BROADWELL(dev))
+                else if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
                         I915_WRITE(GEN6_RPNSWREQ,
                                    HSW_FREQUENCY(val));
                 else
@@ -4495,15 +4813,13 @@ static void gen6_set_rps(struct drm_device *dev, u8 val)
         trace_intel_gpu_freq_change(intel_gpu_freq(dev_priv, val));
 }
 
-static void valleyview_set_rps(struct drm_device *dev, u8 val)
+static void valleyview_set_rps(struct drm_i915_private *dev_priv, u8 val)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
-
         WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock));
         WARN_ON(val > dev_priv->rps.max_freq);
         WARN_ON(val < dev_priv->rps.min_freq);
 
-        if (WARN_ONCE(IS_CHERRYVIEW(dev) && (val & 1),
+        if (WARN_ONCE(IS_CHERRYVIEW(dev_priv) && (val & 1),
                       "Odd GPU freq value\n"))
                 val &= ~1;
 
@@ -4536,7 +4852,7 @@ static void vlv_set_rps_idle(struct drm_i915_private *dev_priv)
         /* Wake up the media well, as that takes a lot less
          * power than the Render well. */
         intel_uncore_forcewake_get(dev_priv, FORCEWAKE_MEDIA);
-        valleyview_set_rps(dev_priv->dev, val);
+        valleyview_set_rps(dev_priv, val);
         intel_uncore_forcewake_put(dev_priv, FORCEWAKE_MEDIA);
 }
 
@@ -4548,20 +4864,33 @@ void gen6_rps_busy(struct drm_i915_private *dev_priv)
                         gen6_rps_reset_ei(dev_priv);
                 I915_WRITE(GEN6_PMINTRMSK,
                            gen6_rps_pm_mask(dev_priv, dev_priv->rps.cur_freq));
+
+                gen6_enable_rps_interrupts(dev_priv);
+
+                /* Ensure we start at the user's desired frequency */
+                intel_set_rps(dev_priv,
+                              clamp(dev_priv->rps.cur_freq,
+                                    dev_priv->rps.min_freq_softlimit,
+                                    dev_priv->rps.max_freq_softlimit));
         }
         mutex_unlock(&dev_priv->rps.hw_lock);
 }
 
 void gen6_rps_idle(struct drm_i915_private *dev_priv)
 {
-        struct drm_device *dev = dev_priv->dev;
+        /* Flush our bottom-half so that it does not race with us
+         * setting the idle frequency and so that it is bounded by
+         * our rpm wakeref. And then disable the interrupts to stop any
+         * futher RPS reclocking whilst we are asleep.
+         */
+        gen6_disable_rps_interrupts(dev_priv);
 
         mutex_lock(&dev_priv->rps.hw_lock);
         if (dev_priv->rps.enabled) {
-                if (IS_VALLEYVIEW(dev) || IS_CHERRYVIEW(dev))
+                if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
                         vlv_set_rps_idle(dev_priv);
                 else
-                        gen6_set_rps(dev_priv->dev, dev_priv->rps.idle_freq);
+                        gen6_set_rps(dev_priv, dev_priv->rps.idle_freq);
                 dev_priv->rps.last_adj = 0;
                 I915_WRITE(GEN6_PMINTRMSK, 0xffffffff);
         }
@@ -4580,7 +4909,7 @@ void gen6_rps_boost(struct drm_i915_private *dev_priv,
         /* This is intentionally racy! We peek at the state here, then
          * validate inside the RPS worker.
          */
-        if (!(dev_priv->mm.busy &&
+        if (!(dev_priv->gt.awake &&
               dev_priv->rps.enabled &&
               dev_priv->rps.cur_freq < dev_priv->rps.max_freq_softlimit))
                 return;
@@ -4596,7 +4925,7 @@ void gen6_rps_boost(struct drm_i915_private *dev_priv,
                 spin_lock_irq(&dev_priv->irq_lock);
                 if (dev_priv->rps.interrupts_enabled) {
                         dev_priv->rps.client_boost = true;
-                        queue_work(dev_priv->wq, &dev_priv->rps.work);
+                        schedule_work(&dev_priv->rps.work);
                 }
                 spin_unlock_irq(&dev_priv->irq_lock);
 
@@ -4609,49 +4938,39 @@ void gen6_rps_boost(struct drm_i915_private *dev_priv,
         spin_unlock(&dev_priv->rps.client_lock);
 }
 
-void intel_set_rps(struct drm_device *dev, u8 val)
+void intel_set_rps(struct drm_i915_private *dev_priv, u8 val)
 {
-        if (IS_VALLEYVIEW(dev) || IS_CHERRYVIEW(dev))
-                valleyview_set_rps(dev, val);
+        if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
+                valleyview_set_rps(dev_priv, val);
         else
-                gen6_set_rps(dev, val);
+                gen6_set_rps(dev_priv, val);
 }
 
-static void gen9_disable_rc6(struct drm_device *dev)
+static void gen9_disable_rc6(struct drm_i915_private *dev_priv)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
-
         I915_WRITE(GEN6_RC_CONTROL, 0);
         I915_WRITE(GEN9_PG_ENABLE, 0);
 }
 
-static void gen9_disable_rps(struct drm_device *dev)
+static void gen9_disable_rps(struct drm_i915_private *dev_priv)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
-
         I915_WRITE(GEN6_RP_CONTROL, 0);
 }
 
-static void gen6_disable_rps(struct drm_device *dev)
+static void gen6_disable_rps(struct drm_i915_private *dev_priv)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
-
         I915_WRITE(GEN6_RC_CONTROL, 0);
         I915_WRITE(GEN6_RPNSWREQ, 1 << 31);
         I915_WRITE(GEN6_RP_CONTROL, 0);
 }
 
-static void cherryview_disable_rps(struct drm_device *dev)
+static void cherryview_disable_rps(struct drm_i915_private *dev_priv)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
-
         I915_WRITE(GEN6_RC_CONTROL, 0);
 }
 
-static void valleyview_disable_rps(struct drm_device *dev)
+static void valleyview_disable_rps(struct drm_i915_private *dev_priv)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
-
         /* we're doing forcewake before Disabling RC6,
          * This what the BIOS expects when going into suspend */
         intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
@@ -4661,34 +4980,45 @@ static void valleyview_disable_rps(struct drm_device *dev)
         intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
 }
 
-static void intel_print_rc6_info(struct drm_device *dev, u32 mode)
+static void intel_print_rc6_info(struct drm_i915_private *dev_priv, u32 mode)
 {
-        if (IS_VALLEYVIEW(dev) || IS_CHERRYVIEW(dev)) {
+        if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) {
                 if (mode & (GEN7_RC_CTL_TO_MODE | GEN6_RC_CTL_EI_MODE(1)))
                         mode = GEN6_RC_CTL_RC6_ENABLE;
                 else
                         mode = 0;
         }
-        if (HAS_RC6p(dev))
-                DRM_DEBUG_KMS("Enabling RC6 states: RC6 %s RC6p %s RC6pp %s\n",
-                              onoff(mode & GEN6_RC_CTL_RC6_ENABLE),
-                              onoff(mode & GEN6_RC_CTL_RC6p_ENABLE),
-                              onoff(mode & GEN6_RC_CTL_RC6pp_ENABLE));
+        if (HAS_RC6p(dev_priv))
+                DRM_DEBUG_DRIVER("Enabling RC6 states: "
+                                 "RC6 %s RC6p %s RC6pp %s\n",
+                                 onoff(mode & GEN6_RC_CTL_RC6_ENABLE),
+                                 onoff(mode & GEN6_RC_CTL_RC6p_ENABLE),
+                                 onoff(mode & GEN6_RC_CTL_RC6pp_ENABLE));
 
         else
-                DRM_DEBUG_KMS("Enabling RC6 states: RC6 %s\n",
-                              onoff(mode & GEN6_RC_CTL_RC6_ENABLE));
+                DRM_DEBUG_DRIVER("Enabling RC6 states: RC6 %s\n",
+                                 onoff(mode & GEN6_RC_CTL_RC6_ENABLE));
 }
 
-static bool bxt_check_bios_rc6_setup(const struct drm_device *dev)
+static bool bxt_check_bios_rc6_setup(struct drm_i915_private *dev_priv)
 {
-        struct drm_i915_private *dev_priv = to_i915(dev);
         struct i915_ggtt *ggtt = &dev_priv->ggtt;
         bool enable_rc6 = true;
         unsigned long rc6_ctx_base;
+        u32 rc_ctl;
+        int rc_sw_target;
+
+        rc_ctl = I915_READ(GEN6_RC_CONTROL);
+        rc_sw_target = (I915_READ(GEN6_RC_STATE) & RC_SW_TARGET_STATE_MASK) >>
+                       RC_SW_TARGET_STATE_SHIFT;
+        DRM_DEBUG_DRIVER("BIOS enabled RC states: "
+                         "HW_CTRL %s HW_RC6 %s SW_TARGET_STATE %x\n",
+                         onoff(rc_ctl & GEN6_RC_CTL_HW_ENABLE),
+                         onoff(rc_ctl & GEN6_RC_CTL_RC6_ENABLE),
+                         rc_sw_target);
 
         if (!(I915_READ(RC6_LOCATION) & RC6_CTX_IN_DRAM)) {
-                DRM_DEBUG_KMS("RC6 Base location not set properly.\n");
+                DRM_DEBUG_DRIVER("RC6 Base location not set properly.\n");
                 enable_rc6 = false;
         }
 
@@ -4700,7 +5030,7 @@ static bool bxt_check_bios_rc6_setup(const struct drm_device *dev)
         if (!((rc6_ctx_base >= ggtt->stolen_reserved_base) &&
               (rc6_ctx_base + PAGE_SIZE <= ggtt->stolen_reserved_base +
                                         ggtt->stolen_reserved_size))) {
-                DRM_DEBUG_KMS("RC6 Base address not as expected.\n");
+                DRM_DEBUG_DRIVER("RC6 Base address not as expected.\n");
                 enable_rc6 = false;
         }
 
@@ -4708,31 +5038,40 @@ static bool bxt_check_bios_rc6_setup(const struct drm_device *dev)
               ((I915_READ(PWRCTX_MAXCNT_VCSUNIT0) & IDLE_TIME_MASK) > 1) &&
               ((I915_READ(PWRCTX_MAXCNT_BCSUNIT) & IDLE_TIME_MASK) > 1) &&
               ((I915_READ(PWRCTX_MAXCNT_VECSUNIT) & IDLE_TIME_MASK) > 1))) {
-                DRM_DEBUG_KMS("Engine Idle wait time not set properly.\n");
+                DRM_DEBUG_DRIVER("Engine Idle wait time not set properly.\n");
+                enable_rc6 = false;
+        }
+
+        if (!I915_READ(GEN8_PUSHBUS_CONTROL) ||
+            !I915_READ(GEN8_PUSHBUS_ENABLE) ||
+            !I915_READ(GEN8_PUSHBUS_SHIFT)) {
+                DRM_DEBUG_DRIVER("Pushbus not setup properly.\n");
+                enable_rc6 = false;
+        }
+
+        if (!I915_READ(GEN6_GFXPAUSE)) {
+                DRM_DEBUG_DRIVER("GFX pause not setup properly.\n");
                 enable_rc6 = false;
         }
 
-        if (!(I915_READ(GEN6_RC_CONTROL) & (GEN6_RC_CTL_RC6_ENABLE |
-                                            GEN6_RC_CTL_HW_ENABLE)) &&
-            ((I915_READ(GEN6_RC_CONTROL) & GEN6_RC_CTL_HW_ENABLE) ||
-             !(I915_READ(GEN6_RC_STATE) & RC6_STATE))) {
-                DRM_DEBUG_KMS("HW/SW RC6 is not enabled by BIOS.\n");
+        if (!I915_READ(GEN8_MISC_CTRL0)) {
+                DRM_DEBUG_DRIVER("GPM control not setup properly.\n");
                 enable_rc6 = false;
         }
 
         return enable_rc6;
 }
 
-int sanitize_rc6_option(const struct drm_device *dev, int enable_rc6)
+int sanitize_rc6_option(struct drm_i915_private *dev_priv, int enable_rc6)
 {
         /* No RC6 before Ironlake and code is gone for ilk. */
-        if (INTEL_INFO(dev)->gen < 6)
+        if (INTEL_INFO(dev_priv)->gen < 6)
                 return 0;
 
         if (!enable_rc6)
                 return 0;
 
-        if (IS_BROXTON(dev) && !bxt_check_bios_rc6_setup(dev)) {
+        if (IS_BROXTON(dev_priv) && !bxt_check_bios_rc6_setup(dev_priv)) {
                 DRM_INFO("RC6 disabled by BIOS\n");
                 return 0;
         }
@@ -4741,33 +5080,28 @@ int sanitize_rc6_option(const struct drm_device *dev, int enable_rc6)
         if (enable_rc6 >= 0) {
                 int mask;
 
-                if (HAS_RC6p(dev))
+                if (HAS_RC6p(dev_priv))
                         mask = INTEL_RC6_ENABLE | INTEL_RC6p_ENABLE |
                                INTEL_RC6pp_ENABLE;
                 else
                         mask = INTEL_RC6_ENABLE;
 
                 if ((enable_rc6 & mask) != enable_rc6)
-                        DRM_DEBUG_KMS("Adjusting RC6 mask to %d (requested %d, valid %d)\n",
-                                      enable_rc6 & mask, enable_rc6, mask);
+                        DRM_DEBUG_DRIVER("Adjusting RC6 mask to %d "
+                                         "(requested %d, valid %d)\n",
+                                         enable_rc6 & mask, enable_rc6, mask);
 
                 return enable_rc6 & mask;
         }
 
-        if (IS_IVYBRIDGE(dev))
+        if (IS_IVYBRIDGE(dev_priv))
                 return (INTEL_RC6_ENABLE | INTEL_RC6p_ENABLE);
 
         return INTEL_RC6_ENABLE;
 }
 
-int intel_enable_rc6(const struct drm_device *dev)
+static void gen6_init_rps_frequencies(struct drm_i915_private *dev_priv)
 {
-        return i915.enable_rc6;
-}
-
-static void gen6_init_rps_frequencies(struct drm_device *dev)
-{
-        struct drm_i915_private *dev_priv = dev->dev_private;
         uint32_t rp_state_cap;
         u32 ddcc_status = 0;
         int ret;
@@ -4775,7 +5109,7 @@ static void gen6_init_rps_frequencies(struct drm_device *dev)
         /* All of these values are in units of 50MHz */
         dev_priv->rps.cur_freq          = 0;
         /* static values from HW: RP0 > RP1 > RPn (min_freq) */
-        if (IS_BROXTON(dev)) {
+        if (IS_BROXTON(dev_priv)) {
                 rp_state_cap = I915_READ(BXT_RP_STATE_CAP);
                 dev_priv->rps.rp0_freq = (rp_state_cap >> 16) & 0xff;
                 dev_priv->rps.rp1_freq = (rp_state_cap >>  8) & 0xff;
@@ -4791,8 +5125,8 @@ static void gen6_init_rps_frequencies(struct drm_device *dev)
         dev_priv->rps.max_freq          = dev_priv->rps.rp0_freq;
 
         dev_priv->rps.efficient_freq = dev_priv->rps.rp1_freq;
-        if (IS_HASWELL(dev) || IS_BROADWELL(dev) ||
-            IS_SKYLAKE(dev) || IS_KABYLAKE(dev)) {
+        if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv) ||
+            IS_SKYLAKE(dev_priv) || IS_KABYLAKE(dev_priv)) {
                 ret = sandybridge_pcode_read(dev_priv,
                                         HSW_PCODE_DYNAMIC_DUTY_CYCLE_CONTROL,
                                         &ddcc_status);
@@ -4804,7 +5138,7 @@ static void gen6_init_rps_frequencies(struct drm_device *dev)
                                         dev_priv->rps.max_freq);
         }
 
-        if (IS_SKYLAKE(dev) || IS_KABYLAKE(dev)) {
+        if (IS_SKYLAKE(dev_priv) || IS_KABYLAKE(dev_priv)) {
                 /* Store the frequency values in 16.66 MHZ units, which is
                    the natural hardware unit for SKL */
                 dev_priv->rps.rp0_freq *= GEN9_FREQ_SCALER;
@@ -4821,7 +5155,7 @@ static void gen6_init_rps_frequencies(struct drm_device *dev)
                 dev_priv->rps.max_freq_softlimit = dev_priv->rps.max_freq;
 
         if (dev_priv->rps.min_freq_softlimit == 0) {
-                if (IS_HASWELL(dev) || IS_BROADWELL(dev))
+                if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv))
                         dev_priv->rps.min_freq_softlimit =
                                 max_t(int, dev_priv->rps.efficient_freq,
                                       intel_freq_opcode(dev_priv, 450));
@@ -4832,16 +5166,14 @@ static void gen6_init_rps_frequencies(struct drm_device *dev)
 }
 
 /* See the Gen9_GT_PM_Programming_Guide doc for the below */
-static void gen9_enable_rps(struct drm_device *dev)
+static void gen9_enable_rps(struct drm_i915_private *dev_priv)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
-
         intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
 
-        gen6_init_rps_frequencies(dev);
+        gen6_init_rps_frequencies(dev_priv);
 
         /* WaGsvDisableTurbo: Workaround to disable turbo on BXT A* */
-        if (IS_BXT_REVID(dev, 0, BXT_REVID_A1)) {
+        if (IS_BXT_REVID(dev_priv, 0, BXT_REVID_A1)) {
                 /*
                  * BIOS could leave the Hw Turbo enabled, so need to explicitly
                  * clear out the Control register just to avoid inconsitency
@@ -4851,7 +5183,7 @@ static void gen9_enable_rps(struct drm_device *dev)
                  * if the Turbo is left enabled in the Control register, as the
                  * Up/Down interrupts would remain masked.
                  */
-                gen9_disable_rps(dev);
+                gen9_disable_rps(dev_priv);
                 intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
                 return;
         }
@@ -4870,14 +5202,13 @@ static void gen9_enable_rps(struct drm_device *dev)
          * Up/Down EI & threshold registers, as well as the RP_CONTROL,
          * RP_INTERRUPT_LIMITS & RPNSWREQ registers */
         dev_priv->rps.power = HIGH_POWER; /* force a reset */
-        gen6_set_rps(dev_priv->dev, dev_priv->rps.idle_freq);
+        gen6_set_rps(dev_priv, dev_priv->rps.idle_freq);
 
         intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
 }
 
-static void gen9_enable_rc6(struct drm_device *dev)
+static void gen9_enable_rc6(struct drm_i915_private *dev_priv)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
         struct intel_engine_cs *engine;
         uint32_t rc6_mask = 0;
 
@@ -4894,7 +5225,7 @@ static void gen9_enable_rc6(struct drm_device *dev)
         /* 2b: Program RC6 thresholds.*/
 
         /* WaRsDoubleRc6WrlWithCoarsePowerGating: Doubling WRL only when CPG is enabled */
-        if (IS_SKYLAKE(dev))
+        if (IS_SKYLAKE(dev_priv))
                 I915_WRITE(GEN6_RC6_WAKE_RATE_LIMIT, 108 << 16);
         else
                 I915_WRITE(GEN6_RC6_WAKE_RATE_LIMIT, 54 << 16);
@@ -4903,7 +5234,7 @@ static void gen9_enable_rc6(struct drm_device *dev)
         for_each_engine(engine, dev_priv)
                 I915_WRITE(RING_MAX_IDLE(engine->mmio_base), 10);
 
-        if (HAS_GUC_UCODE(dev))
+        if (HAS_GUC(dev_priv))
                 I915_WRITE(GUC_MAX_IDLE_COUNT, 0xA);
 
         I915_WRITE(GEN6_RC_SLEEP, 0);
@@ -4913,12 +5244,12 @@ static void gen9_enable_rc6(struct drm_device *dev)
         I915_WRITE(GEN9_RENDER_PG_IDLE_HYSTERESIS, 25);
 
         /* 3a: Enable RC6 */
-        if (intel_enable_rc6(dev) & INTEL_RC6_ENABLE)
+        if (intel_enable_rc6() & INTEL_RC6_ENABLE)
                 rc6_mask = GEN6_RC_CTL_RC6_ENABLE;
         DRM_INFO("RC6 %s\n", onoff(rc6_mask & GEN6_RC_CTL_RC6_ENABLE));
         /* WaRsUseTimeoutMode */
-        if (IS_SKL_REVID(dev, 0, SKL_REVID_D0) ||
-            IS_BXT_REVID(dev, 0, BXT_REVID_A1)) {
+        if (IS_SKL_REVID(dev_priv, 0, SKL_REVID_D0) ||
+            IS_BXT_REVID(dev_priv, 0, BXT_REVID_A1)) {
                 I915_WRITE(GEN6_RC6_THRESHOLD, 625); /* 800us */
                 I915_WRITE(GEN6_RC_CONTROL, GEN6_RC_CTL_HW_ENABLE |
                            GEN7_RC_CTL_TO_MODE |
@@ -4934,19 +5265,17 @@ static void gen9_enable_rc6(struct drm_device *dev)
          * 3b: Enable Coarse Power Gating only when RC6 is enabled.
          * WaRsDisableCoarsePowerGating:skl,bxt - Render/Media PG need to be disabled with RC6.
          */
-        if (NEEDS_WaRsDisableCoarsePowerGating(dev))
+        if (NEEDS_WaRsDisableCoarsePowerGating(dev_priv))
                 I915_WRITE(GEN9_PG_ENABLE, 0);
         else
                 I915_WRITE(GEN9_PG_ENABLE, (rc6_mask & GEN6_RC_CTL_RC6_ENABLE) ?
                                 (GEN9_RENDER_PG_ENABLE | GEN9_MEDIA_PG_ENABLE) : 0);
 
         intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
-
 }
 
-static void gen8_enable_rps(struct drm_device *dev)
+static void gen8_enable_rps(struct drm_i915_private *dev_priv)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
         struct intel_engine_cs *engine;
         uint32_t rc6_mask = 0;
 
@@ -4961,7 +5290,7 @@ static void gen8_enable_rps(struct drm_device *dev)
         I915_WRITE(GEN6_RC_CONTROL, 0);
 
         /* Initialize rps frequencies */
-        gen6_init_rps_frequencies(dev);
+        gen6_init_rps_frequencies(dev_priv);
 
         /* 2b: Program RC6 thresholds.*/
         I915_WRITE(GEN6_RC6_WAKE_RATE_LIMIT, 40 << 16);
@@ -4970,16 +5299,16 @@ static void gen8_enable_rps(struct drm_device *dev)
         for_each_engine(engine, dev_priv)
                 I915_WRITE(RING_MAX_IDLE(engine->mmio_base), 10);
         I915_WRITE(GEN6_RC_SLEEP, 0);
-        if (IS_BROADWELL(dev))
+        if (IS_BROADWELL(dev_priv))
                 I915_WRITE(GEN6_RC6_THRESHOLD, 625); /* 800us/1.28 for TO */
         else
                 I915_WRITE(GEN6_RC6_THRESHOLD, 50000); /* 50/125ms per EI */
 
         /* 3: Enable RC6 */
-        if (intel_enable_rc6(dev) & INTEL_RC6_ENABLE)
+        if (intel_enable_rc6() & INTEL_RC6_ENABLE)
                 rc6_mask = GEN6_RC_CTL_RC6_ENABLE;
-        intel_print_rc6_info(dev, rc6_mask);
-        if (IS_BROADWELL(dev))
+        intel_print_rc6_info(dev_priv, rc6_mask);
+        if (IS_BROADWELL(dev_priv))
                 I915_WRITE(GEN6_RC_CONTROL, GEN6_RC_CTL_HW_ENABLE |
                                 GEN7_RC_CTL_TO_MODE |
                                 rc6_mask);
@@ -5020,14 +5349,13 @@ static void gen8_enable_rps(struct drm_device *dev)
         /* 6: Ring frequency + overclocking (our driver does this later */
 
         dev_priv->rps.power = HIGH_POWER; /* force a reset */
-        gen6_set_rps(dev_priv->dev, dev_priv->rps.idle_freq);
+        gen6_set_rps(dev_priv, dev_priv->rps.idle_freq);
 
         intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
 }
 
-static void gen6_enable_rps(struct drm_device *dev)
+static void gen6_enable_rps(struct drm_i915_private *dev_priv)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
         struct intel_engine_cs *engine;
         u32 rc6vids, pcu_mbox = 0, rc6_mask = 0;
         u32 gtfifodbg;
@@ -5054,7 +5382,7 @@ static void gen6_enable_rps(struct drm_device *dev)
         intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
 
         /* Initialize rps frequencies */
-        gen6_init_rps_frequencies(dev);
+        gen6_init_rps_frequencies(dev_priv);
 
         /* disable the counters and set deterministic thresholds */
         I915_WRITE(GEN6_RC_CONTROL, 0);
@@ -5070,7 +5398,7 @@ static void gen6_enable_rps(struct drm_device *dev)
 
         I915_WRITE(GEN6_RC_SLEEP, 0);
         I915_WRITE(GEN6_RC1e_THRESHOLD, 1000);
-        if (IS_IVYBRIDGE(dev))
+        if (IS_IVYBRIDGE(dev_priv))
                 I915_WRITE(GEN6_RC6_THRESHOLD, 125000);
         else
                 I915_WRITE(GEN6_RC6_THRESHOLD, 50000);
@@ -5078,12 +5406,12 @@ static void gen6_enable_rps(struct drm_device *dev)
         I915_WRITE(GEN6_RC6pp_THRESHOLD, 64000); /* unused */
 
         /* Check if we are enabling RC6 */
-        rc6_mode = intel_enable_rc6(dev_priv->dev);
+        rc6_mode = intel_enable_rc6();
         if (rc6_mode & INTEL_RC6_ENABLE)
                 rc6_mask |= GEN6_RC_CTL_RC6_ENABLE;
 
         /* We don't use those on Haswell */
-        if (!IS_HASWELL(dev)) {
+        if (!IS_HASWELL(dev_priv)) {
                 if (rc6_mode & INTEL_RC6p_ENABLE)
                         rc6_mask |= GEN6_RC_CTL_RC6p_ENABLE;
 
@@ -5091,7 +5419,7 @@ static void gen6_enable_rps(struct drm_device *dev)
                         rc6_mask |= GEN6_RC_CTL_RC6pp_ENABLE;
         }
 
-        intel_print_rc6_info(dev, rc6_mask);
+        intel_print_rc6_info(dev_priv, rc6_mask);
 
         I915_WRITE(GEN6_RC_CONTROL,
                    rc6_mask |
@@ -5115,13 +5443,13 @@ static void gen6_enable_rps(struct drm_device *dev)
         }
 
         dev_priv->rps.power = HIGH_POWER; /* force a reset */
-        gen6_set_rps(dev_priv->dev, dev_priv->rps.idle_freq);
+        gen6_set_rps(dev_priv, dev_priv->rps.idle_freq);
 
         rc6vids = 0;
         ret = sandybridge_pcode_read(dev_priv, GEN6_PCODE_READ_RC6VIDS, &rc6vids);
-        if (IS_GEN6(dev) && ret) {
+        if (IS_GEN6(dev_priv) && ret) {
                 DRM_DEBUG_DRIVER("Couldn't check for BIOS workaround\n");
-        } else if (IS_GEN6(dev) && (GEN6_DECODE_RC6_VID(rc6vids & 0xff) < 450)) {
+        } else if (IS_GEN6(dev_priv) && (GEN6_DECODE_RC6_VID(rc6vids & 0xff) < 450)) {
                 DRM_DEBUG_DRIVER("You should update your BIOS. Correcting minimum rc6 voltage (%dmV->%dmV)\n",
                           GEN6_DECODE_RC6_VID(rc6vids & 0xff), 450);
                 rc6vids &= 0xffff00;
@@ -5134,9 +5462,8 @@ static void gen6_enable_rps(struct drm_device *dev)
         intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
 }
 
-static void __gen6_update_ring_freq(struct drm_device *dev)
+static void __gen6_update_ring_freq(struct drm_i915_private *dev_priv)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
         int min_freq = 15;
         unsigned int gpu_freq;
         unsigned int max_ia_freq, min_ring_freq;
@@ -5165,7 +5492,7 @@ static void __gen6_update_ring_freq(struct drm_device *dev)
         /* convert DDR frequency from units of 266.6MHz to bandwidth */
         min_ring_freq = mult_frac(min_ring_freq, 8, 3);
 
-        if (IS_SKYLAKE(dev) || IS_KABYLAKE(dev)) {
+        if (IS_SKYLAKE(dev_priv) || IS_KABYLAKE(dev_priv)) {
                 /* Convert GT frequency to 50 HZ units */
                 min_gpu_freq = dev_priv->rps.min_freq / GEN9_FREQ_SCALER;
                 max_gpu_freq = dev_priv->rps.max_freq / GEN9_FREQ_SCALER;
@@ -5183,16 +5510,16 @@ static void __gen6_update_ring_freq(struct drm_device *dev)
                 int diff = max_gpu_freq - gpu_freq;
                 unsigned int ia_freq = 0, ring_freq = 0;
 
-                if (IS_SKYLAKE(dev) || IS_KABYLAKE(dev)) {
+                if (IS_SKYLAKE(dev_priv) || IS_KABYLAKE(dev_priv)) {
                         /*
                          * ring_freq = 2 * GT. ring_freq is in 100MHz units
                          * No floor required for ring frequency on SKL.
                          */
                         ring_freq = gpu_freq;
-                } else if (INTEL_INFO(dev)->gen >= 8) {
+                } else if (INTEL_INFO(dev_priv)->gen >= 8) {
                         /* max(2 * GT, DDR). NB: GT is 50MHz units */
                         ring_freq = max(min_ring_freq, gpu_freq);
-                } else if (IS_HASWELL(dev)) {
+                } else if (IS_HASWELL(dev_priv)) {
                         ring_freq = mult_frac(gpu_freq, 5, 4);
                         ring_freq = max(min_ring_freq, ring_freq);
                         /* leave ia_freq as the default, chosen by cpufreq */
@@ -5219,26 +5546,23 @@ static void __gen6_update_ring_freq(struct drm_device *dev)
         }
 }
 
-void gen6_update_ring_freq(struct drm_device *dev)
+void gen6_update_ring_freq(struct drm_i915_private *dev_priv)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
-
-        if (!HAS_CORE_RING_FREQ(dev))
+        if (!HAS_CORE_RING_FREQ(dev_priv))
                 return;
 
         mutex_lock(&dev_priv->rps.hw_lock);
-        __gen6_update_ring_freq(dev);
+        __gen6_update_ring_freq(dev_priv);
         mutex_unlock(&dev_priv->rps.hw_lock);
 }
 
 static int cherryview_rps_max_freq(struct drm_i915_private *dev_priv)
 {
-        struct drm_device *dev = dev_priv->dev;
         u32 val, rp0;
 
         val = vlv_punit_read(dev_priv, FB_GFX_FMAX_AT_VMAX_FUSE);
 
-        switch (INTEL_INFO(dev)->eu_total) {
+        switch (INTEL_INFO(dev_priv)->eu_total) {
         case 8:
                 /* (2 * 4) config */
                 rp0 = (val >> FB_GFX_FMAX_AT_VMAX_2SS4EU_FUSE_SHIFT);
@@ -5349,9 +5673,8 @@ static void cherryview_check_pctx(struct drm_i915_private *dev_priv)
         WARN_ON((pctx_addr >> VLV_PCBR_ADDR_SHIFT) == 0);
 }
 
-static void cherryview_setup_pctx(struct drm_device *dev)
+static void cherryview_setup_pctx(struct drm_i915_private *dev_priv)
 {
-        struct drm_i915_private *dev_priv = to_i915(dev);
         struct i915_ggtt *ggtt = &dev_priv->ggtt;
         unsigned long pctx_paddr, paddr;
         u32 pcbr;
@@ -5370,15 +5693,14 @@ static void cherryview_setup_pctx(struct drm_device *dev)
         DRM_DEBUG_DRIVER("PCBR: 0x%08x\n", I915_READ(VLV_PCBR));
 }
 
-static void valleyview_setup_pctx(struct drm_device *dev)
+static void valleyview_setup_pctx(struct drm_i915_private *dev_priv)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
         struct drm_i915_gem_object *pctx;
         unsigned long pctx_paddr;
         u32 pcbr;
         int pctx_size = 24*1024;
 
-        mutex_lock(&dev->struct_mutex);
+        mutex_lock(&dev_priv->drm.struct_mutex);
 
         pcbr = I915_READ(VLV_PCBR);
         if (pcbr) {
@@ -5386,7 +5708,7 @@ static void valleyview_setup_pctx(struct drm_device *dev)
                 int pcbr_offset;
 
                 pcbr_offset = (pcbr & (~4095)) - dev_priv->mm.stolen_base;
-                pctx = i915_gem_object_create_stolen_for_preallocated(dev_priv->dev,
+                pctx = i915_gem_object_create_stolen_for_preallocated(&dev_priv->drm,
                                                                       pcbr_offset,
                                                                       I915_GTT_OFFSET_NONE,
                                                                       pctx_size);
@@ -5403,7 +5725,7 @@ static void valleyview_setup_pctx(struct drm_device *dev)
          * overlap with other ranges, such as the frame buffer, protected
          * memory, or any other relevant ranges.
          */
-        pctx = i915_gem_object_create_stolen(dev, pctx_size);
+        pctx = i915_gem_object_create_stolen(&dev_priv->drm, pctx_size);
         if (!pctx) {
                 DRM_DEBUG("not enough stolen space for PCTX, disabling\n");
                 goto out;
@@ -5415,13 +5737,11 @@ static void valleyview_setup_pctx(struct drm_device *dev)
 out:
         DRM_DEBUG_DRIVER("PCBR: 0x%08x\n", I915_READ(VLV_PCBR));
         dev_priv->vlv_pctx = pctx;
-        mutex_unlock(&dev->struct_mutex);
+        mutex_unlock(&dev_priv->drm.struct_mutex);
 }
 
-static void valleyview_cleanup_pctx(struct drm_device *dev)
+static void valleyview_cleanup_pctx(struct drm_i915_private *dev_priv)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
-
         if (WARN_ON(!dev_priv->vlv_pctx))
                 return;
 
@@ -5440,12 +5760,11 @@ static void vlv_init_gpll_ref_freq(struct drm_i915_private *dev_priv)
                          dev_priv->rps.gpll_ref_freq);
 }
 
-static void valleyview_init_gt_powersave(struct drm_device *dev)
+static void valleyview_init_gt_powersave(struct drm_i915_private *dev_priv)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
         u32 val;
 
-        valleyview_setup_pctx(dev);
+        valleyview_setup_pctx(dev_priv);
 
         vlv_init_gpll_ref_freq(dev_priv);
 
@@ -5499,12 +5818,11 @@ static void valleyview_init_gt_powersave(struct drm_device *dev)
         mutex_unlock(&dev_priv->rps.hw_lock);
 }
 
-static void cherryview_init_gt_powersave(struct drm_device *dev)
+static void cherryview_init_gt_powersave(struct drm_i915_private *dev_priv)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
         u32 val;
 
-        cherryview_setup_pctx(dev);
+        cherryview_setup_pctx(dev_priv);
 
         vlv_init_gpll_ref_freq(dev_priv);
 
@@ -5564,14 +5882,13 @@ static void cherryview_init_gt_powersave(struct drm_device *dev)
         mutex_unlock(&dev_priv->rps.hw_lock);
 }
 
-static void valleyview_cleanup_gt_powersave(struct drm_device *dev)
+static void valleyview_cleanup_gt_powersave(struct drm_i915_private *dev_priv)
 {
-        valleyview_cleanup_pctx(dev);
+        valleyview_cleanup_pctx(dev_priv);
 }
 
-static void cherryview_enable_rps(struct drm_device *dev)
+static void cherryview_enable_rps(struct drm_i915_private *dev_priv)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
         struct intel_engine_cs *engine;
         u32 gtfifodbg, val, rc6_mode = 0, pcbr;
 
@@ -5616,8 +5933,8 @@ static void cherryview_enable_rps(struct drm_device *dev)
         pcbr = I915_READ(VLV_PCBR);
 
         /* 3: Enable RC6 */
-        if ((intel_enable_rc6(dev) & INTEL_RC6_ENABLE) &&
-                                                (pcbr >> VLV_PCBR_ADDR_SHIFT))
+        if ((intel_enable_rc6() & INTEL_RC6_ENABLE) &&
+            (pcbr >> VLV_PCBR_ADDR_SHIFT))
                 rc6_mode = GEN7_RC_CTL_TO_MODE;
 
         I915_WRITE(GEN6_RC_CONTROL, rc6_mode);
@@ -5662,14 +5979,13 @@ static void cherryview_enable_rps(struct drm_device *dev)
                          intel_gpu_freq(dev_priv, dev_priv->rps.idle_freq),
                          dev_priv->rps.idle_freq);
 
-        valleyview_set_rps(dev_priv->dev, dev_priv->rps.idle_freq);
+        valleyview_set_rps(dev_priv, dev_priv->rps.idle_freq);
 
         intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
 }
 
-static void valleyview_enable_rps(struct drm_device *dev)
+static void valleyview_enable_rps(struct drm_i915_private *dev_priv)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
         struct intel_engine_cs *engine;
         u32 gtfifodbg, val, rc6_mode = 0;
 
@@ -5722,10 +6038,10 @@ static void valleyview_enable_rps(struct drm_device *dev)
                                       VLV_MEDIA_RC6_COUNT_EN |
                                       VLV_RENDER_RC6_COUNT_EN));
 
-        if (intel_enable_rc6(dev) & INTEL_RC6_ENABLE)
+        if (intel_enable_rc6() & INTEL_RC6_ENABLE)
                 rc6_mode = GEN7_RC_CTL_TO_MODE | VLV_RC_CTL_CTX_RST_PARALLEL;
 
-        intel_print_rc6_info(dev, rc6_mode);
+        intel_print_rc6_info(dev_priv, rc6_mode);
 
         I915_WRITE(GEN6_RC_CONTROL, rc6_mode);
 
@@ -5752,7 +6068,7 @@ static void valleyview_enable_rps(struct drm_device *dev)
                          intel_gpu_freq(dev_priv, dev_priv->rps.idle_freq),
                          dev_priv->rps.idle_freq);
 
-        valleyview_set_rps(dev_priv->dev, dev_priv->rps.idle_freq);
+        valleyview_set_rps(dev_priv, dev_priv->rps.idle_freq);
 
         intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
 }
@@ -5842,10 +6158,9 @@ static unsigned long __i915_chipset_val(struct drm_i915_private *dev_priv)
 
 unsigned long i915_chipset_val(struct drm_i915_private *dev_priv)
 {
-        struct drm_device *dev = dev_priv->dev;
         unsigned long val;
 
-        if (INTEL_INFO(dev)->gen != 5)
+        if (INTEL_INFO(dev_priv)->gen != 5)
                 return 0;
 
         spin_lock_irq(&mchdev_lock);
@@ -5885,11 +6200,10 @@ static int _pxvid_to_vd(u8 pxvid)
 
 static u32 pvid_to_extvid(struct drm_i915_private *dev_priv, u8 pxvid)
 {
-        struct drm_device *dev = dev_priv->dev;
         const int vd = _pxvid_to_vd(pxvid);
         const int vm = vd - 1125;
 
-        if (INTEL_INFO(dev)->is_mobile)
+        if (INTEL_INFO(dev_priv)->is_mobile)
                 return vm > 0 ? vm : 0;
 
         return vd;
@@ -5930,9 +6244,7 @@ static void __i915_update_gfx_val(struct drm_i915_private *dev_priv)
 
 void i915_update_gfx_val(struct drm_i915_private *dev_priv)
 {
-        struct drm_device *dev = dev_priv->dev;
-
-        if (INTEL_INFO(dev)->gen != 5)
+        if (INTEL_INFO(dev_priv)->gen != 5)
                 return;
 
         spin_lock_irq(&mchdev_lock);
@@ -5981,10 +6293,9 @@ static unsigned long __i915_gfx_val(struct drm_i915_private *dev_priv)
 
 unsigned long i915_gfx_val(struct drm_i915_private *dev_priv)
 {
-        struct drm_device *dev = dev_priv->dev;
         unsigned long val;
 
-        if (INTEL_INFO(dev)->gen != 5)
+        if (INTEL_INFO(dev_priv)->gen != 5)
                 return 0;
 
         spin_lock_irq(&mchdev_lock);
@@ -6125,7 +6436,7 @@ bool i915_gpu_turbo_disable(void)
 
         dev_priv->ips.max_delay = dev_priv->ips.fstart;
 
-        if (!ironlake_set_drps(dev_priv->dev, dev_priv->ips.fstart))
+        if (!ironlake_set_drps(dev_priv, dev_priv->ips.fstart))
                 ret = false;
 
 out_unlock:
@@ -6173,9 +6484,8 @@ void intel_gpu_ips_teardown(void)
         spin_unlock_irq(&mchdev_lock);
 }
 
-static void intel_init_emon(struct drm_device *dev)
+static void intel_init_emon(struct drm_i915_private *dev_priv)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
         u32 lcfuse;
         u8 pxw[16];
         int i;
@@ -6244,10 +6554,8 @@ static void intel_init_emon(struct drm_device *dev)
         dev_priv->ips.corr = (lcfuse & LCFUSE_HIV_MASK);
 }
 
-void intel_init_gt_powersave(struct drm_device *dev)
+void intel_init_gt_powersave(struct drm_i915_private *dev_priv)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
-
         /*
          * RPM depends on RC6 to save restore the GT HW context, so make RC6 a
          * requirement.
@@ -6257,74 +6565,66 @@ void intel_init_gt_powersave(struct drm_device *dev)
                 intel_runtime_pm_get(dev_priv);
         }
 
-        if (IS_CHERRYVIEW(dev))
-                cherryview_init_gt_powersave(dev);
-        else if (IS_VALLEYVIEW(dev))
-                valleyview_init_gt_powersave(dev);
+        if (IS_CHERRYVIEW(dev_priv))
+                cherryview_init_gt_powersave(dev_priv);
+        else if (IS_VALLEYVIEW(dev_priv))
+                valleyview_init_gt_powersave(dev_priv);
 }
 
-void intel_cleanup_gt_powersave(struct drm_device *dev)
+void intel_cleanup_gt_powersave(struct drm_i915_private *dev_priv)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
-
-        if (IS_CHERRYVIEW(dev))
+        if (IS_CHERRYVIEW(dev_priv))
                 return;
-        else if (IS_VALLEYVIEW(dev))
-                valleyview_cleanup_gt_powersave(dev);
+        else if (IS_VALLEYVIEW(dev_priv))
+                valleyview_cleanup_gt_powersave(dev_priv);
 
         if (!i915.enable_rc6)
                 intel_runtime_pm_put(dev_priv);
 }
 
-static void gen6_suspend_rps(struct drm_device *dev)
+static void gen6_suspend_rps(struct drm_i915_private *dev_priv)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
-
         flush_delayed_work(&dev_priv->rps.delayed_resume_work);
 
-        gen6_disable_rps_interrupts(dev);
+        gen6_disable_rps_interrupts(dev_priv);
 }
 
 /**
  * intel_suspend_gt_powersave - suspend PM work and helper threads
- * @dev: drm device
+ * @dev_priv: i915 device
  *
  * We don't want to disable RC6 or other features here, we just want
  * to make sure any work we've queued has finished and won't bother
  * us while we're suspended.
  */
-void intel_suspend_gt_powersave(struct drm_device *dev)
+void intel_suspend_gt_powersave(struct drm_i915_private *dev_priv)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
-
-        if (INTEL_INFO(dev)->gen < 6)
+        if (INTEL_GEN(dev_priv) < 6)
                 return;
 
-        gen6_suspend_rps(dev);
+        gen6_suspend_rps(dev_priv);
 
         /* Force GPU to min freq during suspend */
         gen6_rps_idle(dev_priv);
 }
 
-void intel_disable_gt_powersave(struct drm_device *dev)
+void intel_disable_gt_powersave(struct drm_i915_private *dev_priv)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
-
-        if (IS_IRONLAKE_M(dev)) {
-                ironlake_disable_drps(dev);
-        } else if (INTEL_INFO(dev)->gen >= 6) {
-                intel_suspend_gt_powersave(dev);
+        if (IS_IRONLAKE_M(dev_priv)) {
+                ironlake_disable_drps(dev_priv);
+        } else if (INTEL_INFO(dev_priv)->gen >= 6) {
+                intel_suspend_gt_powersave(dev_priv);
 
                 mutex_lock(&dev_priv->rps.hw_lock);
-                if (INTEL_INFO(dev)->gen >= 9) {
-                        gen9_disable_rc6(dev);
-                        gen9_disable_rps(dev);
-                } else if (IS_CHERRYVIEW(dev))
-                        cherryview_disable_rps(dev);
-                else if (IS_VALLEYVIEW(dev))
-                        valleyview_disable_rps(dev);
+                if (INTEL_INFO(dev_priv)->gen >= 9) {
+                        gen9_disable_rc6(dev_priv);
+                        gen9_disable_rps(dev_priv);
+                } else if (IS_CHERRYVIEW(dev_priv))
+                        cherryview_disable_rps(dev_priv);
+                else if (IS_VALLEYVIEW(dev_priv))
+                        valleyview_disable_rps(dev_priv);
                 else
-                        gen6_disable_rps(dev);
+                        gen6_disable_rps(dev_priv);
 
                 dev_priv->rps.enabled = false;
                 mutex_unlock(&dev_priv->rps.hw_lock);
@@ -6336,27 +6636,26 @@ static void intel_gen6_powersave_work(struct work_struct *work)
         struct drm_i915_private *dev_priv =
                 container_of(work, struct drm_i915_private,
                              rps.delayed_resume_work.work);
-        struct drm_device *dev = dev_priv->dev;
 
         mutex_lock(&dev_priv->rps.hw_lock);
 
-        gen6_reset_rps_interrupts(dev);
+        gen6_reset_rps_interrupts(dev_priv);
 
-        if (IS_CHERRYVIEW(dev)) {
-                cherryview_enable_rps(dev);
-        } else if (IS_VALLEYVIEW(dev)) {
-                valleyview_enable_rps(dev);
-        } else if (INTEL_INFO(dev)->gen >= 9) {
-                gen9_enable_rc6(dev);
-                gen9_enable_rps(dev);
-                if (IS_SKYLAKE(dev) || IS_KABYLAKE(dev))
-                        __gen6_update_ring_freq(dev);
-        } else if (IS_BROADWELL(dev)) {
-                gen8_enable_rps(dev);
-                __gen6_update_ring_freq(dev);
+        if (IS_CHERRYVIEW(dev_priv)) {
+                cherryview_enable_rps(dev_priv);
+        } else if (IS_VALLEYVIEW(dev_priv)) {
+                valleyview_enable_rps(dev_priv);
+        } else if (INTEL_INFO(dev_priv)->gen >= 9) {
+                gen9_enable_rc6(dev_priv);
+                gen9_enable_rps(dev_priv);
+                if (IS_SKYLAKE(dev_priv) || IS_KABYLAKE(dev_priv))
+                        __gen6_update_ring_freq(dev_priv);
+        } else if (IS_BROADWELL(dev_priv)) {
+                gen8_enable_rps(dev_priv);
+                __gen6_update_ring_freq(dev_priv);
         } else {
-                gen6_enable_rps(dev);
-                __gen6_update_ring_freq(dev);
+                gen6_enable_rps(dev_priv);
+                __gen6_update_ring_freq(dev_priv);
         }
 
         WARN_ON(dev_priv->rps.max_freq < dev_priv->rps.min_freq);
@@ -6367,27 +6666,25 @@ static void intel_gen6_powersave_work(struct work_struct *work)
 
         dev_priv->rps.enabled = true;
 
-        gen6_enable_rps_interrupts(dev);
+        gen6_enable_rps_interrupts(dev_priv);
 
         mutex_unlock(&dev_priv->rps.hw_lock);
 
         intel_runtime_pm_put(dev_priv);
 }
 
-void intel_enable_gt_powersave(struct drm_device *dev)
+void intel_enable_gt_powersave(struct drm_i915_private *dev_priv)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
-
         /* Powersaving is controlled by the host when inside a VM */
-        if (intel_vgpu_active(dev))
+        if (intel_vgpu_active(dev_priv))
                 return;
 
-        if (IS_IRONLAKE_M(dev)) {
-                ironlake_enable_drps(dev);
-                mutex_lock(&dev->struct_mutex);
-                intel_init_emon(dev);
-                mutex_unlock(&dev->struct_mutex);
-        } else if (INTEL_INFO(dev)->gen >= 6) {
+        if (IS_IRONLAKE_M(dev_priv)) {
+                ironlake_enable_drps(dev_priv);
+                mutex_lock(&dev_priv->drm.struct_mutex);
+                intel_init_emon(dev_priv);
+                mutex_unlock(&dev_priv->drm.struct_mutex);
+        } else if (INTEL_INFO(dev_priv)->gen >= 6) {
                 /*
                  * PCU communication is slow and this doesn't need to be
                  * done at any specific time, so do this out of our fast path
@@ -6406,20 +6703,18 @@ void intel_enable_gt_powersave(struct drm_device *dev)
         }
 }
 
-void intel_reset_gt_powersave(struct drm_device *dev)
+void intel_reset_gt_powersave(struct drm_i915_private *dev_priv)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
-
-        if (INTEL_INFO(dev)->gen < 6)
+        if (INTEL_INFO(dev_priv)->gen < 6)
                 return;
 
-        gen6_suspend_rps(dev);
+        gen6_suspend_rps(dev_priv);
         dev_priv->rps.enabled = false;
 }
 
 static void ibx_init_clock_gating(struct drm_device *dev)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
 
         /*
          * On Ibex Peak and Cougar Point, we need to disable clock
@@ -6431,7 +6726,7 @@ static void ibx_init_clock_gating(struct drm_device *dev)
 
 static void g4x_disable_trickle_feed(struct drm_device *dev)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         enum pipe pipe;
 
         for_each_pipe(dev_priv, pipe) {
@@ -6446,7 +6741,7 @@ static void g4x_disable_trickle_feed(struct drm_device *dev)
 
 static void ilk_init_lp_watermarks(struct drm_device *dev)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
 
         I915_WRITE(WM3_LP_ILK, I915_READ(WM3_LP_ILK) & ~WM1_LP_SR_EN);
         I915_WRITE(WM2_LP_ILK, I915_READ(WM2_LP_ILK) & ~WM1_LP_SR_EN);
@@ -6460,7 +6755,7 @@ static void ilk_init_lp_watermarks(struct drm_device *dev)
 
 static void ironlake_init_clock_gating(struct drm_device *dev)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         uint32_t dspclk_gate = ILK_VRHUNIT_CLOCK_GATE_DISABLE;
 
         /*
@@ -6534,7 +6829,7 @@ static void ironlake_init_clock_gating(struct drm_device *dev)
 
 static void cpt_init_clock_gating(struct drm_device *dev)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         int pipe;
         uint32_t val;
 
@@ -6571,7 +6866,7 @@ static void cpt_init_clock_gating(struct drm_device *dev)
 
 static void gen6_check_mch_setup(struct drm_device *dev)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         uint32_t tmp;
 
         tmp = I915_READ(MCH_SSKPD);
@@ -6582,7 +6877,7 @@ static void gen6_check_mch_setup(struct drm_device *dev)
 
 static void gen6_init_clock_gating(struct drm_device *dev)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         uint32_t dspclk_gate = ILK_VRHUNIT_CLOCK_GATE_DISABLE;
 
         I915_WRITE(ILK_DSPCLK_GATE_D, dspclk_gate);
@@ -6697,7 +6992,7 @@ static void gen7_setup_fixed_func_scheduler(struct drm_i915_private *dev_priv)
 
 static void lpt_init_clock_gating(struct drm_device *dev)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
 
         /*
          * TODO: this bit should only be enabled when really needed, then
@@ -6716,7 +7011,7 @@ static void lpt_init_clock_gating(struct drm_device *dev)
 
 static void lpt_suspend_hw(struct drm_device *dev)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
 
         if (HAS_PCH_LPT_LP(dev)) {
                 uint32_t val = I915_READ(SOUTH_DSPCLK_GATE_D);
@@ -6726,6 +7021,29 @@ static void lpt_suspend_hw(struct drm_device *dev)
         }
 }
 
+static void gen8_set_l3sqc_credits(struct drm_i915_private *dev_priv,
+                                   int general_prio_credits,
+                                   int high_prio_credits)
+{
+        u32 misccpctl;
+
+        /* WaTempDisableDOPClkGating:bdw */
+        misccpctl = I915_READ(GEN7_MISCCPCTL);
+        I915_WRITE(GEN7_MISCCPCTL, misccpctl & ~GEN7_DOP_CLOCK_GATE_ENABLE);
+
+        I915_WRITE(GEN8_L3SQCREG1,
+                   L3_GENERAL_PRIO_CREDITS(general_prio_credits) |
+                   L3_HIGH_PRIO_CREDITS(high_prio_credits));
+
+        /*
+         * Wait at least 100 clocks before re-enabling clock gating.
+         * See the definition of L3SQCREG1 in BSpec.
+         */
+        POSTING_READ(GEN8_L3SQCREG1);
+        udelay(1);
+        I915_WRITE(GEN7_MISCCPCTL, misccpctl);
+}
+
 static void kabylake_init_clock_gating(struct drm_device *dev)
 {
         struct drm_i915_private *dev_priv = dev->dev_private;
@@ -6753,6 +7071,10 @@ static void skylake_init_clock_gating(struct drm_device *dev)
 
         gen9_init_clock_gating(dev);
 
+        /* WAC6entrylatency:skl */
+        I915_WRITE(FBC_LLC_READ_CTRL, I915_READ(FBC_LLC_READ_CTRL) |
+                   FBC_LLC_FULLY_OPEN);
+
         /* WaFbcNukeOnHostModify:skl */
         I915_WRITE(ILK_DPFC_CHICKEN, I915_READ(ILK_DPFC_CHICKEN) |
                    ILK_DPFC_NUKE_ON_ANY_MODIFICATION);
@@ -6760,9 +7082,8 @@ static void skylake_init_clock_gating(struct drm_device *dev)
 
 static void broadwell_init_clock_gating(struct drm_device *dev)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         enum pipe pipe;
-        uint32_t misccpctl;
 
         ilk_init_lp_watermarks(dev);
 
@@ -6793,20 +7114,8 @@ static void broadwell_init_clock_gating(struct drm_device *dev)
         I915_WRITE(GEN8_UCGCTL6, I915_READ(GEN8_UCGCTL6) |
                    GEN8_SDEUNIT_CLOCK_GATE_DISABLE);
 
-        /*
-         * WaProgramL3SqcReg1Default:bdw
-         * WaTempDisableDOPClkGating:bdw
-         */
-        misccpctl = I915_READ(GEN7_MISCCPCTL);
-        I915_WRITE(GEN7_MISCCPCTL, misccpctl & ~GEN7_DOP_CLOCK_GATE_ENABLE);
-        I915_WRITE(GEN8_L3SQCREG1, BDW_WA_L3SQCREG1_DEFAULT);
-        /*
-         * Wait at least 100 clocks before re-enabling clock gating. See
-         * the definition of L3SQCREG1 in BSpec.
-         */
-        POSTING_READ(GEN8_L3SQCREG1);
-        udelay(1);
-        I915_WRITE(GEN7_MISCCPCTL, misccpctl);
+        /* WaProgramL3SqcReg1Default:bdw */
+        gen8_set_l3sqc_credits(dev_priv, 30, 2);
 
         /*
          * WaGttCachingOffByDefault:bdw
@@ -6815,12 +7124,16 @@ static void broadwell_init_clock_gating(struct drm_device *dev)
          */
         I915_WRITE(HSW_GTT_CACHE_EN, GTT_CACHE_EN_ALL);
 
+        /* WaKVMNotificationOnConfigChange:bdw */
+        I915_WRITE(CHICKEN_PAR2_1, I915_READ(CHICKEN_PAR2_1)
+                   | KVM_CONFIG_CHANGE_NOTIFICATION_SELECT);
+
         lpt_init_clock_gating(dev);
 }
 
 static void haswell_init_clock_gating(struct drm_device *dev)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
 
         ilk_init_lp_watermarks(dev);
 
@@ -6876,7 +7189,7 @@ static void haswell_init_clock_gating(struct drm_device *dev)
 
 static void ivybridge_init_clock_gating(struct drm_device *dev)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         uint32_t snpcr;
 
         ilk_init_lp_watermarks(dev);
@@ -6974,7 +7287,7 @@ static void ivybridge_init_clock_gating(struct drm_device *dev)
 
 static void valleyview_init_clock_gating(struct drm_device *dev)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
 
         /* WaDisableEarlyCull:vlv */
         I915_WRITE(_3D_CHICKEN3,
@@ -7056,7 +7369,7 @@ static void valleyview_init_clock_gating(struct drm_device *dev)
 
 static void cherryview_init_clock_gating(struct drm_device *dev)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
 
         /* WaVSRefCountFullforceMissDisable:chv */
         /* WaDSRefCountFullforceMissDisable:chv */
@@ -7077,6 +7390,13 @@ static void cherryview_init_clock_gating(struct drm_device *dev)
                    GEN8_SDEUNIT_CLOCK_GATE_DISABLE);
 
         /*
+         * WaProgramL3SqcReg1Default:chv
+         * See gfxspecs/Related Documents/Performance Guide/
+         * LSQC Setting Recommendations.
+         */
+        gen8_set_l3sqc_credits(dev_priv, 38, 2);
+
+        /*
          * GTT cache may not work with big pages, so if those
          * are ever enabled GTT cache may need to be disabled.
          */
@@ -7085,7 +7405,7 @@ static void cherryview_init_clock_gating(struct drm_device *dev)
 
 static void g4x_init_clock_gating(struct drm_device *dev)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         uint32_t dspclk_gate;
 
         I915_WRITE(RENCLK_GATE_D1, 0);
@@ -7112,7 +7432,7 @@ static void g4x_init_clock_gating(struct drm_device *dev)
 
 static void crestline_init_clock_gating(struct drm_device *dev)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
 
         I915_WRITE(RENCLK_GATE_D1, I965_RCC_CLOCK_GATE_DISABLE);
         I915_WRITE(RENCLK_GATE_D2, 0);
@@ -7128,7 +7448,7 @@ static void crestline_init_clock_gating(struct drm_device *dev)
 
 static void broadwater_init_clock_gating(struct drm_device *dev)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
 
         I915_WRITE(RENCLK_GATE_D1, I965_RCZ_CLOCK_GATE_DISABLE |
                    I965_RCC_CLOCK_GATE_DISABLE |
@@ -7145,7 +7465,7 @@ static void broadwater_init_clock_gating(struct drm_device *dev)
 
 static void gen3_init_clock_gating(struct drm_device *dev)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         u32 dstate = I915_READ(D_STATE);
 
         dstate |= DSTATE_PLL_D3_OFF | DSTATE_GFX_CLOCK_GATING |
@@ -7170,7 +7490,7 @@ static void gen3_init_clock_gating(struct drm_device *dev)
 
 static void i85x_init_clock_gating(struct drm_device *dev)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
 
         I915_WRITE(RENCLK_GATE_D1, SV_CLOCK_GATE_DISABLE);
 
@@ -7184,7 +7504,7 @@ static void i85x_init_clock_gating(struct drm_device *dev)
 
 static void i830_init_clock_gating(struct drm_device *dev)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
 
         I915_WRITE(DSPCLK_GATE_D, OVRUNIT_CLOCK_GATE_DISABLE);
 
@@ -7195,7 +7515,7 @@ static void i830_init_clock_gating(struct drm_device *dev)
 
 void intel_init_clock_gating(struct drm_device *dev)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
 
         dev_priv->display.init_clock_gating(dev);
 }
@@ -7263,7 +7583,7 @@ void intel_init_clock_gating_hooks(struct drm_i915_private *dev_priv)
 /* Set up chip specific power management-related functions */
 void intel_init_pm(struct drm_device *dev)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
 
         intel_fbc_init(dev_priv);
 
@@ -7277,6 +7597,7 @@ void intel_init_pm(struct drm_device *dev)
         if (INTEL_INFO(dev)->gen >= 9) {
                 skl_setup_wm_latency(dev);
                 dev_priv->display.update_wm = skl_update_wm;
+                dev_priv->display.compute_global_watermarks = skl_compute_wm;
         } else if (HAS_PCH_SPLIT(dev)) {
                 ilk_setup_wm_latency(dev);
 
@@ -7340,46 +7661,59 @@ int sandybridge_pcode_read(struct drm_i915_private *dev_priv, u32 mbox, u32 *val
 {
         WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock));
 
-        if (I915_READ(GEN6_PCODE_MAILBOX) & GEN6_PCODE_READY) {
+        /* GEN6_PCODE_* are outside of the forcewake domain, we can
+         * use te fw I915_READ variants to reduce the amount of work
+         * required when reading/writing.
+         */
+
+        if (I915_READ_FW(GEN6_PCODE_MAILBOX) & GEN6_PCODE_READY) {
                 DRM_DEBUG_DRIVER("warning: pcode (read) mailbox access failed\n");
                 return -EAGAIN;
         }
 
-        I915_WRITE(GEN6_PCODE_DATA, *val);
-        I915_WRITE(GEN6_PCODE_DATA1, 0);
-        I915_WRITE(GEN6_PCODE_MAILBOX, GEN6_PCODE_READY | mbox);
+        I915_WRITE_FW(GEN6_PCODE_DATA, *val);
+        I915_WRITE_FW(GEN6_PCODE_DATA1, 0);
+        I915_WRITE_FW(GEN6_PCODE_MAILBOX, GEN6_PCODE_READY | mbox);
 
-        if (wait_for((I915_READ(GEN6_PCODE_MAILBOX) & GEN6_PCODE_READY) == 0,
-                     500)) {
+        if (intel_wait_for_register_fw(dev_priv,
+                                       GEN6_PCODE_MAILBOX, GEN6_PCODE_READY, 0,
+                                       500)) {
                 DRM_ERROR("timeout waiting for pcode read (%d) to finish\n", mbox);
                 return -ETIMEDOUT;
         }
 
-        *val = I915_READ(GEN6_PCODE_DATA);
-        I915_WRITE(GEN6_PCODE_DATA, 0);
+        *val = I915_READ_FW(GEN6_PCODE_DATA);
+        I915_WRITE_FW(GEN6_PCODE_DATA, 0);
 
         return 0;
 }
 
-int sandybridge_pcode_write(struct drm_i915_private *dev_priv, u32 mbox, u32 val)
+int sandybridge_pcode_write(struct drm_i915_private *dev_priv,
+                               u32 mbox, u32 val)
 {
         WARN_ON(!mutex_is_locked(&dev_priv->rps.hw_lock));
 
-        if (I915_READ(GEN6_PCODE_MAILBOX) & GEN6_PCODE_READY) {
+        /* GEN6_PCODE_* are outside of the forcewake domain, we can
+         * use te fw I915_READ variants to reduce the amount of work
+         * required when reading/writing.
+         */
+
+        if (I915_READ_FW(GEN6_PCODE_MAILBOX) & GEN6_PCODE_READY) {
                 DRM_DEBUG_DRIVER("warning: pcode (write) mailbox access failed\n");
                 return -EAGAIN;
         }
 
-        I915_WRITE(GEN6_PCODE_DATA, val);
-        I915_WRITE(GEN6_PCODE_MAILBOX, GEN6_PCODE_READY | mbox);
+        I915_WRITE_FW(GEN6_PCODE_DATA, val);
+        I915_WRITE_FW(GEN6_PCODE_MAILBOX, GEN6_PCODE_READY | mbox);
 
-        if (wait_for((I915_READ(GEN6_PCODE_MAILBOX) & GEN6_PCODE_READY) == 0,
-                     500)) {
+        if (intel_wait_for_register_fw(dev_priv,
+                                       GEN6_PCODE_MAILBOX, GEN6_PCODE_READY, 0,
+                                       500)) {
                 DRM_ERROR("timeout waiting for pcode write (%d) to finish\n", mbox);
                 return -ETIMEDOUT;
         }
 
-        I915_WRITE(GEN6_PCODE_DATA, 0);
+        I915_WRITE_FW(GEN6_PCODE_DATA, 0);
 
         return 0;
 }
@@ -7449,23 +7783,21 @@ static void __intel_rps_boost_work(struct work_struct *work)
         struct request_boost *boost = container_of(work, struct request_boost, work);
         struct drm_i915_gem_request *req = boost->req;
 
-        if (!i915_gem_request_completed(req, true))
-                gen6_rps_boost(to_i915(req->engine->dev), NULL,
-                               req->emitted_jiffies);
+        if (!i915_gem_request_completed(req))
+                gen6_rps_boost(req->i915, NULL, req->emitted_jiffies);
 
-        i915_gem_request_unreference__unlocked(req);
+        i915_gem_request_unreference(req);
         kfree(boost);
 }
 
-void intel_queue_rps_boost_for_request(struct drm_device *dev,
-                                       struct drm_i915_gem_request *req)
+void intel_queue_rps_boost_for_request(struct drm_i915_gem_request *req)
 {
         struct request_boost *boost;
 
-        if (req == NULL || INTEL_INFO(dev)->gen < 6)
+        if (req == NULL || INTEL_GEN(req->i915) < 6)
                 return;
 
-        if (i915_gem_request_completed(req, true))
+        if (i915_gem_request_completed(req))
                 return;
 
         boost = kmalloc(sizeof(*boost), GFP_ATOMIC);
@@ -7476,12 +7808,12 @@ void intel_queue_rps_boost_for_request(struct drm_device *dev,
         boost->req = req;
 
         INIT_WORK(&boost->work, __intel_rps_boost_work);
-        queue_work(to_i915(dev)->wq, &boost->work);
+        queue_work(req->i915->wq, &boost->work);
 }
 
 void intel_pm_setup(struct drm_device *dev)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
 
         mutex_init(&dev_priv->rps.hw_lock);
         spin_lock_init(&dev_priv->rps.client_lock);
diff --git a/drivers/gpu/drm/i915/intel_psr.c b/drivers/gpu/drm/i915/intel_psr.c
index a788d1e9589b..68bd0bb34817 100644
--- a/drivers/gpu/drm/i915/intel_psr.c
+++ b/drivers/gpu/drm/i915/intel_psr.c
@@ -63,7 +63,7 @@ static bool is_edp_psr(struct intel_dp *intel_dp)
 
 static bool vlv_is_psr_active_on_pipe(struct drm_device *dev, int pipe)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         uint32_t val;
 
         val = I915_READ(VLV_PSRSTAT(pipe)) &
@@ -77,7 +77,7 @@ static void intel_psr_write_vsc(struct intel_dp *intel_dp,
 {
         struct intel_digital_port *dig_port = dp_to_dig_port(intel_dp);
         struct drm_device *dev = dig_port->base.base.dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         struct intel_crtc *crtc = to_intel_crtc(dig_port->base.base.crtc);
         enum transcoder cpu_transcoder = crtc->config->cpu_transcoder;
         i915_reg_t ctl_reg = HSW_TVIDEO_DIP_CTL(cpu_transcoder);
@@ -107,7 +107,7 @@ static void vlv_psr_setup_vsc(struct intel_dp *intel_dp)
 {
         struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp);
         struct drm_device *dev = intel_dig_port->base.base.dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         struct drm_crtc *crtc = intel_dig_port->base.base.crtc;
         enum pipe pipe = to_intel_crtc(crtc)->pipe;
         uint32_t val;
@@ -173,10 +173,9 @@ static void hsw_psr_enable_sink(struct intel_dp *intel_dp)
 {
         struct intel_digital_port *dig_port = dp_to_dig_port(intel_dp);
         struct drm_device *dev = dig_port->base.base.dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         uint32_t aux_clock_divider;
         i915_reg_t aux_ctl_reg;
-        int precharge = 0x3;
         static const uint8_t aux_msg[] = {
                 [0] = DP_AUX_NATIVE_WRITE << 4,
                 [1] = DP_SET_POWER >> 8,
@@ -185,6 +184,7 @@ static void hsw_psr_enable_sink(struct intel_dp *intel_dp)
                 [4] = DP_SET_POWER_D0,
         };
         enum port port = dig_port->port;
+        u32 aux_ctl;
         int i;
 
         BUILD_BUG_ON(sizeof(aux_msg) > 20);
@@ -197,6 +197,13 @@ static void hsw_psr_enable_sink(struct intel_dp *intel_dp)
                                 DP_SINK_DEVICE_AUX_FRAME_SYNC_CONF,
                                 DP_AUX_FRAME_SYNC_ENABLE);
 
+        if (dev_priv->psr.link_standby)
+                drm_dp_dpcd_writeb(&intel_dp->aux, DP_PSR_EN_CFG,
+                                   DP_PSR_ENABLE | DP_PSR_MAIN_LINK_ACTIVE);
+        else
+                drm_dp_dpcd_writeb(&intel_dp->aux, DP_PSR_EN_CFG,
+                                   DP_PSR_ENABLE);
+
         aux_ctl_reg = psr_aux_ctl_reg(dev_priv, port);
 
         /* Setup AUX registers */
@@ -204,40 +211,16 @@ static void hsw_psr_enable_sink(struct intel_dp *intel_dp)
                 I915_WRITE(psr_aux_data_reg(dev_priv, port, i >> 2),
                            intel_dp_pack_aux(&aux_msg[i], sizeof(aux_msg) - i));
 
-        if (INTEL_INFO(dev)->gen >= 9) {
-                uint32_t val;
-
-                val = I915_READ(aux_ctl_reg);
-                val &= ~DP_AUX_CH_CTL_TIME_OUT_MASK;
-                val |= DP_AUX_CH_CTL_TIME_OUT_1600us;
-                val &= ~DP_AUX_CH_CTL_MESSAGE_SIZE_MASK;
-                val |= (sizeof(aux_msg) << DP_AUX_CH_CTL_MESSAGE_SIZE_SHIFT);
-                /* Use hardcoded data values for PSR, frame sync and GTC */
-                val &= ~DP_AUX_CH_CTL_PSR_DATA_AUX_REG_SKL;
-                val &= ~DP_AUX_CH_CTL_FS_DATA_AUX_REG_SKL;
-                val &= ~DP_AUX_CH_CTL_GTC_DATA_AUX_REG_SKL;
-                I915_WRITE(aux_ctl_reg, val);
-        } else {
-                I915_WRITE(aux_ctl_reg,
-                   DP_AUX_CH_CTL_TIME_OUT_400us |
-                   (sizeof(aux_msg) << DP_AUX_CH_CTL_MESSAGE_SIZE_SHIFT) |
-                   (precharge << DP_AUX_CH_CTL_PRECHARGE_2US_SHIFT) |
-                   (aux_clock_divider << DP_AUX_CH_CTL_BIT_CLOCK_2X_SHIFT));
-        }
-
-        if (dev_priv->psr.link_standby)
-                drm_dp_dpcd_writeb(&intel_dp->aux, DP_PSR_EN_CFG,
-                                   DP_PSR_ENABLE | DP_PSR_MAIN_LINK_ACTIVE);
-        else
-                drm_dp_dpcd_writeb(&intel_dp->aux, DP_PSR_EN_CFG,
-                                   DP_PSR_ENABLE);
+        aux_ctl = intel_dp->get_aux_send_ctl(intel_dp, 0, sizeof(aux_msg),
+                                             aux_clock_divider);
+        I915_WRITE(aux_ctl_reg, aux_ctl);
 }
 
 static void vlv_psr_enable_source(struct intel_dp *intel_dp)
 {
         struct intel_digital_port *dig_port = dp_to_dig_port(intel_dp);
         struct drm_device *dev = dig_port->base.base.dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         struct drm_crtc *crtc = dig_port->base.base.crtc;
         enum pipe pipe = to_intel_crtc(crtc)->pipe;
 
@@ -252,7 +235,7 @@ static void vlv_psr_activate(struct intel_dp *intel_dp)
 {
         struct intel_digital_port *dig_port = dp_to_dig_port(intel_dp);
         struct drm_device *dev = dig_port->base.base.dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         struct drm_crtc *crtc = dig_port->base.base.crtc;
         enum pipe pipe = to_intel_crtc(crtc)->pipe;
 
@@ -269,17 +252,17 @@ static void hsw_psr_enable_source(struct intel_dp *intel_dp)
 {
         struct intel_digital_port *dig_port = dp_to_dig_port(intel_dp);
         struct drm_device *dev = dig_port->base.base.dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
 
         uint32_t max_sleep_time = 0x1f;
-        /*
-         * Let's respect VBT in case VBT asks a higher idle_frame value.
-         * Let's use 6 as the minimum to cover all known cases including
-         * the off-by-one issue that HW has in some cases. Also there are
-         * cases where sink should be able to train
-         * with the 5 or 6 idle patterns.
+        /* Lately it was identified that depending on panel idle frame count
+         * calculated at HW can be off by 1. So let's use what came
+         * from VBT + 1.
+         * There are also other cases where panel demands at least 4
+         * but VBT is not being set. To cover these 2 cases lets use
+         * at least 5 when VBT isn't set to be on the safest side.
          */
-        uint32_t idle_frames = max(6, dev_priv->vbt.psr.idle_frames);
+        uint32_t idle_frames = dev_priv->vbt.psr.idle_frames + 1;
         uint32_t val = EDP_PSR_ENABLE;
 
         val |= max_sleep_time << EDP_PSR_MAX_SLEEP_TIME_SHIFT;
@@ -341,7 +324,7 @@ static bool intel_psr_match_conditions(struct intel_dp *intel_dp)
 {
         struct intel_digital_port *dig_port = dp_to_dig_port(intel_dp);
         struct drm_device *dev = dig_port->base.base.dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         struct drm_crtc *crtc = dig_port->base.base.crtc;
         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
 
@@ -395,7 +378,7 @@ static void intel_psr_activate(struct intel_dp *intel_dp)
 {
         struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp);
         struct drm_device *dev = intel_dig_port->base.base.dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
 
         WARN_ON(I915_READ(EDP_PSR_CTL) & EDP_PSR_ENABLE);
         WARN_ON(dev_priv->psr.active);
@@ -424,7 +407,7 @@ void intel_psr_enable(struct intel_dp *intel_dp)
 {
         struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp);
         struct drm_device *dev = intel_dig_port->base.base.dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         struct intel_crtc *crtc = to_intel_crtc(intel_dig_port->base.base.crtc);
 
         if (!HAS_PSR(dev)) {
@@ -511,15 +494,18 @@ static void vlv_psr_disable(struct intel_dp *intel_dp)
 {
         struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp);
         struct drm_device *dev = intel_dig_port->base.base.dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         struct intel_crtc *intel_crtc =
                 to_intel_crtc(intel_dig_port->base.base.crtc);
         uint32_t val;
 
         if (dev_priv->psr.active) {
                 /* Put VLV PSR back to PSR_state 0 that is PSR Disabled. */
-                if (wait_for((I915_READ(VLV_PSRSTAT(intel_crtc->pipe)) &
-                              VLV_EDP_PSR_IN_TRANS) == 0, 1))
+                if (intel_wait_for_register(dev_priv,
+                                            VLV_PSRSTAT(intel_crtc->pipe),
+                                            VLV_EDP_PSR_IN_TRANS,
+                                            0,
+                                            1))
                         WARN(1, "PSR transition took longer than expected\n");
 
                 val = I915_READ(VLV_PSRCTL(intel_crtc->pipe));
@@ -538,16 +524,18 @@ static void hsw_psr_disable(struct intel_dp *intel_dp)
 {
         struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp);
         struct drm_device *dev = intel_dig_port->base.base.dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
 
         if (dev_priv->psr.active) {
                 I915_WRITE(EDP_PSR_CTL,
                            I915_READ(EDP_PSR_CTL) & ~EDP_PSR_ENABLE);
 
                 /* Wait till PSR is idle */
-                if (_wait_for((I915_READ(EDP_PSR_STATUS_CTL) &
-                               EDP_PSR_STATUS_STATE_MASK) == 0,
-                               2 * USEC_PER_SEC, 10 * USEC_PER_MSEC))
+                if (intel_wait_for_register(dev_priv,
+                                            EDP_PSR_STATUS_CTL,
+                                            EDP_PSR_STATUS_STATE_MASK,
+                                            0,
+                                            2000))
                         DRM_ERROR("Timed out waiting for PSR Idle State\n");
 
                 dev_priv->psr.active = false;
@@ -566,7 +554,7 @@ void intel_psr_disable(struct intel_dp *intel_dp)
 {
         struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp);
         struct drm_device *dev = intel_dig_port->base.base.dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
 
         mutex_lock(&dev_priv->psr.lock);
         if (!dev_priv->psr.enabled) {
@@ -603,14 +591,20 @@ static void intel_psr_work(struct work_struct *work)
          * and be ready for re-enable.
          */
         if (HAS_DDI(dev_priv)) {
-                if (wait_for((I915_READ(EDP_PSR_STATUS_CTL) &
-                              EDP_PSR_STATUS_STATE_MASK) == 0, 50)) {
+                if (intel_wait_for_register(dev_priv,
+                                            EDP_PSR_STATUS_CTL,
+                                            EDP_PSR_STATUS_STATE_MASK,
+                                            0,
+                                            50)) {
                         DRM_ERROR("Timed out waiting for PSR Idle for re-enable\n");
                         return;
                 }
         } else {
-                if (wait_for((I915_READ(VLV_PSRSTAT(pipe)) &
-                              VLV_EDP_PSR_IN_TRANS) == 0, 1)) {
+                if (intel_wait_for_register(dev_priv,
+                                            VLV_PSRSTAT(pipe),
+                                            VLV_EDP_PSR_IN_TRANS,
+                                            0,
+                                            1)) {
                         DRM_ERROR("Timed out waiting for PSR Idle for re-enable\n");
                         return;
                 }
@@ -636,7 +630,7 @@ unlock:
 
 static void intel_psr_exit(struct drm_device *dev)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         struct intel_dp *intel_dp = dev_priv->psr.enabled;
         struct drm_crtc *crtc = dp_to_dig_port(intel_dp)->base.base.crtc;
         enum pipe pipe = to_intel_crtc(crtc)->pipe;
@@ -691,7 +685,7 @@ static void intel_psr_exit(struct drm_device *dev)
 void intel_psr_single_frame_update(struct drm_device *dev,
                                    unsigned frontbuffer_bits)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         struct drm_crtc *crtc;
         enum pipe pipe;
         u32 val;
@@ -739,7 +733,7 @@ void intel_psr_single_frame_update(struct drm_device *dev,
 void intel_psr_invalidate(struct drm_device *dev,
                           unsigned frontbuffer_bits)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         struct drm_crtc *crtc;
         enum pipe pipe;
 
@@ -777,7 +771,7 @@ void intel_psr_invalidate(struct drm_device *dev,
 void intel_psr_flush(struct drm_device *dev,
                      unsigned frontbuffer_bits, enum fb_op_origin origin)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         struct drm_crtc *crtc;
         enum pipe pipe;
 
@@ -813,7 +807,7 @@ void intel_psr_flush(struct drm_device *dev,
  */
 void intel_psr_init(struct drm_device *dev)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
 
         dev_priv->psr_mmio_base = IS_HASWELL(dev_priv) ?
                 HSW_EDP_PSR_BASE : BDW_EDP_PSR_BASE;
diff --git a/drivers/gpu/drm/i915/intel_ringbuffer.c b/drivers/gpu/drm/i915/intel_ringbuffer.c
index 68c5af079ef8..cca7792f26d5 100644
--- a/drivers/gpu/drm/i915/intel_ringbuffer.c
+++ b/drivers/gpu/drm/i915/intel_ringbuffer.c
@@ -34,6 +34,11 @@
 #include "i915_trace.h"
 #include "intel_drv.h"
 
+/* Rough estimate of the typical request size, performing a flush,
+ * set-context and then emitting the batch.
+ */
+#define LEGACY_REQUEST_SIZE 200
+
 int __intel_ring_space(int head, int tail, int size)
 {
         int space = head - tail;
@@ -53,18 +58,10 @@ void intel_ring_update_space(struct intel_ringbuffer *ringbuf)
                                             ringbuf->tail, ringbuf->size);
 }
 
-bool intel_engine_stopped(struct intel_engine_cs *engine)
-{
-        struct drm_i915_private *dev_priv = engine->dev->dev_private;
-        return dev_priv->gpu_error.stop_rings & intel_engine_flag(engine);
-}
-
 static void __intel_ring_advance(struct intel_engine_cs *engine)
 {
         struct intel_ringbuffer *ringbuf = engine->buffer;
         ringbuf->tail &= ringbuf->size - 1;
-        if (intel_engine_stopped(engine))
-                return;
         engine->write_tail(engine, ringbuf->tail);
 }
 
@@ -101,7 +98,6 @@ gen4_render_ring_flush(struct drm_i915_gem_request *req,
                        u32      flush_domains)
 {
         struct intel_engine_cs *engine = req->engine;
-        struct drm_device *dev = engine->dev;
         u32 cmd;
         int ret;
 
@@ -140,7 +136,7 @@ gen4_render_ring_flush(struct drm_i915_gem_request *req,
                 cmd |= MI_EXE_FLUSH;
 
         if (invalidate_domains & I915_GEM_DOMAIN_COMMAND &&
-            (IS_G4X(dev) || IS_GEN5(dev)))
+            (IS_G4X(req->i915) || IS_GEN5(req->i915)))
                 cmd |= MI_INVALIDATE_ISP;
 
         ret = intel_ring_begin(req, 2);
@@ -426,19 +422,19 @@ gen8_render_ring_flush(struct drm_i915_gem_request *req,
 static void ring_write_tail(struct intel_engine_cs *engine,
                             u32 value)
 {
-        struct drm_i915_private *dev_priv = engine->dev->dev_private;
+        struct drm_i915_private *dev_priv = engine->i915;
         I915_WRITE_TAIL(engine, value);
 }
 
 u64 intel_ring_get_active_head(struct intel_engine_cs *engine)
 {
-        struct drm_i915_private *dev_priv = engine->dev->dev_private;
+        struct drm_i915_private *dev_priv = engine->i915;
         u64 acthd;
 
-        if (INTEL_INFO(engine->dev)->gen >= 8)
+        if (INTEL_GEN(dev_priv) >= 8)
                 acthd = I915_READ64_2x32(RING_ACTHD(engine->mmio_base),
                                          RING_ACTHD_UDW(engine->mmio_base));
-        else if (INTEL_INFO(engine->dev)->gen >= 4)
+        else if (INTEL_GEN(dev_priv) >= 4)
                 acthd = I915_READ(RING_ACTHD(engine->mmio_base));
         else
                 acthd = I915_READ(ACTHD);
@@ -448,25 +444,24 @@ u64 intel_ring_get_active_head(struct intel_engine_cs *engine)
 
 static void ring_setup_phys_status_page(struct intel_engine_cs *engine)
 {
-        struct drm_i915_private *dev_priv = engine->dev->dev_private;
+        struct drm_i915_private *dev_priv = engine->i915;
         u32 addr;
 
         addr = dev_priv->status_page_dmah->busaddr;
-        if (INTEL_INFO(engine->dev)->gen >= 4)
+        if (INTEL_GEN(dev_priv) >= 4)
                 addr |= (dev_priv->status_page_dmah->busaddr >> 28) & 0xf0;
         I915_WRITE(HWS_PGA, addr);
 }
 
 static void intel_ring_setup_status_page(struct intel_engine_cs *engine)
 {
-        struct drm_device *dev = engine->dev;
-        struct drm_i915_private *dev_priv = engine->dev->dev_private;
+        struct drm_i915_private *dev_priv = engine->i915;
         i915_reg_t mmio;
 
         /* The ring status page addresses are no longer next to the rest of
          * the ring registers as of gen7.
          */
-        if (IS_GEN7(dev)) {
+        if (IS_GEN7(dev_priv)) {
                 switch (engine->id) {
                 case RCS:
                         mmio = RENDER_HWS_PGA_GEN7;
@@ -486,7 +481,7 @@ static void intel_ring_setup_status_page(struct intel_engine_cs *engine)
                         mmio = VEBOX_HWS_PGA_GEN7;
                         break;
                 }
-        } else if (IS_GEN6(engine->dev)) {
+        } else if (IS_GEN6(dev_priv)) {
                 mmio = RING_HWS_PGA_GEN6(engine->mmio_base);
         } else {
                 /* XXX: gen8 returns to sanity */
@@ -503,7 +498,7 @@ static void intel_ring_setup_status_page(struct intel_engine_cs *engine)
          * arises: do we still need this and if so how should we go about
          * invalidating the TLB?
          */
-        if (INTEL_INFO(dev)->gen >= 6 && INTEL_INFO(dev)->gen < 8) {
+        if (IS_GEN(dev_priv, 6, 7)) {
                 i915_reg_t reg = RING_INSTPM(engine->mmio_base);
 
                 /* ring should be idle before issuing a sync flush*/
@@ -512,8 +507,9 @@ static void intel_ring_setup_status_page(struct intel_engine_cs *engine)
                 I915_WRITE(reg,
                            _MASKED_BIT_ENABLE(INSTPM_TLB_INVALIDATE |
                                               INSTPM_SYNC_FLUSH));
-                if (wait_for((I915_READ(reg) & INSTPM_SYNC_FLUSH) == 0,
-                             1000))
+                if (intel_wait_for_register(dev_priv,
+                                            reg, INSTPM_SYNC_FLUSH, 0,
+                                            1000))
                         DRM_ERROR("%s: wait for SyncFlush to complete for TLB invalidation timed out\n",
                                   engine->name);
         }
@@ -521,11 +517,15 @@ static void intel_ring_setup_status_page(struct intel_engine_cs *engine)
 
 static bool stop_ring(struct intel_engine_cs *engine)
 {
-        struct drm_i915_private *dev_priv = to_i915(engine->dev);
+        struct drm_i915_private *dev_priv = engine->i915;
 
-        if (!IS_GEN2(engine->dev)) {
+        if (!IS_GEN2(dev_priv)) {
                 I915_WRITE_MODE(engine, _MASKED_BIT_ENABLE(STOP_RING));
-                if (wait_for((I915_READ_MODE(engine) & MODE_IDLE) != 0, 1000)) {
+                if (intel_wait_for_register(dev_priv,
+                                            RING_MI_MODE(engine->mmio_base),
+                                            MODE_IDLE,
+                                            MODE_IDLE,
+                                            1000)) {
                         DRM_ERROR("%s : timed out trying to stop ring\n",
                                   engine->name);
                         /* Sometimes we observe that the idle flag is not
@@ -541,7 +541,7 @@ static bool stop_ring(struct intel_engine_cs *engine)
         I915_WRITE_HEAD(engine, 0);
         engine->write_tail(engine, 0);
 
-        if (!IS_GEN2(engine->dev)) {
+        if (!IS_GEN2(dev_priv)) {
                 (void)I915_READ_CTL(engine);
                 I915_WRITE_MODE(engine, _MASKED_BIT_DISABLE(STOP_RING));
         }
@@ -556,8 +556,7 @@ void intel_engine_init_hangcheck(struct intel_engine_cs *engine)
 
 static int init_ring_common(struct intel_engine_cs *engine)
 {
-        struct drm_device *dev = engine->dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = engine->i915;
         struct intel_ringbuffer *ringbuf = engine->buffer;
         struct drm_i915_gem_object *obj = ringbuf->obj;
         int ret = 0;
@@ -587,7 +586,7 @@ static int init_ring_common(struct intel_engine_cs *engine)
                 }
         }
 
-        if (I915_NEED_GFX_HWS(dev))
+        if (I915_NEED_GFX_HWS(dev_priv))
                 intel_ring_setup_status_page(engine);
         else
                 ring_setup_phys_status_page(engine);
@@ -641,59 +640,42 @@ out:
         return ret;
 }
 
-void
-intel_fini_pipe_control(struct intel_engine_cs *engine)
+void intel_fini_pipe_control(struct intel_engine_cs *engine)
 {
-        struct drm_device *dev = engine->dev;
-
         if (engine->scratch.obj == NULL)
                 return;
 
-        if (INTEL_INFO(dev)->gen >= 5) {
-                kunmap(sg_page(engine->scratch.obj->pages->sgl));
-                i915_gem_object_ggtt_unpin(engine->scratch.obj);
-        }
-
+        i915_gem_object_ggtt_unpin(engine->scratch.obj);
         drm_gem_object_unreference(&engine->scratch.obj->base);
         engine->scratch.obj = NULL;
 }
 
-int
-intel_init_pipe_control(struct intel_engine_cs *engine)
+int intel_init_pipe_control(struct intel_engine_cs *engine, int size)
 {
+        struct drm_i915_gem_object *obj;
         int ret;
 
         WARN_ON(engine->scratch.obj);
 
-        engine->scratch.obj = i915_gem_alloc_object(engine->dev, 4096);
-        if (engine->scratch.obj == NULL) {
-                DRM_ERROR("Failed to allocate seqno page\n");
-                ret = -ENOMEM;
+        obj = i915_gem_object_create_stolen(&engine->i915->drm, size);
+        if (!obj)
+                obj = i915_gem_object_create(&engine->i915->drm, size);
+        if (IS_ERR(obj)) {
+                DRM_ERROR("Failed to allocate scratch page\n");
+                ret = PTR_ERR(obj);
                 goto err;
         }
 
-        ret = i915_gem_object_set_cache_level(engine->scratch.obj,
-                                              I915_CACHE_LLC);
-        if (ret)
-                goto err_unref;
-
-        ret = i915_gem_obj_ggtt_pin(engine->scratch.obj, 4096, 0);
+        ret = i915_gem_obj_ggtt_pin(obj, 4096, PIN_HIGH);
         if (ret)
                 goto err_unref;
 
-        engine->scratch.gtt_offset = i915_gem_obj_ggtt_offset(engine->scratch.obj);
-        engine->scratch.cpu_page = kmap(sg_page(engine->scratch.obj->pages->sgl));
-        if (engine->scratch.cpu_page == NULL) {
-                ret = -ENOMEM;
-                goto err_unpin;
-        }
-
+        engine->scratch.obj = obj;
+        engine->scratch.gtt_offset = i915_gem_obj_ggtt_offset(obj);
         DRM_DEBUG_DRIVER("%s pipe control offset: 0x%08x\n",
                          engine->name, engine->scratch.gtt_offset);
         return 0;
 
-err_unpin:
-        i915_gem_object_ggtt_unpin(engine->scratch.obj);
 err_unref:
         drm_gem_object_unreference(&engine->scratch.obj->base);
 err:
@@ -702,11 +684,9 @@ err:
 
 static int intel_ring_workarounds_emit(struct drm_i915_gem_request *req)
 {
-        int ret, i;
         struct intel_engine_cs *engine = req->engine;
-        struct drm_device *dev = engine->dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
-        struct i915_workarounds *w = &dev_priv->workarounds;
+        struct i915_workarounds *w = &req->i915->workarounds;
+        int ret, i;
 
         if (w->count == 0)
                 return 0;
@@ -795,7 +775,7 @@ static int wa_add(struct drm_i915_private *dev_priv,
 static int wa_ring_whitelist_reg(struct intel_engine_cs *engine,
                                  i915_reg_t reg)
 {
-        struct drm_i915_private *dev_priv = engine->dev->dev_private;
+        struct drm_i915_private *dev_priv = engine->i915;
         struct i915_workarounds *wa = &dev_priv->workarounds;
         const uint32_t index = wa->hw_whitelist_count[engine->id];
 
@@ -811,8 +791,7 @@ static int wa_ring_whitelist_reg(struct intel_engine_cs *engine,
 
 static int gen8_init_workarounds(struct intel_engine_cs *engine)
 {
-        struct drm_device *dev = engine->dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = engine->i915;
 
         WA_SET_BIT_MASKED(INSTPM, INSTPM_FORCE_ORDERING);
 
@@ -863,9 +842,8 @@ static int gen8_init_workarounds(struct intel_engine_cs *engine)
 
 static int bdw_init_workarounds(struct intel_engine_cs *engine)
 {
+        struct drm_i915_private *dev_priv = engine->i915;
         int ret;
-        struct drm_device *dev = engine->dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
 
         ret = gen8_init_workarounds(engine);
         if (ret)
@@ -885,16 +863,15 @@ static int bdw_init_workarounds(struct intel_engine_cs *engine)
                           /* WaForceContextSaveRestoreNonCoherent:bdw */
                           HDC_FORCE_CONTEXT_SAVE_RESTORE_NON_COHERENT |
                           /* WaDisableFenceDestinationToSLM:bdw (pre-prod) */
-                          (IS_BDW_GT3(dev) ? HDC_FENCE_DEST_SLM_DISABLE : 0));
+                          (IS_BDW_GT3(dev_priv) ? HDC_FENCE_DEST_SLM_DISABLE : 0));
 
         return 0;
 }
 
 static int chv_init_workarounds(struct intel_engine_cs *engine)
 {
+        struct drm_i915_private *dev_priv = engine->i915;
         int ret;
-        struct drm_device *dev = engine->dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
 
         ret = gen8_init_workarounds(engine);
         if (ret)
@@ -911,8 +888,7 @@ static int chv_init_workarounds(struct intel_engine_cs *engine)
 
 static int gen9_init_workarounds(struct intel_engine_cs *engine)
 {
-        struct drm_device *dev = engine->dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = engine->i915;
         int ret;
 
         /* WaConextSwitchWithConcurrentTLBInvalidate:skl,bxt,kbl */
@@ -937,14 +913,14 @@ static int gen9_init_workarounds(struct intel_engine_cs *engine)
                           GEN9_DISABLE_OCL_OOB_SUPPRESS_LOGIC);
 
         /* WaDisableDgMirrorFixInHalfSliceChicken5:skl,bxt */
-        if (IS_SKL_REVID(dev, 0, SKL_REVID_B0) ||
-            IS_BXT_REVID(dev, 0, BXT_REVID_A1))
+        if (IS_SKL_REVID(dev_priv, 0, SKL_REVID_B0) ||
+            IS_BXT_REVID(dev_priv, 0, BXT_REVID_A1))
                 WA_CLR_BIT_MASKED(GEN9_HALF_SLICE_CHICKEN5,
                                   GEN9_DG_MIRROR_FIX_ENABLE);
 
         /* WaSetDisablePixMaskCammingAndRhwoInCommonSliceChicken:skl,bxt */
-        if (IS_SKL_REVID(dev, 0, SKL_REVID_B0) ||
-            IS_BXT_REVID(dev, 0, BXT_REVID_A1)) {
+        if (IS_SKL_REVID(dev_priv, 0, SKL_REVID_B0) ||
+            IS_BXT_REVID(dev_priv, 0, BXT_REVID_A1)) {
                 WA_SET_BIT_MASKED(GEN7_COMMON_SLICE_CHICKEN1,
                                   GEN9_RHWO_OPTIMIZATION_DISABLE);
                 /*
@@ -970,8 +946,8 @@ static int gen9_init_workarounds(struct intel_engine_cs *engine)
                           GEN9_CCS_TLB_PREFETCH_ENABLE);
 
         /* WaDisableMaskBasedCammingInRCC:skl,bxt */
-        if (IS_SKL_REVID(dev, SKL_REVID_C0, SKL_REVID_C0) ||
-            IS_BXT_REVID(dev, 0, BXT_REVID_A1))
+        if (IS_SKL_REVID(dev_priv, SKL_REVID_C0, SKL_REVID_C0) ||
+            IS_BXT_REVID(dev_priv, 0, BXT_REVID_A1))
                 WA_SET_BIT_MASKED(SLICE_ECO_CHICKEN0,
                                   PIXEL_MASK_CAMMING_DISABLE);
 
@@ -1035,8 +1011,7 @@ static int gen9_init_workarounds(struct intel_engine_cs *engine)
 
 static int skl_tune_iz_hashing(struct intel_engine_cs *engine)
 {
-        struct drm_device *dev = engine->dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = engine->i915;
         u8 vals[3] = { 0, 0, 0 };
         unsigned int i;
 
@@ -1077,9 +1052,8 @@ static int skl_tune_iz_hashing(struct intel_engine_cs *engine)
 
 static int skl_init_workarounds(struct intel_engine_cs *engine)
 {
+        struct drm_i915_private *dev_priv = engine->i915;
         int ret;
-        struct drm_device *dev = engine->dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
 
         ret = gen9_init_workarounds(engine);
         if (ret)
@@ -1090,12 +1064,12 @@ static int skl_init_workarounds(struct intel_engine_cs *engine)
          * until D0 which is the default case so this is equivalent to
          * !WaDisablePerCtxtPreemptionGranularityControl:skl
          */
-        if (IS_SKL_REVID(dev, SKL_REVID_E0, REVID_FOREVER)) {
+        if (IS_SKL_REVID(dev_priv, SKL_REVID_E0, REVID_FOREVER)) {
                 I915_WRITE(GEN7_FF_SLICE_CS_CHICKEN1,
                            _MASKED_BIT_ENABLE(GEN9_FFSC_PERCTX_PREEMPT_CTRL));
         }
 
-        if (IS_SKL_REVID(dev, 0, SKL_REVID_D0)) {
+        if (IS_SKL_REVID(dev_priv, 0, SKL_REVID_E0)) {
                 /* WaDisableChickenBitTSGBarrierAckForFFSliceCS:skl */
                 I915_WRITE(FF_SLICE_CS_CHICKEN2,
                            _MASKED_BIT_ENABLE(GEN9_TSG_BARRIER_ACK_DISABLE));
@@ -1104,30 +1078,30 @@ static int skl_init_workarounds(struct intel_engine_cs *engine)
         /* GEN8_L3SQCREG4 has a dependency with WA batch so any new changes
          * involving this register should also be added to WA batch as required.
          */
-        if (IS_SKL_REVID(dev, 0, SKL_REVID_E0))
+        if (IS_SKL_REVID(dev_priv, 0, SKL_REVID_E0))
                 /* WaDisableLSQCROPERFforOCL:skl */
                 I915_WRITE(GEN8_L3SQCREG4, I915_READ(GEN8_L3SQCREG4) |
                            GEN8_LQSC_RO_PERF_DIS);
 
         /* WaEnableGapsTsvCreditFix:skl */
-        if (IS_SKL_REVID(dev, SKL_REVID_C0, REVID_FOREVER)) {
+        if (IS_SKL_REVID(dev_priv, SKL_REVID_C0, REVID_FOREVER)) {
                 I915_WRITE(GEN8_GARBCNTL, (I915_READ(GEN8_GARBCNTL) |
                                            GEN9_GAPS_TSV_CREDIT_DISABLE));
         }
 
         /* WaDisablePowerCompilerClockGating:skl */
-        if (IS_SKL_REVID(dev, SKL_REVID_B0, SKL_REVID_B0))
+        if (IS_SKL_REVID(dev_priv, SKL_REVID_B0, SKL_REVID_B0))
                 WA_SET_BIT_MASKED(HIZ_CHICKEN,
                                   BDW_HIZ_POWER_COMPILER_CLOCK_GATING_DISABLE);
 
         /* WaBarrierPerformanceFixDisable:skl */
-        if (IS_SKL_REVID(dev, SKL_REVID_C0, SKL_REVID_D0))
+        if (IS_SKL_REVID(dev_priv, SKL_REVID_C0, SKL_REVID_D0))
                 WA_SET_BIT_MASKED(HDC_CHICKEN0,
                                   HDC_FENCE_DEST_SLM_DISABLE |
                                   HDC_BARRIER_PERFORMANCE_DISABLE);
 
         /* WaDisableSbeCacheDispatchPortSharing:skl */
-        if (IS_SKL_REVID(dev, 0, SKL_REVID_F0))
+        if (IS_SKL_REVID(dev_priv, 0, SKL_REVID_F0))
                 WA_SET_BIT_MASKED(
                         GEN7_HALF_SLICE_CHICKEN1,
                         GEN7_SBE_SS_CACHE_DISPATCH_PORT_SHARING_DISABLE);
@@ -1135,6 +1109,11 @@ static int skl_init_workarounds(struct intel_engine_cs *engine)
         /* WaDisableGafsUnitClkGating:skl */
         WA_SET_BIT(GEN7_UCGCTL4, GEN8_EU_GAUNIT_CLOCK_GATE_DISABLE);
 
+        /* WaInPlaceDecompressionHang:skl */
+        if (IS_SKL_REVID(dev_priv, SKL_REVID_H0, REVID_FOREVER))
+                WA_SET_BIT(GEN9_GAMT_ECO_REG_RW_IA,
+                           GAMT_ECO_ENABLE_IN_PLACE_DECOMPRESS);
+
         /* WaDisableLSQCROPERFforOCL:skl */
         ret = wa_ring_whitelist_reg(engine, GEN8_L3SQCREG4);
         if (ret)
@@ -1145,9 +1124,8 @@ static int skl_init_workarounds(struct intel_engine_cs *engine)
 
 static int bxt_init_workarounds(struct intel_engine_cs *engine)
 {
+        struct drm_i915_private *dev_priv = engine->i915;
         int ret;
-        struct drm_device *dev = engine->dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
 
         ret = gen9_init_workarounds(engine);
         if (ret)
@@ -1155,11 +1133,11 @@ static int bxt_init_workarounds(struct intel_engine_cs *engine)
 
         /* WaStoreMultiplePTEenable:bxt */
         /* This is a requirement according to Hardware specification */
-        if (IS_BXT_REVID(dev, 0, BXT_REVID_A1))
+        if (IS_BXT_REVID(dev_priv, 0, BXT_REVID_A1))
                 I915_WRITE(TILECTL, I915_READ(TILECTL) | TILECTL_TLBPF);
 
         /* WaSetClckGatingDisableMedia:bxt */
-        if (IS_BXT_REVID(dev, 0, BXT_REVID_A1)) {
+        if (IS_BXT_REVID(dev_priv, 0, BXT_REVID_A1)) {
                 I915_WRITE(GEN7_MISCCPCTL, (I915_READ(GEN7_MISCCPCTL) &
                                             ~GEN8_DOP_CLOCK_GATE_MEDIA_ENABLE));
         }
@@ -1168,8 +1146,14 @@ static int bxt_init_workarounds(struct intel_engine_cs *engine)
         WA_SET_BIT_MASKED(GEN8_ROW_CHICKEN,
                           STALL_DOP_GATING_DISABLE);
 
+        /* WaDisablePooledEuLoadBalancingFix:bxt */
+        if (IS_BXT_REVID(dev_priv, BXT_REVID_B0, REVID_FOREVER)) {
+                WA_SET_BIT_MASKED(FF_SLICE_CS_CHICKEN2,
+                                  GEN9_POOLED_EU_LOAD_BALANCING_FIX_DISABLE);
+        }
+
         /* WaDisableSbeCacheDispatchPortSharing:bxt */
-        if (IS_BXT_REVID(dev, 0, BXT_REVID_B0)) {
+        if (IS_BXT_REVID(dev_priv, 0, BXT_REVID_B0)) {
                 WA_SET_BIT_MASKED(
                         GEN7_HALF_SLICE_CHICKEN1,
                         GEN7_SBE_SS_CACHE_DISPATCH_PORT_SHARING_DISABLE);
@@ -1179,7 +1163,7 @@ static int bxt_init_workarounds(struct intel_engine_cs *engine)
         /* WaDisableObjectLevelPreemptionForInstancedDraw:bxt */
         /* WaDisableObjectLevelPreemtionForInstanceId:bxt */
         /* WaDisableLSQCROPERFforOCL:bxt */
-        if (IS_BXT_REVID(dev, 0, BXT_REVID_A1)) {
+        if (IS_BXT_REVID(dev_priv, 0, BXT_REVID_A1)) {
                 ret = wa_ring_whitelist_reg(engine, GEN9_CS_DEBUG_MODE1);
                 if (ret)
                         return ret;
@@ -1189,17 +1173,27 @@ static int bxt_init_workarounds(struct intel_engine_cs *engine)
                         return ret;
         }
 
+        /* WaProgramL3SqcReg1DefaultForPerf:bxt */
+        if (IS_BXT_REVID(dev_priv, BXT_REVID_B0, REVID_FOREVER))
+                I915_WRITE(GEN8_L3SQCREG1, L3_GENERAL_PRIO_CREDITS(62) |
+                                           L3_HIGH_PRIO_CREDITS(2));
+
         /* WaInsertDummyPushConstPs:bxt */
         if (IS_BXT_REVID(dev_priv, 0, BXT_REVID_B0))
                 WA_SET_BIT_MASKED(COMMON_SLICE_CHICKEN2,
                                   GEN8_SBE_DISABLE_REPLAY_BUF_OPTIMIZATION);
 
+        /* WaInPlaceDecompressionHang:bxt */
+        if (IS_BXT_REVID(dev_priv, BXT_REVID_C0, REVID_FOREVER))
+                WA_SET_BIT(GEN9_GAMT_ECO_REG_RW_IA,
+                           GAMT_ECO_ENABLE_IN_PLACE_DECOMPRESS);
+
         return 0;
 }
 
 static int kbl_init_workarounds(struct intel_engine_cs *engine)
 {
-        struct drm_i915_private *dev_priv = engine->dev->dev_private;
+        struct drm_i915_private *dev_priv = engine->i915;
         int ret;
 
         ret = gen9_init_workarounds(engine);
@@ -1241,6 +1235,10 @@ static int kbl_init_workarounds(struct intel_engine_cs *engine)
                 GEN7_HALF_SLICE_CHICKEN1,
                 GEN7_SBE_SS_CACHE_DISPATCH_PORT_SHARING_DISABLE);
 
+        /* WaInPlaceDecompressionHang:kbl */
+        WA_SET_BIT(GEN9_GAMT_ECO_REG_RW_IA,
+                   GAMT_ECO_ENABLE_IN_PLACE_DECOMPRESS);
+
         /* WaDisableLSQCROPERFforOCL:kbl */
         ret = wa_ring_whitelist_reg(engine, GEN8_L3SQCREG4);
         if (ret)
@@ -1251,24 +1249,23 @@ static int kbl_init_workarounds(struct intel_engine_cs *engine)
 
 int init_workarounds_ring(struct intel_engine_cs *engine)
 {
-        struct drm_device *dev = engine->dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = engine->i915;
 
         WARN_ON(engine->id != RCS);
 
         dev_priv->workarounds.count = 0;
         dev_priv->workarounds.hw_whitelist_count[RCS] = 0;
 
-        if (IS_BROADWELL(dev))
+        if (IS_BROADWELL(dev_priv))
                 return bdw_init_workarounds(engine);
 
-        if (IS_CHERRYVIEW(dev))
+        if (IS_CHERRYVIEW(dev_priv))
                 return chv_init_workarounds(engine);
 
-        if (IS_SKYLAKE(dev))
+        if (IS_SKYLAKE(dev_priv))
                 return skl_init_workarounds(engine);
 
-        if (IS_BROXTON(dev))
+        if (IS_BROXTON(dev_priv))
                 return bxt_init_workarounds(engine);
 
         if (IS_KABYLAKE(dev_priv))
@@ -1279,14 +1276,13 @@ int init_workarounds_ring(struct intel_engine_cs *engine)
 
 static int init_render_ring(struct intel_engine_cs *engine)
 {
-        struct drm_device *dev = engine->dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = engine->i915;
         int ret = init_ring_common(engine);
         if (ret)
                 return ret;
 
         /* WaTimedSingleVertexDispatch:cl,bw,ctg,elk,ilk,snb */
-        if (INTEL_INFO(dev)->gen >= 4 && INTEL_INFO(dev)->gen < 7)
+        if (IS_GEN(dev_priv, 4, 6))
                 I915_WRITE(MI_MODE, _MASKED_BIT_ENABLE(VS_TIMER_DISPATCH));
 
         /* We need to disable the AsyncFlip performance optimisations in order
@@ -1295,22 +1291,22 @@ static int init_render_ring(struct intel_engine_cs *engine)
          *
          * WaDisableAsyncFlipPerfMode:snb,ivb,hsw,vlv
          */
-        if (INTEL_INFO(dev)->gen >= 6 && INTEL_INFO(dev)->gen < 8)
+        if (IS_GEN(dev_priv, 6, 7))
                 I915_WRITE(MI_MODE, _MASKED_BIT_ENABLE(ASYNC_FLIP_PERF_DISABLE));
 
         /* Required for the hardware to program scanline values for waiting */
         /* WaEnableFlushTlbInvalidationMode:snb */
-        if (INTEL_INFO(dev)->gen == 6)
+        if (IS_GEN6(dev_priv))
                 I915_WRITE(GFX_MODE,
                            _MASKED_BIT_ENABLE(GFX_TLB_INVALIDATE_EXPLICIT));
 
         /* WaBCSVCSTlbInvalidationMode:ivb,vlv,hsw */
-        if (IS_GEN7(dev))
+        if (IS_GEN7(dev_priv))
                 I915_WRITE(GFX_MODE_GEN7,
                            _MASKED_BIT_ENABLE(GFX_TLB_INVALIDATE_EXPLICIT) |
                            _MASKED_BIT_ENABLE(GFX_REPLAY_MODE));
 
-        if (IS_GEN6(dev)) {
+        if (IS_GEN6(dev_priv)) {
                 /* From the Sandybridge PRM, volume 1 part 3, page 24:
                  * "If this bit is set, STCunit will have LRA as replacement
                  *  policy. [...] This bit must be reset.  LRA replacement
@@ -1320,19 +1316,18 @@ static int init_render_ring(struct intel_engine_cs *engine)
                            _MASKED_BIT_DISABLE(CM0_STC_EVICT_DISABLE_LRA_SNB));
         }
 
-        if (INTEL_INFO(dev)->gen >= 6 && INTEL_INFO(dev)->gen < 8)
+        if (IS_GEN(dev_priv, 6, 7))
                 I915_WRITE(INSTPM, _MASKED_BIT_ENABLE(INSTPM_FORCE_ORDERING));
 
-        if (HAS_L3_DPF(dev))
-                I915_WRITE_IMR(engine, ~GT_PARITY_ERROR(dev));
+        if (INTEL_INFO(dev_priv)->gen >= 6)
+                I915_WRITE_IMR(engine, ~engine->irq_keep_mask);
 
         return init_workarounds_ring(engine);
 }
 
 static void render_ring_cleanup(struct intel_engine_cs *engine)
 {
-        struct drm_device *dev = engine->dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = engine->i915;
 
         if (dev_priv->semaphore_obj) {
                 i915_gem_object_ggtt_unpin(dev_priv->semaphore_obj);
@@ -1348,13 +1343,12 @@ static int gen8_rcs_signal(struct drm_i915_gem_request *signaller_req,
 {
 #define MBOX_UPDATE_DWORDS 8
         struct intel_engine_cs *signaller = signaller_req->engine;
-        struct drm_device *dev = signaller->dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = signaller_req->i915;
         struct intel_engine_cs *waiter;
         enum intel_engine_id id;
         int ret, num_rings;
 
-        num_rings = hweight32(INTEL_INFO(dev)->ring_mask);
+        num_rings = hweight32(INTEL_INFO(dev_priv)->ring_mask);
         num_dwords += (num_rings-1) * MBOX_UPDATE_DWORDS;
 #undef MBOX_UPDATE_DWORDS
 
@@ -1363,19 +1357,17 @@ static int gen8_rcs_signal(struct drm_i915_gem_request *signaller_req,
                 return ret;
 
         for_each_engine_id(waiter, dev_priv, id) {
-                u32 seqno;
                 u64 gtt_offset = signaller->semaphore.signal_ggtt[id];
                 if (gtt_offset == MI_SEMAPHORE_SYNC_INVALID)
                         continue;
 
-                seqno = i915_gem_request_get_seqno(signaller_req);
                 intel_ring_emit(signaller, GFX_OP_PIPE_CONTROL(6));
                 intel_ring_emit(signaller, PIPE_CONTROL_GLOBAL_GTT_IVB |
                                            PIPE_CONTROL_QW_WRITE |
-                                           PIPE_CONTROL_FLUSH_ENABLE);
+                                           PIPE_CONTROL_CS_STALL);
                 intel_ring_emit(signaller, lower_32_bits(gtt_offset));
                 intel_ring_emit(signaller, upper_32_bits(gtt_offset));
-                intel_ring_emit(signaller, seqno);
+                intel_ring_emit(signaller, signaller_req->seqno);
                 intel_ring_emit(signaller, 0);
                 intel_ring_emit(signaller, MI_SEMAPHORE_SIGNAL |
                                            MI_SEMAPHORE_TARGET(waiter->hw_id));
@@ -1390,13 +1382,12 @@ static int gen8_xcs_signal(struct drm_i915_gem_request *signaller_req,
 {
 #define MBOX_UPDATE_DWORDS 6
         struct intel_engine_cs *signaller = signaller_req->engine;
-        struct drm_device *dev = signaller->dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = signaller_req->i915;
         struct intel_engine_cs *waiter;
         enum intel_engine_id id;
         int ret, num_rings;
 
-        num_rings = hweight32(INTEL_INFO(dev)->ring_mask);
+        num_rings = hweight32(INTEL_INFO(dev_priv)->ring_mask);
         num_dwords += (num_rings-1) * MBOX_UPDATE_DWORDS;
 #undef MBOX_UPDATE_DWORDS
 
@@ -1405,18 +1396,16 @@ static int gen8_xcs_signal(struct drm_i915_gem_request *signaller_req,
                 return ret;
 
         for_each_engine_id(waiter, dev_priv, id) {
-                u32 seqno;
                 u64 gtt_offset = signaller->semaphore.signal_ggtt[id];
                 if (gtt_offset == MI_SEMAPHORE_SYNC_INVALID)
                         continue;
 
-                seqno = i915_gem_request_get_seqno(signaller_req);
                 intel_ring_emit(signaller, (MI_FLUSH_DW + 1) |
                                            MI_FLUSH_DW_OP_STOREDW);
                 intel_ring_emit(signaller, lower_32_bits(gtt_offset) |
                                            MI_FLUSH_DW_USE_GTT);
                 intel_ring_emit(signaller, upper_32_bits(gtt_offset));
-                intel_ring_emit(signaller, seqno);
+                intel_ring_emit(signaller, signaller_req->seqno);
                 intel_ring_emit(signaller, MI_SEMAPHORE_SIGNAL |
                                            MI_SEMAPHORE_TARGET(waiter->hw_id));
                 intel_ring_emit(signaller, 0);
@@ -1429,14 +1418,13 @@ static int gen6_signal(struct drm_i915_gem_request *signaller_req,
                        unsigned int num_dwords)
 {
         struct intel_engine_cs *signaller = signaller_req->engine;
-        struct drm_device *dev = signaller->dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = signaller_req->i915;
         struct intel_engine_cs *useless;
         enum intel_engine_id id;
         int ret, num_rings;
 
 #define MBOX_UPDATE_DWORDS 3
-        num_rings = hweight32(INTEL_INFO(dev)->ring_mask);
+        num_rings = hweight32(INTEL_INFO(dev_priv)->ring_mask);
         num_dwords += round_up((num_rings-1) * MBOX_UPDATE_DWORDS, 2);
 #undef MBOX_UPDATE_DWORDS
 
@@ -1448,11 +1436,9 @@ static int gen6_signal(struct drm_i915_gem_request *signaller_req,
                 i915_reg_t mbox_reg = signaller->semaphore.mbox.signal[id];
 
                 if (i915_mmio_reg_valid(mbox_reg)) {
-                        u32 seqno = i915_gem_request_get_seqno(signaller_req);
-
                         intel_ring_emit(signaller, MI_LOAD_REGISTER_IMM(1));
                         intel_ring_emit_reg(signaller, mbox_reg);
-                        intel_ring_emit(signaller, seqno);
+                        intel_ring_emit(signaller, signaller_req->seqno);
                 }
         }
 
@@ -1488,17 +1474,45 @@ gen6_add_request(struct drm_i915_gem_request *req)
         intel_ring_emit(engine, MI_STORE_DWORD_INDEX);
         intel_ring_emit(engine,
                         I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT);
+        intel_ring_emit(engine, req->seqno);
+        intel_ring_emit(engine, MI_USER_INTERRUPT);
+        __intel_ring_advance(engine);
+
+        return 0;
+}
+
+static int
+gen8_render_add_request(struct drm_i915_gem_request *req)
+{
+        struct intel_engine_cs *engine = req->engine;
+        int ret;
+
+        if (engine->semaphore.signal)
+                ret = engine->semaphore.signal(req, 8);
+        else
+                ret = intel_ring_begin(req, 8);
+        if (ret)
+                return ret;
+
+        intel_ring_emit(engine, GFX_OP_PIPE_CONTROL(6));
+        intel_ring_emit(engine, (PIPE_CONTROL_GLOBAL_GTT_IVB |
+                                 PIPE_CONTROL_CS_STALL |
+                                 PIPE_CONTROL_QW_WRITE));
+        intel_ring_emit(engine, intel_hws_seqno_address(req->engine));
+        intel_ring_emit(engine, 0);
         intel_ring_emit(engine, i915_gem_request_get_seqno(req));
+        /* We're thrashing one dword of HWS. */
+        intel_ring_emit(engine, 0);
         intel_ring_emit(engine, MI_USER_INTERRUPT);
+        intel_ring_emit(engine, MI_NOOP);
         __intel_ring_advance(engine);
 
         return 0;
 }
 
-static inline bool i915_gem_has_seqno_wrapped(struct drm_device *dev,
+static inline bool i915_gem_has_seqno_wrapped(struct drm_i915_private *dev_priv,
                                               u32 seqno)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
         return dev_priv->last_seqno < seqno;
 }
 
@@ -1516,7 +1530,9 @@ gen8_ring_sync(struct drm_i915_gem_request *waiter_req,
                u32 seqno)
 {
         struct intel_engine_cs *waiter = waiter_req->engine;
-        struct drm_i915_private *dev_priv = waiter->dev->dev_private;
+        struct drm_i915_private *dev_priv = waiter_req->i915;
+        u64 offset = GEN8_WAIT_OFFSET(waiter, signaller->id);
+        struct i915_hw_ppgtt *ppgtt;
         int ret;
 
         ret = intel_ring_begin(waiter_req, 4);
@@ -1525,14 +1541,20 @@ gen8_ring_sync(struct drm_i915_gem_request *waiter_req,
 
         intel_ring_emit(waiter, MI_SEMAPHORE_WAIT |
                                 MI_SEMAPHORE_GLOBAL_GTT |
-                                MI_SEMAPHORE_POLL |
                                 MI_SEMAPHORE_SAD_GTE_SDD);
         intel_ring_emit(waiter, seqno);
-        intel_ring_emit(waiter,
-                        lower_32_bits(GEN8_WAIT_OFFSET(waiter, signaller->id)));
-        intel_ring_emit(waiter,
-                        upper_32_bits(GEN8_WAIT_OFFSET(waiter, signaller->id)));
+        intel_ring_emit(waiter, lower_32_bits(offset));
+        intel_ring_emit(waiter, upper_32_bits(offset));
         intel_ring_advance(waiter);
+
+        /* When the !RCS engines idle waiting upon a semaphore, they lose their
+         * pagetables and we must reload them before executing the batch.
+         * We do this on the i915_switch_context() following the wait and
+         * before the dispatch.
+         */
+        ppgtt = waiter_req->ctx->ppgtt;
+        if (ppgtt && waiter_req->engine->id != RCS)
+                ppgtt->pd_dirty_rings |= intel_engine_flag(waiter_req->engine);
         return 0;
 }
 
@@ -1561,7 +1583,7 @@ gen6_ring_sync(struct drm_i915_gem_request *waiter_req,
                 return ret;
 
         /* If seqno wrap happened, omit the wait with no-ops */
-        if (likely(!i915_gem_has_seqno_wrapped(waiter->dev, seqno))) {
+        if (likely(!i915_gem_has_seqno_wrapped(waiter_req->i915, seqno))) {
                 intel_ring_emit(waiter, dw1 | wait_mbox);
                 intel_ring_emit(waiter, seqno);
                 intel_ring_emit(waiter, 0);
@@ -1577,72 +1599,28 @@ gen6_ring_sync(struct drm_i915_gem_request *waiter_req,
         return 0;
 }
 
-#define PIPE_CONTROL_FLUSH(ring__, addr__)                                      \
-do {                                                                    \
-        intel_ring_emit(ring__, GFX_OP_PIPE_CONTROL(4) | PIPE_CONTROL_QW_WRITE |                \
-                 PIPE_CONTROL_DEPTH_STALL);                             \
-        intel_ring_emit(ring__, (addr__) | PIPE_CONTROL_GLOBAL_GTT);                    \
-        intel_ring_emit(ring__, 0);                                                     \
-        intel_ring_emit(ring__, 0);                                                     \
-} while (0)
-
-static int
-pc_render_add_request(struct drm_i915_gem_request *req)
+static void
+gen5_seqno_barrier(struct intel_engine_cs *ring)
 {
-        struct intel_engine_cs *engine = req->engine;
-        u32 scratch_addr = engine->scratch.gtt_offset + 2 * CACHELINE_BYTES;
-        int ret;
-
-        /* For Ironlake, MI_USER_INTERRUPT was deprecated and apparently
-         * incoherent with writes to memory, i.e. completely fubar,
-         * so we need to use PIPE_NOTIFY instead.
+        /* MI_STORE are internally buffered by the GPU and not flushed
+         * either by MI_FLUSH or SyncFlush or any other combination of
+         * MI commands.
          *
-         * However, we also need to workaround the qword write
-         * incoherence by flushing the 6 PIPE_NOTIFY buffers out to
-         * memory before requesting an interrupt.
+         * "Only the submission of the store operation is guaranteed.
+         * The write result will be complete (coherent) some time later
+         * (this is practically a finite period but there is no guaranteed
+         * latency)."
+         *
+         * Empirically, we observe that we need a delay of at least 75us to
+         * be sure that the seqno write is visible by the CPU.
          */
-        ret = intel_ring_begin(req, 32);
-        if (ret)
-                return ret;
-
-        intel_ring_emit(engine,
-                        GFX_OP_PIPE_CONTROL(4) | PIPE_CONTROL_QW_WRITE |
-                        PIPE_CONTROL_WRITE_FLUSH |
-                        PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE);
-        intel_ring_emit(engine,
-                        engine->scratch.gtt_offset | PIPE_CONTROL_GLOBAL_GTT);
-        intel_ring_emit(engine, i915_gem_request_get_seqno(req));
-        intel_ring_emit(engine, 0);
-        PIPE_CONTROL_FLUSH(engine, scratch_addr);
-        scratch_addr += 2 * CACHELINE_BYTES; /* write to separate cachelines */
-        PIPE_CONTROL_FLUSH(engine, scratch_addr);
-        scratch_addr += 2 * CACHELINE_BYTES;
-        PIPE_CONTROL_FLUSH(engine, scratch_addr);
-        scratch_addr += 2 * CACHELINE_BYTES;
-        PIPE_CONTROL_FLUSH(engine, scratch_addr);
-        scratch_addr += 2 * CACHELINE_BYTES;
-        PIPE_CONTROL_FLUSH(engine, scratch_addr);
-        scratch_addr += 2 * CACHELINE_BYTES;
-        PIPE_CONTROL_FLUSH(engine, scratch_addr);
-
-        intel_ring_emit(engine,
-                        GFX_OP_PIPE_CONTROL(4) | PIPE_CONTROL_QW_WRITE |
-                        PIPE_CONTROL_WRITE_FLUSH |
-                        PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE |
-                        PIPE_CONTROL_NOTIFY);
-        intel_ring_emit(engine,
-                        engine->scratch.gtt_offset | PIPE_CONTROL_GLOBAL_GTT);
-        intel_ring_emit(engine, i915_gem_request_get_seqno(req));
-        intel_ring_emit(engine, 0);
-        __intel_ring_advance(engine);
-
-        return 0;
+        usleep_range(125, 250);
 }
 
 static void
 gen6_seqno_barrier(struct intel_engine_cs *engine)
 {
-        struct drm_i915_private *dev_priv = engine->dev->dev_private;
+        struct drm_i915_private *dev_priv = engine->i915;
 
         /* Workaround to force correct ordering between irq and seqno writes on
          * ivb (and maybe also on snb) by reading from a CS register (like
@@ -1664,133 +1642,54 @@ gen6_seqno_barrier(struct intel_engine_cs *engine)
         spin_unlock_irq(&dev_priv->uncore.lock);
 }
 
-static u32
-ring_get_seqno(struct intel_engine_cs *engine)
-{
-        return intel_read_status_page(engine, I915_GEM_HWS_INDEX);
-}
-
 static void
-ring_set_seqno(struct intel_engine_cs *engine, u32 seqno)
-{
-        intel_write_status_page(engine, I915_GEM_HWS_INDEX, seqno);
-}
-
-static u32
-pc_render_get_seqno(struct intel_engine_cs *engine)
+gen5_irq_enable(struct intel_engine_cs *engine)
 {
-        return engine->scratch.cpu_page[0];
+        gen5_enable_gt_irq(engine->i915, engine->irq_enable_mask);
 }
 
 static void
-pc_render_set_seqno(struct intel_engine_cs *engine, u32 seqno)
-{
-        engine->scratch.cpu_page[0] = seqno;
-}
-
-static bool
-gen5_ring_get_irq(struct intel_engine_cs *engine)
+gen5_irq_disable(struct intel_engine_cs *engine)
 {
-        struct drm_device *dev = engine->dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
-        unsigned long flags;
-
-        if (WARN_ON(!intel_irqs_enabled(dev_priv)))
-                return false;
-
-        spin_lock_irqsave(&dev_priv->irq_lock, flags);
-        if (engine->irq_refcount++ == 0)
-                gen5_enable_gt_irq(dev_priv, engine->irq_enable_mask);
-        spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
-
-        return true;
+        gen5_disable_gt_irq(engine->i915, engine->irq_enable_mask);
 }
 
 static void
-gen5_ring_put_irq(struct intel_engine_cs *engine)
+i9xx_irq_enable(struct intel_engine_cs *engine)
 {
-        struct drm_device *dev = engine->dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
-        unsigned long flags;
-
-        spin_lock_irqsave(&dev_priv->irq_lock, flags);
-        if (--engine->irq_refcount == 0)
-                gen5_disable_gt_irq(dev_priv, engine->irq_enable_mask);
-        spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
-}
-
-static bool
-i9xx_ring_get_irq(struct intel_engine_cs *engine)
-{
-        struct drm_device *dev = engine->dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
-        unsigned long flags;
-
-        if (!intel_irqs_enabled(dev_priv))
-                return false;
-
-        spin_lock_irqsave(&dev_priv->irq_lock, flags);
-        if (engine->irq_refcount++ == 0) {
-                dev_priv->irq_mask &= ~engine->irq_enable_mask;
-                I915_WRITE(IMR, dev_priv->irq_mask);
-                POSTING_READ(IMR);
-        }
-        spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
+        struct drm_i915_private *dev_priv = engine->i915;
 
-        return true;
+        dev_priv->irq_mask &= ~engine->irq_enable_mask;
+        I915_WRITE(IMR, dev_priv->irq_mask);
+        POSTING_READ_FW(RING_IMR(engine->mmio_base));
 }
 
 static void
-i9xx_ring_put_irq(struct intel_engine_cs *engine)
+i9xx_irq_disable(struct intel_engine_cs *engine)
 {
-        struct drm_device *dev = engine->dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
-        unsigned long flags;
+        struct drm_i915_private *dev_priv = engine->i915;
 
-        spin_lock_irqsave(&dev_priv->irq_lock, flags);
-        if (--engine->irq_refcount == 0) {
-                dev_priv->irq_mask |= engine->irq_enable_mask;
-                I915_WRITE(IMR, dev_priv->irq_mask);
-                POSTING_READ(IMR);
-        }
-        spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
+        dev_priv->irq_mask |= engine->irq_enable_mask;
+        I915_WRITE(IMR, dev_priv->irq_mask);
 }
 
-static bool
-i8xx_ring_get_irq(struct intel_engine_cs *engine)
+static void
+i8xx_irq_enable(struct intel_engine_cs *engine)
 {
-        struct drm_device *dev = engine->dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
-        unsigned long flags;
-
-        if (!intel_irqs_enabled(dev_priv))
-                return false;
-
-        spin_lock_irqsave(&dev_priv->irq_lock, flags);
-        if (engine->irq_refcount++ == 0) {
-                dev_priv->irq_mask &= ~engine->irq_enable_mask;
-                I915_WRITE16(IMR, dev_priv->irq_mask);
-                POSTING_READ16(IMR);
-        }
-        spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
+        struct drm_i915_private *dev_priv = engine->i915;
 
-        return true;
+        dev_priv->irq_mask &= ~engine->irq_enable_mask;
+        I915_WRITE16(IMR, dev_priv->irq_mask);
+        POSTING_READ16(RING_IMR(engine->mmio_base));
 }
 
 static void
-i8xx_ring_put_irq(struct intel_engine_cs *engine)
+i8xx_irq_disable(struct intel_engine_cs *engine)
 {
-        struct drm_device *dev = engine->dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
-        unsigned long flags;
+        struct drm_i915_private *dev_priv = engine->i915;
 
-        spin_lock_irqsave(&dev_priv->irq_lock, flags);
-        if (--engine->irq_refcount == 0) {
-                dev_priv->irq_mask |= engine->irq_enable_mask;
-                I915_WRITE16(IMR, dev_priv->irq_mask);
-                POSTING_READ16(IMR);
-        }
-        spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
+        dev_priv->irq_mask |= engine->irq_enable_mask;
+        I915_WRITE16(IMR, dev_priv->irq_mask);
 }
 
 static int
@@ -1824,135 +1723,68 @@ i9xx_add_request(struct drm_i915_gem_request *req)
         intel_ring_emit(engine, MI_STORE_DWORD_INDEX);
         intel_ring_emit(engine,
                         I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT);
-        intel_ring_emit(engine, i915_gem_request_get_seqno(req));
+        intel_ring_emit(engine, req->seqno);
         intel_ring_emit(engine, MI_USER_INTERRUPT);
         __intel_ring_advance(engine);
 
         return 0;
 }
 
-static bool
-gen6_ring_get_irq(struct intel_engine_cs *engine)
+static void
+gen6_irq_enable(struct intel_engine_cs *engine)
 {
-        struct drm_device *dev = engine->dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
-        unsigned long flags;
-
-        if (WARN_ON(!intel_irqs_enabled(dev_priv)))
-                return false;
-
-        spin_lock_irqsave(&dev_priv->irq_lock, flags);
-        if (engine->irq_refcount++ == 0) {
-                if (HAS_L3_DPF(dev) && engine->id == RCS)
-                        I915_WRITE_IMR(engine,
-                                       ~(engine->irq_enable_mask |
-                                         GT_PARITY_ERROR(dev)));
-                else
-                        I915_WRITE_IMR(engine, ~engine->irq_enable_mask);
-                gen5_enable_gt_irq(dev_priv, engine->irq_enable_mask);
-        }
-        spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
+        struct drm_i915_private *dev_priv = engine->i915;
 
-        return true;
+        I915_WRITE_IMR(engine,
+                       ~(engine->irq_enable_mask |
+                         engine->irq_keep_mask));
+        gen5_enable_gt_irq(dev_priv, engine->irq_enable_mask);
 }
 
 static void
-gen6_ring_put_irq(struct intel_engine_cs *engine)
+gen6_irq_disable(struct intel_engine_cs *engine)
 {
-        struct drm_device *dev = engine->dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
-        unsigned long flags;
+        struct drm_i915_private *dev_priv = engine->i915;
 
-        spin_lock_irqsave(&dev_priv->irq_lock, flags);
-        if (--engine->irq_refcount == 0) {
-                if (HAS_L3_DPF(dev) && engine->id == RCS)
-                        I915_WRITE_IMR(engine, ~GT_PARITY_ERROR(dev));
-                else
-                        I915_WRITE_IMR(engine, ~0);
-                gen5_disable_gt_irq(dev_priv, engine->irq_enable_mask);
-        }
-        spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
+        I915_WRITE_IMR(engine, ~engine->irq_keep_mask);
+        gen5_disable_gt_irq(dev_priv, engine->irq_enable_mask);
 }
 
-static bool
-hsw_vebox_get_irq(struct intel_engine_cs *engine)
+static void
+hsw_vebox_irq_enable(struct intel_engine_cs *engine)
 {
-        struct drm_device *dev = engine->dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
-        unsigned long flags;
-
-        if (WARN_ON(!intel_irqs_enabled(dev_priv)))
-                return false;
+        struct drm_i915_private *dev_priv = engine->i915;
 
-        spin_lock_irqsave(&dev_priv->irq_lock, flags);
-        if (engine->irq_refcount++ == 0) {
-                I915_WRITE_IMR(engine, ~engine->irq_enable_mask);
-                gen6_enable_pm_irq(dev_priv, engine->irq_enable_mask);
-        }
-        spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
-
-        return true;
+        I915_WRITE_IMR(engine, ~engine->irq_enable_mask);
+        gen6_enable_pm_irq(dev_priv, engine->irq_enable_mask);
 }
 
 static void
-hsw_vebox_put_irq(struct intel_engine_cs *engine)
+hsw_vebox_irq_disable(struct intel_engine_cs *engine)
 {
-        struct drm_device *dev = engine->dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
-        unsigned long flags;
+        struct drm_i915_private *dev_priv = engine->i915;
 
-        spin_lock_irqsave(&dev_priv->irq_lock, flags);
-        if (--engine->irq_refcount == 0) {
-                I915_WRITE_IMR(engine, ~0);
-                gen6_disable_pm_irq(dev_priv, engine->irq_enable_mask);
-        }
-        spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
+        I915_WRITE_IMR(engine, ~0);
+        gen6_disable_pm_irq(dev_priv, engine->irq_enable_mask);
 }
 
-static bool
-gen8_ring_get_irq(struct intel_engine_cs *engine)
+static void
+gen8_irq_enable(struct intel_engine_cs *engine)
 {
-        struct drm_device *dev = engine->dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
-        unsigned long flags;
-
-        if (WARN_ON(!intel_irqs_enabled(dev_priv)))
-                return false;
+        struct drm_i915_private *dev_priv = engine->i915;
 
-        spin_lock_irqsave(&dev_priv->irq_lock, flags);
-        if (engine->irq_refcount++ == 0) {
-                if (HAS_L3_DPF(dev) && engine->id == RCS) {
-                        I915_WRITE_IMR(engine,
-                                       ~(engine->irq_enable_mask |
-                                         GT_RENDER_L3_PARITY_ERROR_INTERRUPT));
-                } else {
-                        I915_WRITE_IMR(engine, ~engine->irq_enable_mask);
-                }
-                POSTING_READ(RING_IMR(engine->mmio_base));
-        }
-        spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
-
-        return true;
+        I915_WRITE_IMR(engine,
+                       ~(engine->irq_enable_mask |
+                         engine->irq_keep_mask));
+        POSTING_READ_FW(RING_IMR(engine->mmio_base));
 }
 
 static void
-gen8_ring_put_irq(struct intel_engine_cs *engine)
+gen8_irq_disable(struct intel_engine_cs *engine)
 {
-        struct drm_device *dev = engine->dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
-        unsigned long flags;
+        struct drm_i915_private *dev_priv = engine->i915;
 
-        spin_lock_irqsave(&dev_priv->irq_lock, flags);
-        if (--engine->irq_refcount == 0) {
-                if (HAS_L3_DPF(dev) && engine->id == RCS) {
-                        I915_WRITE_IMR(engine,
-                                       ~GT_RENDER_L3_PARITY_ERROR_INTERRUPT);
-                } else {
-                        I915_WRITE_IMR(engine, ~0);
-                }
-                POSTING_READ(RING_IMR(engine->mmio_base));
-        }
-        spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
+        I915_WRITE_IMR(engine, ~engine->irq_keep_mask);
 }
 
 static int
@@ -2066,12 +1898,12 @@ i915_dispatch_execbuffer(struct drm_i915_gem_request *req,
 
 static void cleanup_phys_status_page(struct intel_engine_cs *engine)
 {
-        struct drm_i915_private *dev_priv = to_i915(engine->dev);
+        struct drm_i915_private *dev_priv = engine->i915;
 
         if (!dev_priv->status_page_dmah)
                 return;
 
-        drm_pci_free(engine->dev, dev_priv->status_page_dmah);
+        drm_pci_free(&dev_priv->drm, dev_priv->status_page_dmah);
         engine->status_page.page_addr = NULL;
 }
 
@@ -2097,10 +1929,10 @@ static int init_status_page(struct intel_engine_cs *engine)
                 unsigned flags;
                 int ret;
 
-                obj = i915_gem_alloc_object(engine->dev, 4096);
-                if (obj == NULL) {
+                obj = i915_gem_object_create(&engine->i915->drm, 4096);
+                if (IS_ERR(obj)) {
                         DRM_ERROR("Failed to allocate status page\n");
-                        return -ENOMEM;
+                        return PTR_ERR(obj);
                 }
 
                 ret = i915_gem_object_set_cache_level(obj, I915_CACHE_LLC);
@@ -2108,7 +1940,7 @@ static int init_status_page(struct intel_engine_cs *engine)
                         goto err_unref;
 
                 flags = 0;
-                if (!HAS_LLC(engine->dev))
+                if (!HAS_LLC(engine->i915))
                         /* On g33, we cannot place HWS above 256MiB, so
                          * restrict its pinning to the low mappable arena.
                          * Though this restriction is not documented for
@@ -2142,11 +1974,11 @@ err_unref:
 
 static int init_phys_status_page(struct intel_engine_cs *engine)
 {
-        struct drm_i915_private *dev_priv = engine->dev->dev_private;
+        struct drm_i915_private *dev_priv = engine->i915;
 
         if (!dev_priv->status_page_dmah) {
                 dev_priv->status_page_dmah =
-                        drm_pci_alloc(engine->dev, PAGE_SIZE, PAGE_SIZE);
+                        drm_pci_alloc(&dev_priv->drm, PAGE_SIZE, PAGE_SIZE);
                 if (!dev_priv->status_page_dmah)
                         return -ENOMEM;
         }
@@ -2159,20 +1991,22 @@ static int init_phys_status_page(struct intel_engine_cs *engine)
 
 void intel_unpin_ringbuffer_obj(struct intel_ringbuffer *ringbuf)
 {
+        GEM_BUG_ON(ringbuf->vma == NULL);
+        GEM_BUG_ON(ringbuf->virtual_start == NULL);
+
         if (HAS_LLC(ringbuf->obj->base.dev) && !ringbuf->obj->stolen)
                 i915_gem_object_unpin_map(ringbuf->obj);
         else
-                iounmap(ringbuf->virtual_start);
+                i915_vma_unpin_iomap(ringbuf->vma);
         ringbuf->virtual_start = NULL;
-        ringbuf->vma = NULL;
+
         i915_gem_object_ggtt_unpin(ringbuf->obj);
+        ringbuf->vma = NULL;
 }
 
-int intel_pin_and_map_ringbuffer_obj(struct drm_device *dev,
+int intel_pin_and_map_ringbuffer_obj(struct drm_i915_private *dev_priv,
                                      struct intel_ringbuffer *ringbuf)
 {
-        struct drm_i915_private *dev_priv = to_i915(dev);
-        struct i915_ggtt *ggtt = &dev_priv->ggtt;
         struct drm_i915_gem_object *obj = ringbuf->obj;
         /* Ring wraparound at offset 0 sometimes hangs. No idea why. */
         unsigned flags = PIN_OFFSET_BIAS | 4096;
@@ -2206,10 +2040,9 @@ int intel_pin_and_map_ringbuffer_obj(struct drm_device *dev,
                 /* Access through the GTT requires the device to be awake. */
                 assert_rpm_wakelock_held(dev_priv);
 
-                addr = ioremap_wc(ggtt->mappable_base +
-                                  i915_gem_obj_ggtt_offset(obj), ringbuf->size);
-                if (addr == NULL) {
-                        ret = -ENOMEM;
+                addr = i915_vma_pin_iomap(i915_gem_obj_to_ggtt(obj));
+                if (IS_ERR(addr)) {
+                        ret = PTR_ERR(addr);
                         goto err_unpin;
                 }
         }
@@ -2238,9 +2071,9 @@ static int intel_alloc_ringbuffer_obj(struct drm_device *dev,
         if (!HAS_LLC(dev))
                 obj = i915_gem_object_create_stolen(dev, ringbuf->size);
         if (obj == NULL)
-                obj = i915_gem_alloc_object(dev, ringbuf->size);
-        if (obj == NULL)
-                return -ENOMEM;
+                obj = i915_gem_object_create(dev, ringbuf->size);
+        if (IS_ERR(obj))
+                return PTR_ERR(obj);
 
         /* mark ring buffers as read-only from GPU side by default */
         obj->gt_ro = 1;
@@ -2272,13 +2105,13 @@ intel_engine_create_ringbuffer(struct intel_engine_cs *engine, int size)
          * of the buffer.
          */
         ring->effective_size = size;
-        if (IS_I830(engine->dev) || IS_845G(engine->dev))
+        if (IS_I830(engine->i915) || IS_845G(engine->i915))
                 ring->effective_size -= 2 * CACHELINE_BYTES;
 
         ring->last_retired_head = -1;
         intel_ring_update_space(ring);
 
-        ret = intel_alloc_ringbuffer_obj(engine->dev, ring);
+        ret = intel_alloc_ringbuffer_obj(&engine->i915->drm, ring);
         if (ret) {
                 DRM_DEBUG_DRIVER("Failed to allocate ringbuffer %s: %d\n",
                                  engine->name, ret);
@@ -2298,15 +2131,67 @@ intel_ringbuffer_free(struct intel_ringbuffer *ring)
         kfree(ring);
 }
 
+static int intel_ring_context_pin(struct i915_gem_context *ctx,
+                                  struct intel_engine_cs *engine)
+{
+        struct intel_context *ce = &ctx->engine[engine->id];
+        int ret;
+
+        lockdep_assert_held(&ctx->i915->drm.struct_mutex);
+
+        if (ce->pin_count++)
+                return 0;
+
+        if (ce->state) {
+                ret = i915_gem_obj_ggtt_pin(ce->state, ctx->ggtt_alignment, 0);
+                if (ret)
+                        goto error;
+        }
+
+        /* The kernel context is only used as a placeholder for flushing the
+         * active context. It is never used for submitting user rendering and
+         * as such never requires the golden render context, and so we can skip
+         * emitting it when we switch to the kernel context. This is required
+         * as during eviction we cannot allocate and pin the renderstate in
+         * order to initialise the context.
+         */
+        if (ctx == ctx->i915->kernel_context)
+                ce->initialised = true;
+
+        i915_gem_context_reference(ctx);
+        return 0;
+
+error:
+        ce->pin_count = 0;
+        return ret;
+}
+
+static void intel_ring_context_unpin(struct i915_gem_context *ctx,
+                                     struct intel_engine_cs *engine)
+{
+        struct intel_context *ce = &ctx->engine[engine->id];
+
+        lockdep_assert_held(&ctx->i915->drm.struct_mutex);
+
+        if (--ce->pin_count)
+                return;
+
+        if (ce->state)
+                i915_gem_object_ggtt_unpin(ce->state);
+
+        i915_gem_context_unreference(ctx);
+}
+
 static int intel_init_ring_buffer(struct drm_device *dev,
                                   struct intel_engine_cs *engine)
 {
+        struct drm_i915_private *dev_priv = to_i915(dev);
         struct intel_ringbuffer *ringbuf;
         int ret;
 
         WARN_ON(engine->buffer);
 
-        engine->dev = dev;
+        engine->i915 = dev_priv;
         INIT_LIST_HEAD(&engine->active_list);
         INIT_LIST_HEAD(&engine->request_list);
         INIT_LIST_HEAD(&engine->execlist_queue);
@@ -2315,7 +2200,20 @@ static int intel_init_ring_buffer(struct drm_device *dev,
         memset(engine->semaphore.sync_seqno, 0,
                sizeof(engine->semaphore.sync_seqno));
 
-        init_waitqueue_head(&engine->irq_queue);
+        ret = intel_engine_init_breadcrumbs(engine);
+        if (ret)
+                goto error;
+
+        /* We may need to do things with the shrinker which
+         * require us to immediately switch back to the default
+         * context. This can cause a problem as pinning the
+         * default context also requires GTT space which may not
+         * be available. To avoid this we always pin the default
+         * context.
+         */
+        ret = intel_ring_context_pin(dev_priv->kernel_context, engine);
+        if (ret)
+                goto error;
 
         ringbuf = intel_engine_create_ringbuffer(engine, 32 * PAGE_SIZE);
         if (IS_ERR(ringbuf)) {
@@ -2324,7 +2222,7 @@ static int intel_init_ring_buffer(struct drm_device *dev,
         }
         engine->buffer = ringbuf;
 
-        if (I915_NEED_GFX_HWS(dev)) {
+        if (I915_NEED_GFX_HWS(dev_priv)) {
                 ret = init_status_page(engine);
                 if (ret)
                         goto error;
@@ -2335,7 +2233,7 @@ static int intel_init_ring_buffer(struct drm_device *dev,
                         goto error;
         }
 
-        ret = intel_pin_and_map_ringbuffer_obj(dev, ringbuf);
+        ret = intel_pin_and_map_ringbuffer_obj(dev_priv, ringbuf);
         if (ret) {
                 DRM_ERROR("Failed to pin and map ringbuffer %s: %d\n",
                                 engine->name, ret);
@@ -2361,11 +2259,11 @@ void intel_cleanup_engine(struct intel_engine_cs *engine)
         if (!intel_engine_initialized(engine))
                 return;
 
-        dev_priv = to_i915(engine->dev);
+        dev_priv = engine->i915;
 
         if (engine->buffer) {
                 intel_stop_engine(engine);
-                WARN_ON(!IS_GEN2(engine->dev) && (I915_READ_MODE(engine) & MODE_IDLE) == 0);
+                WARN_ON(!IS_GEN2(dev_priv) && (I915_READ_MODE(engine) & MODE_IDLE) == 0);
 
                 intel_unpin_ringbuffer_obj(engine->buffer);
                 intel_ringbuffer_free(engine->buffer);
@@ -2375,7 +2273,7 @@ void intel_cleanup_engine(struct intel_engine_cs *engine)
         if (engine->cleanup)
                 engine->cleanup(engine);
 
-        if (I915_NEED_GFX_HWS(engine->dev)) {
+        if (I915_NEED_GFX_HWS(dev_priv)) {
                 cleanup_status_page(engine);
         } else {
                 WARN_ON(engine->id != RCS);
@@ -2384,7 +2282,11 @@ void intel_cleanup_engine(struct intel_engine_cs *engine)
 
         i915_cmd_parser_fini_ring(engine);
         i915_gem_batch_pool_fini(&engine->batch_pool);
-        engine->dev = NULL;
+        intel_engine_fini_breadcrumbs(engine);
+
+        intel_ring_context_unpin(dev_priv->kernel_context, engine);
+
+        engine->i915 = NULL;
 }
 
 int intel_engine_idle(struct intel_engine_cs *engine)
@@ -2407,46 +2309,22 @@ int intel_engine_idle(struct intel_engine_cs *engine)
 
 int intel_ring_alloc_request_extras(struct drm_i915_gem_request *request)
 {
-        request->ringbuf = request->engine->buffer;
-        return 0;
-}
+        int ret;
 
-int intel_ring_reserve_space(struct drm_i915_gem_request *request)
-{
-        /*
-         * The first call merely notes the reserve request and is common for
-         * all back ends. The subsequent localised _begin() call actually
-         * ensures that the reservation is available. Without the begin, if
-         * the request creator immediately submitted the request without
-         * adding any commands to it then there might not actually be
-         * sufficient room for the submission commands.
+        /* Flush enough space to reduce the likelihood of waiting after
+         * we start building the request - in which case we will just
+         * have to repeat work.
          */
-        intel_ring_reserved_space_reserve(request->ringbuf, MIN_SPACE_FOR_ADD_REQUEST);
-
-        return intel_ring_begin(request, 0);
-}
+        request->reserved_space += LEGACY_REQUEST_SIZE;
 
-void intel_ring_reserved_space_reserve(struct intel_ringbuffer *ringbuf, int size)
-{
-        GEM_BUG_ON(ringbuf->reserved_size);
-        ringbuf->reserved_size = size;
-}
-
-void intel_ring_reserved_space_cancel(struct intel_ringbuffer *ringbuf)
-{
-        GEM_BUG_ON(!ringbuf->reserved_size);
-        ringbuf->reserved_size   = 0;
-}
+        request->ringbuf = request->engine->buffer;
 
-void intel_ring_reserved_space_use(struct intel_ringbuffer *ringbuf)
-{
-        GEM_BUG_ON(!ringbuf->reserved_size);
-        ringbuf->reserved_size   = 0;
-}
+        ret = intel_ring_begin(request, 0);
+        if (ret)
+                return ret;
 
-void intel_ring_reserved_space_end(struct intel_ringbuffer *ringbuf)
-{
-        GEM_BUG_ON(ringbuf->reserved_size);
+        request->reserved_space -= LEGACY_REQUEST_SIZE;
+        return 0;
 }
 
 static int wait_for_space(struct drm_i915_gem_request *req, int bytes)
@@ -2468,7 +2346,7 @@ static int wait_for_space(struct drm_i915_gem_request *req, int bytes)
          *
          * See also i915_gem_request_alloc() and i915_add_request().
          */
-        GEM_BUG_ON(!ringbuf->reserved_size);
+        GEM_BUG_ON(!req->reserved_space);
 
         list_for_each_entry(target, &engine->request_list, list) {
                 unsigned space;
@@ -2503,7 +2381,7 @@ int intel_ring_begin(struct drm_i915_gem_request *req, int num_dwords)
         int total_bytes, wait_bytes;
         bool need_wrap = false;
 
-        total_bytes = bytes + ringbuf->reserved_size;
+        total_bytes = bytes + req->reserved_space;
 
         if (unlikely(bytes > remain_usable)) {
                 /*
@@ -2519,7 +2397,7 @@ int intel_ring_begin(struct drm_i915_gem_request *req, int num_dwords)
                  * and only need to effectively wait for the reserved
                  * size space from the start of ringbuffer.
                  */
-                wait_bytes = remain_actual + ringbuf->reserved_size;
+                wait_bytes = remain_actual + req->reserved_space;
         } else {
                 /* No wrapping required, just waiting. */
                 wait_bytes = total_bytes;
@@ -2576,7 +2454,7 @@ int intel_ring_cacheline_align(struct drm_i915_gem_request *req)
 
 void intel_ring_init_seqno(struct intel_engine_cs *engine, u32 seqno)
 {
-        struct drm_i915_private *dev_priv = to_i915(engine->dev);
+        struct drm_i915_private *dev_priv = engine->i915;
 
         /* Our semaphore implementation is strictly monotonic (i.e. we proceed
          * so long as the semaphore value in the register/page is greater
@@ -2586,7 +2464,7 @@ void intel_ring_init_seqno(struct intel_engine_cs *engine, u32 seqno)
          * the semaphore value, then when the seqno moves backwards all
          * future waits will complete instantly (causing rendering corruption).
          */
-        if (INTEL_INFO(dev_priv)->gen == 6 || INTEL_INFO(dev_priv)->gen == 7) {
+        if (IS_GEN6(dev_priv) || IS_GEN7(dev_priv)) {
                 I915_WRITE(RING_SYNC_0(engine->mmio_base), 0);
                 I915_WRITE(RING_SYNC_1(engine->mmio_base), 0);
                 if (HAS_VEBOX(dev_priv))
@@ -2603,43 +2481,58 @@ void intel_ring_init_seqno(struct intel_engine_cs *engine, u32 seqno)
         memset(engine->semaphore.sync_seqno, 0,
                sizeof(engine->semaphore.sync_seqno));
 
-        engine->set_seqno(engine, seqno);
+        intel_write_status_page(engine, I915_GEM_HWS_INDEX, seqno);
+        if (engine->irq_seqno_barrier)
+                engine->irq_seqno_barrier(engine);
         engine->last_submitted_seqno = seqno;
 
         engine->hangcheck.seqno = seqno;
+
+        /* After manually advancing the seqno, fake the interrupt in case
+         * there are any waiters for that seqno.
+         */
+        rcu_read_lock();
+        intel_engine_wakeup(engine);
+        rcu_read_unlock();
 }
 
 static void gen6_bsd_ring_write_tail(struct intel_engine_cs *engine,
                                      u32 value)
 {
-        struct drm_i915_private *dev_priv = engine->dev->dev_private;
+        struct drm_i915_private *dev_priv = engine->i915;
+
+        intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
 
        /* Every tail move must follow the sequence below */
 
         /* Disable notification that the ring is IDLE. The GT
          * will then assume that it is busy and bring it out of rc6.
          */
-        I915_WRITE(GEN6_BSD_SLEEP_PSMI_CONTROL,
-                   _MASKED_BIT_ENABLE(GEN6_BSD_SLEEP_MSG_DISABLE));
+        I915_WRITE_FW(GEN6_BSD_SLEEP_PSMI_CONTROL,
+                      _MASKED_BIT_ENABLE(GEN6_BSD_SLEEP_MSG_DISABLE));
 
         /* Clear the context id. Here be magic! */
-        I915_WRITE64(GEN6_BSD_RNCID, 0x0);
+        I915_WRITE64_FW(GEN6_BSD_RNCID, 0x0);
 
         /* Wait for the ring not to be idle, i.e. for it to wake up. */
-        if (wait_for((I915_READ(GEN6_BSD_SLEEP_PSMI_CONTROL) &
-                      GEN6_BSD_SLEEP_INDICATOR) == 0,
-                     50))
+        if (intel_wait_for_register_fw(dev_priv,
+                                       GEN6_BSD_SLEEP_PSMI_CONTROL,
+                                       GEN6_BSD_SLEEP_INDICATOR,
+                                       0,
+                                       50))
                 DRM_ERROR("timed out waiting for the BSD ring to wake up\n");
 
         /* Now that the ring is fully powered up, update the tail */
-        I915_WRITE_TAIL(engine, value);
-        POSTING_READ(RING_TAIL(engine->mmio_base));
+        I915_WRITE_FW(RING_TAIL(engine->mmio_base), value);
+        POSTING_READ_FW(RING_TAIL(engine->mmio_base));
 
         /* Let the ring send IDLE messages to the GT again,
          * and so let it sleep to conserve power when idle.
          */
-        I915_WRITE(GEN6_BSD_SLEEP_PSMI_CONTROL,
-                   _MASKED_BIT_DISABLE(GEN6_BSD_SLEEP_MSG_DISABLE));
+        I915_WRITE_FW(GEN6_BSD_SLEEP_PSMI_CONTROL,
+                      _MASKED_BIT_DISABLE(GEN6_BSD_SLEEP_MSG_DISABLE));
+
+        intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
 }
 
 static int gen6_bsd_ring_flush(struct drm_i915_gem_request *req,
@@ -2654,7 +2547,7 @@ static int gen6_bsd_ring_flush(struct drm_i915_gem_request *req,
                 return ret;
 
         cmd = MI_FLUSH_DW;
-        if (INTEL_INFO(engine->dev)->gen >= 8)
+        if (INTEL_GEN(req->i915) >= 8)
                 cmd += 1;
 
         /* We always require a command barrier so that subsequent
@@ -2676,7 +2569,7 @@ static int gen6_bsd_ring_flush(struct drm_i915_gem_request *req,
         intel_ring_emit(engine, cmd);
         intel_ring_emit(engine,
                         I915_GEM_HWS_SCRATCH_ADDR | MI_FLUSH_DW_USE_GTT);
-        if (INTEL_INFO(engine->dev)->gen >= 8) {
+        if (INTEL_GEN(req->i915) >= 8) {
                 intel_ring_emit(engine, 0); /* upper addr */
                 intel_ring_emit(engine, 0); /* value */
         } else  {
@@ -2767,7 +2660,6 @@ static int gen6_ring_flush(struct drm_i915_gem_request *req,
                            u32 invalidate, u32 flush)
 {
         struct intel_engine_cs *engine = req->engine;
-        struct drm_device *dev = engine->dev;
         uint32_t cmd;
         int ret;
 
@@ -2776,7 +2668,7 @@ static int gen6_ring_flush(struct drm_i915_gem_request *req,
                 return ret;
 
         cmd = MI_FLUSH_DW;
-        if (INTEL_INFO(dev)->gen >= 8)
+        if (INTEL_GEN(req->i915) >= 8)
                 cmd += 1;
 
         /* We always require a command barrier so that subsequent
@@ -2797,7 +2689,7 @@ static int gen6_ring_flush(struct drm_i915_gem_request *req,
         intel_ring_emit(engine, cmd);
         intel_ring_emit(engine,
                         I915_GEM_HWS_SCRATCH_ADDR | MI_FLUSH_DW_USE_GTT);
-        if (INTEL_INFO(dev)->gen >= 8) {
+        if (INTEL_GEN(req->i915) >= 8) {
                 intel_ring_emit(engine, 0); /* upper addr */
                 intel_ring_emit(engine, 0); /* value */
         } else  {
@@ -2809,11 +2701,159 @@ static int gen6_ring_flush(struct drm_i915_gem_request *req,
         return 0;
 }
 
+static void intel_ring_init_semaphores(struct drm_i915_private *dev_priv,
+                                       struct intel_engine_cs *engine)
+{
+        struct drm_i915_gem_object *obj;
+        int ret, i;
+
+        if (!i915_semaphore_is_enabled(dev_priv))
+                return;
+
+        if (INTEL_GEN(dev_priv) >= 8 && !dev_priv->semaphore_obj) {
+                obj = i915_gem_object_create(&dev_priv->drm, 4096);
+                if (IS_ERR(obj)) {
+                        DRM_ERROR("Failed to allocate semaphore bo. Disabling semaphores\n");
+                        i915.semaphores = 0;
+                } else {
+                        i915_gem_object_set_cache_level(obj, I915_CACHE_LLC);
+                        ret = i915_gem_obj_ggtt_pin(obj, 0, PIN_NONBLOCK);
+                        if (ret != 0) {
+                                drm_gem_object_unreference(&obj->base);
+                                DRM_ERROR("Failed to pin semaphore bo. Disabling semaphores\n");
+                                i915.semaphores = 0;
+                        } else {
+                                dev_priv->semaphore_obj = obj;
+                        }
+                }
+        }
+
+        if (!i915_semaphore_is_enabled(dev_priv))
+                return;
+
+        if (INTEL_GEN(dev_priv) >= 8) {
+                u64 offset = i915_gem_obj_ggtt_offset(dev_priv->semaphore_obj);
+
+                engine->semaphore.sync_to = gen8_ring_sync;
+                engine->semaphore.signal = gen8_xcs_signal;
+
+                for (i = 0; i < I915_NUM_ENGINES; i++) {
+                        u64 ring_offset;
+
+                        if (i != engine->id)
+                                ring_offset = offset + GEN8_SEMAPHORE_OFFSET(engine->id, i);
+                        else
+                                ring_offset = MI_SEMAPHORE_SYNC_INVALID;
+
+                        engine->semaphore.signal_ggtt[i] = ring_offset;
+                }
+        } else if (INTEL_GEN(dev_priv) >= 6) {
+                engine->semaphore.sync_to = gen6_ring_sync;
+                engine->semaphore.signal = gen6_signal;
+
+                /*
+                 * The current semaphore is only applied on pre-gen8
+                 * platform.  And there is no VCS2 ring on the pre-gen8
+                 * platform. So the semaphore between RCS and VCS2 is
+                 * initialized as INVALID.  Gen8 will initialize the
+                 * sema between VCS2 and RCS later.
+                 */
+                for (i = 0; i < I915_NUM_ENGINES; i++) {
+                        static const struct {
+                                u32 wait_mbox;
+                                i915_reg_t mbox_reg;
+                        } sem_data[I915_NUM_ENGINES][I915_NUM_ENGINES] = {
+                                [RCS] = {
+                                        [VCS] =  { .wait_mbox = MI_SEMAPHORE_SYNC_RV,  .mbox_reg = GEN6_VRSYNC },
+                                        [BCS] =  { .wait_mbox = MI_SEMAPHORE_SYNC_RB,  .mbox_reg = GEN6_BRSYNC },
+                                        [VECS] = { .wait_mbox = MI_SEMAPHORE_SYNC_RVE, .mbox_reg = GEN6_VERSYNC },
+                                },
+                                [VCS] = {
+                                        [RCS] =  { .wait_mbox = MI_SEMAPHORE_SYNC_VR,  .mbox_reg = GEN6_RVSYNC },
+                                        [BCS] =  { .wait_mbox = MI_SEMAPHORE_SYNC_VB,  .mbox_reg = GEN6_BVSYNC },
+                                        [VECS] = { .wait_mbox = MI_SEMAPHORE_SYNC_VVE, .mbox_reg = GEN6_VEVSYNC },
+                                },
+                                [BCS] = {
+                                        [RCS] =  { .wait_mbox = MI_SEMAPHORE_SYNC_BR,  .mbox_reg = GEN6_RBSYNC },
+                                        [VCS] =  { .wait_mbox = MI_SEMAPHORE_SYNC_BV,  .mbox_reg = GEN6_VBSYNC },
+                                        [VECS] = { .wait_mbox = MI_SEMAPHORE_SYNC_BVE, .mbox_reg = GEN6_VEBSYNC },
+                                },
+                                [VECS] = {
+                                        [RCS] =  { .wait_mbox = MI_SEMAPHORE_SYNC_VER, .mbox_reg = GEN6_RVESYNC },
+                                        [VCS] =  { .wait_mbox = MI_SEMAPHORE_SYNC_VEV, .mbox_reg = GEN6_VVESYNC },
+                                        [BCS] =  { .wait_mbox = MI_SEMAPHORE_SYNC_VEB, .mbox_reg = GEN6_BVESYNC },
+                                },
+                        };
+                        u32 wait_mbox;
+                        i915_reg_t mbox_reg;
+
+                        if (i == engine->id || i == VCS2) {
+                                wait_mbox = MI_SEMAPHORE_SYNC_INVALID;
+                                mbox_reg = GEN6_NOSYNC;
+                        } else {
+                                wait_mbox = sem_data[engine->id][i].wait_mbox;
+                                mbox_reg = sem_data[engine->id][i].mbox_reg;
+                        }
+
+                        engine->semaphore.mbox.wait[i] = wait_mbox;
+                        engine->semaphore.mbox.signal[i] = mbox_reg;
+                }
+        }
+}
+
+static void intel_ring_init_irq(struct drm_i915_private *dev_priv,
+                                struct intel_engine_cs *engine)
+{
+        if (INTEL_GEN(dev_priv) >= 8) {
+                engine->irq_enable = gen8_irq_enable;
+                engine->irq_disable = gen8_irq_disable;
+                engine->irq_seqno_barrier = gen6_seqno_barrier;
+        } else if (INTEL_GEN(dev_priv) >= 6) {
+                engine->irq_enable = gen6_irq_enable;
+                engine->irq_disable = gen6_irq_disable;
+                engine->irq_seqno_barrier = gen6_seqno_barrier;
+        } else if (INTEL_GEN(dev_priv) >= 5) {
+                engine->irq_enable = gen5_irq_enable;
+                engine->irq_disable = gen5_irq_disable;
+                engine->irq_seqno_barrier = gen5_seqno_barrier;
+        } else if (INTEL_GEN(dev_priv) >= 3) {
+                engine->irq_enable = i9xx_irq_enable;
+                engine->irq_disable = i9xx_irq_disable;
+        } else {
+                engine->irq_enable = i8xx_irq_enable;
+                engine->irq_disable = i8xx_irq_disable;
+        }
+}
+
+static void intel_ring_default_vfuncs(struct drm_i915_private *dev_priv,
+                                      struct intel_engine_cs *engine)
+{
+        engine->init_hw = init_ring_common;
+        engine->write_tail = ring_write_tail;
+
+        engine->add_request = i9xx_add_request;
+        if (INTEL_GEN(dev_priv) >= 6)
+                engine->add_request = gen6_add_request;
+
+        if (INTEL_GEN(dev_priv) >= 8)
+                engine->dispatch_execbuffer = gen8_ring_dispatch_execbuffer;
+        else if (INTEL_GEN(dev_priv) >= 6)
+                engine->dispatch_execbuffer = gen6_ring_dispatch_execbuffer;
+        else if (INTEL_GEN(dev_priv) >= 4)
+                engine->dispatch_execbuffer = i965_dispatch_execbuffer;
+        else if (IS_I830(dev_priv) || IS_845G(dev_priv))
+                engine->dispatch_execbuffer = i830_dispatch_execbuffer;
+        else
+                engine->dispatch_execbuffer = i915_dispatch_execbuffer;
+
+        intel_ring_init_irq(dev_priv, engine);
+        intel_ring_init_semaphores(dev_priv, engine);
+}
+
 int intel_init_render_ring_buffer(struct drm_device *dev)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         struct intel_engine_cs *engine = &dev_priv->engine[RCS];
-        struct drm_i915_gem_object *obj;
         int ret;
 
         engine->name = "render ring";
@@ -2822,140 +2862,49 @@ int intel_init_render_ring_buffer(struct drm_device *dev)
         engine->hw_id = 0;
         engine->mmio_base = RENDER_RING_BASE;
 
-        if (INTEL_INFO(dev)->gen >= 8) {
-                if (i915_semaphore_is_enabled(dev)) {
-                        obj = i915_gem_alloc_object(dev, 4096);
-                        if (obj == NULL) {
-                                DRM_ERROR("Failed to allocate semaphore bo. Disabling semaphores\n");
-                                i915.semaphores = 0;
-                        } else {
-                                i915_gem_object_set_cache_level(obj, I915_CACHE_LLC);
-                                ret = i915_gem_obj_ggtt_pin(obj, 0, PIN_NONBLOCK);
-                                if (ret != 0) {
-                                        drm_gem_object_unreference(&obj->base);
-                                        DRM_ERROR("Failed to pin semaphore bo. Disabling semaphores\n");
-                                        i915.semaphores = 0;
-                                } else
-                                        dev_priv->semaphore_obj = obj;
-                        }
-                }
+        intel_ring_default_vfuncs(dev_priv, engine);
 
+        engine->irq_enable_mask = GT_RENDER_USER_INTERRUPT;
+        if (HAS_L3_DPF(dev_priv))
+                engine->irq_keep_mask = GT_RENDER_L3_PARITY_ERROR_INTERRUPT;
+
+        if (INTEL_GEN(dev_priv) >= 8) {
                 engine->init_context = intel_rcs_ctx_init;
-                engine->add_request = gen6_add_request;
+                engine->add_request = gen8_render_add_request;
                 engine->flush = gen8_render_ring_flush;
-                engine->irq_get = gen8_ring_get_irq;
-                engine->irq_put = gen8_ring_put_irq;
-                engine->irq_enable_mask = GT_RENDER_USER_INTERRUPT;
-                engine->irq_seqno_barrier = gen6_seqno_barrier;
-                engine->get_seqno = ring_get_seqno;
-                engine->set_seqno = ring_set_seqno;
-                if (i915_semaphore_is_enabled(dev)) {
-                        WARN_ON(!dev_priv->semaphore_obj);
-                        engine->semaphore.sync_to = gen8_ring_sync;
+                if (i915_semaphore_is_enabled(dev_priv))
                         engine->semaphore.signal = gen8_rcs_signal;
-                        GEN8_RING_SEMAPHORE_INIT(engine);
-                }
-        } else if (INTEL_INFO(dev)->gen >= 6) {
+        } else if (INTEL_GEN(dev_priv) >= 6) {
                 engine->init_context = intel_rcs_ctx_init;
-                engine->add_request = gen6_add_request;
                 engine->flush = gen7_render_ring_flush;
-                if (INTEL_INFO(dev)->gen == 6)
+                if (IS_GEN6(dev_priv))
                         engine->flush = gen6_render_ring_flush;
-                engine->irq_get = gen6_ring_get_irq;
-                engine->irq_put = gen6_ring_put_irq;
-                engine->irq_enable_mask = GT_RENDER_USER_INTERRUPT;
-                engine->irq_seqno_barrier = gen6_seqno_barrier;
-                engine->get_seqno = ring_get_seqno;
-                engine->set_seqno = ring_set_seqno;
-                if (i915_semaphore_is_enabled(dev)) {
-                        engine->semaphore.sync_to = gen6_ring_sync;
-                        engine->semaphore.signal = gen6_signal;
-                        /*
-                         * The current semaphore is only applied on pre-gen8
-                         * platform.  And there is no VCS2 ring on the pre-gen8
-                         * platform. So the semaphore between RCS and VCS2 is
-                         * initialized as INVALID.  Gen8 will initialize the
-                         * sema between VCS2 and RCS later.
-                         */
-                        engine->semaphore.mbox.wait[RCS] = MI_SEMAPHORE_SYNC_INVALID;
-                        engine->semaphore.mbox.wait[VCS] = MI_SEMAPHORE_SYNC_RV;
-                        engine->semaphore.mbox.wait[BCS] = MI_SEMAPHORE_SYNC_RB;
-                        engine->semaphore.mbox.wait[VECS] = MI_SEMAPHORE_SYNC_RVE;
-                        engine->semaphore.mbox.wait[VCS2] = MI_SEMAPHORE_SYNC_INVALID;
-                        engine->semaphore.mbox.signal[RCS] = GEN6_NOSYNC;
-                        engine->semaphore.mbox.signal[VCS] = GEN6_VRSYNC;
-                        engine->semaphore.mbox.signal[BCS] = GEN6_BRSYNC;
-                        engine->semaphore.mbox.signal[VECS] = GEN6_VERSYNC;
-                        engine->semaphore.mbox.signal[VCS2] = GEN6_NOSYNC;
-                }
-        } else if (IS_GEN5(dev)) {
-                engine->add_request = pc_render_add_request;
+        } else if (IS_GEN5(dev_priv)) {
                 engine->flush = gen4_render_ring_flush;
-                engine->get_seqno = pc_render_get_seqno;
-                engine->set_seqno = pc_render_set_seqno;
-                engine->irq_get = gen5_ring_get_irq;
-                engine->irq_put = gen5_ring_put_irq;
-                engine->irq_enable_mask = GT_RENDER_USER_INTERRUPT |
-                                        GT_RENDER_PIPECTL_NOTIFY_INTERRUPT;
         } else {
-                engine->add_request = i9xx_add_request;
-                if (INTEL_INFO(dev)->gen < 4)
+                if (INTEL_GEN(dev_priv) < 4)
                         engine->flush = gen2_render_ring_flush;
                 else
                         engine->flush = gen4_render_ring_flush;
-                engine->get_seqno = ring_get_seqno;
-                engine->set_seqno = ring_set_seqno;
-                if (IS_GEN2(dev)) {
-                        engine->irq_get = i8xx_ring_get_irq;
-                        engine->irq_put = i8xx_ring_put_irq;
-                } else {
-                        engine->irq_get = i9xx_ring_get_irq;
-                        engine->irq_put = i9xx_ring_put_irq;
-                }
                 engine->irq_enable_mask = I915_USER_INTERRUPT;
         }
-        engine->write_tail = ring_write_tail;
 
-        if (IS_HASWELL(dev))
+        if (IS_HASWELL(dev_priv))
                 engine->dispatch_execbuffer = hsw_ring_dispatch_execbuffer;
-        else if (IS_GEN8(dev))
-                engine->dispatch_execbuffer = gen8_ring_dispatch_execbuffer;
-        else if (INTEL_INFO(dev)->gen >= 6)
-                engine->dispatch_execbuffer = gen6_ring_dispatch_execbuffer;
-        else if (INTEL_INFO(dev)->gen >= 4)
-                engine->dispatch_execbuffer = i965_dispatch_execbuffer;
-        else if (IS_I830(dev) || IS_845G(dev))
-                engine->dispatch_execbuffer = i830_dispatch_execbuffer;
-        else
-                engine->dispatch_execbuffer = i915_dispatch_execbuffer;
+
         engine->init_hw = init_render_ring;
         engine->cleanup = render_ring_cleanup;
 
-        /* Workaround batchbuffer to combat CS tlb bug. */
-        if (HAS_BROKEN_CS_TLB(dev)) {
-                obj = i915_gem_alloc_object(dev, I830_WA_SIZE);
-                if (obj == NULL) {
-                        DRM_ERROR("Failed to allocate batch bo\n");
-                        return -ENOMEM;
-                }
-
-                ret = i915_gem_obj_ggtt_pin(obj, 0, 0);
-                if (ret != 0) {
-                        drm_gem_object_unreference(&obj->base);
-                        DRM_ERROR("Failed to ping batch bo\n");
-                        return ret;
-                }
-
-                engine->scratch.obj = obj;
-                engine->scratch.gtt_offset = i915_gem_obj_ggtt_offset(obj);
-        }
-
         ret = intel_init_ring_buffer(dev, engine);
         if (ret)
                 return ret;
 
-        if (INTEL_INFO(dev)->gen >= 5) {
-                ret = intel_init_pipe_control(engine);
+        if (INTEL_GEN(dev_priv) >= 6) {
+                ret = intel_init_pipe_control(engine, 4096);
+                if (ret)
+                        return ret;
+        } else if (HAS_BROKEN_CS_TLB(dev_priv)) {
+                ret = intel_init_pipe_control(engine, I830_WA_SIZE);
                 if (ret)
                         return ret;
         }
@@ -2965,7 +2914,7 @@ int intel_init_render_ring_buffer(struct drm_device *dev)
 
 int intel_init_bsd_ring_buffer(struct drm_device *dev)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         struct intel_engine_cs *engine = &dev_priv->engine[VCS];
 
         engine->name = "bsd ring";
@@ -2973,68 +2922,27 @@ int intel_init_bsd_ring_buffer(struct drm_device *dev)
         engine->exec_id = I915_EXEC_BSD;
         engine->hw_id = 1;
 
-        engine->write_tail = ring_write_tail;
-        if (INTEL_INFO(dev)->gen >= 6) {
+        intel_ring_default_vfuncs(dev_priv, engine);
+
+        if (INTEL_GEN(dev_priv) >= 6) {
                 engine->mmio_base = GEN6_BSD_RING_BASE;
                 /* gen6 bsd needs a special wa for tail updates */
-                if (IS_GEN6(dev))
+                if (IS_GEN6(dev_priv))
                         engine->write_tail = gen6_bsd_ring_write_tail;
                 engine->flush = gen6_bsd_ring_flush;
-                engine->add_request = gen6_add_request;
-                engine->irq_seqno_barrier = gen6_seqno_barrier;
-                engine->get_seqno = ring_get_seqno;
-                engine->set_seqno = ring_set_seqno;
-                if (INTEL_INFO(dev)->gen >= 8) {
+                if (INTEL_GEN(dev_priv) >= 8)
                         engine->irq_enable_mask =
                                 GT_RENDER_USER_INTERRUPT << GEN8_VCS1_IRQ_SHIFT;
-                        engine->irq_get = gen8_ring_get_irq;
-                        engine->irq_put = gen8_ring_put_irq;
-                        engine->dispatch_execbuffer =
-                                gen8_ring_dispatch_execbuffer;
-                        if (i915_semaphore_is_enabled(dev)) {
-                                engine->semaphore.sync_to = gen8_ring_sync;
-                                engine->semaphore.signal = gen8_xcs_signal;
-                                GEN8_RING_SEMAPHORE_INIT(engine);
-                        }
-                } else {
+                else
                         engine->irq_enable_mask = GT_BSD_USER_INTERRUPT;
-                        engine->irq_get = gen6_ring_get_irq;
-                        engine->irq_put = gen6_ring_put_irq;
-                        engine->dispatch_execbuffer =
-                                gen6_ring_dispatch_execbuffer;
-                        if (i915_semaphore_is_enabled(dev)) {
-                                engine->semaphore.sync_to = gen6_ring_sync;
-                                engine->semaphore.signal = gen6_signal;
-                                engine->semaphore.mbox.wait[RCS] = MI_SEMAPHORE_SYNC_VR;
-                                engine->semaphore.mbox.wait[VCS] = MI_SEMAPHORE_SYNC_INVALID;
-                                engine->semaphore.mbox.wait[BCS] = MI_SEMAPHORE_SYNC_VB;
-                                engine->semaphore.mbox.wait[VECS] = MI_SEMAPHORE_SYNC_VVE;
-                                engine->semaphore.mbox.wait[VCS2] = MI_SEMAPHORE_SYNC_INVALID;
-                                engine->semaphore.mbox.signal[RCS] = GEN6_RVSYNC;
-                                engine->semaphore.mbox.signal[VCS] = GEN6_NOSYNC;
-                                engine->semaphore.mbox.signal[BCS] = GEN6_BVSYNC;
-                                engine->semaphore.mbox.signal[VECS] = GEN6_VEVSYNC;
-                                engine->semaphore.mbox.signal[VCS2] = GEN6_NOSYNC;
-                        }
-                }
         } else {
                 engine->mmio_base = BSD_RING_BASE;
                 engine->flush = bsd_ring_flush;
-                engine->add_request = i9xx_add_request;
-                engine->get_seqno = ring_get_seqno;
-                engine->set_seqno = ring_set_seqno;
-                if (IS_GEN5(dev)) {
+                if (IS_GEN5(dev_priv))
                         engine->irq_enable_mask = ILK_BSD_USER_INTERRUPT;
-                        engine->irq_get = gen5_ring_get_irq;
-                        engine->irq_put = gen5_ring_put_irq;
-                } else {
+                else
                         engine->irq_enable_mask = I915_BSD_USER_INTERRUPT;
-                        engine->irq_get = i9xx_ring_get_irq;
-                        engine->irq_put = i9xx_ring_put_irq;
-                }
-                engine->dispatch_execbuffer = i965_dispatch_execbuffer;
         }
-        engine->init_hw = init_ring_common;
 
         return intel_init_ring_buffer(dev, engine);
 }
@@ -3044,147 +2952,70 @@ int intel_init_bsd_ring_buffer(struct drm_device *dev)
  */
 int intel_init_bsd2_ring_buffer(struct drm_device *dev)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         struct intel_engine_cs *engine = &dev_priv->engine[VCS2];
 
         engine->name = "bsd2 ring";
         engine->id = VCS2;
         engine->exec_id = I915_EXEC_BSD;
         engine->hw_id = 4;
-
-        engine->write_tail = ring_write_tail;
         engine->mmio_base = GEN8_BSD2_RING_BASE;
+
+        intel_ring_default_vfuncs(dev_priv, engine);
+
         engine->flush = gen6_bsd_ring_flush;
-        engine->add_request = gen6_add_request;
-        engine->irq_seqno_barrier = gen6_seqno_barrier;
-        engine->get_seqno = ring_get_seqno;
-        engine->set_seqno = ring_set_seqno;
         engine->irq_enable_mask =
                         GT_RENDER_USER_INTERRUPT << GEN8_VCS2_IRQ_SHIFT;
-        engine->irq_get = gen8_ring_get_irq;
-        engine->irq_put = gen8_ring_put_irq;
-        engine->dispatch_execbuffer =
-                        gen8_ring_dispatch_execbuffer;
-        if (i915_semaphore_is_enabled(dev)) {
-                engine->semaphore.sync_to = gen8_ring_sync;
-                engine->semaphore.signal = gen8_xcs_signal;
-                GEN8_RING_SEMAPHORE_INIT(engine);
-        }
-        engine->init_hw = init_ring_common;
 
         return intel_init_ring_buffer(dev, engine);
 }
 
 int intel_init_blt_ring_buffer(struct drm_device *dev)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         struct intel_engine_cs *engine = &dev_priv->engine[BCS];
 
         engine->name = "blitter ring";
         engine->id = BCS;
         engine->exec_id = I915_EXEC_BLT;
         engine->hw_id = 2;
-
         engine->mmio_base = BLT_RING_BASE;
-        engine->write_tail = ring_write_tail;
+
+        intel_ring_default_vfuncs(dev_priv, engine);
+
         engine->flush = gen6_ring_flush;
-        engine->add_request = gen6_add_request;
-        engine->irq_seqno_barrier = gen6_seqno_barrier;
-        engine->get_seqno = ring_get_seqno;
-        engine->set_seqno = ring_set_seqno;
-        if (INTEL_INFO(dev)->gen >= 8) {
+        if (INTEL_GEN(dev_priv) >= 8)
                 engine->irq_enable_mask =
                         GT_RENDER_USER_INTERRUPT << GEN8_BCS_IRQ_SHIFT;
-                engine->irq_get = gen8_ring_get_irq;
-                engine->irq_put = gen8_ring_put_irq;
-                engine->dispatch_execbuffer = gen8_ring_dispatch_execbuffer;
-                if (i915_semaphore_is_enabled(dev)) {
-                        engine->semaphore.sync_to = gen8_ring_sync;
-                        engine->semaphore.signal = gen8_xcs_signal;
-                        GEN8_RING_SEMAPHORE_INIT(engine);
-                }
-        } else {
+        else
                 engine->irq_enable_mask = GT_BLT_USER_INTERRUPT;
-                engine->irq_get = gen6_ring_get_irq;
-                engine->irq_put = gen6_ring_put_irq;
-                engine->dispatch_execbuffer = gen6_ring_dispatch_execbuffer;
-                if (i915_semaphore_is_enabled(dev)) {
-                        engine->semaphore.signal = gen6_signal;
-                        engine->semaphore.sync_to = gen6_ring_sync;
-                        /*
-                         * The current semaphore is only applied on pre-gen8
-                         * platform.  And there is no VCS2 ring on the pre-gen8
-                         * platform. So the semaphore between BCS and VCS2 is
-                         * initialized as INVALID.  Gen8 will initialize the
-                         * sema between BCS and VCS2 later.
-                         */
-                        engine->semaphore.mbox.wait[RCS] = MI_SEMAPHORE_SYNC_BR;
-                        engine->semaphore.mbox.wait[VCS] = MI_SEMAPHORE_SYNC_BV;
-                        engine->semaphore.mbox.wait[BCS] = MI_SEMAPHORE_SYNC_INVALID;
-                        engine->semaphore.mbox.wait[VECS] = MI_SEMAPHORE_SYNC_BVE;
-                        engine->semaphore.mbox.wait[VCS2] = MI_SEMAPHORE_SYNC_INVALID;
-                        engine->semaphore.mbox.signal[RCS] = GEN6_RBSYNC;
-                        engine->semaphore.mbox.signal[VCS] = GEN6_VBSYNC;
-                        engine->semaphore.mbox.signal[BCS] = GEN6_NOSYNC;
-                        engine->semaphore.mbox.signal[VECS] = GEN6_VEBSYNC;
-                        engine->semaphore.mbox.signal[VCS2] = GEN6_NOSYNC;
-                }
-        }
-        engine->init_hw = init_ring_common;
 
         return intel_init_ring_buffer(dev, engine);
 }
 
 int intel_init_vebox_ring_buffer(struct drm_device *dev)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         struct intel_engine_cs *engine = &dev_priv->engine[VECS];
 
         engine->name = "video enhancement ring";
         engine->id = VECS;
         engine->exec_id = I915_EXEC_VEBOX;
         engine->hw_id = 3;
-
         engine->mmio_base = VEBOX_RING_BASE;
-        engine->write_tail = ring_write_tail;
+
+        intel_ring_default_vfuncs(dev_priv, engine);
+
         engine->flush = gen6_ring_flush;
-        engine->add_request = gen6_add_request;
-        engine->irq_seqno_barrier = gen6_seqno_barrier;
-        engine->get_seqno = ring_get_seqno;
-        engine->set_seqno = ring_set_seqno;
 
-        if (INTEL_INFO(dev)->gen >= 8) {
+        if (INTEL_GEN(dev_priv) >= 8) {
                 engine->irq_enable_mask =
                         GT_RENDER_USER_INTERRUPT << GEN8_VECS_IRQ_SHIFT;
-                engine->irq_get = gen8_ring_get_irq;
-                engine->irq_put = gen8_ring_put_irq;
-                engine->dispatch_execbuffer = gen8_ring_dispatch_execbuffer;
-                if (i915_semaphore_is_enabled(dev)) {
-                        engine->semaphore.sync_to = gen8_ring_sync;
-                        engine->semaphore.signal = gen8_xcs_signal;
-                        GEN8_RING_SEMAPHORE_INIT(engine);
-                }
         } else {
                 engine->irq_enable_mask = PM_VEBOX_USER_INTERRUPT;
-                engine->irq_get = hsw_vebox_get_irq;
-                engine->irq_put = hsw_vebox_put_irq;
-                engine->dispatch_execbuffer = gen6_ring_dispatch_execbuffer;
-                if (i915_semaphore_is_enabled(dev)) {
-                        engine->semaphore.sync_to = gen6_ring_sync;
-                        engine->semaphore.signal = gen6_signal;
-                        engine->semaphore.mbox.wait[RCS] = MI_SEMAPHORE_SYNC_VER;
-                        engine->semaphore.mbox.wait[VCS] = MI_SEMAPHORE_SYNC_VEV;
-                        engine->semaphore.mbox.wait[BCS] = MI_SEMAPHORE_SYNC_VEB;
-                        engine->semaphore.mbox.wait[VECS] = MI_SEMAPHORE_SYNC_INVALID;
-                        engine->semaphore.mbox.wait[VCS2] = MI_SEMAPHORE_SYNC_INVALID;
-                        engine->semaphore.mbox.signal[RCS] = GEN6_RVESYNC;
-                        engine->semaphore.mbox.signal[VCS] = GEN6_VVESYNC;
-                        engine->semaphore.mbox.signal[BCS] = GEN6_BVESYNC;
-                        engine->semaphore.mbox.signal[VECS] = GEN6_NOSYNC;
-                        engine->semaphore.mbox.signal[VCS2] = GEN6_NOSYNC;
-                }
+                engine->irq_enable = hsw_vebox_irq_enable;
+                engine->irq_disable = hsw_vebox_irq_disable;
         }
-        engine->init_hw = init_ring_common;
 
         return intel_init_ring_buffer(dev, engine);
 }
diff --git a/drivers/gpu/drm/i915/intel_ringbuffer.h b/drivers/gpu/drm/i915/intel_ringbuffer.h
index ff126485d398..12cb7ed90014 100644
--- a/drivers/gpu/drm/i915/intel_ringbuffer.h
+++ b/drivers/gpu/drm/i915/intel_ringbuffer.h
@@ -62,18 +62,6 @@ struct  intel_hw_status_page {
         (i915_gem_obj_ggtt_offset(dev_priv->semaphore_obj) + \
          GEN8_SEMAPHORE_OFFSET(from, (__ring)->id))
 
-#define GEN8_RING_SEMAPHORE_INIT(e) do { \
-        if (!dev_priv->semaphore_obj) { \
-                break; \
-        } \
-        (e)->semaphore.signal_ggtt[RCS] = GEN8_SIGNAL_OFFSET((e), RCS); \
-        (e)->semaphore.signal_ggtt[VCS] = GEN8_SIGNAL_OFFSET((e), VCS); \
-        (e)->semaphore.signal_ggtt[BCS] = GEN8_SIGNAL_OFFSET((e), BCS); \
-        (e)->semaphore.signal_ggtt[VECS] = GEN8_SIGNAL_OFFSET((e), VECS); \
-        (e)->semaphore.signal_ggtt[VCS2] = GEN8_SIGNAL_OFFSET((e), VCS2); \
-        (e)->semaphore.signal_ggtt[(e)->id] = MI_SEMAPHORE_SYNC_INVALID; \
-        } while(0)
-
 enum intel_ring_hangcheck_action {
         HANGCHECK_IDLE = 0,
         HANGCHECK_WAIT,
@@ -86,8 +74,8 @@ enum intel_ring_hangcheck_action {
 
 struct intel_ring_hangcheck {
         u64 acthd;
+        unsigned long user_interrupts;
         u32 seqno;
-        unsigned user_interrupts;
         int score;
         enum intel_ring_hangcheck_action action;
         int deadlock;
@@ -107,7 +95,6 @@ struct intel_ringbuffer {
         int space;
         int size;
         int effective_size;
-        int reserved_size;
 
         /** We track the position of the requests in the ring buffer, and
          * when each is retired we increment last_retired_head as the GPU
@@ -120,7 +107,7 @@ struct intel_ringbuffer {
         u32 last_retired_head;
 };
 
-struct  intel_context;
+struct i915_gem_context;
 struct drm_i915_reg_table;
 
 /*
@@ -142,7 +129,10 @@ struct  i915_ctx_workarounds {
         struct drm_i915_gem_object *obj;
 };
 
-struct  intel_engine_cs {
+struct drm_i915_gem_request;
+
+struct intel_engine_cs {
+        struct drm_i915_private *i915;
         const char      *name;
         enum intel_engine_id {
                 RCS = 0,
@@ -157,10 +147,42 @@ struct  intel_engine_cs {
         unsigned int hw_id;
         unsigned int guc_id; /* XXX same as hw_id? */
         u32             mmio_base;
-        struct          drm_device *dev;
         struct intel_ringbuffer *buffer;
         struct list_head buffers;
 
+        /* Rather than have every client wait upon all user interrupts,
+         * with the herd waking after every interrupt and each doing the
+         * heavyweight seqno dance, we delegate the task (of being the
+         * bottom-half of the user interrupt) to the first client. After
+         * every interrupt, we wake up one client, who does the heavyweight
+         * coherent seqno read and either goes back to sleep (if incomplete),
+         * or wakes up all the completed clients in parallel, before then
+         * transferring the bottom-half status to the next client in the queue.
+         *
+         * Compared to walking the entire list of waiters in a single dedicated
+         * bottom-half, we reduce the latency of the first waiter by avoiding
+         * a context switch, but incur additional coherent seqno reads when
+         * following the chain of request breadcrumbs. Since it is most likely
+         * that we have a single client waiting on each seqno, then reducing
+         * the overhead of waking that client is much preferred.
+         */
+        struct intel_breadcrumbs {
+                struct task_struct *irq_seqno_bh; /* bh for user interrupts */
+                unsigned long irq_wakeups;
+                bool irq_posted;
+
+                spinlock_t lock; /* protects the lists of requests */
+                struct rb_root waiters; /* sorted by retirement, priority */
+                struct rb_root signals; /* sorted by retirement */
+                struct intel_wait *first_wait; /* oldest waiter by retirement */
+                struct task_struct *signaler; /* used for fence signalling */
+                struct drm_i915_gem_request *first_signal;
+                struct timer_list fake_irq; /* used after a missed interrupt */
+
+                bool irq_enabled : 1;
+                bool rpm_wakelock : 1;
+        } breadcrumbs;
+
         /*
          * A pool of objects to use as shadow copies of client batch buffers
          * when the command parser is enabled. Prevents the client from
@@ -171,11 +193,10 @@ struct  intel_engine_cs {
         struct intel_hw_status_page status_page;
         struct i915_ctx_workarounds wa_ctx;
 
-        unsigned irq_refcount; /* protected by dev_priv->irq_lock */
-        u32             irq_enable_mask;        /* bitmask to enable ring interrupt */
-        struct drm_i915_gem_request *trace_irq_req;
-        bool __must_check (*irq_get)(struct intel_engine_cs *ring);
-        void            (*irq_put)(struct intel_engine_cs *ring);
+        u32             irq_keep_mask; /* always keep these interrupts */
+        u32             irq_enable_mask; /* bitmask to enable ring interrupt */
+        void            (*irq_enable)(struct intel_engine_cs *ring);
+        void            (*irq_disable)(struct intel_engine_cs *ring);
 
         int             (*init_hw)(struct intel_engine_cs *ring);
 
@@ -194,9 +215,6 @@ struct  intel_engine_cs {
          * monotonic, even if not coherent.
          */
         void            (*irq_seqno_barrier)(struct intel_engine_cs *ring);
-        u32             (*get_seqno)(struct intel_engine_cs *ring);
-        void            (*set_seqno)(struct intel_engine_cs *ring,
-                                     u32 seqno);
         int             (*dispatch_execbuffer)(struct drm_i915_gem_request *req,
                                                u64 offset, u32 length,
                                                unsigned dispatch_flags);
@@ -268,13 +286,11 @@ struct  intel_engine_cs {
         struct tasklet_struct irq_tasklet;
         spinlock_t execlist_lock; /* used inside tasklet, use spin_lock_bh */
         struct list_head execlist_queue;
-        struct list_head execlist_retired_req_list;
         unsigned int fw_domains;
         unsigned int next_context_status_buffer;
         unsigned int idle_lite_restore_wa;
         bool disable_lite_restore_wa;
         u32 ctx_desc_template;
-        u32             irq_keep_mask; /* bitmask for interrupts that should not be masked */
         int             (*emit_request)(struct drm_i915_gem_request *request);
         int             (*emit_flush)(struct drm_i915_gem_request *request,
                                       u32 invalidate_domains,
@@ -306,20 +322,16 @@ struct  intel_engine_cs {
          * inspecting request list.
          */
         u32 last_submitted_seqno;
-        unsigned user_interrupts;
 
         bool gpu_caches_dirty;
 
-        wait_queue_head_t irq_queue;
-
-        struct intel_context *last_context;
+        struct i915_gem_context *last_context;
 
         struct intel_ring_hangcheck hangcheck;
 
         struct {
                 struct drm_i915_gem_object *obj;
                 u32 gtt_offset;
-                volatile u32 *cpu_page;
         } scratch;
 
         bool needs_cmd_parser;
@@ -350,13 +362,13 @@ struct  intel_engine_cs {
 };
 
 static inline bool
-intel_engine_initialized(struct intel_engine_cs *engine)
+intel_engine_initialized(const struct intel_engine_cs *engine)
 {
-        return engine->dev != NULL;
+        return engine->i915 != NULL;
 }
 
 static inline unsigned
-intel_engine_flag(struct intel_engine_cs *engine)
+intel_engine_flag(const struct intel_engine_cs *engine)
 {
         return 1 << engine->id;
 }
@@ -427,7 +439,7 @@ intel_write_status_page(struct intel_engine_cs *engine,
 
 struct intel_ringbuffer *
 intel_engine_create_ringbuffer(struct intel_engine_cs *engine, int size);
-int intel_pin_and_map_ringbuffer_obj(struct drm_device *dev,
+int intel_pin_and_map_ringbuffer_obj(struct drm_i915_private *dev_priv,
                                      struct intel_ringbuffer *ringbuf);
 void intel_unpin_ringbuffer_obj(struct intel_ringbuffer *ringbuf);
 void intel_ringbuffer_free(struct intel_ringbuffer *ring);
@@ -458,15 +470,14 @@ static inline void intel_ring_advance(struct intel_engine_cs *engine)
 }
 int __intel_ring_space(int head, int tail, int size);
 void intel_ring_update_space(struct intel_ringbuffer *ringbuf);
-bool intel_engine_stopped(struct intel_engine_cs *engine);
 
 int __must_check intel_engine_idle(struct intel_engine_cs *engine);
 void intel_ring_init_seqno(struct intel_engine_cs *engine, u32 seqno);
 int intel_ring_flush_all_caches(struct drm_i915_gem_request *req);
 int intel_ring_invalidate_all_caches(struct drm_i915_gem_request *req);
 
+int intel_init_pipe_control(struct intel_engine_cs *engine, int size);
 void intel_fini_pipe_control(struct intel_engine_cs *engine);
-int intel_init_pipe_control(struct intel_engine_cs *engine);
 
 int intel_init_render_ring_buffer(struct drm_device *dev);
 int intel_init_bsd_ring_buffer(struct drm_device *dev);
@@ -475,6 +486,10 @@ int intel_init_blt_ring_buffer(struct drm_device *dev);
 int intel_init_vebox_ring_buffer(struct drm_device *dev);
 
 u64 intel_ring_get_active_head(struct intel_engine_cs *engine);
+static inline u32 intel_engine_get_seqno(struct intel_engine_cs *engine)
+{
+        return intel_read_status_page(engine, I915_GEM_HWS_INDEX);
+}
 
 int init_workarounds_ring(struct intel_engine_cs *engine);
 
@@ -486,26 +501,73 @@ static inline u32 intel_ring_get_tail(struct intel_ringbuffer *ringbuf)
 /*
  * Arbitrary size for largest possible 'add request' sequence. The code paths
  * are complex and variable. Empirical measurement shows that the worst case
- * is ILK at 136 words. Reserving too much is better than reserving too little
- * as that allows for corner cases that might have been missed. So the figure
- * has been rounded up to 160 words.
+ * is BDW at 192 bytes (6 + 6 + 36 dwords), then ILK at 136 bytes. However,
+ * we need to allocate double the largest single packet within that emission
+ * to account for tail wraparound (so 6 + 6 + 72 dwords for BDW).
  */
-#define MIN_SPACE_FOR_ADD_REQUEST       160
+#define MIN_SPACE_FOR_ADD_REQUEST 336
 
-/*
- * Reserve space in the ring to guarantee that the i915_add_request() call
- * will always have sufficient room to do its stuff. The request creation
- * code calls this automatically.
- */
-void intel_ring_reserved_space_reserve(struct intel_ringbuffer *ringbuf, int size);
-/* Cancel the reservation, e.g. because the request is being discarded. */
-void intel_ring_reserved_space_cancel(struct intel_ringbuffer *ringbuf);
-/* Use the reserved space - for use by i915_add_request() only. */
-void intel_ring_reserved_space_use(struct intel_ringbuffer *ringbuf);
-/* Finish with the reserved space - for use by i915_add_request() only. */
-void intel_ring_reserved_space_end(struct intel_ringbuffer *ringbuf);
-
-/* Legacy ringbuffer specific portion of reservation code: */
-int intel_ring_reserve_space(struct drm_i915_gem_request *request);
+static inline u32 intel_hws_seqno_address(struct intel_engine_cs *engine)
+{
+        return engine->status_page.gfx_addr + I915_GEM_HWS_INDEX_ADDR;
+}
+
+/* intel_breadcrumbs.c -- user interrupt bottom-half for waiters */
+struct intel_wait {
+        struct rb_node node;
+        struct task_struct *tsk;
+        u32 seqno;
+};
+
+struct intel_signal_node {
+        struct rb_node node;
+        struct intel_wait wait;
+};
+
+int intel_engine_init_breadcrumbs(struct intel_engine_cs *engine);
+
+static inline void intel_wait_init(struct intel_wait *wait, u32 seqno)
+{
+        wait->tsk = current;
+        wait->seqno = seqno;
+}
+
+static inline bool intel_wait_complete(const struct intel_wait *wait)
+{
+        return RB_EMPTY_NODE(&wait->node);
+}
+
+bool intel_engine_add_wait(struct intel_engine_cs *engine,
+                           struct intel_wait *wait);
+void intel_engine_remove_wait(struct intel_engine_cs *engine,
+                              struct intel_wait *wait);
+void intel_engine_enable_signaling(struct drm_i915_gem_request *request);
+
+static inline bool intel_engine_has_waiter(struct intel_engine_cs *engine)
+{
+        return READ_ONCE(engine->breadcrumbs.irq_seqno_bh);
+}
+
+static inline bool intel_engine_wakeup(struct intel_engine_cs *engine)
+{
+        bool wakeup = false;
+        struct task_struct *tsk = READ_ONCE(engine->breadcrumbs.irq_seqno_bh);
+        /* Note that for this not to dangerously chase a dangling pointer,
+         * the caller is responsible for ensure that the task remain valid for
+         * wake_up_process() i.e. that the RCU grace period cannot expire.
+         *
+         * Also note that tsk is likely to be in !TASK_RUNNING state so an
+         * early test for tsk->state != TASK_RUNNING before wake_up_process()
+         * is unlikely to be beneficial.
+         */
+        if (tsk)
+                wakeup = wake_up_process(tsk);
+        return wakeup;
+}
+
+void intel_engine_enable_fake_irq(struct intel_engine_cs *engine);
+void intel_engine_fini_breadcrumbs(struct intel_engine_cs *engine);
+unsigned int intel_kick_waiters(struct drm_i915_private *i915);
+unsigned int intel_kick_signalers(struct drm_i915_private *i915);
 
 #endif /* _INTEL_RINGBUFFER_H_ */
diff --git a/drivers/gpu/drm/i915/intel_runtime_pm.c b/drivers/gpu/drm/i915/intel_runtime_pm.c
index 7fb1da4e7fc3..1c603bbe5784 100644
--- a/drivers/gpu/drm/i915/intel_runtime_pm.c
+++ b/drivers/gpu/drm/i915/intel_runtime_pm.c
@@ -65,6 +65,9 @@
 bool intel_display_power_well_is_enabled(struct drm_i915_private *dev_priv,
                                     int power_well_id);
 
+static struct i915_power_well *
+lookup_power_well(struct drm_i915_private *dev_priv, int power_well_id);
+
 const char *
 intel_display_power_domain_str(enum intel_display_power_domain domain)
 {
@@ -151,6 +154,23 @@ static void intel_power_well_disable(struct drm_i915_private *dev_priv,
         power_well->ops->disable(dev_priv, power_well);
 }
 
+static void intel_power_well_get(struct drm_i915_private *dev_priv,
+                                 struct i915_power_well *power_well)
+{
+        if (!power_well->count++)
+                intel_power_well_enable(dev_priv, power_well);
+}
+
+static void intel_power_well_put(struct drm_i915_private *dev_priv,
+                                 struct i915_power_well *power_well)
+{
+        WARN(!power_well->count, "Use count on power well %s is already zero",
+             power_well->name);
+
+        if (!--power_well->count)
+                intel_power_well_disable(dev_priv, power_well);
+}
+
 /*
  * We should only use the power well if we explicitly asked the hardware to
  * enable it, so check if it's enabled and also check if we've requested it to
@@ -267,7 +287,7 @@ void intel_display_set_init_power(struct drm_i915_private *dev_priv,
  */
 static void hsw_power_well_post_enable(struct drm_i915_private *dev_priv)
 {
-        struct drm_device *dev = dev_priv->dev;
+        struct drm_device *dev = &dev_priv->drm;
 
         /*
          * After we re-enable the power well, if we touch VGA register 0x3d5
@@ -298,7 +318,7 @@ static void hsw_power_well_pre_disable(struct drm_i915_private *dev_priv)
 static void skl_power_well_post_enable(struct drm_i915_private *dev_priv,
                                        struct i915_power_well *power_well)
 {
-        struct drm_device *dev = dev_priv->dev;
+        struct drm_device *dev = &dev_priv->drm;
 
         /*
          * After we re-enable the power well, if we touch VGA register 0x3d5
@@ -345,8 +365,11 @@ static void hsw_set_power_well(struct drm_i915_private *dev_priv,
 
                 if (!is_enabled) {
                         DRM_DEBUG_KMS("Enabling power well\n");
-                        if (wait_for((I915_READ(HSW_PWR_WELL_DRIVER) &
-                                      HSW_PWR_WELL_STATE_ENABLED), 20))
+                        if (intel_wait_for_register(dev_priv,
+                                                    HSW_PWR_WELL_DRIVER,
+                                                    HSW_PWR_WELL_STATE_ENABLED,
+                                                    HSW_PWR_WELL_STATE_ENABLED,
+                                                    20))
                                 DRM_ERROR("Timeout enabling power well\n");
                         hsw_power_well_post_enable(dev_priv);
                 }
@@ -419,6 +442,16 @@ static void hsw_set_power_well(struct drm_i915_private *dev_priv,
         BIT(POWER_DOMAIN_MODESET) |                     \
         BIT(POWER_DOMAIN_AUX_A) |                       \
         BIT(POWER_DOMAIN_INIT))
+#define BXT_DPIO_CMN_A_POWER_DOMAINS (                  \
+        BIT(POWER_DOMAIN_PORT_DDI_A_LANES) |            \
+        BIT(POWER_DOMAIN_AUX_A) |                       \
+        BIT(POWER_DOMAIN_INIT))
+#define BXT_DPIO_CMN_BC_POWER_DOMAINS (                 \
+        BIT(POWER_DOMAIN_PORT_DDI_B_LANES) |            \
+        BIT(POWER_DOMAIN_PORT_DDI_C_LANES) |            \
+        BIT(POWER_DOMAIN_AUX_B) |                       \
+        BIT(POWER_DOMAIN_AUX_C) |                       \
+        BIT(POWER_DOMAIN_INIT))
 
 static void assert_can_enable_dc9(struct drm_i915_private *dev_priv)
 {
@@ -548,6 +581,7 @@ void bxt_enable_dc9(struct drm_i915_private *dev_priv)
 
         DRM_DEBUG_KMS("Enabling DC9\n");
 
+        intel_power_sequencer_reset(dev_priv);
         gen9_set_dc_state(dev_priv, DC_STATE_EN_DC9);
 }
 
@@ -669,8 +703,11 @@ static void skl_set_power_well(struct drm_i915_private *dev_priv,
 
         switch (power_well->data) {
         case SKL_DISP_PW_1:
-                if (wait_for((I915_READ(SKL_FUSE_STATUS) &
-                        SKL_FUSE_PG0_DIST_STATUS), 1)) {
+                if (intel_wait_for_register(dev_priv,
+                                            SKL_FUSE_STATUS,
+                                            SKL_FUSE_PG0_DIST_STATUS,
+                                            SKL_FUSE_PG0_DIST_STATUS,
+                                            1)) {
                         DRM_ERROR("PG0 not enabled\n");
                         return;
                 }
@@ -731,12 +768,18 @@ static void skl_set_power_well(struct drm_i915_private *dev_priv,
 
         if (check_fuse_status) {
                 if (power_well->data == SKL_DISP_PW_1) {
-                        if (wait_for((I915_READ(SKL_FUSE_STATUS) &
-                                SKL_FUSE_PG1_DIST_STATUS), 1))
+                        if (intel_wait_for_register(dev_priv,
+                                                    SKL_FUSE_STATUS,
+                                                    SKL_FUSE_PG1_DIST_STATUS,
+                                                    SKL_FUSE_PG1_DIST_STATUS,
+                                                    1))
                                 DRM_ERROR("PG1 distributing status timeout\n");
                 } else if (power_well->data == SKL_DISP_PW_2) {
-                        if (wait_for((I915_READ(SKL_FUSE_STATUS) &
-                                SKL_FUSE_PG2_DIST_STATUS), 1))
+                        if (intel_wait_for_register(dev_priv,
+                                                    SKL_FUSE_STATUS,
+                                                    SKL_FUSE_PG2_DIST_STATUS,
+                                                    SKL_FUSE_PG2_DIST_STATUS,
+                                                    1))
                                 DRM_ERROR("PG2 distributing status timeout\n");
                 }
         }
@@ -800,21 +843,99 @@ static void skl_power_well_disable(struct drm_i915_private *dev_priv,
         skl_set_power_well(dev_priv, power_well, false);
 }
 
+static enum dpio_phy bxt_power_well_to_phy(struct i915_power_well *power_well)
+{
+        enum skl_disp_power_wells power_well_id = power_well->data;
+
+        return power_well_id == BXT_DPIO_CMN_A ? DPIO_PHY1 : DPIO_PHY0;
+}
+
+static void bxt_dpio_cmn_power_well_enable(struct drm_i915_private *dev_priv,
+                                           struct i915_power_well *power_well)
+{
+        enum skl_disp_power_wells power_well_id = power_well->data;
+        struct i915_power_well *cmn_a_well;
+
+        if (power_well_id == BXT_DPIO_CMN_BC) {
+                /*
+                 * We need to copy the GRC calibration value from the eDP PHY,
+                 * so make sure it's powered up.
+                 */
+                cmn_a_well = lookup_power_well(dev_priv, BXT_DPIO_CMN_A);
+                intel_power_well_get(dev_priv, cmn_a_well);
+        }
+
+        bxt_ddi_phy_init(dev_priv, bxt_power_well_to_phy(power_well));
+
+        if (power_well_id == BXT_DPIO_CMN_BC)
+                intel_power_well_put(dev_priv, cmn_a_well);
+}
+
+static void bxt_dpio_cmn_power_well_disable(struct drm_i915_private *dev_priv,
+                                            struct i915_power_well *power_well)
+{
+        bxt_ddi_phy_uninit(dev_priv, bxt_power_well_to_phy(power_well));
+}
+
+static bool bxt_dpio_cmn_power_well_enabled(struct drm_i915_private *dev_priv,
+                                            struct i915_power_well *power_well)
+{
+        return bxt_ddi_phy_is_enabled(dev_priv,
+                                      bxt_power_well_to_phy(power_well));
+}
+
+static void bxt_dpio_cmn_power_well_sync_hw(struct drm_i915_private *dev_priv,
+                                            struct i915_power_well *power_well)
+{
+        if (power_well->count > 0)
+                bxt_dpio_cmn_power_well_enable(dev_priv, power_well);
+        else
+                bxt_dpio_cmn_power_well_disable(dev_priv, power_well);
+}
+
+
+static void bxt_verify_ddi_phy_power_wells(struct drm_i915_private *dev_priv)
+{
+        struct i915_power_well *power_well;
+
+        power_well = lookup_power_well(dev_priv, BXT_DPIO_CMN_A);
+        if (power_well->count > 0)
+                bxt_ddi_phy_verify_state(dev_priv,
+                                         bxt_power_well_to_phy(power_well));
+
+        power_well = lookup_power_well(dev_priv, BXT_DPIO_CMN_BC);
+        if (power_well->count > 0)
+                bxt_ddi_phy_verify_state(dev_priv,
+                                         bxt_power_well_to_phy(power_well));
+}
+
 static bool gen9_dc_off_power_well_enabled(struct drm_i915_private *dev_priv,
                                            struct i915_power_well *power_well)
 {
         return (I915_READ(DC_STATE_EN) & DC_STATE_EN_UPTO_DC5_DC6_MASK) == 0;
 }
 
+static void gen9_assert_dbuf_enabled(struct drm_i915_private *dev_priv)
+{
+        u32 tmp = I915_READ(DBUF_CTL);
+
+        WARN((tmp & (DBUF_POWER_STATE | DBUF_POWER_REQUEST)) !=
+             (DBUF_POWER_STATE | DBUF_POWER_REQUEST),
+             "Unexpected DBuf power power state (0x%08x)\n", tmp);
+}
+
 static void gen9_dc_off_power_well_enable(struct drm_i915_private *dev_priv,
                                           struct i915_power_well *power_well)
 {
         gen9_set_dc_state(dev_priv, DC_STATE_DISABLE);
 
-        if (IS_BROXTON(dev_priv)) {
-                broxton_cdclk_verify_state(dev_priv);
-                broxton_ddi_phy_verify_state(dev_priv);
-        }
+        WARN_ON(dev_priv->cdclk_freq !=
+                dev_priv->display.get_display_clock_speed(&dev_priv->drm));
+
+        gen9_assert_dbuf_enabled(dev_priv);
+
+        if (IS_BROXTON(dev_priv))
+                bxt_verify_ddi_phy_power_wells(dev_priv);
 }
 
 static void gen9_dc_off_power_well_disable(struct drm_i915_private *dev_priv,
@@ -948,10 +1069,16 @@ static void vlv_init_display_clock_gating(struct drm_i915_private *dev_priv)
          */
         I915_WRITE(MI_ARB_VLV, MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE);
         I915_WRITE(CBR1_VLV, 0);
+
+        WARN_ON(dev_priv->rawclk_freq == 0);
+
+        I915_WRITE(RAWCLK_FREQ_VLV,
+                   DIV_ROUND_CLOSEST(dev_priv->rawclk_freq, 1000));
 }
 
 static void vlv_display_power_well_init(struct drm_i915_private *dev_priv)
 {
+        struct intel_encoder *encoder;
         enum pipe pipe;
 
         /*
@@ -962,7 +1089,7 @@ static void vlv_display_power_well_init(struct drm_i915_private *dev_priv)
          *
          * CHV DPLL B/C have some issues if VGA mode is enabled.
          */
-        for_each_pipe(dev_priv->dev, pipe) {
+        for_each_pipe(&dev_priv->drm, pipe) {
                 u32 val = I915_READ(DPLL(pipe));
 
                 val |= DPLL_REF_CLK_ENABLE_VLV | DPLL_VGA_MODE_DIS;
@@ -987,7 +1114,13 @@ static void vlv_display_power_well_init(struct drm_i915_private *dev_priv)
 
         intel_hpd_init(dev_priv);
 
-        i915_redisable_vga_power_on(dev_priv->dev);
+        /* Re-enable the ADPA, if we have one */
+        for_each_intel_encoder(&dev_priv->drm, encoder) {
+                if (encoder->type == INTEL_OUTPUT_ANALOG)
+                        intel_crt_reset(&encoder->base);
+        }
+
+        i915_redisable_vga_power_on(&dev_priv->drm);
 }
 
 static void vlv_display_power_well_deinit(struct drm_i915_private *dev_priv)
@@ -997,9 +1130,11 @@ static void vlv_display_power_well_deinit(struct drm_i915_private *dev_priv)
         spin_unlock_irq(&dev_priv->irq_lock);
 
         /* make sure we're done processing display irqs */
-        synchronize_irq(dev_priv->dev->irq);
+        synchronize_irq(dev_priv->drm.irq);
+
+        intel_power_sequencer_reset(dev_priv);
 
-        vlv_power_sequencer_reset(dev_priv);
+        intel_hpd_poll_init(dev_priv);
 }
 
 static void vlv_display_power_well_enable(struct drm_i915_private *dev_priv,
@@ -1092,7 +1227,6 @@ static void assert_chv_phy_status(struct drm_i915_private *dev_priv)
         u32 phy_control = dev_priv->chv_phy_control;
         u32 phy_status = 0;
         u32 phy_status_mask = 0xffffffff;
-        u32 tmp;
 
         /*
          * The BIOS can leave the PHY is some weird state
@@ -1180,10 +1314,14 @@ static void assert_chv_phy_status(struct drm_i915_private *dev_priv)
          * The PHY may be busy with some initial calibration and whatnot,
          * so the power state can take a while to actually change.
          */
-        if (wait_for((tmp = I915_READ(DISPLAY_PHY_STATUS) & phy_status_mask) == phy_status, 10))
-                WARN(phy_status != tmp,
-                     "Unexpected PHY_STATUS 0x%08x, expected 0x%08x (PHY_CONTROL=0x%08x)\n",
-                     tmp, phy_status, dev_priv->chv_phy_control);
+        if (intel_wait_for_register(dev_priv,
+                                    DISPLAY_PHY_STATUS,
+                                    phy_status_mask,
+                                    phy_status,
+                                    10))
+                DRM_ERROR("Unexpected PHY_STATUS 0x%08x, expected 0x%08x (PHY_CONTROL=0x%08x)\n",
+                          I915_READ(DISPLAY_PHY_STATUS) & phy_status_mask,
+                           phy_status, dev_priv->chv_phy_control);
 }
 
 #undef BITS_SET
@@ -1211,7 +1349,11 @@ static void chv_dpio_cmn_power_well_enable(struct drm_i915_private *dev_priv,
         vlv_set_power_well(dev_priv, power_well, true);
 
         /* Poll for phypwrgood signal */
-        if (wait_for(I915_READ(DISPLAY_PHY_STATUS) & PHY_POWERGOOD(phy), 1))
+        if (intel_wait_for_register(dev_priv,
+                                    DISPLAY_PHY_STATUS,
+                                    PHY_POWERGOOD(phy),
+                                    PHY_POWERGOOD(phy),
+                                    1))
                 DRM_ERROR("Display PHY %d is not power up\n", phy);
 
         mutex_lock(&dev_priv->sb_lock);
@@ -1501,10 +1643,8 @@ __intel_display_power_get_domain(struct drm_i915_private *dev_priv,
         struct i915_power_well *power_well;
         int i;
 
-        for_each_power_well(i, power_well, BIT(domain), power_domains) {
-                if (!power_well->count++)
-                        intel_power_well_enable(dev_priv, power_well);
-        }
+        for_each_power_well(i, power_well, BIT(domain), power_domains)
+                intel_power_well_get(dev_priv, power_well);
 
         power_domains->domain_use_count[domain]++;
 }
@@ -1598,14 +1738,8 @@ void intel_display_power_put(struct drm_i915_private *dev_priv,
              intel_display_power_domain_str(domain));
         power_domains->domain_use_count[domain]--;
 
-        for_each_power_well_rev(i, power_well, BIT(domain), power_domains) {
-                WARN(!power_well->count,
-                     "Use count on power well %s is already zero",
-                     power_well->name);
-
-                if (!--power_well->count)
-                        intel_power_well_disable(dev_priv, power_well);
-        }
+        for_each_power_well_rev(i, power_well, BIT(domain), power_domains)
+                intel_power_well_put(dev_priv, power_well);
 
         mutex_unlock(&power_domains->lock);
 
@@ -1776,6 +1910,13 @@ static const struct i915_power_well_ops gen9_dc_off_power_well_ops = {
         .is_enabled = gen9_dc_off_power_well_enabled,
 };
 
+static const struct i915_power_well_ops bxt_dpio_cmn_power_well_ops = {
+        .sync_hw = bxt_dpio_cmn_power_well_sync_hw,
+        .enable = bxt_dpio_cmn_power_well_enable,
+        .disable = bxt_dpio_cmn_power_well_disable,
+        .is_enabled = bxt_dpio_cmn_power_well_enabled,
+};
+
 static struct i915_power_well hsw_power_wells[] = {
         {
                 .name = "always-on",
@@ -2012,6 +2153,18 @@ static struct i915_power_well bxt_power_wells[] = {
                 .ops = &skl_power_well_ops,
                 .data = SKL_DISP_PW_2,
         },
+        {
+                .name = "dpio-common-a",
+                .domains = BXT_DPIO_CMN_A_POWER_DOMAINS,
+                .ops = &bxt_dpio_cmn_power_well_ops,
+                .data = BXT_DPIO_CMN_A,
+        },
+        {
+                .name = "dpio-common-bc",
+                .domains = BXT_DPIO_CMN_BC_POWER_DOMAINS,
+                .ops = &bxt_dpio_cmn_power_well_ops,
+                .data = BXT_DPIO_CMN_BC,
+        },
 };
 
 static int
@@ -2131,7 +2284,7 @@ int intel_power_domains_init(struct drm_i915_private *dev_priv)
  */
 void intel_power_domains_fini(struct drm_i915_private *dev_priv)
 {
-        struct device *device = &dev_priv->dev->pdev->dev;
+        struct device *device = &dev_priv->drm.pdev->dev;
 
         /*
          * The i915.ko module is still not prepared to be loaded when
@@ -2171,6 +2324,28 @@ static void intel_power_domains_sync_hw(struct drm_i915_private *dev_priv)
         mutex_unlock(&power_domains->lock);
 }
 
+static void gen9_dbuf_enable(struct drm_i915_private *dev_priv)
+{
+        I915_WRITE(DBUF_CTL, I915_READ(DBUF_CTL) | DBUF_POWER_REQUEST);
+        POSTING_READ(DBUF_CTL);
+
+        udelay(10);
+
+        if (!(I915_READ(DBUF_CTL) & DBUF_POWER_STATE))
+                DRM_ERROR("DBuf power enable timeout\n");
+}
+
+static void gen9_dbuf_disable(struct drm_i915_private *dev_priv)
+{
+        I915_WRITE(DBUF_CTL, I915_READ(DBUF_CTL) & ~DBUF_POWER_REQUEST);
+        POSTING_READ(DBUF_CTL);
+
+        udelay(10);
+
+        if (I915_READ(DBUF_CTL) & DBUF_POWER_STATE)
+                DRM_ERROR("DBuf power disable timeout!\n");
+}
+
 static void skl_display_core_init(struct drm_i915_private *dev_priv,
                                    bool resume)
 {
@@ -2195,12 +2370,11 @@ static void skl_display_core_init(struct drm_i915_private *dev_priv,
 
         mutex_unlock(&power_domains->lock);
 
-        if (!resume)
-                return;
-
         skl_init_cdclk(dev_priv);
 
-        if (dev_priv->csr.dmc_payload)
+        gen9_dbuf_enable(dev_priv);
+
+        if (resume && dev_priv->csr.dmc_payload)
                 intel_csr_load_program(dev_priv);
 }
 
@@ -2211,6 +2385,8 @@ static void skl_display_core_uninit(struct drm_i915_private *dev_priv)
 
         gen9_set_dc_state(dev_priv, DC_STATE_DISABLE);
 
+        gen9_dbuf_disable(dev_priv);
+
         skl_uninit_cdclk(dev_priv);
 
         /* The spec doesn't call for removing the reset handshake flag */
@@ -2254,11 +2430,9 @@ void bxt_display_core_init(struct drm_i915_private *dev_priv,
 
         mutex_unlock(&power_domains->lock);
 
-        broxton_init_cdclk(dev_priv);
-        broxton_ddi_phy_init(dev_priv);
+        bxt_init_cdclk(dev_priv);
 
-        broxton_cdclk_verify_state(dev_priv);
-        broxton_ddi_phy_verify_state(dev_priv);
+        gen9_dbuf_enable(dev_priv);
 
         if (resume && dev_priv->csr.dmc_payload)
                 intel_csr_load_program(dev_priv);
@@ -2271,8 +2445,9 @@ void bxt_display_core_uninit(struct drm_i915_private *dev_priv)
 
         gen9_set_dc_state(dev_priv, DC_STATE_DISABLE);
 
-        broxton_ddi_phy_uninit(dev_priv);
-        broxton_uninit_cdclk(dev_priv);
+        gen9_dbuf_disable(dev_priv);
+
+        bxt_uninit_cdclk(dev_priv);
 
         /* The spec doesn't call for removing the reset handshake flag */
 
@@ -2403,13 +2578,14 @@ static void vlv_cmnlane_wa(struct drm_i915_private *dev_priv)
 /**
  * intel_power_domains_init_hw - initialize hardware power domain state
  * @dev_priv: i915 device instance
+ * @resume: Called from resume code paths or not
  *
  * This function initializes the hardware power domain state and enables all
  * power domains using intel_display_set_init_power().
  */
 void intel_power_domains_init_hw(struct drm_i915_private *dev_priv, bool resume)
 {
-        struct drm_device *dev = dev_priv->dev;
+        struct drm_device *dev = &dev_priv->drm;
         struct i915_power_domains *power_domains = &dev_priv->power_domains;
 
         power_domains->initializing = true;
@@ -2471,7 +2647,7 @@ void intel_power_domains_suspend(struct drm_i915_private *dev_priv)
  */
 void intel_runtime_pm_get(struct drm_i915_private *dev_priv)
 {
-        struct drm_device *dev = dev_priv->dev;
+        struct drm_device *dev = &dev_priv->drm;
         struct device *device = &dev->pdev->dev;
 
         pm_runtime_get_sync(device);
@@ -2492,7 +2668,7 @@ void intel_runtime_pm_get(struct drm_i915_private *dev_priv)
  */
 bool intel_runtime_pm_get_if_in_use(struct drm_i915_private *dev_priv)
 {
-        struct drm_device *dev = dev_priv->dev;
+        struct drm_device *dev = &dev_priv->drm;
         struct device *device = &dev->pdev->dev;
 
         if (IS_ENABLED(CONFIG_PM)) {
@@ -2534,7 +2710,7 @@ bool intel_runtime_pm_get_if_in_use(struct drm_i915_private *dev_priv)
  */
 void intel_runtime_pm_get_noresume(struct drm_i915_private *dev_priv)
 {
-        struct drm_device *dev = dev_priv->dev;
+        struct drm_device *dev = &dev_priv->drm;
         struct device *device = &dev->pdev->dev;
 
         assert_rpm_wakelock_held(dev_priv);
@@ -2553,7 +2729,7 @@ void intel_runtime_pm_get_noresume(struct drm_i915_private *dev_priv)
  */
 void intel_runtime_pm_put(struct drm_i915_private *dev_priv)
 {
-        struct drm_device *dev = dev_priv->dev;
+        struct drm_device *dev = &dev_priv->drm;
         struct device *device = &dev->pdev->dev;
 
         assert_rpm_wakelock_held(dev_priv);
@@ -2576,7 +2752,7 @@ void intel_runtime_pm_put(struct drm_i915_private *dev_priv)
  */
 void intel_runtime_pm_enable(struct drm_i915_private *dev_priv)
 {
-        struct drm_device *dev = dev_priv->dev;
+        struct drm_device *dev = &dev_priv->drm;
         struct device *device = &dev->pdev->dev;
 
         pm_runtime_set_autosuspend_delay(device, 10000); /* 10s */
diff --git a/drivers/gpu/drm/i915/intel_sdvo.c b/drivers/gpu/drm/i915/intel_sdvo.c
index 2128fae5687d..e378f35365a2 100644
--- a/drivers/gpu/drm/i915/intel_sdvo.c
+++ b/drivers/gpu/drm/i915/intel_sdvo.c
@@ -240,7 +240,7 @@ intel_sdvo_create_enhance_property(struct intel_sdvo *intel_sdvo,
 static void intel_sdvo_write_sdvox(struct intel_sdvo *intel_sdvo, u32 val)
 {
         struct drm_device *dev = intel_sdvo->base.base.dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         u32 bval = val, cval = val;
         int i;
 
@@ -1195,7 +1195,7 @@ static bool intel_sdvo_compute_config(struct intel_encoder *encoder,
 static void intel_sdvo_pre_enable(struct intel_encoder *intel_encoder)
 {
         struct drm_device *dev = intel_encoder->base.dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         struct intel_crtc *crtc = to_intel_crtc(intel_encoder->base.crtc);
         const struct drm_display_mode *adjusted_mode = &crtc->config->base.adjusted_mode;
         struct drm_display_mode *mode = &crtc->config->base.mode;
@@ -1330,7 +1330,7 @@ static bool intel_sdvo_get_hw_state(struct intel_encoder *encoder,
                                     enum pipe *pipe)
 {
         struct drm_device *dev = encoder->base.dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         struct intel_sdvo *intel_sdvo = to_sdvo(encoder);
         u16 active_outputs = 0;
         u32 tmp;
@@ -1353,7 +1353,7 @@ static void intel_sdvo_get_config(struct intel_encoder *encoder,
                                   struct intel_crtc_state *pipe_config)
 {
         struct drm_device *dev = encoder->base.dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         struct intel_sdvo *intel_sdvo = to_sdvo(encoder);
         struct intel_sdvo_dtd dtd;
         int encoder_pixel_multiplier = 0;
@@ -1436,7 +1436,7 @@ static void intel_sdvo_get_config(struct intel_encoder *encoder,
 
 static void intel_disable_sdvo(struct intel_encoder *encoder)
 {
-        struct drm_i915_private *dev_priv = encoder->base.dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
         struct intel_sdvo *intel_sdvo = to_sdvo(encoder);
         struct intel_crtc *crtc = to_intel_crtc(encoder->base.crtc);
         u32 temp;
@@ -1471,7 +1471,7 @@ static void intel_disable_sdvo(struct intel_encoder *encoder)
                 temp &= ~SDVO_ENABLE;
                 intel_sdvo_write_sdvox(intel_sdvo, temp);
 
-                intel_wait_for_vblank_if_active(dev_priv->dev, PIPE_A);
+                intel_wait_for_vblank_if_active(&dev_priv->drm, PIPE_A);
                 intel_set_cpu_fifo_underrun_reporting(dev_priv, PIPE_A, true);
                 intel_set_pch_fifo_underrun_reporting(dev_priv, PIPE_A, true);
         }
@@ -1489,7 +1489,7 @@ static void pch_post_disable_sdvo(struct intel_encoder *encoder)
 static void intel_enable_sdvo(struct intel_encoder *encoder)
 {
         struct drm_device *dev = encoder->base.dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         struct intel_sdvo *intel_sdvo = to_sdvo(encoder);
         struct intel_crtc *intel_crtc = to_intel_crtc(encoder->base.crtc);
         u32 temp;
@@ -1633,7 +1633,7 @@ intel_sdvo_get_edid(struct drm_connector *connector)
 static struct edid *
 intel_sdvo_get_analog_edid(struct drm_connector *connector)
 {
-        struct drm_i915_private *dev_priv = connector->dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(connector->dev);
 
         return drm_get_edid(connector,
                             intel_gmbus_get_adapter(dev_priv,
@@ -1916,7 +1916,7 @@ static void intel_sdvo_get_tv_modes(struct drm_connector *connector)
 static void intel_sdvo_get_lvds_modes(struct drm_connector *connector)
 {
         struct intel_sdvo *intel_sdvo = intel_attached_sdvo(connector);
-        struct drm_i915_private *dev_priv = connector->dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(connector->dev);
         struct drm_display_mode *newmode;
 
         DRM_DEBUG_KMS("[CONNECTOR:%d:%s]\n",
@@ -2001,7 +2001,7 @@ intel_sdvo_set_property(struct drm_connector *connector,
 {
         struct intel_sdvo *intel_sdvo = intel_attached_sdvo(connector);
         struct intel_sdvo_connector *intel_sdvo_connector = to_intel_sdvo_connector(connector);
-        struct drm_i915_private *dev_priv = connector->dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(connector->dev);
         uint16_t temp_value;
         uint8_t cmd;
         int ret;
@@ -2177,12 +2177,39 @@ done:
 #undef CHECK_PROPERTY
 }
 
+static int
+intel_sdvo_connector_register(struct drm_connector *connector)
+{
+        struct intel_sdvo *sdvo = intel_attached_sdvo(connector);
+        int ret;
+
+        ret = intel_connector_register(connector);
+        if (ret)
+                return ret;
+
+        return sysfs_create_link(&connector->kdev->kobj,
+                                 &sdvo->ddc.dev.kobj,
+                                 sdvo->ddc.dev.kobj.name);
+}
+
+static void
+intel_sdvo_connector_unregister(struct drm_connector *connector)
+{
+        struct intel_sdvo *sdvo = intel_attached_sdvo(connector);
+
+        sysfs_remove_link(&connector->kdev->kobj,
+                          sdvo->ddc.dev.kobj.name);
+        intel_connector_unregister(connector);
+}
+
 static const struct drm_connector_funcs intel_sdvo_connector_funcs = {
         .dpms = drm_atomic_helper_connector_dpms,
         .detect = intel_sdvo_detect,
         .fill_modes = drm_helper_probe_single_connector_modes,
         .set_property = intel_sdvo_set_property,
         .atomic_get_property = intel_connector_atomic_get_property,
+        .late_register = intel_sdvo_connector_register,
+        .early_unregister = intel_sdvo_connector_unregister,
         .destroy = intel_sdvo_destroy,
         .atomic_destroy_state = drm_atomic_helper_connector_destroy_state,
         .atomic_duplicate_state = drm_atomic_helper_connector_duplicate_state,
@@ -2191,7 +2218,6 @@ static const struct drm_connector_funcs intel_sdvo_connector_funcs = {
 static const struct drm_connector_helper_funcs intel_sdvo_connector_helper_funcs = {
         .get_modes = intel_sdvo_get_modes,
         .mode_valid = intel_sdvo_mode_valid,
-        .best_encoder = intel_best_encoder,
 };
 
 static void intel_sdvo_enc_destroy(struct drm_encoder *encoder)
@@ -2312,7 +2338,7 @@ intel_sdvo_is_hdmi_connector(struct intel_sdvo *intel_sdvo, int device)
 static u8
 intel_sdvo_get_slave_addr(struct drm_device *dev, struct intel_sdvo *sdvo)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         struct sdvo_device_mapping *my_mapping, *other_mapping;
 
         if (sdvo->port == PORT_B) {
@@ -2346,20 +2372,6 @@ intel_sdvo_get_slave_addr(struct drm_device *dev, struct intel_sdvo *sdvo)
                 return 0x72;
 }
 
-static void
-intel_sdvo_connector_unregister(struct intel_connector *intel_connector)
-{
-        struct drm_connector *drm_connector;
-        struct intel_sdvo *sdvo_encoder;
-
-        drm_connector = &intel_connector->base;
-        sdvo_encoder = intel_attached_sdvo(&intel_connector->base);
-
-        sysfs_remove_link(&drm_connector->kdev->kobj,
-                          sdvo_encoder->ddc.dev.kobj.name);
-        intel_connector_unregister(intel_connector);
-}
-
 static int
 intel_sdvo_connector_init(struct intel_sdvo_connector *connector,
                           struct intel_sdvo *encoder)
@@ -2382,27 +2394,10 @@ intel_sdvo_connector_init(struct intel_sdvo_connector *connector,
         connector->base.base.doublescan_allowed = 0;
         connector->base.base.display_info.subpixel_order = SubPixelHorizontalRGB;
         connector->base.get_hw_state = intel_sdvo_connector_get_hw_state;
-        connector->base.unregister = intel_sdvo_connector_unregister;
 
         intel_connector_attach_encoder(&connector->base, &encoder->base);
-        ret = drm_connector_register(drm_connector);
-        if (ret < 0)
-                goto err1;
-
-        ret = sysfs_create_link(&drm_connector->kdev->kobj,
-                                &encoder->ddc.dev.kobj,
-                                encoder->ddc.dev.kobj.name);
-        if (ret < 0)
-                goto err2;
 
         return 0;
-
-err2:
-        drm_connector_unregister(drm_connector);
-err1:
-        drm_connector_cleanup(drm_connector);
-
-        return ret;
 }
 
 static void
@@ -2529,7 +2524,6 @@ intel_sdvo_tv_init(struct intel_sdvo *intel_sdvo, int type)
         return true;
 
 err:
-        drm_connector_unregister(connector);
         intel_sdvo_destroy(connector);
         return false;
 }
@@ -2608,7 +2602,6 @@ intel_sdvo_lvds_init(struct intel_sdvo *intel_sdvo, int device)
         return true;
 
 err:
-        drm_connector_unregister(connector);
         intel_sdvo_destroy(connector);
         return false;
 }
@@ -2959,7 +2952,7 @@ static void assert_sdvo_port_valid(const struct drm_i915_private *dev_priv,
 bool intel_sdvo_init(struct drm_device *dev,
                      i915_reg_t sdvo_reg, enum port port)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         struct intel_encoder *intel_encoder;
         struct intel_sdvo *intel_sdvo;
         int i;
@@ -2981,7 +2974,7 @@ bool intel_sdvo_init(struct drm_device *dev,
         intel_encoder = &intel_sdvo->base;
         intel_encoder->type = INTEL_OUTPUT_SDVO;
         drm_encoder_init(dev, &intel_encoder->base, &intel_sdvo_enc_funcs, 0,
-                         NULL);
+                         "SDVO %c", port_name(port));
 
         /* Read the regs to test if we can talk to the device */
         for (i = 0; i < 0x40; i++) {
diff --git a/drivers/gpu/drm/i915/intel_sideband.c b/drivers/gpu/drm/i915/intel_sideband.c
index c3998188cf35..1a840bf92eea 100644
--- a/drivers/gpu/drm/i915/intel_sideband.c
+++ b/drivers/gpu/drm/i915/intel_sideband.c
@@ -51,7 +51,9 @@ static int vlv_sideband_rw(struct drm_i915_private *dev_priv, u32 devfn,
 
         WARN_ON(!mutex_is_locked(&dev_priv->sb_lock));
 
-        if (wait_for((I915_READ(VLV_IOSF_DOORBELL_REQ) & IOSF_SB_BUSY) == 0, 5)) {
+        if (intel_wait_for_register(dev_priv,
+                                    VLV_IOSF_DOORBELL_REQ, IOSF_SB_BUSY, 0,
+                                    5)) {
                 DRM_DEBUG_DRIVER("IOSF sideband idle wait (%s) timed out\n",
                                  is_read ? "read" : "write");
                 return -EAGAIN;
@@ -62,7 +64,9 @@ static int vlv_sideband_rw(struct drm_i915_private *dev_priv, u32 devfn,
                 I915_WRITE(VLV_IOSF_DATA, *val);
         I915_WRITE(VLV_IOSF_DOORBELL_REQ, cmd);
 
-        if (wait_for((I915_READ(VLV_IOSF_DOORBELL_REQ) & IOSF_SB_BUSY) == 0, 5)) {
+        if (intel_wait_for_register(dev_priv,
+                                    VLV_IOSF_DOORBELL_REQ, IOSF_SB_BUSY, 0,
+                                    5)) {
                 DRM_DEBUG_DRIVER("IOSF sideband finish wait (%s) timed out\n",
                                  is_read ? "read" : "write");
                 return -ETIMEDOUT;
@@ -202,8 +206,9 @@ u32 intel_sbi_read(struct drm_i915_private *dev_priv, u16 reg,
         u32 value = 0;
         WARN_ON(!mutex_is_locked(&dev_priv->sb_lock));
 
-        if (wait_for((I915_READ(SBI_CTL_STAT) & SBI_BUSY) == 0,
-                                100)) {
+        if (intel_wait_for_register(dev_priv,
+                                    SBI_CTL_STAT, SBI_BUSY, 0,
+                                    100)) {
                 DRM_ERROR("timeout waiting for SBI to become ready\n");
                 return 0;
         }
@@ -216,8 +221,11 @@ u32 intel_sbi_read(struct drm_i915_private *dev_priv, u16 reg,
                 value = SBI_CTL_DEST_MPHY | SBI_CTL_OP_IORD;
         I915_WRITE(SBI_CTL_STAT, value | SBI_BUSY);
 
-        if (wait_for((I915_READ(SBI_CTL_STAT) & (SBI_BUSY | SBI_RESPONSE_FAIL)) == 0,
-                                100)) {
+        if (intel_wait_for_register(dev_priv,
+                                    SBI_CTL_STAT,
+                                    SBI_BUSY | SBI_RESPONSE_FAIL,
+                                    0,
+                                    100)) {
                 DRM_ERROR("timeout waiting for SBI to complete read transaction\n");
                 return 0;
         }
@@ -232,8 +240,9 @@ void intel_sbi_write(struct drm_i915_private *dev_priv, u16 reg, u32 value,
 
         WARN_ON(!mutex_is_locked(&dev_priv->sb_lock));
 
-        if (wait_for((I915_READ(SBI_CTL_STAT) & SBI_BUSY) == 0,
-                                100)) {
+        if (intel_wait_for_register(dev_priv,
+                                    SBI_CTL_STAT, SBI_BUSY, 0,
+                                    100)) {
                 DRM_ERROR("timeout waiting for SBI to become ready\n");
                 return;
         }
@@ -247,8 +256,11 @@ void intel_sbi_write(struct drm_i915_private *dev_priv, u16 reg, u32 value,
                 tmp = SBI_CTL_DEST_MPHY | SBI_CTL_OP_IOWR;
         I915_WRITE(SBI_CTL_STAT, SBI_BUSY | tmp);
 
-        if (wait_for((I915_READ(SBI_CTL_STAT) & (SBI_BUSY | SBI_RESPONSE_FAIL)) == 0,
-                                100)) {
+        if (intel_wait_for_register(dev_priv,
+                                    SBI_CTL_STAT,
+                                    SBI_BUSY | SBI_RESPONSE_FAIL,
+                                    0,
+                                    100)) {
                 DRM_ERROR("timeout waiting for SBI to complete write transaction\n");
                 return;
         }
diff --git a/drivers/gpu/drm/i915/intel_sprite.c b/drivers/gpu/drm/i915/intel_sprite.c
index 0f3e2303e0e9..0de935ad01c2 100644
--- a/drivers/gpu/drm/i915/intel_sprite.c
+++ b/drivers/gpu/drm/i915/intel_sprite.c
@@ -80,9 +80,7 @@ static int usecs_to_scanlines(const struct drm_display_mode *adjusted_mode,
  */
 void intel_pipe_update_start(struct intel_crtc *crtc)
 {
-        struct drm_device *dev = crtc->base.dev;
         const struct drm_display_mode *adjusted_mode = &crtc->config->base.adjusted_mode;
-        enum pipe pipe = crtc->pipe;
         long timeout = msecs_to_jiffies_timeout(1);
         int scanline, min, max, vblank_start;
         wait_queue_head_t *wq = drm_crtc_vblank_waitqueue(&crtc->base);
@@ -139,8 +137,7 @@ void intel_pipe_update_start(struct intel_crtc *crtc)
 
         crtc->debug.scanline_start = scanline;
         crtc->debug.start_vbl_time = ktime_get();
-        crtc->debug.start_vbl_count =
-                dev->driver->get_vblank_counter(dev, pipe);
+        crtc->debug.start_vbl_count = intel_crtc_get_vblank_counter(crtc);
 
         trace_i915_pipe_update_vblank_evaded(crtc);
 }
@@ -154,16 +151,35 @@ void intel_pipe_update_start(struct intel_crtc *crtc)
  * re-enables interrupts and verifies the update was actually completed
  * before a vblank using the value of @start_vbl_count.
  */
-void intel_pipe_update_end(struct intel_crtc *crtc)
+void intel_pipe_update_end(struct intel_crtc *crtc, struct intel_flip_work *work)
 {
-        struct drm_device *dev = crtc->base.dev;
         enum pipe pipe = crtc->pipe;
         int scanline_end = intel_get_crtc_scanline(crtc);
-        u32 end_vbl_count = dev->driver->get_vblank_counter(dev, pipe);
+        u32 end_vbl_count = intel_crtc_get_vblank_counter(crtc);
         ktime_t end_vbl_time = ktime_get();
 
+        if (work) {
+                work->flip_queued_vblank = end_vbl_count;
+                smp_mb__before_atomic();
+                atomic_set(&work->pending, 1);
+        }
+
         trace_i915_pipe_update_end(crtc, end_vbl_count, scanline_end);
 
+        /* We're still in the vblank-evade critical section, this can't race.
+         * Would be slightly nice to just grab the vblank count and arm the
+         * event outside of the critical section - the spinlock might spin for a
+         * while ... */
+        if (crtc->base.state->event) {
+                WARN_ON(drm_crtc_vblank_get(&crtc->base) != 0);
+
+                spin_lock(&crtc->base.dev->event_lock);
+                drm_crtc_arm_vblank_event(&crtc->base, crtc->base.state->event);
+                spin_unlock(&crtc->base.dev->event_lock);
+
+                crtc->base.state->event = NULL;
+        }
+
         local_irq_enable();
 
         if (crtc->debug.start_vbl_count &&
@@ -183,7 +199,7 @@ skl_update_plane(struct drm_plane *drm_plane,
                  const struct intel_plane_state *plane_state)
 {
         struct drm_device *dev = drm_plane->dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         struct intel_plane *intel_plane = to_intel_plane(drm_plane);
         struct drm_framebuffer *fb = plane_state->base.fb;
         struct drm_i915_gem_object *obj = intel_fb_obj(fb);
@@ -203,8 +219,6 @@ skl_update_plane(struct drm_plane *drm_plane,
         uint32_t y = plane_state->src.y1 >> 16;
         uint32_t src_w = drm_rect_width(&plane_state->src) >> 16;
         uint32_t src_h = drm_rect_height(&plane_state->src) >> 16;
-        const struct intel_scaler *scaler =
-                &crtc_state->scaler_state.scalers[plane_state->scaler_id];
 
         plane_ctl = PLANE_CTL_ENABLE |
                 PLANE_CTL_PIPE_GAMMA_ENABLE |
@@ -260,13 +274,16 @@ skl_update_plane(struct drm_plane *drm_plane,
 
         /* program plane scaler */
         if (plane_state->scaler_id >= 0) {
-                uint32_t ps_ctrl = 0;
                 int scaler_id = plane_state->scaler_id;
+                const struct intel_scaler *scaler;
 
                 DRM_DEBUG_KMS("plane = %d PS_PLANE_SEL(plane) = 0x%x\n", plane,
                         PS_PLANE_SEL(plane));
-                ps_ctrl = PS_SCALER_EN | PS_PLANE_SEL(plane) | scaler->mode;
-                I915_WRITE(SKL_PS_CTRL(pipe, scaler_id), ps_ctrl);
+
+                scaler = &crtc_state->scaler_state.scalers[scaler_id];
+
+                I915_WRITE(SKL_PS_CTRL(pipe, scaler_id),
+                           PS_SCALER_EN | PS_PLANE_SEL(plane) | scaler->mode);
                 I915_WRITE(SKL_PS_PWR_GATE(pipe, scaler_id), 0);
                 I915_WRITE(SKL_PS_WIN_POS(pipe, scaler_id), (crtc_x << 16) | crtc_y);
                 I915_WRITE(SKL_PS_WIN_SZ(pipe, scaler_id),
@@ -286,7 +303,7 @@ static void
 skl_disable_plane(struct drm_plane *dplane, struct drm_crtc *crtc)
 {
         struct drm_device *dev = dplane->dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         struct intel_plane *intel_plane = to_intel_plane(dplane);
         const int pipe = intel_plane->pipe;
         const int plane = intel_plane->plane + 1;
@@ -300,7 +317,7 @@ skl_disable_plane(struct drm_plane *dplane, struct drm_crtc *crtc)
 static void
 chv_update_csc(struct intel_plane *intel_plane, uint32_t format)
 {
-        struct drm_i915_private *dev_priv = intel_plane->base.dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(intel_plane->base.dev);
         int plane = intel_plane->plane;
 
         /* Seems RGB data bypasses the CSC always */
@@ -342,7 +359,7 @@ vlv_update_plane(struct drm_plane *dplane,
                  const struct intel_plane_state *plane_state)
 {
         struct drm_device *dev = dplane->dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         struct intel_plane *intel_plane = to_intel_plane(dplane);
         struct drm_framebuffer *fb = plane_state->base.fb;
         struct drm_i915_gem_object *obj = intel_fb_obj(fb);
@@ -468,7 +485,7 @@ static void
 vlv_disable_plane(struct drm_plane *dplane, struct drm_crtc *crtc)
 {
         struct drm_device *dev = dplane->dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         struct intel_plane *intel_plane = to_intel_plane(dplane);
         int pipe = intel_plane->pipe;
         int plane = intel_plane->plane;
@@ -485,7 +502,7 @@ ivb_update_plane(struct drm_plane *plane,
                  const struct intel_plane_state *plane_state)
 {
         struct drm_device *dev = plane->dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         struct intel_plane *intel_plane = to_intel_plane(plane);
         struct drm_framebuffer *fb = plane_state->base.fb;
         struct drm_i915_gem_object *obj = intel_fb_obj(fb);
@@ -607,7 +624,7 @@ static void
 ivb_disable_plane(struct drm_plane *plane, struct drm_crtc *crtc)
 {
         struct drm_device *dev = plane->dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         struct intel_plane *intel_plane = to_intel_plane(plane);
         int pipe = intel_plane->pipe;
 
@@ -626,7 +643,7 @@ ilk_update_plane(struct drm_plane *plane,
                  const struct intel_plane_state *plane_state)
 {
         struct drm_device *dev = plane->dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         struct intel_plane *intel_plane = to_intel_plane(plane);
         struct drm_framebuffer *fb = plane_state->base.fb;
         struct drm_i915_gem_object *obj = intel_fb_obj(fb);
@@ -736,7 +753,7 @@ static void
 ilk_disable_plane(struct drm_plane *plane, struct drm_crtc *crtc)
 {
         struct drm_device *dev = plane->dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         struct intel_plane *intel_plane = to_intel_plane(plane);
         int pipe = intel_plane->pipe;
 
@@ -1111,10 +1128,18 @@ intel_plane_init(struct drm_device *dev, enum pipe pipe, int plane)
 
         possible_crtcs = (1 << pipe);
 
-        ret = drm_universal_plane_init(dev, &intel_plane->base, possible_crtcs,
-                                       &intel_plane_funcs,
-                                       plane_formats, num_plane_formats,
-                                       DRM_PLANE_TYPE_OVERLAY, NULL);
+        if (INTEL_INFO(dev)->gen >= 9)
+                ret = drm_universal_plane_init(dev, &intel_plane->base, possible_crtcs,
+                                               &intel_plane_funcs,
+                                               plane_formats, num_plane_formats,
+                                               DRM_PLANE_TYPE_OVERLAY,
+                                               "plane %d%c", plane + 2, pipe_name(pipe));
+        else
+                ret = drm_universal_plane_init(dev, &intel_plane->base, possible_crtcs,
+                                               &intel_plane_funcs,
+                                               plane_formats, num_plane_formats,
+                                               DRM_PLANE_TYPE_OVERLAY,
+                                               "sprite %c", sprite_name(pipe, plane));
         if (ret)
                 goto fail;
 
diff --git a/drivers/gpu/drm/i915/intel_tv.c b/drivers/gpu/drm/i915/intel_tv.c
index 223129d3c765..49136ad5473e 100644
--- a/drivers/gpu/drm/i915/intel_tv.c
+++ b/drivers/gpu/drm/i915/intel_tv.c
@@ -826,7 +826,7 @@ static bool
 intel_tv_get_hw_state(struct intel_encoder *encoder, enum pipe *pipe)
 {
         struct drm_device *dev = encoder->base.dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         u32 tmp = I915_READ(TV_CTL);
 
         if (!(tmp & TV_ENC_ENABLE))
@@ -841,7 +841,7 @@ static void
 intel_enable_tv(struct intel_encoder *encoder)
 {
         struct drm_device *dev = encoder->base.dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
 
         /* Prevents vblank waits from timing out in intel_tv_detect_type() */
         intel_wait_for_vblank(encoder->base.dev,
@@ -854,7 +854,7 @@ static void
 intel_disable_tv(struct intel_encoder *encoder)
 {
         struct drm_device *dev = encoder->base.dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
 
         I915_WRITE(TV_CTL, I915_READ(TV_CTL) & ~TV_ENC_ENABLE);
 }
@@ -1013,7 +1013,7 @@ static void set_color_conversion(struct drm_i915_private *dev_priv,
 static void intel_tv_pre_enable(struct intel_encoder *encoder)
 {
         struct drm_device *dev = encoder->base.dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         struct intel_crtc *intel_crtc = to_intel_crtc(encoder->base.crtc);
         struct intel_tv *intel_tv = enc_to_tv(encoder);
         const struct tv_mode *tv_mode = intel_tv_mode_find(intel_tv);
@@ -1173,7 +1173,7 @@ intel_tv_detect_type(struct intel_tv *intel_tv,
         struct drm_crtc *crtc = connector->state->crtc;
         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
         struct drm_device *dev = connector->dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         u32 tv_ctl, save_tv_ctl;
         u32 tv_dac, save_tv_dac;
         int type;
@@ -1501,6 +1501,8 @@ out:
 static const struct drm_connector_funcs intel_tv_connector_funcs = {
         .dpms = drm_atomic_helper_connector_dpms,
         .detect = intel_tv_detect,
+        .late_register = intel_connector_register,
+        .early_unregister = intel_connector_unregister,
         .destroy = intel_tv_destroy,
         .set_property = intel_tv_set_property,
         .atomic_get_property = intel_connector_atomic_get_property,
@@ -1512,7 +1514,6 @@ static const struct drm_connector_funcs intel_tv_connector_funcs = {
 static const struct drm_connector_helper_funcs intel_tv_connector_helper_funcs = {
         .mode_valid = intel_tv_mode_valid,
         .get_modes = intel_tv_get_modes,
-        .best_encoder = intel_best_encoder,
 };
 
 static const struct drm_encoder_funcs intel_tv_enc_funcs = {
@@ -1522,7 +1523,7 @@ static const struct drm_encoder_funcs intel_tv_enc_funcs = {
 void
 intel_tv_init(struct drm_device *dev)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         struct drm_connector *connector;
         struct intel_tv *intel_tv;
         struct intel_encoder *intel_encoder;
@@ -1591,7 +1592,7 @@ intel_tv_init(struct drm_device *dev)
                            DRM_MODE_CONNECTOR_SVIDEO);
 
         drm_encoder_init(dev, &intel_encoder->base, &intel_tv_enc_funcs,
-                         DRM_MODE_ENCODER_TVDAC, NULL);
+                         DRM_MODE_ENCODER_TVDAC, "TV");
 
         intel_encoder->compute_config = intel_tv_compute_config;
         intel_encoder->get_config = intel_tv_get_config;
@@ -1600,7 +1601,6 @@ intel_tv_init(struct drm_device *dev)
         intel_encoder->disable = intel_disable_tv;
         intel_encoder->get_hw_state = intel_tv_get_hw_state;
         intel_connector->get_hw_state = intel_connector_get_hw_state;
-        intel_connector->unregister = intel_connector_unregister;
 
         intel_connector_attach_encoder(intel_connector, intel_encoder);
         intel_encoder->type = INTEL_OUTPUT_TVOUT;
@@ -1642,5 +1642,4 @@ intel_tv_init(struct drm_device *dev)
         drm_object_attach_property(&connector->base,
                                    dev->mode_config.tv_bottom_margin_property,
                                    intel_tv->margin[TV_MARGIN_BOTTOM]);
-        drm_connector_register(connector);
 }
diff --git a/drivers/gpu/drm/i915/intel_uncore.c b/drivers/gpu/drm/i915/intel_uncore.c
index 4f1dfe616856..ff80a81b1a84 100644
--- a/drivers/gpu/drm/i915/intel_uncore.c
+++ b/drivers/gpu/drm/i915/intel_uncore.c
@@ -248,9 +248,9 @@ intel_uncore_fw_release_timer(struct hrtimer *timer)
         return HRTIMER_NORESTART;
 }
 
-void intel_uncore_forcewake_reset(struct drm_device *dev, bool restore)
+void intel_uncore_forcewake_reset(struct drm_i915_private *dev_priv,
+                                  bool restore)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
         unsigned long irqflags;
         struct intel_uncore_forcewake_domain *domain;
         int retry_count = 100;
@@ -304,7 +304,7 @@ void intel_uncore_forcewake_reset(struct drm_device *dev, bool restore)
                 if (fw)
                         dev_priv->uncore.funcs.force_wake_get(dev_priv, fw);
 
-                if (IS_GEN6(dev) || IS_GEN7(dev))
+                if (IS_GEN6(dev_priv) || IS_GEN7(dev_priv))
                         dev_priv->uncore.fifo_count =
                                 fifo_free_entries(dev_priv);
         }
@@ -400,43 +400,42 @@ check_for_unclaimed_mmio(struct drm_i915_private *dev_priv)
         return false;
 }
 
-static void __intel_uncore_early_sanitize(struct drm_device *dev,
+static void __intel_uncore_early_sanitize(struct drm_i915_private *dev_priv,
                                           bool restore_forcewake)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
-
         /* clear out unclaimed reg detection bit */
         if (check_for_unclaimed_mmio(dev_priv))
                 DRM_DEBUG("unclaimed mmio detected on uncore init, clearing\n");
 
         /* clear out old GT FIFO errors */
-        if (IS_GEN6(dev) || IS_GEN7(dev))
+        if (IS_GEN6(dev_priv) || IS_GEN7(dev_priv))
                 __raw_i915_write32(dev_priv, GTFIFODBG,
                                    __raw_i915_read32(dev_priv, GTFIFODBG));
 
         /* WaDisableShadowRegForCpd:chv */
-        if (IS_CHERRYVIEW(dev)) {
+        if (IS_CHERRYVIEW(dev_priv)) {
                 __raw_i915_write32(dev_priv, GTFIFOCTL,
                                    __raw_i915_read32(dev_priv, GTFIFOCTL) |
                                    GT_FIFO_CTL_BLOCK_ALL_POLICY_STALL |
                                    GT_FIFO_CTL_RC6_POLICY_STALL);
         }
 
-        intel_uncore_forcewake_reset(dev, restore_forcewake);
+        intel_uncore_forcewake_reset(dev_priv, restore_forcewake);
 }
 
-void intel_uncore_early_sanitize(struct drm_device *dev, bool restore_forcewake)
+void intel_uncore_early_sanitize(struct drm_i915_private *dev_priv,
+                                 bool restore_forcewake)
 {
-        __intel_uncore_early_sanitize(dev, restore_forcewake);
-        i915_check_and_clear_faults(dev);
+        __intel_uncore_early_sanitize(dev_priv, restore_forcewake);
+        i915_check_and_clear_faults(dev_priv);
 }
 
-void intel_uncore_sanitize(struct drm_device *dev)
+void intel_uncore_sanitize(struct drm_i915_private *dev_priv)
 {
-        i915.enable_rc6 = sanitize_rc6_option(dev, i915.enable_rc6);
+        i915.enable_rc6 = sanitize_rc6_option(dev_priv, i915.enable_rc6);
 
         /* BIOS often leaves RC6 enabled, but disable it for hw init */
-        intel_disable_gt_powersave(dev);
+        intel_disable_gt_powersave(dev_priv);
 }
 
 static void __intel_uncore_forcewake_get(struct drm_i915_private *dev_priv,
@@ -1233,14 +1232,12 @@ static void fw_domain_init(struct drm_i915_private *dev_priv,
         fw_domain_reset(d);
 }
 
-static void intel_uncore_fw_domains_init(struct drm_device *dev)
+static void intel_uncore_fw_domains_init(struct drm_i915_private *dev_priv)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
-
         if (INTEL_INFO(dev_priv)->gen <= 5)
                 return;
 
-        if (IS_GEN9(dev)) {
+        if (IS_GEN9(dev_priv)) {
                 dev_priv->uncore.funcs.force_wake_get = fw_domains_get;
                 dev_priv->uncore.funcs.force_wake_put = fw_domains_put;
                 fw_domain_init(dev_priv, FW_DOMAIN_ID_RENDER,
@@ -1251,9 +1248,9 @@ static void intel_uncore_fw_domains_init(struct drm_device *dev)
                                FORCEWAKE_ACK_BLITTER_GEN9);
                 fw_domain_init(dev_priv, FW_DOMAIN_ID_MEDIA,
                                FORCEWAKE_MEDIA_GEN9, FORCEWAKE_ACK_MEDIA_GEN9);
-        } else if (IS_VALLEYVIEW(dev) || IS_CHERRYVIEW(dev)) {
+        } else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) {
                 dev_priv->uncore.funcs.force_wake_get = fw_domains_get;
-                if (!IS_CHERRYVIEW(dev))
+                if (!IS_CHERRYVIEW(dev_priv))
                         dev_priv->uncore.funcs.force_wake_put =
                                 fw_domains_put_with_fifo;
                 else
@@ -1262,17 +1259,17 @@ static void intel_uncore_fw_domains_init(struct drm_device *dev)
                                FORCEWAKE_VLV, FORCEWAKE_ACK_VLV);
                 fw_domain_init(dev_priv, FW_DOMAIN_ID_MEDIA,
                                FORCEWAKE_MEDIA_VLV, FORCEWAKE_ACK_MEDIA_VLV);
-        } else if (IS_HASWELL(dev) || IS_BROADWELL(dev)) {
+        } else if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) {
                 dev_priv->uncore.funcs.force_wake_get =
                         fw_domains_get_with_thread_status;
-                if (IS_HASWELL(dev))
+                if (IS_HASWELL(dev_priv))
                         dev_priv->uncore.funcs.force_wake_put =
                                 fw_domains_put_with_fifo;
                 else
                         dev_priv->uncore.funcs.force_wake_put = fw_domains_put;
                 fw_domain_init(dev_priv, FW_DOMAIN_ID_RENDER,
                                FORCEWAKE_MT, FORCEWAKE_ACK_HSW);
-        } else if (IS_IVYBRIDGE(dev)) {
+        } else if (IS_IVYBRIDGE(dev_priv)) {
                 u32 ecobus;
 
                 /* IVB configs may use multi-threaded forcewake */
@@ -1302,11 +1299,11 @@ static void intel_uncore_fw_domains_init(struct drm_device *dev)
                 fw_domain_init(dev_priv, FW_DOMAIN_ID_RENDER,
                                FORCEWAKE_MT, FORCEWAKE_MT_ACK);
 
-                mutex_lock(&dev->struct_mutex);
+                spin_lock_irq(&dev_priv->uncore.lock);
                 fw_domains_get_with_thread_status(dev_priv, FORCEWAKE_ALL);
                 ecobus = __raw_i915_read32(dev_priv, ECOBUS);
                 fw_domains_put_with_fifo(dev_priv, FORCEWAKE_ALL);
-                mutex_unlock(&dev->struct_mutex);
+                spin_unlock_irq(&dev_priv->uncore.lock);
 
                 if (!(ecobus & FORCEWAKE_MT_ENABLE)) {
                         DRM_INFO("No MT forcewake available on Ivybridge, this can result in issues\n");
@@ -1314,7 +1311,7 @@ static void intel_uncore_fw_domains_init(struct drm_device *dev)
                         fw_domain_init(dev_priv, FW_DOMAIN_ID_RENDER,
                                        FORCEWAKE, FORCEWAKE_ACK);
                 }
-        } else if (IS_GEN6(dev)) {
+        } else if (IS_GEN6(dev_priv)) {
                 dev_priv->uncore.funcs.force_wake_get =
                         fw_domains_get_with_thread_status;
                 dev_priv->uncore.funcs.force_wake_put =
@@ -1327,26 +1324,24 @@ static void intel_uncore_fw_domains_init(struct drm_device *dev)
         WARN_ON(dev_priv->uncore.fw_domains == 0);
 }
 
-void intel_uncore_init(struct drm_device *dev)
+void intel_uncore_init(struct drm_i915_private *dev_priv)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
-
-        i915_check_vgpu(dev);
+        i915_check_vgpu(dev_priv);
 
         intel_uncore_edram_detect(dev_priv);
-        intel_uncore_fw_domains_init(dev);
-        __intel_uncore_early_sanitize(dev, false);
+        intel_uncore_fw_domains_init(dev_priv);
+        __intel_uncore_early_sanitize(dev_priv, false);
 
         dev_priv->uncore.unclaimed_mmio_check = 1;
 
-        switch (INTEL_INFO(dev)->gen) {
+        switch (INTEL_INFO(dev_priv)->gen) {
         default:
         case 9:
                 ASSIGN_WRITE_MMIO_VFUNCS(gen9);
                 ASSIGN_READ_MMIO_VFUNCS(gen9);
                 break;
         case 8:
-                if (IS_CHERRYVIEW(dev)) {
+                if (IS_CHERRYVIEW(dev_priv)) {
                         ASSIGN_WRITE_MMIO_VFUNCS(chv);
                         ASSIGN_READ_MMIO_VFUNCS(chv);
 
@@ -1357,13 +1352,13 @@ void intel_uncore_init(struct drm_device *dev)
                 break;
         case 7:
         case 6:
-                if (IS_HASWELL(dev)) {
+                if (IS_HASWELL(dev_priv)) {
                         ASSIGN_WRITE_MMIO_VFUNCS(hsw);
                 } else {
                         ASSIGN_WRITE_MMIO_VFUNCS(gen6);
                 }
 
-                if (IS_VALLEYVIEW(dev)) {
+                if (IS_VALLEYVIEW(dev_priv)) {
                         ASSIGN_READ_MMIO_VFUNCS(vlv);
                 } else {
                         ASSIGN_READ_MMIO_VFUNCS(gen6);
@@ -1381,24 +1376,24 @@ void intel_uncore_init(struct drm_device *dev)
                 break;
         }
 
-        if (intel_vgpu_active(dev)) {
+        if (intel_vgpu_active(dev_priv)) {
                 ASSIGN_WRITE_MMIO_VFUNCS(vgpu);
                 ASSIGN_READ_MMIO_VFUNCS(vgpu);
         }
 
-        i915_check_and_clear_faults(dev);
+        i915_check_and_clear_faults(dev_priv);
 }
 #undef ASSIGN_WRITE_MMIO_VFUNCS
 #undef ASSIGN_READ_MMIO_VFUNCS
 
-void intel_uncore_fini(struct drm_device *dev)
+void intel_uncore_fini(struct drm_i915_private *dev_priv)
 {
         /* Paranoia: make sure we have disabled everything before we exit. */
-        intel_uncore_sanitize(dev);
-        intel_uncore_forcewake_reset(dev, false);
+        intel_uncore_sanitize(dev_priv);
+        intel_uncore_forcewake_reset(dev_priv, false);
 }
 
-#define GEN_RANGE(l, h) GENMASK(h, l)
+#define GEN_RANGE(l, h) GENMASK((h) - 1, (l) - 1)
 
 static const struct register_whitelist {
         i915_reg_t offset_ldw, offset_udw;
@@ -1414,7 +1409,7 @@ static const struct register_whitelist {
 int i915_reg_read_ioctl(struct drm_device *dev,
                         void *data, struct drm_file *file)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = to_i915(dev);
         struct drm_i915_reg_read *reg = data;
         struct register_whitelist const *entry = whitelist;
         unsigned size;
@@ -1423,7 +1418,7 @@ int i915_reg_read_ioctl(struct drm_device *dev,
 
         for (i = 0; i < ARRAY_SIZE(whitelist); i++, entry++) {
                 if (i915_mmio_reg_offset(entry->offset_ldw) == (reg->offset & -entry->size) &&
-                    (1 << INTEL_INFO(dev)->gen & entry->gen_bitmask))
+                    (INTEL_INFO(dev)->gen_mask & entry->gen_bitmask))
                         break;
         }
 
@@ -1467,83 +1462,47 @@ out:
         return ret;
 }
 
-int i915_get_reset_stats_ioctl(struct drm_device *dev,
-                               void *data, struct drm_file *file)
-{
-        struct drm_i915_private *dev_priv = dev->dev_private;
-        struct drm_i915_reset_stats *args = data;
-        struct i915_ctx_hang_stats *hs;
-        struct intel_context *ctx;
-        int ret;
-
-        if (args->flags || args->pad)
-                return -EINVAL;
-
-        if (args->ctx_id == DEFAULT_CONTEXT_HANDLE && !capable(CAP_SYS_ADMIN))
-                return -EPERM;
-
-        ret = mutex_lock_interruptible(&dev->struct_mutex);
-        if (ret)
-                return ret;
-
-        ctx = i915_gem_context_get(file->driver_priv, args->ctx_id);
-        if (IS_ERR(ctx)) {
-                mutex_unlock(&dev->struct_mutex);
-                return PTR_ERR(ctx);
-        }
-        hs = &ctx->hang_stats;
-
-        if (capable(CAP_SYS_ADMIN))
-                args->reset_count = i915_reset_count(&dev_priv->gpu_error);
-        else
-                args->reset_count = 0;
-
-        args->batch_active = hs->batch_active;
-        args->batch_pending = hs->batch_pending;
-
-        mutex_unlock(&dev->struct_mutex);
-
-        return 0;
-}
-
-static int i915_reset_complete(struct drm_device *dev)
+static int i915_reset_complete(struct pci_dev *pdev)
 {
         u8 gdrst;
-        pci_read_config_byte(dev->pdev, I915_GDRST, &gdrst);
+        pci_read_config_byte(pdev, I915_GDRST, &gdrst);
         return (gdrst & GRDOM_RESET_STATUS) == 0;
 }
 
-static int i915_do_reset(struct drm_device *dev, unsigned engine_mask)
+static int i915_do_reset(struct drm_i915_private *dev_priv, unsigned engine_mask)
 {
+        struct pci_dev *pdev = dev_priv->drm.pdev;
+
         /* assert reset for at least 20 usec */
-        pci_write_config_byte(dev->pdev, I915_GDRST, GRDOM_RESET_ENABLE);
+        pci_write_config_byte(pdev, I915_GDRST, GRDOM_RESET_ENABLE);
         udelay(20);
-        pci_write_config_byte(dev->pdev, I915_GDRST, 0);
+        pci_write_config_byte(pdev, I915_GDRST, 0);
 
-        return wait_for(i915_reset_complete(dev), 500);
+        return wait_for(i915_reset_complete(pdev), 500);
 }
 
-static int g4x_reset_complete(struct drm_device *dev)
+static int g4x_reset_complete(struct pci_dev *pdev)
 {
         u8 gdrst;
-        pci_read_config_byte(dev->pdev, I915_GDRST, &gdrst);
+        pci_read_config_byte(pdev, I915_GDRST, &gdrst);
         return (gdrst & GRDOM_RESET_ENABLE) == 0;
 }
 
-static int g33_do_reset(struct drm_device *dev, unsigned engine_mask)
+static int g33_do_reset(struct drm_i915_private *dev_priv, unsigned engine_mask)
 {
-        pci_write_config_byte(dev->pdev, I915_GDRST, GRDOM_RESET_ENABLE);
-        return wait_for(g4x_reset_complete(dev), 500);
+        struct pci_dev *pdev = dev_priv->drm.pdev;
+        pci_write_config_byte(pdev, I915_GDRST, GRDOM_RESET_ENABLE);
+        return wait_for(g4x_reset_complete(pdev), 500);
 }
 
-static int g4x_do_reset(struct drm_device *dev, unsigned engine_mask)
+static int g4x_do_reset(struct drm_i915_private *dev_priv, unsigned engine_mask)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct pci_dev *pdev = dev_priv->drm.pdev;
         int ret;
 
-        pci_write_config_byte(dev->pdev, I915_GDRST,
+        pci_write_config_byte(pdev, I915_GDRST,
                               GRDOM_RENDER | GRDOM_RESET_ENABLE);
-        ret =  wait_for(g4x_reset_complete(dev), 500);
+        ret =  wait_for(g4x_reset_complete(pdev), 500);
         if (ret)
                 return ret;
 
@@ -1551,9 +1510,9 @@ static int g4x_do_reset(struct drm_device *dev, unsigned engine_mask)
         I915_WRITE(VDECCLK_GATE_D, I915_READ(VDECCLK_GATE_D) | VCP_UNIT_CLOCK_GATE_DISABLE);
         POSTING_READ(VDECCLK_GATE_D);
 
-        pci_write_config_byte(dev->pdev, I915_GDRST,
+        pci_write_config_byte(pdev, I915_GDRST,
                               GRDOM_MEDIA | GRDOM_RESET_ENABLE);
-        ret =  wait_for(g4x_reset_complete(dev), 500);
+        ret =  wait_for(g4x_reset_complete(pdev), 500);
         if (ret)
                 return ret;
 
@@ -1561,27 +1520,29 @@ static int g4x_do_reset(struct drm_device *dev, unsigned engine_mask)
         I915_WRITE(VDECCLK_GATE_D, I915_READ(VDECCLK_GATE_D) & ~VCP_UNIT_CLOCK_GATE_DISABLE);
         POSTING_READ(VDECCLK_GATE_D);
 
-        pci_write_config_byte(dev->pdev, I915_GDRST, 0);
+        pci_write_config_byte(pdev, I915_GDRST, 0);
 
         return 0;
 }
 
-static int ironlake_do_reset(struct drm_device *dev, unsigned engine_mask)
+static int ironlake_do_reset(struct drm_i915_private *dev_priv,
+                             unsigned engine_mask)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
         int ret;
 
         I915_WRITE(ILK_GDSR,
                    ILK_GRDOM_RENDER | ILK_GRDOM_RESET_ENABLE);
-        ret = wait_for((I915_READ(ILK_GDSR) &
-                        ILK_GRDOM_RESET_ENABLE) == 0, 500);
+        ret = intel_wait_for_register(dev_priv,
+                                      ILK_GDSR, ILK_GRDOM_RESET_ENABLE, 0,
+                                      500);
         if (ret)
                 return ret;
 
         I915_WRITE(ILK_GDSR,
                    ILK_GRDOM_MEDIA | ILK_GRDOM_RESET_ENABLE);
-        ret = wait_for((I915_READ(ILK_GDSR) &
-                        ILK_GRDOM_RESET_ENABLE) == 0, 500);
+        ret = intel_wait_for_register(dev_priv,
+                                      ILK_GDSR, ILK_GRDOM_RESET_ENABLE, 0,
+                                      500);
         if (ret)
                 return ret;
 
@@ -1594,25 +1555,21 @@ static int ironlake_do_reset(struct drm_device *dev, unsigned engine_mask)
 static int gen6_hw_domain_reset(struct drm_i915_private *dev_priv,
                                 u32 hw_domain_mask)
 {
-        int ret;
-
         /* GEN6_GDRST is not in the gt power well, no need to check
          * for fifo space for the write or forcewake the chip for
          * the read
          */
         __raw_i915_write32(dev_priv, GEN6_GDRST, hw_domain_mask);
 
-#define ACKED ((__raw_i915_read32(dev_priv, GEN6_GDRST) & hw_domain_mask) == 0)
         /* Spin waiting for the device to ack the reset requests */
-        ret = wait_for(ACKED, 500);
-#undef ACKED
-
-        return ret;
+        return intel_wait_for_register_fw(dev_priv,
+                                          GEN6_GDRST, hw_domain_mask, 0,
+                                          500);
 }
 
 /**
  * gen6_reset_engines - reset individual engines
- * @dev: DRM device
+ * @dev_priv: i915 device
  * @engine_mask: mask of intel_ring_flag() engines or ALL_ENGINES for full reset
  *
  * This function will reset the individual engines that are set in engine_mask.
@@ -1623,9 +1580,9 @@ static int gen6_hw_domain_reset(struct drm_i915_private *dev_priv,
  *
  * Returns 0 on success, nonzero on error.
  */
-static int gen6_reset_engines(struct drm_device *dev, unsigned engine_mask)
+static int gen6_reset_engines(struct drm_i915_private *dev_priv,
+                              unsigned engine_mask)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
         struct intel_engine_cs *engine;
         const u32 hw_engine_mask[I915_NUM_ENGINES] = {
                 [RCS] = GEN6_GRDOM_RENDER,
@@ -1647,33 +1604,94 @@ static int gen6_reset_engines(struct drm_device *dev, unsigned engine_mask)
 
         ret = gen6_hw_domain_reset(dev_priv, hw_mask);
 
-        intel_uncore_forcewake_reset(dev, true);
+        intel_uncore_forcewake_reset(dev_priv, true);
 
         return ret;
 }
 
-static int wait_for_register_fw(struct drm_i915_private *dev_priv,
-                                i915_reg_t reg,
-                                const u32 mask,
-                                const u32 value,
-                                const unsigned long timeout_ms)
+/**
+ * intel_wait_for_register_fw - wait until register matches expected state
+ * @dev_priv: the i915 device
+ * @reg: the register to read
+ * @mask: mask to apply to register value
+ * @value: expected value
+ * @timeout_ms: timeout in millisecond
+ *
+ * This routine waits until the target register @reg contains the expected
+ * @value after applying the @mask, i.e. it waits until
+ *   (I915_READ_FW(@reg) & @mask) == @value
+ * Otherwise, the wait will timeout after @timeout_ms milliseconds.
+ *
+ * Note that this routine assumes the caller holds forcewake asserted, it is
+ * not suitable for very long waits. See intel_wait_for_register() if you
+ * wish to wait without holding forcewake for the duration (i.e. you expect
+ * the wait to be slow).
+ *
+ * Returns 0 if the register matches the desired condition, or -ETIMEOUT.
+ */
+int intel_wait_for_register_fw(struct drm_i915_private *dev_priv,
+                               i915_reg_t reg,
+                               const u32 mask,
+                               const u32 value,
+                               const unsigned long timeout_ms)
+{
+#define done ((I915_READ_FW(reg) & mask) == value)
+        int ret = wait_for_us(done, 2);
+        if (ret)
+                ret = wait_for(done, timeout_ms);
+        return ret;
+#undef done
+}
+
+/**
+ * intel_wait_for_register - wait until register matches expected state
+ * @dev_priv: the i915 device
+ * @reg: the register to read
+ * @mask: mask to apply to register value
+ * @value: expected value
+ * @timeout_ms: timeout in millisecond
+ *
+ * This routine waits until the target register @reg contains the expected
+ * @value after applying the @mask, i.e. it waits until
+ *   (I915_READ(@reg) & @mask) == @value
+ * Otherwise, the wait will timeout after @timeout_ms milliseconds.
+ *
+ * Returns 0 if the register matches the desired condition, or -ETIMEOUT.
+ */
+int intel_wait_for_register(struct drm_i915_private *dev_priv,
+                            i915_reg_t reg,
+                            const u32 mask,
+                            const u32 value,
+                            const unsigned long timeout_ms)
 {
-        return wait_for((I915_READ_FW(reg) & mask) == value, timeout_ms);
+
+        unsigned fw =
+                intel_uncore_forcewake_for_reg(dev_priv, reg, FW_REG_READ);
+        int ret;
+
+        intel_uncore_forcewake_get(dev_priv, fw);
+        ret = wait_for_us((I915_READ_FW(reg) & mask) == value, 2);
+        intel_uncore_forcewake_put(dev_priv, fw);
+        if (ret)
+                ret = wait_for((I915_READ_NOTRACE(reg) & mask) == value,
+                               timeout_ms);
+
+        return ret;
 }
 
 static int gen8_request_engine_reset(struct intel_engine_cs *engine)
 {
+        struct drm_i915_private *dev_priv = engine->i915;
         int ret;
-        struct drm_i915_private *dev_priv = engine->dev->dev_private;
 
         I915_WRITE_FW(RING_RESET_CTL(engine->mmio_base),
                       _MASKED_BIT_ENABLE(RESET_CTL_REQUEST_RESET));
 
-        ret = wait_for_register_fw(dev_priv,
-                                   RING_RESET_CTL(engine->mmio_base),
-                                   RESET_CTL_READY_TO_RESET,
-                                   RESET_CTL_READY_TO_RESET,
-                                   700);
+        ret = intel_wait_for_register_fw(dev_priv,
+                                         RING_RESET_CTL(engine->mmio_base),
+                                         RESET_CTL_READY_TO_RESET,
+                                         RESET_CTL_READY_TO_RESET,
+                                         700);
         if (ret)
                 DRM_ERROR("%s: reset request timeout\n", engine->name);
 
@@ -1682,22 +1700,22 @@ static int gen8_request_engine_reset(struct intel_engine_cs *engine)
 
 static void gen8_unrequest_engine_reset(struct intel_engine_cs *engine)
 {
-        struct drm_i915_private *dev_priv = engine->dev->dev_private;
+        struct drm_i915_private *dev_priv = engine->i915;
 
         I915_WRITE_FW(RING_RESET_CTL(engine->mmio_base),
                       _MASKED_BIT_DISABLE(RESET_CTL_REQUEST_RESET));
 }
 
-static int gen8_reset_engines(struct drm_device *dev, unsigned engine_mask)
+static int gen8_reset_engines(struct drm_i915_private *dev_priv,
+                              unsigned engine_mask)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
         struct intel_engine_cs *engine;
 
         for_each_engine_masked(engine, dev_priv, engine_mask)
                 if (gen8_request_engine_reset(engine))
                         goto not_ready;
 
-        return gen6_reset_engines(dev, engine_mask);
+        return gen6_reset_engines(dev_priv, engine_mask);
 
 not_ready:
         for_each_engine_masked(engine, dev_priv, engine_mask)
@@ -1706,35 +1724,35 @@ not_ready:
         return -EIO;
 }
 
-static int (*intel_get_gpu_reset(struct drm_device *dev))(struct drm_device *,
-                                                          unsigned engine_mask)
+typedef int (*reset_func)(struct drm_i915_private *, unsigned engine_mask);
+
+static reset_func intel_get_gpu_reset(struct drm_i915_private *dev_priv)
 {
         if (!i915.reset)
                 return NULL;
 
-        if (INTEL_INFO(dev)->gen >= 8)
+        if (INTEL_INFO(dev_priv)->gen >= 8)
                 return gen8_reset_engines;
-        else if (INTEL_INFO(dev)->gen >= 6)
+        else if (INTEL_INFO(dev_priv)->gen >= 6)
                 return gen6_reset_engines;
-        else if (IS_GEN5(dev))
+        else if (IS_GEN5(dev_priv))
                 return ironlake_do_reset;
-        else if (IS_G4X(dev))
+        else if (IS_G4X(dev_priv))
                 return g4x_do_reset;
-        else if (IS_G33(dev))
+        else if (IS_G33(dev_priv))
                 return g33_do_reset;
-        else if (INTEL_INFO(dev)->gen >= 3)
+        else if (INTEL_INFO(dev_priv)->gen >= 3)
                 return i915_do_reset;
         else
                 return NULL;
 }
 
-int intel_gpu_reset(struct drm_device *dev, unsigned engine_mask)
+int intel_gpu_reset(struct drm_i915_private *dev_priv, unsigned engine_mask)
 {
-        struct drm_i915_private *dev_priv = to_i915(dev);
-        int (*reset)(struct drm_device *, unsigned);
+        reset_func reset;
         int ret;
 
-        reset = intel_get_gpu_reset(dev);
+        reset = intel_get_gpu_reset(dev_priv);
         if (reset == NULL)
                 return -ENODEV;
 
@@ -1742,15 +1760,15 @@ int intel_gpu_reset(struct drm_device *dev, unsigned engine_mask)
          * request may be dropped and never completes (causing -EIO).
          */
         intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
-        ret = reset(dev, engine_mask);
+        ret = reset(dev_priv, engine_mask);
         intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
 
         return ret;
 }
 
-bool intel_has_gpu_reset(struct drm_device *dev)
+bool intel_has_gpu_reset(struct drm_i915_private *dev_priv)
 {
-        return intel_get_gpu_reset(dev) != NULL;
+        return intel_get_gpu_reset(dev_priv) != NULL;
 }
 
 int intel_guc_reset(struct drm_i915_private *dev_priv)
@@ -1758,7 +1776,7 @@ int intel_guc_reset(struct drm_i915_private *dev_priv)
         int ret;
         unsigned long irqflags;
 
-        if (!i915.enable_guc_submission)
+        if (!HAS_GUC(dev_priv))
                 return -EINVAL;
 
         intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
@@ -1802,10 +1820,10 @@ intel_uncore_forcewake_for_read(struct drm_i915_private *dev_priv,
 {
         enum forcewake_domains fw_domains;
 
-        if (intel_vgpu_active(dev_priv->dev))
+        if (intel_vgpu_active(dev_priv))
                 return 0;
 
-        switch (INTEL_INFO(dev_priv)->gen) {
+        switch (INTEL_GEN(dev_priv)) {
         case 9:
                 fw_domains = __gen9_reg_read_fw_domains(i915_mmio_reg_offset(reg));
                 break;
@@ -1842,10 +1860,10 @@ intel_uncore_forcewake_for_write(struct drm_i915_private *dev_priv,
 {
         enum forcewake_domains fw_domains;
 
-        if (intel_vgpu_active(dev_priv->dev))
+        if (intel_vgpu_active(dev_priv))
                 return 0;
 
-        switch (INTEL_INFO(dev_priv)->gen) {
+        switch (INTEL_GEN(dev_priv)) {
         case 9:
                 fw_domains = __gen9_reg_write_fw_domains(i915_mmio_reg_offset(reg));
                 break;
diff --git a/drivers/gpu/drm/i915/intel_vbt_defs.h b/drivers/gpu/drm/i915/intel_vbt_defs.h
index 44fb0b35eed3..68db9621f1f0 100644
--- a/drivers/gpu/drm/i915/intel_vbt_defs.h
+++ b/drivers/gpu/drm/i915/intel_vbt_defs.h
@@ -447,10 +447,16 @@ struct bdb_lfp_backlight_data_entry {
         u8 obsolete3;
 } __packed;
 
+struct bdb_lfp_backlight_control_method {
+        u8 type:4;
+        u8 controller:4;
+} __packed;
+
 struct bdb_lfp_backlight_data {
         u8 entry_size;
         struct bdb_lfp_backlight_data_entry data[16];
         u8 level[16];
+        struct bdb_lfp_backlight_control_method backlight_control[16];
 } __packed;
 
 struct aimdb_header {
diff --git a/drivers/gpu/drm/imx/Kconfig b/drivers/gpu/drm/imx/Kconfig
index a1844b50546c..f2c9ae822149 100644
--- a/drivers/gpu/drm/imx/Kconfig
+++ b/drivers/gpu/drm/imx/Kconfig
@@ -1,7 +1,6 @@
 config DRM_IMX
         tristate "DRM Support for Freescale i.MX"
         select DRM_KMS_HELPER
-        select DRM_KMS_FB_HELPER
         select VIDEOMODE_HELPERS
         select DRM_GEM_CMA_HELPER
         select DRM_KMS_CMA_HELPER
diff --git a/drivers/gpu/drm/imx/dw_hdmi-imx.c b/drivers/gpu/drm/imx/dw_hdmi-imx.c
index a24631fdf4ad..359cd2765552 100644
--- a/drivers/gpu/drm/imx/dw_hdmi-imx.c
+++ b/drivers/gpu/drm/imx/dw_hdmi-imx.c
@@ -28,6 +28,11 @@ struct imx_hdmi {
         struct regmap *regmap;
 };
 
+static inline struct imx_hdmi *enc_to_imx_hdmi(struct drm_encoder *e)
+{
+        return container_of(e, struct imx_hdmi, encoder);
+}
+
 static const struct dw_hdmi_mpll_config imx_mpll_cfg[] = {
         {
                 45250000, {
@@ -109,15 +114,9 @@ static void dw_hdmi_imx_encoder_disable(struct drm_encoder *encoder)
 {
 }
 
-static void dw_hdmi_imx_encoder_mode_set(struct drm_encoder *encoder,
-                                         struct drm_display_mode *mode,
-                                         struct drm_display_mode *adj_mode)
+static void dw_hdmi_imx_encoder_enable(struct drm_encoder *encoder)
 {
-}
-
-static void dw_hdmi_imx_encoder_commit(struct drm_encoder *encoder)
-{
-        struct imx_hdmi *hdmi = container_of(encoder, struct imx_hdmi, encoder);
+        struct imx_hdmi *hdmi = enc_to_imx_hdmi(encoder);
         int mux = drm_of_encoder_active_port_id(hdmi->dev->of_node, encoder);
 
         regmap_update_bits(hdmi->regmap, IOMUXC_GPR3,
@@ -125,16 +124,23 @@ static void dw_hdmi_imx_encoder_commit(struct drm_encoder *encoder)
                            mux << IMX6Q_GPR3_HDMI_MUX_CTL_SHIFT);
 }
 
-static void dw_hdmi_imx_encoder_prepare(struct drm_encoder *encoder)
+static int dw_hdmi_imx_atomic_check(struct drm_encoder *encoder,
+                                    struct drm_crtc_state *crtc_state,
+                                    struct drm_connector_state *conn_state)
 {
-        imx_drm_set_bus_format(encoder, MEDIA_BUS_FMT_RGB888_1X24);
+        struct imx_crtc_state *imx_crtc_state = to_imx_crtc_state(crtc_state);
+
+        imx_crtc_state->bus_format = MEDIA_BUS_FMT_RGB888_1X24;
+        imx_crtc_state->di_hsync_pin = 2;
+        imx_crtc_state->di_vsync_pin = 3;
+
+        return 0;
 }
 
 static const struct drm_encoder_helper_funcs dw_hdmi_imx_encoder_helper_funcs = {
-        .mode_set   = dw_hdmi_imx_encoder_mode_set,
-        .prepare    = dw_hdmi_imx_encoder_prepare,
-        .commit     = dw_hdmi_imx_encoder_commit,
+        .enable     = dw_hdmi_imx_encoder_enable,
         .disable    = dw_hdmi_imx_encoder_disable,
+        .atomic_check = dw_hdmi_imx_atomic_check,
 };
 
 static const struct drm_encoder_funcs dw_hdmi_imx_encoder_funcs = {
diff --git a/drivers/gpu/drm/imx/imx-drm-core.c b/drivers/gpu/drm/imx/imx-drm-core.c
index 82656654fb21..9f7dafce3a4c 100644
--- a/drivers/gpu/drm/imx/imx-drm-core.c
+++ b/drivers/gpu/drm/imx/imx-drm-core.c
@@ -15,10 +15,14 @@
  */
 #include <linux/component.h>
 #include <linux/device.h>
+#include <linux/dma-buf.h>
 #include <linux/fb.h>
 #include <linux/module.h>
 #include <linux/platform_device.h>
+#include <linux/reservation.h>
 #include <drm/drmP.h>
+#include <drm/drm_atomic.h>
+#include <drm/drm_atomic_helper.h>
 #include <drm/drm_fb_helper.h>
 #include <drm/drm_crtc_helper.h>
 #include <drm/drm_gem_cma_helper.h>
@@ -41,6 +45,7 @@ struct imx_drm_device {
         struct imx_drm_crtc                     *crtc[MAX_CRTC];
         unsigned int                            pipes;
         struct drm_fbdev_cma                    *fbhelper;
+        struct drm_atomic_state                 *state;
 };
 
 struct imx_drm_crtc {
@@ -85,45 +90,6 @@ static int imx_drm_driver_unload(struct drm_device *drm)
         return 0;
 }
 
-static struct imx_drm_crtc *imx_drm_find_crtc(struct drm_crtc *crtc)
-{
-        struct imx_drm_device *imxdrm = crtc->dev->dev_private;
-        unsigned i;
-
-        for (i = 0; i < MAX_CRTC; i++)
-                if (imxdrm->crtc[i] && imxdrm->crtc[i]->crtc == crtc)
-                        return imxdrm->crtc[i];
-
-        return NULL;
-}
-
-int imx_drm_set_bus_config(struct drm_encoder *encoder, u32 bus_format,
-                int hsync_pin, int vsync_pin, u32 bus_flags)
-{
-        struct imx_drm_crtc_helper_funcs *helper;
-        struct imx_drm_crtc *imx_crtc;
-
-        imx_crtc = imx_drm_find_crtc(encoder->crtc);
-        if (!imx_crtc)
-                return -EINVAL;
-
-        helper = &imx_crtc->imx_drm_helper_funcs;
-        if (helper->set_interface_pix_fmt)
-                return helper->set_interface_pix_fmt(encoder->crtc,
-                                        bus_format, hsync_pin, vsync_pin,
-                                        bus_flags);
-        return 0;
-}
-EXPORT_SYMBOL_GPL(imx_drm_set_bus_config);
-
-int imx_drm_set_bus_format(struct drm_encoder *encoder, u32 bus_format)
-{
-        return imx_drm_set_bus_config(encoder, bus_format, 2, 3,
-                                      DRM_BUS_FLAG_DE_HIGH |
-                                      DRM_BUS_FLAG_PIXDATA_NEGEDGE);
-}
-EXPORT_SYMBOL_GPL(imx_drm_set_bus_format);
-
 int imx_drm_crtc_vblank_get(struct imx_drm_crtc *imx_drm_crtc)
 {
         return drm_crtc_vblank_get(imx_drm_crtc->crtc);
@@ -208,6 +174,63 @@ static void imx_drm_output_poll_changed(struct drm_device *drm)
 static const struct drm_mode_config_funcs imx_drm_mode_config_funcs = {
         .fb_create = drm_fb_cma_create,
         .output_poll_changed = imx_drm_output_poll_changed,
+        .atomic_check = drm_atomic_helper_check,
+        .atomic_commit = drm_atomic_helper_commit,
+};
+
+static void imx_drm_atomic_commit_tail(struct drm_atomic_state *state)
+{
+        struct drm_device *dev = state->dev;
+        struct drm_crtc *crtc;
+        struct drm_crtc_state *crtc_state;
+        struct drm_plane_state *plane_state;
+        struct drm_gem_cma_object *cma_obj;
+        struct fence *excl;
+        unsigned shared_count;
+        struct fence **shared;
+        unsigned int i, j;
+        int ret;
+
+        /* Wait for fences. */
+        for_each_crtc_in_state(state, crtc, crtc_state, i) {
+                plane_state = crtc->primary->state;
+                if (plane_state->fb) {
+                        cma_obj = drm_fb_cma_get_gem_obj(plane_state->fb, 0);
+                        if (cma_obj->base.dma_buf) {
+                                ret = reservation_object_get_fences_rcu(
+                                        cma_obj->base.dma_buf->resv, &excl,
+                                        &shared_count, &shared);
+                                if (unlikely(ret))
+                                        DRM_ERROR("failed to get fences "
+                                                  "for buffer\n");
+
+                                if (excl) {
+                                        fence_wait(excl, false);
+                                        fence_put(excl);
+                                }
+                                for (j = 0; j < shared_count; i++) {
+                                        fence_wait(shared[j], false);
+                                        fence_put(shared[j]);
+                                }
+                        }
+                }
+        }
+
+        drm_atomic_helper_commit_modeset_disables(dev, state);
+
+        drm_atomic_helper_commit_planes(dev, state, true);
+
+        drm_atomic_helper_commit_modeset_enables(dev, state);
+
+        drm_atomic_helper_commit_hw_done(state);
+
+        drm_atomic_helper_wait_for_vblanks(dev, state);
+
+        drm_atomic_helper_cleanup_planes(dev, state);
+}
+
+static struct drm_mode_config_helper_funcs imx_drm_mode_config_helpers = {
+        .atomic_commit_tail = imx_drm_atomic_commit_tail,
 };
 
 /*
@@ -249,6 +272,7 @@ static int imx_drm_driver_load(struct drm_device *drm, unsigned long flags)
         drm->mode_config.max_width = 4096;
         drm->mode_config.max_height = 4096;
         drm->mode_config.funcs = &imx_drm_mode_config_funcs;
+        drm->mode_config.helper_private = &imx_drm_mode_config_helpers;
 
         drm_mode_config_init(drm);
 
@@ -279,6 +303,8 @@ static int imx_drm_driver_load(struct drm_device *drm, unsigned long flags)
                 }
         }
 
+        drm_mode_config_reset(drm);
+
         /*
          * All components are now initialised, so setup the fb helper.
          * The fb helper takes copies of key hardware information, so the
@@ -289,7 +315,6 @@ static int imx_drm_driver_load(struct drm_device *drm, unsigned long flags)
                 dev_warn(drm->dev, "Invalid legacyfb_depth.  Defaulting to 16bpp\n");
                 legacyfb_depth = 16;
         }
-        drm_helper_disable_unused_functions(drm);
         imxdrm->fbhelper = drm_fbdev_cma_init(drm, legacyfb_depth,
                                 drm->mode_config.num_crtc, MAX_CRTC);
         if (IS_ERR(imxdrm->fbhelper)) {
@@ -403,11 +428,11 @@ static const struct drm_ioctl_desc imx_drm_ioctls[] = {
 };
 
 static struct drm_driver imx_drm_driver = {
-        .driver_features        = DRIVER_MODESET | DRIVER_GEM | DRIVER_PRIME,
+        .driver_features        = DRIVER_MODESET | DRIVER_GEM | DRIVER_PRIME |
+                                  DRIVER_ATOMIC,
         .load                   = imx_drm_driver_load,
         .unload                 = imx_drm_driver_unload,
         .lastclose              = imx_drm_driver_lastclose,
-        .set_busid              = drm_platform_set_busid,
         .gem_free_object_unlocked = drm_gem_cma_free_object,
         .gem_vm_ops             = &drm_gem_cma_vm_ops,
         .dumb_create            = drm_gem_cma_dumb_create,
@@ -492,6 +517,7 @@ static int imx_drm_platform_remove(struct platform_device *pdev)
 static int imx_drm_suspend(struct device *dev)
 {
         struct drm_device *drm_dev = dev_get_drvdata(dev);
+        struct imx_drm_device *imxdrm;
 
         /* The drm_dev is NULL before .load hook is called */
         if (drm_dev == NULL)
@@ -499,17 +525,26 @@ static int imx_drm_suspend(struct device *dev)
 
         drm_kms_helper_poll_disable(drm_dev);
 
+        imxdrm = drm_dev->dev_private;
+        imxdrm->state = drm_atomic_helper_suspend(drm_dev);
+        if (IS_ERR(imxdrm->state)) {
+                drm_kms_helper_poll_enable(drm_dev);
+                return PTR_ERR(imxdrm->state);
+        }
+
         return 0;
 }
 
 static int imx_drm_resume(struct device *dev)
 {
         struct drm_device *drm_dev = dev_get_drvdata(dev);
+        struct imx_drm_device *imx_drm;
 
         if (drm_dev == NULL)
                 return 0;
 
-        drm_helper_resume_force_mode(drm_dev);
+        imx_drm = drm_dev->dev_private;
+        drm_atomic_helper_resume(drm_dev, imx_drm->state);
         drm_kms_helper_poll_enable(drm_dev);
 
         return 0;
diff --git a/drivers/gpu/drm/imx/imx-drm.h b/drivers/gpu/drm/imx/imx-drm.h
index 74320a1723b7..07d33e45f90f 100644
--- a/drivers/gpu/drm/imx/imx-drm.h
+++ b/drivers/gpu/drm/imx/imx-drm.h
@@ -15,12 +15,22 @@ struct platform_device;
 
 unsigned int imx_drm_crtc_id(struct imx_drm_crtc *crtc);
 
+struct imx_crtc_state {
+        struct drm_crtc_state                   base;
+        u32                                     bus_format;
+        u32                                     bus_flags;
+        int                                     di_hsync_pin;
+        int                                     di_vsync_pin;
+};
+
+static inline struct imx_crtc_state *to_imx_crtc_state(struct drm_crtc_state *s)
+{
+        return container_of(s, struct imx_crtc_state, base);
+}
+
 struct imx_drm_crtc_helper_funcs {
         int (*enable_vblank)(struct drm_crtc *crtc);
         void (*disable_vblank)(struct drm_crtc *crtc);
-        int (*set_interface_pix_fmt)(struct drm_crtc *crtc,
-                        u32 bus_format, int hsync_pin, int vsync_pin,
-                        u32 bus_flags);
         const struct drm_crtc_helper_funcs *crtc_helper_funcs;
         const struct drm_crtc_funcs *crtc_funcs;
 };
@@ -42,11 +52,6 @@ void imx_drm_mode_config_init(struct drm_device *drm);
 
 struct drm_gem_cma_object *imx_drm_fb_get_obj(struct drm_framebuffer *fb);
 
-int imx_drm_set_bus_config(struct drm_encoder *encoder, u32 bus_format,
-                int hsync_pin, int vsync_pin, u32 bus_flags);
-int imx_drm_set_bus_format(struct drm_encoder *encoder,
-                u32 bus_format);
-
 int imx_drm_encoder_parse_of(struct drm_device *drm,
         struct drm_encoder *encoder, struct device_node *np);
 
diff --git a/drivers/gpu/drm/imx/imx-ldb.c b/drivers/gpu/drm/imx/imx-ldb.c
index beff793bb717..b03919ed60ba 100644
--- a/drivers/gpu/drm/imx/imx-ldb.c
+++ b/drivers/gpu/drm/imx/imx-ldb.c
@@ -17,6 +17,8 @@
 #include <linux/clk.h>
 #include <linux/component.h>
 #include <drm/drmP.h>
+#include <drm/drm_atomic.h>
+#include <drm/drm_atomic_helper.h>
 #include <drm/drm_fb_helper.h>
 #include <drm/drm_crtc_helper.h>
 #include <drm/drm_of.h>
@@ -49,9 +51,6 @@
 #define LDB_DI1_VS_POL_ACT_LOW          (1 << 10)
 #define LDB_BGREF_RMODE_INT             (1 << 15)
 
-#define con_to_imx_ldb_ch(x) container_of(x, struct imx_ldb_channel, connector)
-#define enc_to_imx_ldb_ch(x) container_of(x, struct imx_ldb_channel, encoder)
-
 struct imx_ldb;
 
 struct imx_ldb_channel {
@@ -66,9 +65,19 @@ struct imx_ldb_channel {
         int edid_len;
         struct drm_display_mode mode;
         int mode_valid;
-        int bus_format;
+        u32 bus_format;
 };
 
+static inline struct imx_ldb_channel *con_to_imx_ldb_ch(struct drm_connector *c)
+{
+        return container_of(c, struct imx_ldb_channel, connector);
+}
+
+static inline struct imx_ldb_channel *enc_to_imx_ldb_ch(struct drm_encoder *e)
+{
+        return container_of(e, struct imx_ldb_channel, encoder);
+}
+
 struct bus_mux {
         int reg;
         int shift;
@@ -93,6 +102,32 @@ static enum drm_connector_status imx_ldb_connector_detect(
         return connector_status_connected;
 }
 
+static void imx_ldb_ch_set_bus_format(struct imx_ldb_channel *imx_ldb_ch,
+                                      u32 bus_format)
+{
+        struct imx_ldb *ldb = imx_ldb_ch->ldb;
+        int dual = ldb->ldb_ctrl & LDB_SPLIT_MODE_EN;
+
+        switch (bus_format) {
+        case MEDIA_BUS_FMT_RGB666_1X7X3_SPWG:
+                break;
+        case MEDIA_BUS_FMT_RGB888_1X7X4_SPWG:
+                if (imx_ldb_ch->chno == 0 || dual)
+                        ldb->ldb_ctrl |= LDB_DATA_WIDTH_CH0_24;
+                if (imx_ldb_ch->chno == 1 || dual)
+                        ldb->ldb_ctrl |= LDB_DATA_WIDTH_CH1_24;
+                break;
+        case MEDIA_BUS_FMT_RGB888_1X7X4_JEIDA:
+                if (imx_ldb_ch->chno == 0 || dual)
+                        ldb->ldb_ctrl |= LDB_DATA_WIDTH_CH0_24 |
+                                         LDB_BIT_MAP_CH0_JEIDA;
+                if (imx_ldb_ch->chno == 1 || dual)
+                        ldb->ldb_ctrl |= LDB_DATA_WIDTH_CH1_24 |
+                                         LDB_BIT_MAP_CH1_JEIDA;
+                break;
+        }
+}
+
 static int imx_ldb_connector_get_modes(struct drm_connector *connector)
 {
         struct imx_ldb_channel *imx_ldb_ch = con_to_imx_ldb_ch(connector);
@@ -100,11 +135,7 @@ static int imx_ldb_connector_get_modes(struct drm_connector *connector)
 
         if (imx_ldb_ch->panel && imx_ldb_ch->panel->funcs &&
             imx_ldb_ch->panel->funcs->get_modes) {
-                struct drm_display_info *di = &connector->display_info;
-
                 num_modes = imx_ldb_ch->panel->funcs->get_modes(imx_ldb_ch->panel);
-                if (!imx_ldb_ch->bus_format && di->num_bus_formats)
-                        imx_ldb_ch->bus_format = di->bus_formats[0];
                 if (num_modes > 0)
                         return num_modes;
         }
@@ -141,10 +172,6 @@ static struct drm_encoder *imx_ldb_connector_best_encoder(
         return &imx_ldb_ch->encoder;
 }
 
-static void imx_ldb_encoder_dpms(struct drm_encoder *encoder, int mode)
-{
-}
-
 static void imx_ldb_set_clock(struct imx_ldb *ldb, int mux, int chno,
                 unsigned long serial_clk, unsigned long di_clk)
 {
@@ -173,43 +200,7 @@ static void imx_ldb_set_clock(struct imx_ldb *ldb, int mux, int chno,
                         chno);
 }
 
-static void imx_ldb_encoder_prepare(struct drm_encoder *encoder)
-{
-        struct imx_ldb_channel *imx_ldb_ch = enc_to_imx_ldb_ch(encoder);
-        struct imx_ldb *ldb = imx_ldb_ch->ldb;
-        int dual = ldb->ldb_ctrl & LDB_SPLIT_MODE_EN;
-        u32 bus_format;
-
-        switch (imx_ldb_ch->bus_format) {
-        default:
-                dev_warn(ldb->dev,
-                         "could not determine data mapping, default to 18-bit \"spwg\"\n");
-                /* fallthrough */
-        case MEDIA_BUS_FMT_RGB666_1X7X3_SPWG:
-                bus_format = MEDIA_BUS_FMT_RGB666_1X18;
-                break;
-        case MEDIA_BUS_FMT_RGB888_1X7X4_SPWG:
-                bus_format = MEDIA_BUS_FMT_RGB888_1X24;
-                if (imx_ldb_ch->chno == 0 || dual)
-                        ldb->ldb_ctrl |= LDB_DATA_WIDTH_CH0_24;
-                if (imx_ldb_ch->chno == 1 || dual)
-                        ldb->ldb_ctrl |= LDB_DATA_WIDTH_CH1_24;
-                break;
-        case MEDIA_BUS_FMT_RGB888_1X7X4_JEIDA:
-                bus_format = MEDIA_BUS_FMT_RGB888_1X24;
-                if (imx_ldb_ch->chno == 0 || dual)
-                        ldb->ldb_ctrl |= LDB_DATA_WIDTH_CH0_24 |
-                                         LDB_BIT_MAP_CH0_JEIDA;
-                if (imx_ldb_ch->chno == 1 || dual)
-                        ldb->ldb_ctrl |= LDB_DATA_WIDTH_CH1_24 |
-                                         LDB_BIT_MAP_CH1_JEIDA;
-                break;
-        }
-
-        imx_drm_set_bus_format(encoder, bus_format);
-}
-
-static void imx_ldb_encoder_commit(struct drm_encoder *encoder)
+static void imx_ldb_encoder_enable(struct drm_encoder *encoder)
 {
         struct imx_ldb_channel *imx_ldb_ch = enc_to_imx_ldb_ch(encoder);
         struct imx_ldb *ldb = imx_ldb_ch->ldb;
@@ -219,8 +210,13 @@ static void imx_ldb_encoder_commit(struct drm_encoder *encoder)
         drm_panel_prepare(imx_ldb_ch->panel);
 
         if (dual) {
+                clk_set_parent(ldb->clk_sel[mux], ldb->clk[0]);
+                clk_set_parent(ldb->clk_sel[mux], ldb->clk[1]);
+
                 clk_prepare_enable(ldb->clk[0]);
                 clk_prepare_enable(ldb->clk[1]);
+        } else {
+                clk_set_parent(ldb->clk_sel[mux], ldb->clk[imx_ldb_ch->chno]);
         }
 
         if (imx_ldb_ch == &ldb->channel[0] || dual) {
@@ -265,6 +261,7 @@ static void imx_ldb_encoder_mode_set(struct drm_encoder *encoder,
         unsigned long serial_clk;
         unsigned long di_clk = mode->clock * 1000;
         int mux = drm_of_encoder_active_port_id(imx_ldb_ch->child, encoder);
+        u32 bus_format = imx_ldb_ch->bus_format;
 
         if (mode->clock > 170000) {
                 dev_warn(ldb->dev,
@@ -286,18 +283,33 @@ static void imx_ldb_encoder_mode_set(struct drm_encoder *encoder,
         }
 
         /* FIXME - assumes straight connections DI0 --> CH0, DI1 --> CH1 */
-        if (imx_ldb_ch == &ldb->channel[0]) {
+        if (imx_ldb_ch == &ldb->channel[0] || dual) {
                 if (mode->flags & DRM_MODE_FLAG_NVSYNC)
                         ldb->ldb_ctrl |= LDB_DI0_VS_POL_ACT_LOW;
                 else if (mode->flags & DRM_MODE_FLAG_PVSYNC)
                         ldb->ldb_ctrl &= ~LDB_DI0_VS_POL_ACT_LOW;
         }
-        if (imx_ldb_ch == &ldb->channel[1]) {
+        if (imx_ldb_ch == &ldb->channel[1] || dual) {
                 if (mode->flags & DRM_MODE_FLAG_NVSYNC)
                         ldb->ldb_ctrl |= LDB_DI1_VS_POL_ACT_LOW;
                 else if (mode->flags & DRM_MODE_FLAG_PVSYNC)
                         ldb->ldb_ctrl &= ~LDB_DI1_VS_POL_ACT_LOW;
         }
+
+        if (!bus_format) {
+                struct drm_connector *connector;
+
+                drm_for_each_connector(connector, encoder->dev) {
+                        struct drm_display_info *di = &connector->display_info;
+
+                        if (connector->encoder == encoder &&
+                            di->num_bus_formats) {
+                                bus_format = di->bus_formats[0];
+                                break;
+                        }
+                }
+        }
+        imx_ldb_ch_set_bus_format(imx_ldb_ch, bus_format);
 }
 
 static void imx_ldb_encoder_disable(struct drm_encoder *encoder)
@@ -357,11 +369,45 @@ static void imx_ldb_encoder_disable(struct drm_encoder *encoder)
         drm_panel_unprepare(imx_ldb_ch->panel);
 }
 
+static int imx_ldb_encoder_atomic_check(struct drm_encoder *encoder,
+                                        struct drm_crtc_state *crtc_state,
+                                        struct drm_connector_state *conn_state)
+{
+        struct imx_crtc_state *imx_crtc_state = to_imx_crtc_state(crtc_state);
+        struct imx_ldb_channel *imx_ldb_ch = enc_to_imx_ldb_ch(encoder);
+        struct drm_display_info *di = &conn_state->connector->display_info;
+        u32 bus_format = imx_ldb_ch->bus_format;
+
+        /* Bus format description in DT overrides connector display info. */
+        if (!bus_format && di->num_bus_formats)
+                bus_format = di->bus_formats[0];
+        switch (bus_format) {
+        case MEDIA_BUS_FMT_RGB666_1X7X3_SPWG:
+                imx_crtc_state->bus_format = MEDIA_BUS_FMT_RGB666_1X18;
+                break;
+        case MEDIA_BUS_FMT_RGB888_1X7X4_SPWG:
+        case MEDIA_BUS_FMT_RGB888_1X7X4_JEIDA:
+                imx_crtc_state->bus_format = MEDIA_BUS_FMT_RGB888_1X24;
+                break;
+        default:
+                return -EINVAL;
+        }
+
+        imx_crtc_state->di_hsync_pin = 2;
+        imx_crtc_state->di_vsync_pin = 3;
+
+        return 0;
+}
+
+
 static const struct drm_connector_funcs imx_ldb_connector_funcs = {
-        .dpms = drm_helper_connector_dpms,
+        .dpms = drm_atomic_helper_connector_dpms,
         .fill_modes = drm_helper_probe_single_connector_modes,
         .detect = imx_ldb_connector_detect,
         .destroy = imx_drm_connector_destroy,
+        .reset = drm_atomic_helper_connector_reset,
+        .atomic_duplicate_state = drm_atomic_helper_connector_duplicate_state,
+        .atomic_destroy_state = drm_atomic_helper_connector_destroy_state,
 };
 
 static const struct drm_connector_helper_funcs imx_ldb_connector_helper_funcs = {
@@ -374,11 +420,10 @@ static const struct drm_encoder_funcs imx_ldb_encoder_funcs = {
 };
 
 static const struct drm_encoder_helper_funcs imx_ldb_encoder_helper_funcs = {
-        .dpms = imx_ldb_encoder_dpms,
-        .prepare = imx_ldb_encoder_prepare,
-        .commit = imx_ldb_encoder_commit,
         .mode_set = imx_ldb_encoder_mode_set,
+        .enable = imx_ldb_encoder_enable,
         .disable = imx_ldb_encoder_disable,
+        .atomic_check = imx_ldb_encoder_atomic_check,
 };
 
 static int imx_ldb_get_clk(struct imx_ldb *ldb, int chno)
@@ -400,10 +445,10 @@ static int imx_ldb_register(struct drm_device *drm,
         struct imx_ldb_channel *imx_ldb_ch)
 {
         struct imx_ldb *ldb = imx_ldb_ch->ldb;
+        struct drm_encoder *encoder = &imx_ldb_ch->encoder;
         int ret;
 
-        ret = imx_drm_encoder_parse_of(drm, &imx_ldb_ch->encoder,
-                                       imx_ldb_ch->child);
+        ret = imx_drm_encoder_parse_of(drm, encoder, imx_ldb_ch->child);
         if (ret)
                 return ret;
 
@@ -417,9 +462,8 @@ static int imx_ldb_register(struct drm_device *drm,
                         return ret;
         }
 
-        drm_encoder_helper_add(&imx_ldb_ch->encoder,
-                        &imx_ldb_encoder_helper_funcs);
-        drm_encoder_init(drm, &imx_ldb_ch->encoder, &imx_ldb_encoder_funcs,
+        drm_encoder_helper_add(encoder, &imx_ldb_encoder_helper_funcs);
+        drm_encoder_init(drm, encoder, &imx_ldb_encoder_funcs,
                          DRM_MODE_ENCODER_LVDS, NULL);
 
         drm_connector_helper_add(&imx_ldb_ch->connector,
@@ -427,11 +471,14 @@ static int imx_ldb_register(struct drm_device *drm,
         drm_connector_init(drm, &imx_ldb_ch->connector,
                            &imx_ldb_connector_funcs, DRM_MODE_CONNECTOR_LVDS);
 
-        if (imx_ldb_ch->panel)
-                drm_panel_attach(imx_ldb_ch->panel, &imx_ldb_ch->connector);
+        if (imx_ldb_ch->panel) {
+                ret = drm_panel_attach(imx_ldb_ch->panel,
+                                       &imx_ldb_ch->connector);
+                if (ret)
+                        return ret;
+        }
 
-        drm_mode_connector_attach_encoder(&imx_ldb_ch->connector,
-                        &imx_ldb_ch->encoder);
+        drm_mode_connector_attach_encoder(&imx_ldb_ch->connector, encoder);
 
         return 0;
 }
@@ -560,6 +607,7 @@ static int imx_ldb_bind(struct device *dev, struct device *master, void *data)
                 struct imx_ldb_channel *channel;
                 struct device_node *ddc_node;
                 struct device_node *ep;
+                int bus_format;
 
                 ret = of_property_read_u32(child, "reg", &i);
                 if (ret || i < 0 || i > 1)
@@ -632,21 +680,22 @@ static int imx_ldb_bind(struct device *dev, struct device *master, void *data)
                         }
                 }
 
-                channel->bus_format = of_get_bus_format(dev, child);
-                if (channel->bus_format == -EINVAL) {
+                bus_format = of_get_bus_format(dev, child);
+                if (bus_format == -EINVAL) {
                         /*
                          * If no bus format was specified in the device tree,
                          * we can still get it from the connected panel later.
                          */
                         if (channel->panel && channel->panel->funcs &&
                             channel->panel->funcs->get_modes)
-                                channel->bus_format = 0;
+                                bus_format = 0;
                 }
-                if (channel->bus_format < 0) {
+                if (bus_format < 0) {
                         dev_err(dev, "could not determine data mapping: %d\n",
-                                channel->bus_format);
-                        return channel->bus_format;
+                                bus_format);
+                        return bus_format;
                 }
+                channel->bus_format = bus_format;
 
                 ret = imx_ldb_register(drm, channel);
                 if (ret)
diff --git a/drivers/gpu/drm/imx/imx-tve.c b/drivers/gpu/drm/imx/imx-tve.c
index baf788121287..5e875944ffa2 100644
--- a/drivers/gpu/drm/imx/imx-tve.c
+++ b/drivers/gpu/drm/imx/imx-tve.c
@@ -23,6 +23,7 @@
 #include <linux/spinlock.h>
 #include <linux/videodev2.h>
 #include <drm/drmP.h>
+#include <drm/drm_atomic_helper.h>
 #include <drm/drm_fb_helper.h>
 #include <drm/drm_crtc_helper.h>
 #include <video/imx-ipu-v3.h>
@@ -97,9 +98,6 @@
 /* TVE_TST_MODE_REG */
 #define TVE_TVDAC_TEST_MODE_MASK        (0x7 << 0)
 
-#define con_to_tve(x) container_of(x, struct imx_tve, connector)
-#define enc_to_tve(x) container_of(x, struct imx_tve, encoder)
-
 enum {
         TVE_MODE_TVOUT,
         TVE_MODE_VGA,
@@ -112,6 +110,8 @@ struct imx_tve {
         spinlock_t lock;        /* register lock */
         bool enabled;
         int mode;
+        int di_hsync_pin;
+        int di_vsync_pin;
 
         struct regmap *regmap;
         struct regulator *dac_reg;
@@ -120,10 +120,18 @@ struct imx_tve {
         struct clk *di_sel_clk;
         struct clk_hw clk_hw_di;
         struct clk *di_clk;
-        int vsync_pin;
-        int hsync_pin;
 };
 
+static inline struct imx_tve *con_to_tve(struct drm_connector *c)
+{
+        return container_of(c, struct imx_tve, connector);
+}
+
+static inline struct imx_tve *enc_to_tve(struct drm_encoder *e)
+{
+        return container_of(e, struct imx_tve, encoder);
+}
+
 static void tve_lock(void *__tve)
 __acquires(&tve->lock)
 {
@@ -148,8 +156,7 @@ static void tve_enable(struct imx_tve *tve)
                 tve->enabled = true;
                 clk_prepare_enable(tve->clk);
                 ret = regmap_update_bits(tve->regmap, TVE_COM_CONF_REG,
-                                         TVE_IPU_CLK_EN | TVE_EN,
-                                         TVE_IPU_CLK_EN | TVE_EN);
+                                         TVE_EN, TVE_EN);
         }
 
         /* clear interrupt status register */
@@ -172,7 +179,7 @@ static void tve_disable(struct imx_tve *tve)
         if (tve->enabled) {
                 tve->enabled = false;
                 ret = regmap_update_bits(tve->regmap, TVE_COM_CONF_REG,
-                                         TVE_IPU_CLK_EN | TVE_EN, 0);
+                                         TVE_EN, 0);
                 clk_disable_unprepare(tve->clk);
         }
 }
@@ -275,36 +282,6 @@ static struct drm_encoder *imx_tve_connector_best_encoder(
         return &tve->encoder;
 }
 
-static void imx_tve_encoder_dpms(struct drm_encoder *encoder, int mode)
-{
-        struct imx_tve *tve = enc_to_tve(encoder);
-        int ret;
-
-        ret = regmap_update_bits(tve->regmap, TVE_COM_CONF_REG,
-                                 TVE_TV_OUT_MODE_MASK, TVE_TV_OUT_DISABLE);
-        if (ret < 0)
-                dev_err(tve->dev, "failed to disable TVOUT: %d\n", ret);
-}
-
-static void imx_tve_encoder_prepare(struct drm_encoder *encoder)
-{
-        struct imx_tve *tve = enc_to_tve(encoder);
-
-        tve_disable(tve);
-
-        switch (tve->mode) {
-        case TVE_MODE_VGA:
-                imx_drm_set_bus_config(encoder, MEDIA_BUS_FMT_GBR888_1X24,
-                                       tve->hsync_pin, tve->vsync_pin,
-                                       DRM_BUS_FLAG_DE_HIGH |
-                                       DRM_BUS_FLAG_PIXDATA_NEGEDGE);
-                break;
-        case TVE_MODE_TVOUT:
-                imx_drm_set_bus_format(encoder, MEDIA_BUS_FMT_YUV8_1X24);
-                break;
-        }
-}
-
 static void imx_tve_encoder_mode_set(struct drm_encoder *encoder,
                                      struct drm_display_mode *orig_mode,
                                      struct drm_display_mode *mode)
@@ -333,6 +310,9 @@ static void imx_tve_encoder_mode_set(struct drm_encoder *encoder,
                         ret);
         }
 
+        regmap_update_bits(tve->regmap, TVE_COM_CONF_REG,
+                           TVE_IPU_CLK_EN, TVE_IPU_CLK_EN);
+
         if (tve->mode == TVE_MODE_VGA)
                 ret = tve_setup_vga(tve);
         else
@@ -341,7 +321,7 @@ static void imx_tve_encoder_mode_set(struct drm_encoder *encoder,
                 dev_err(tve->dev, "failed to set configuration: %d\n", ret);
 }
 
-static void imx_tve_encoder_commit(struct drm_encoder *encoder)
+static void imx_tve_encoder_enable(struct drm_encoder *encoder)
 {
         struct imx_tve *tve = enc_to_tve(encoder);
 
@@ -355,11 +335,28 @@ static void imx_tve_encoder_disable(struct drm_encoder *encoder)
         tve_disable(tve);
 }
 
+static int imx_tve_atomic_check(struct drm_encoder *encoder,
+                                struct drm_crtc_state *crtc_state,
+                                struct drm_connector_state *conn_state)
+{
+        struct imx_crtc_state *imx_crtc_state = to_imx_crtc_state(crtc_state);
+        struct imx_tve *tve = enc_to_tve(encoder);
+
+        imx_crtc_state->bus_format = MEDIA_BUS_FMT_GBR888_1X24;
+        imx_crtc_state->di_hsync_pin = tve->di_hsync_pin;
+        imx_crtc_state->di_vsync_pin = tve->di_vsync_pin;
+
+        return 0;
+}
+
 static const struct drm_connector_funcs imx_tve_connector_funcs = {
-        .dpms = drm_helper_connector_dpms,
+        .dpms = drm_atomic_helper_connector_dpms,
         .fill_modes = drm_helper_probe_single_connector_modes,
         .detect = imx_tve_connector_detect,
         .destroy = imx_drm_connector_destroy,
+        .reset = drm_atomic_helper_connector_reset,
+        .atomic_duplicate_state = drm_atomic_helper_connector_duplicate_state,
+        .atomic_destroy_state = drm_atomic_helper_connector_destroy_state,
 };
 
 static const struct drm_connector_helper_funcs imx_tve_connector_helper_funcs = {
@@ -373,11 +370,10 @@ static const struct drm_encoder_funcs imx_tve_encoder_funcs = {
 };
 
 static const struct drm_encoder_helper_funcs imx_tve_encoder_helper_funcs = {
-        .dpms = imx_tve_encoder_dpms,
-        .prepare = imx_tve_encoder_prepare,
         .mode_set = imx_tve_encoder_mode_set,
-        .commit = imx_tve_encoder_commit,
+        .enable = imx_tve_encoder_enable,
         .disable = imx_tve_encoder_disable,
+        .atomic_check = imx_tve_atomic_check,
 };
 
 static irqreturn_t imx_tve_irq_handler(int irq, void *data)
@@ -495,8 +491,7 @@ static int imx_tve_register(struct drm_device *drm, struct imx_tve *tve)
         encoder_type = tve->mode == TVE_MODE_VGA ?
                                 DRM_MODE_ENCODER_DAC : DRM_MODE_ENCODER_TVDAC;
 
-        ret = imx_drm_encoder_parse_of(drm, &tve->encoder,
-                                       tve->dev->of_node);
+        ret = imx_drm_encoder_parse_of(drm, &tve->encoder, tve->dev->of_node);
         if (ret)
                 return ret;
 
@@ -587,15 +582,15 @@ static int imx_tve_bind(struct device *dev, struct device *master, void *data)
 
         if (tve->mode == TVE_MODE_VGA) {
                 ret = of_property_read_u32(np, "fsl,hsync-pin",
-                                           &tve->hsync_pin);
+                                           &tve->di_hsync_pin);
 
                 if (ret < 0) {
-                        dev_err(dev, "failed to get vsync pin\n");
+                        dev_err(dev, "failed to get hsync pin\n");
                         return ret;
                 }
 
-                ret |= of_property_read_u32(np, "fsl,vsync-pin",
-                                            &tve->vsync_pin);
+                ret = of_property_read_u32(np, "fsl,vsync-pin",
+                                           &tve->di_vsync_pin);
 
                 if (ret < 0) {
                         dev_err(dev, "failed to get vsync pin\n");
@@ -633,7 +628,9 @@ static int imx_tve_bind(struct device *dev, struct device *master, void *data)
 
         tve->dac_reg = devm_regulator_get(dev, "dac");
         if (!IS_ERR(tve->dac_reg)) {
-                regulator_set_voltage(tve->dac_reg, 2750000, 2750000);
+                ret = regulator_set_voltage(tve->dac_reg, 2750000, 2750000);
+                if (ret)
+                        return ret;
                 ret = regulator_enable(tve->dac_reg);
                 if (ret)
                         return ret;
diff --git a/drivers/gpu/drm/imx/ipuv3-crtc.c b/drivers/gpu/drm/imx/ipuv3-crtc.c
index fc040417e1e8..08e188bc10fc 100644
--- a/drivers/gpu/drm/imx/ipuv3-crtc.c
+++ b/drivers/gpu/drm/imx/ipuv3-crtc.c
@@ -18,12 +18,12 @@
 #include <linux/device.h>
 #include <linux/platform_device.h>
 #include <drm/drmP.h>
+#include <drm/drm_atomic.h>
+#include <drm/drm_atomic_helper.h>
 #include <drm/drm_crtc_helper.h>
 #include <linux/fb.h>
 #include <linux/clk.h>
 #include <linux/errno.h>
-#include <linux/reservation.h>
-#include <linux/dma-buf.h>
 #include <drm/drm_gem_cma_helper.h>
 #include <drm/drm_fb_cma_helper.h>
 
@@ -33,23 +33,6 @@
 
 #define DRIVER_DESC             "i.MX IPUv3 Graphics"
 
-enum ipu_flip_status {
-        IPU_FLIP_NONE,
-        IPU_FLIP_PENDING,
-        IPU_FLIP_SUBMITTED,
-};
-
-struct ipu_flip_work {
-        struct work_struct              unref_work;
-        struct drm_gem_object           *bo;
-        struct drm_pending_vblank_event *page_flip_event;
-        struct work_struct              fence_work;
-        struct ipu_crtc                 *crtc;
-        struct fence                    *excl;
-        unsigned                        shared_count;
-        struct fence                    **shared;
-};
-
 struct ipu_crtc {
         struct device           *dev;
         struct drm_crtc         base;
@@ -60,201 +43,166 @@ struct ipu_crtc {
 
         struct ipu_dc           *dc;
         struct ipu_di           *di;
-        int                     enabled;
-        enum ipu_flip_status    flip_state;
-        struct workqueue_struct *flip_queue;
-        struct ipu_flip_work    *flip_work;
         int                     irq;
-        u32                     bus_format;
-        u32                     bus_flags;
-        int                     di_hsync_pin;
-        int                     di_vsync_pin;
 };
 
-#define to_ipu_crtc(x) container_of(x, struct ipu_crtc, base)
+static inline struct ipu_crtc *to_ipu_crtc(struct drm_crtc *crtc)
+{
+        return container_of(crtc, struct ipu_crtc, base);
+}
 
-static void ipu_fb_enable(struct ipu_crtc *ipu_crtc)
+static void ipu_crtc_enable(struct drm_crtc *crtc)
 {
+        struct ipu_crtc *ipu_crtc = to_ipu_crtc(crtc);
         struct ipu_soc *ipu = dev_get_drvdata(ipu_crtc->dev->parent);
 
-        if (ipu_crtc->enabled)
-                return;
-
         ipu_dc_enable(ipu);
-        ipu_plane_enable(ipu_crtc->plane[0]);
-        /* Start DC channel and DI after IDMAC */
         ipu_dc_enable_channel(ipu_crtc->dc);
         ipu_di_enable(ipu_crtc->di);
-        drm_crtc_vblank_on(&ipu_crtc->base);
-
-        ipu_crtc->enabled = 1;
 }
 
-static void ipu_fb_disable(struct ipu_crtc *ipu_crtc)
+static void ipu_crtc_disable(struct drm_crtc *crtc)
 {
+        struct ipu_crtc *ipu_crtc = to_ipu_crtc(crtc);
         struct ipu_soc *ipu = dev_get_drvdata(ipu_crtc->dev->parent);
 
-        if (!ipu_crtc->enabled)
-                return;
-
-        /* Stop DC channel and DI before IDMAC */
         ipu_dc_disable_channel(ipu_crtc->dc);
         ipu_di_disable(ipu_crtc->di);
-        ipu_plane_disable(ipu_crtc->plane[0]);
         ipu_dc_disable(ipu);
-        drm_crtc_vblank_off(&ipu_crtc->base);
 
-        ipu_crtc->enabled = 0;
+        spin_lock_irq(&crtc->dev->event_lock);
+        if (crtc->state->event) {
+                drm_crtc_send_vblank_event(crtc, crtc->state->event);
+                crtc->state->event = NULL;
+        }
+        spin_unlock_irq(&crtc->dev->event_lock);
 }
 
-static void ipu_crtc_dpms(struct drm_crtc *crtc, int mode)
+static void imx_drm_crtc_reset(struct drm_crtc *crtc)
 {
-        struct ipu_crtc *ipu_crtc = to_ipu_crtc(crtc);
+        struct imx_crtc_state *state;
 
-        dev_dbg(ipu_crtc->dev, "%s mode: %d\n", __func__, mode);
-
-        switch (mode) {
-        case DRM_MODE_DPMS_ON:
-                ipu_fb_enable(ipu_crtc);
-                break;
-        case DRM_MODE_DPMS_STANDBY:
-        case DRM_MODE_DPMS_SUSPEND:
-        case DRM_MODE_DPMS_OFF:
-                ipu_fb_disable(ipu_crtc);
-                break;
+        if (crtc->state) {
+                if (crtc->state->mode_blob)
+                        drm_property_unreference_blob(crtc->state->mode_blob);
+
+                state = to_imx_crtc_state(crtc->state);
+                memset(state, 0, sizeof(*state));
+        } else {
+                state = kzalloc(sizeof(*state), GFP_KERNEL);
+                if (!state)
+                        return;
+                crtc->state = &state->base;
         }
+
+        state->base.crtc = crtc;
 }
 
-static void ipu_flip_unref_work_func(struct work_struct *__work)
+static struct drm_crtc_state *imx_drm_crtc_duplicate_state(struct drm_crtc *crtc)
 {
-        struct ipu_flip_work *work =
-                        container_of(__work, struct ipu_flip_work, unref_work);
+        struct imx_crtc_state *state;
+
+        state = kzalloc(sizeof(*state), GFP_KERNEL);
+        if (!state)
+                return NULL;
 
-        drm_gem_object_unreference_unlocked(work->bo);
-        kfree(work);
+        __drm_atomic_helper_crtc_duplicate_state(crtc, &state->base);
+
+        WARN_ON(state->base.crtc != crtc);
+        state->base.crtc = crtc;
+
+        return &state->base;
 }
 
-static void ipu_flip_fence_work_func(struct work_struct *__work)
+static void imx_drm_crtc_destroy_state(struct drm_crtc *crtc,
+                                       struct drm_crtc_state *state)
 {
-        struct ipu_flip_work *work =
-                        container_of(__work, struct ipu_flip_work, fence_work);
-        int i;
-
-        /* wait for all fences attached to the FB obj to signal */
-        if (work->excl) {
-                fence_wait(work->excl, false);
-                fence_put(work->excl);
-        }
-        for (i = 0; i < work->shared_count; i++) {
-                fence_wait(work->shared[i], false);
-                fence_put(work->shared[i]);
-        }
+        __drm_atomic_helper_crtc_destroy_state(state);
+        kfree(to_imx_crtc_state(state));
+}
+
+static const struct drm_crtc_funcs ipu_crtc_funcs = {
+        .set_config = drm_atomic_helper_set_config,
+        .destroy = drm_crtc_cleanup,
+        .page_flip = drm_atomic_helper_page_flip,
+        .reset = imx_drm_crtc_reset,
+        .atomic_duplicate_state = imx_drm_crtc_duplicate_state,
+        .atomic_destroy_state = imx_drm_crtc_destroy_state,
+};
 
-        work->crtc->flip_state = IPU_FLIP_SUBMITTED;
+static irqreturn_t ipu_irq_handler(int irq, void *dev_id)
+{
+        struct ipu_crtc *ipu_crtc = dev_id;
+
+        imx_drm_handle_vblank(ipu_crtc->imx_crtc);
+
+        return IRQ_HANDLED;
 }
 
-static int ipu_page_flip(struct drm_crtc *crtc,
-                struct drm_framebuffer *fb,
-                struct drm_pending_vblank_event *event,
-                uint32_t page_flip_flags)
+static bool ipu_crtc_mode_fixup(struct drm_crtc *crtc,
+                                  const struct drm_display_mode *mode,
+                                  struct drm_display_mode *adjusted_mode)
 {
-        struct drm_gem_cma_object *cma_obj = drm_fb_cma_get_gem_obj(fb, 0);
         struct ipu_crtc *ipu_crtc = to_ipu_crtc(crtc);
-        struct ipu_flip_work *flip_work;
+        struct videomode vm;
         int ret;
 
-        if (ipu_crtc->flip_state != IPU_FLIP_NONE)
-                return -EBUSY;
-
-        ret = imx_drm_crtc_vblank_get(ipu_crtc->imx_crtc);
-        if (ret) {
-                dev_dbg(ipu_crtc->dev, "failed to acquire vblank counter\n");
-                list_del(&event->base.link);
-
-                return ret;
-        }
+        drm_display_mode_to_videomode(adjusted_mode, &vm);
 
-        flip_work = kzalloc(sizeof *flip_work, GFP_KERNEL);
-        if (!flip_work) {
-                ret = -ENOMEM;
-                goto put_vblank;
-        }
-        INIT_WORK(&flip_work->unref_work, ipu_flip_unref_work_func);
-        flip_work->page_flip_event = event;
+        ret = ipu_di_adjust_videomode(ipu_crtc->di, &vm);
+        if (ret)
+                return false;
 
-        /* get BO backing the old framebuffer and take a reference */
-        flip_work->bo = &drm_fb_cma_get_gem_obj(crtc->primary->fb, 0)->base;
-        drm_gem_object_reference(flip_work->bo);
+        if ((vm.vsync_len == 0) || (vm.hsync_len == 0))
+                return false;
 
-        ipu_crtc->flip_work = flip_work;
-        /*
-         * If the object has a DMABUF attached, we need to wait on its fences
-         * if there are any.
-         */
-        if (cma_obj->base.dma_buf) {
-                INIT_WORK(&flip_work->fence_work, ipu_flip_fence_work_func);
-                flip_work->crtc = ipu_crtc;
+        drm_display_mode_from_videomode(&vm, adjusted_mode);
 
-                ret = reservation_object_get_fences_rcu(
-                                cma_obj->base.dma_buf->resv, &flip_work->excl,
-                                &flip_work->shared_count, &flip_work->shared);
+        return true;
+}
 
-                if (unlikely(ret)) {
-                        DRM_ERROR("failed to get fences for buffer\n");
-                        goto free_flip_work;
-                }
+static int ipu_crtc_atomic_check(struct drm_crtc *crtc,
+                                 struct drm_crtc_state *state)
+{
+        u32 primary_plane_mask = 1 << drm_plane_index(crtc->primary);
 
-                /* No need to queue the worker if the are no fences */
-                if (!flip_work->excl && !flip_work->shared_count) {
-                        ipu_crtc->flip_state = IPU_FLIP_SUBMITTED;
-                } else {
-                        ipu_crtc->flip_state = IPU_FLIP_PENDING;
-                        queue_work(ipu_crtc->flip_queue,
-                                   &flip_work->fence_work);
-                }
-        } else {
-                ipu_crtc->flip_state = IPU_FLIP_SUBMITTED;
-        }
+        if (state->active && (primary_plane_mask & state->plane_mask) == 0)
+                return -EINVAL;
 
         return 0;
-
-free_flip_work:
-        drm_gem_object_unreference_unlocked(flip_work->bo);
-        kfree(flip_work);
-        ipu_crtc->flip_work = NULL;
-put_vblank:
-        imx_drm_crtc_vblank_put(ipu_crtc->imx_crtc);
-
-        return ret;
 }
 
-static const struct drm_crtc_funcs ipu_crtc_funcs = {
-        .set_config = drm_crtc_helper_set_config,
-        .destroy = drm_crtc_cleanup,
-        .page_flip = ipu_page_flip,
-};
+static void ipu_crtc_atomic_begin(struct drm_crtc *crtc,
+                                  struct drm_crtc_state *old_crtc_state)
+{
+        spin_lock_irq(&crtc->dev->event_lock);
+        if (crtc->state->event) {
+                WARN_ON(drm_crtc_vblank_get(crtc));
+                drm_crtc_arm_vblank_event(crtc, crtc->state->event);
+                crtc->state->event = NULL;
+        }
+        spin_unlock_irq(&crtc->dev->event_lock);
+}
 
-static int ipu_crtc_mode_set(struct drm_crtc *crtc,
-                               struct drm_display_mode *orig_mode,
-                               struct drm_display_mode *mode,
-                               int x, int y,
-                               struct drm_framebuffer *old_fb)
+static void ipu_crtc_mode_set_nofb(struct drm_crtc *crtc)
 {
         struct drm_device *dev = crtc->dev;
         struct drm_encoder *encoder;
         struct ipu_crtc *ipu_crtc = to_ipu_crtc(crtc);
+        struct drm_display_mode *mode = &crtc->state->adjusted_mode;
+        struct imx_crtc_state *imx_crtc_state = to_imx_crtc_state(crtc->state);
         struct ipu_di_signal_cfg sig_cfg = {};
         unsigned long encoder_types = 0;
-        int ret;
 
         dev_dbg(ipu_crtc->dev, "%s: mode->hdisplay: %d\n", __func__,
                         mode->hdisplay);
         dev_dbg(ipu_crtc->dev, "%s: mode->vdisplay: %d\n", __func__,
                         mode->vdisplay);
 
-        list_for_each_entry(encoder, &dev->mode_config.encoder_list, head)
+        list_for_each_entry(encoder, &dev->mode_config.encoder_list, head) {
                 if (encoder->crtc == crtc)
                         encoder_types |= BIT(encoder->encoder_type);
+        }
 
         dev_dbg(ipu_crtc->dev, "%s: attached to encoder types 0x%lx\n",
                 __func__, encoder_types);
@@ -272,114 +220,30 @@ static int ipu_crtc_mode_set(struct drm_crtc *crtc,
         else
                 sig_cfg.clkflags = 0;
 
-        sig_cfg.enable_pol = !(ipu_crtc->bus_flags & DRM_BUS_FLAG_DE_LOW);
+        sig_cfg.enable_pol = !(imx_crtc_state->bus_flags & DRM_BUS_FLAG_DE_LOW);
         /* Default to driving pixel data on negative clock edges */
-        sig_cfg.clk_pol = !!(ipu_crtc->bus_flags &
+        sig_cfg.clk_pol = !!(imx_crtc_state->bus_flags &
                              DRM_BUS_FLAG_PIXDATA_POSEDGE);
-        sig_cfg.bus_format = ipu_crtc->bus_format;
+        sig_cfg.bus_format = imx_crtc_state->bus_format;
         sig_cfg.v_to_h_sync = 0;
-        sig_cfg.hsync_pin = ipu_crtc->di_hsync_pin;
-        sig_cfg.vsync_pin = ipu_crtc->di_vsync_pin;
+        sig_cfg.hsync_pin = imx_crtc_state->di_hsync_pin;
+        sig_cfg.vsync_pin = imx_crtc_state->di_vsync_pin;
 
         drm_display_mode_to_videomode(mode, &sig_cfg.mode);
 
-        ret = ipu_dc_init_sync(ipu_crtc->dc, ipu_crtc->di,
-                               mode->flags & DRM_MODE_FLAG_INTERLACE,
-                               ipu_crtc->bus_format, mode->hdisplay);
-        if (ret) {
-                dev_err(ipu_crtc->dev,
-                                "initializing display controller failed with %d\n",
-                                ret);
-                return ret;
-        }
-
-        ret = ipu_di_init_sync_panel(ipu_crtc->di, &sig_cfg);
-        if (ret) {
-                dev_err(ipu_crtc->dev,
-                                "initializing panel failed with %d\n", ret);
-                return ret;
-        }
-
-        return ipu_plane_mode_set(ipu_crtc->plane[0], crtc, mode,
-                                  crtc->primary->fb,
-                                  0, 0, mode->hdisplay, mode->vdisplay,
-                                  x, y, mode->hdisplay, mode->vdisplay,
-                                  mode->flags & DRM_MODE_FLAG_INTERLACE);
-}
-
-static void ipu_crtc_handle_pageflip(struct ipu_crtc *ipu_crtc)
-{
-        unsigned long flags;
-        struct drm_device *drm = ipu_crtc->base.dev;
-        struct ipu_flip_work *work = ipu_crtc->flip_work;
-
-        spin_lock_irqsave(&drm->event_lock, flags);
-        if (work->page_flip_event)
-                drm_crtc_send_vblank_event(&ipu_crtc->base,
-                                           work->page_flip_event);
-        imx_drm_crtc_vblank_put(ipu_crtc->imx_crtc);
-        spin_unlock_irqrestore(&drm->event_lock, flags);
-}
-
-static irqreturn_t ipu_irq_handler(int irq, void *dev_id)
-{
-        struct ipu_crtc *ipu_crtc = dev_id;
-
-        imx_drm_handle_vblank(ipu_crtc->imx_crtc);
-
-        if (ipu_crtc->flip_state == IPU_FLIP_SUBMITTED) {
-                struct ipu_plane *plane = ipu_crtc->plane[0];
-
-                ipu_plane_set_base(plane, ipu_crtc->base.primary->fb,
-                                   plane->x, plane->y);
-                ipu_crtc_handle_pageflip(ipu_crtc);
-                queue_work(ipu_crtc->flip_queue,
-                           &ipu_crtc->flip_work->unref_work);
-                ipu_crtc->flip_state = IPU_FLIP_NONE;
-        }
-
-        return IRQ_HANDLED;
-}
-
-static bool ipu_crtc_mode_fixup(struct drm_crtc *crtc,
-                                  const struct drm_display_mode *mode,
-                                  struct drm_display_mode *adjusted_mode)
-{
-        struct ipu_crtc *ipu_crtc = to_ipu_crtc(crtc);
-        struct videomode vm;
-        int ret;
-
-        drm_display_mode_to_videomode(adjusted_mode, &vm);
-
-        ret = ipu_di_adjust_videomode(ipu_crtc->di, &vm);
-        if (ret)
-                return false;
-
-        drm_display_mode_from_videomode(&vm, adjusted_mode);
-
-        return true;
-}
-
-static void ipu_crtc_prepare(struct drm_crtc *crtc)
-{
-        struct ipu_crtc *ipu_crtc = to_ipu_crtc(crtc);
-
-        ipu_fb_disable(ipu_crtc);
-}
-
-static void ipu_crtc_commit(struct drm_crtc *crtc)
-{
-        struct ipu_crtc *ipu_crtc = to_ipu_crtc(crtc);
-
-        ipu_fb_enable(ipu_crtc);
+        ipu_dc_init_sync(ipu_crtc->dc, ipu_crtc->di,
+                         mode->flags & DRM_MODE_FLAG_INTERLACE,
+                         imx_crtc_state->bus_format, mode->hdisplay);
+        ipu_di_init_sync_panel(ipu_crtc->di, &sig_cfg);
 }
 
 static const struct drm_crtc_helper_funcs ipu_helper_funcs = {
-        .dpms = ipu_crtc_dpms,
         .mode_fixup = ipu_crtc_mode_fixup,
-        .mode_set = ipu_crtc_mode_set,
-        .prepare = ipu_crtc_prepare,
-        .commit = ipu_crtc_commit,
+        .mode_set_nofb = ipu_crtc_mode_set_nofb,
+        .atomic_check = ipu_crtc_atomic_check,
+        .atomic_begin = ipu_crtc_atomic_begin,
+        .disable = ipu_crtc_disable,
+        .enable = ipu_crtc_enable,
 };
 
 static int ipu_enable_vblank(struct drm_crtc *crtc)
@@ -398,23 +262,9 @@ static void ipu_disable_vblank(struct drm_crtc *crtc)
         disable_irq_nosync(ipu_crtc->irq);
 }
 
-static int ipu_set_interface_pix_fmt(struct drm_crtc *crtc,
-                u32 bus_format, int hsync_pin, int vsync_pin, u32 bus_flags)
-{
-        struct ipu_crtc *ipu_crtc = to_ipu_crtc(crtc);
-
-        ipu_crtc->bus_format = bus_format;
-        ipu_crtc->bus_flags = bus_flags;
-        ipu_crtc->di_hsync_pin = hsync_pin;
-        ipu_crtc->di_vsync_pin = vsync_pin;
-
-        return 0;
-}
-
 static const struct imx_drm_crtc_helper_funcs ipu_crtc_helper_funcs = {
         .enable_vblank = ipu_enable_vblank,
         .disable_vblank = ipu_disable_vblank,
-        .set_interface_pix_fmt = ipu_set_interface_pix_fmt,
         .crtc_funcs = &ipu_crtc_funcs,
         .crtc_helper_funcs = &ipu_helper_funcs,
 };
@@ -496,8 +346,16 @@ static int ipu_crtc_init(struct ipu_crtc *ipu_crtc,
                                                 IPU_DP_FLOW_SYNC_FG,
                                                 drm_crtc_mask(&ipu_crtc->base),
                                                 DRM_PLANE_TYPE_OVERLAY);
-                if (IS_ERR(ipu_crtc->plane[1]))
+                if (IS_ERR(ipu_crtc->plane[1])) {
                         ipu_crtc->plane[1] = NULL;
+                } else {
+                        ret = ipu_plane_get_resources(ipu_crtc->plane[1]);
+                        if (ret) {
+                                dev_err(ipu_crtc->dev, "getting plane 1 "
+                                        "resources failed with %d.\n", ret);
+                                goto err_put_plane0_res;
+                        }
+                }
         }
 
         ipu_crtc->irq = ipu_plane_irq(ipu_crtc->plane[0]);
@@ -505,16 +363,17 @@ static int ipu_crtc_init(struct ipu_crtc *ipu_crtc,
                         "imx_drm", ipu_crtc);
         if (ret < 0) {
                 dev_err(ipu_crtc->dev, "irq request failed with %d.\n", ret);
-                goto err_put_plane_res;
+                goto err_put_plane1_res;
         }
         /* Only enable IRQ when we actually need it to trigger work. */
         disable_irq(ipu_crtc->irq);
 
-        ipu_crtc->flip_queue = create_singlethread_workqueue("ipu-crtc-flip");
-
         return 0;
 
-err_put_plane_res:
+err_put_plane1_res:
+        if (ipu_crtc->plane[1])
+                ipu_plane_put_resources(ipu_crtc->plane[1]);
+err_put_plane0_res:
         ipu_plane_put_resources(ipu_crtc->plane[0]);
 err_remove_crtc:
         imx_drm_remove_crtc(ipu_crtc->imx_crtc);
@@ -553,9 +412,10 @@ static void ipu_drm_unbind(struct device *dev, struct device *master,
 
         imx_drm_remove_crtc(ipu_crtc->imx_crtc);
 
-        destroy_workqueue(ipu_crtc->flip_queue);
-        ipu_plane_put_resources(ipu_crtc->plane[0]);
         ipu_put_resources(ipu_crtc);
+        if (ipu_crtc->plane[1])
+                ipu_plane_put_resources(ipu_crtc->plane[1]);
+        ipu_plane_put_resources(ipu_crtc->plane[0]);
 }
 
 static const struct component_ops ipu_crtc_ops = {
diff --git a/drivers/gpu/drm/imx/ipuv3-plane.c b/drivers/gpu/drm/imx/ipuv3-plane.c
index a4bb44118d33..4ad67d015ec7 100644
--- a/drivers/gpu/drm/imx/ipuv3-plane.c
+++ b/drivers/gpu/drm/imx/ipuv3-plane.c
@@ -14,13 +14,19 @@
  */
 
 #include <drm/drmP.h>
+#include <drm/drm_atomic.h>
+#include <drm/drm_atomic_helper.h>
 #include <drm/drm_fb_cma_helper.h>
 #include <drm/drm_gem_cma_helper.h>
+#include <drm/drm_plane_helper.h>
 
 #include "video/imx-ipu-v3.h"
 #include "ipuv3-plane.h"
 
-#define to_ipu_plane(x) container_of(x, struct ipu_plane, base)
+static inline struct ipu_plane *to_ipu_plane(struct drm_plane *p)
+{
+        return container_of(p, struct ipu_plane, base);
+}
 
 static const uint32_t ipu_plane_formats[] = {
         DRM_FORMAT_ARGB1555,
@@ -53,62 +59,67 @@ int ipu_plane_irq(struct ipu_plane *ipu_plane)
                                      IPU_IRQ_EOF);
 }
 
-static int calc_vref(struct drm_display_mode *mode)
+static inline unsigned long
+drm_plane_state_to_eba(struct drm_plane_state *state)
 {
-        unsigned long htotal, vtotal;
+        struct drm_framebuffer *fb = state->fb;
+        struct drm_gem_cma_object *cma_obj;
 
-        htotal = mode->htotal;
-        vtotal = mode->vtotal;
+        cma_obj = drm_fb_cma_get_gem_obj(fb, 0);
+        BUG_ON(!cma_obj);
 
-        if (!htotal || !vtotal)
-                return 60;
-
-        return DIV_ROUND_UP(mode->clock * 1000, vtotal * htotal);
+        return cma_obj->paddr + fb->offsets[0] +
+               fb->pitches[0] * (state->src_y >> 16) +
+               (fb->bits_per_pixel >> 3) * (state->src_x >> 16);
 }
 
-static inline int calc_bandwidth(int width, int height, unsigned int vref)
+static inline unsigned long
+drm_plane_state_to_ubo(struct drm_plane_state *state)
 {
-        return width * height * vref;
-}
+        struct drm_framebuffer *fb = state->fb;
+        struct drm_gem_cma_object *cma_obj;
+        unsigned long eba = drm_plane_state_to_eba(state);
 
-int ipu_plane_set_base(struct ipu_plane *ipu_plane, struct drm_framebuffer *fb,
-                       int x, int y)
-{
-        struct drm_gem_cma_object *cma_obj[3];
-        unsigned long eba, ubo, vbo;
-        int active, i;
+        cma_obj = drm_fb_cma_get_gem_obj(fb, 1);
+        BUG_ON(!cma_obj);
 
-        for (i = 0; i < drm_format_num_planes(fb->pixel_format); i++) {
-                cma_obj[i] = drm_fb_cma_get_gem_obj(fb, i);
-                if (!cma_obj[i]) {
-                        DRM_DEBUG_KMS("plane %d entry is null.\n", i);
-                        return -EFAULT;
-                }
-        }
+        return cma_obj->paddr + fb->offsets[1] +
+               fb->pitches[1] * (state->src_y >> 16) / 2 +
+               (state->src_x >> 16) / 2 - eba;
+}
 
-        eba = cma_obj[0]->paddr + fb->offsets[0] +
-              fb->pitches[0] * y + (fb->bits_per_pixel >> 3) * x;
+static inline unsigned long
+drm_plane_state_to_vbo(struct drm_plane_state *state)
+{
+        struct drm_framebuffer *fb = state->fb;
+        struct drm_gem_cma_object *cma_obj;
+        unsigned long eba = drm_plane_state_to_eba(state);
 
-        if (eba & 0x7) {
-                DRM_DEBUG_KMS("base address must be a multiple of 8.\n");
-                return -EINVAL;
-        }
+        cma_obj = drm_fb_cma_get_gem_obj(fb, 2);
+        BUG_ON(!cma_obj);
 
-        if (fb->pitches[0] < 1 || fb->pitches[0] > 16384) {
-                DRM_DEBUG_KMS("pitches out of range.\n");
-                return -EINVAL;
-        }
+        return cma_obj->paddr + fb->offsets[2] +
+               fb->pitches[2] * (state->src_y >> 16) / 2 +
+               (state->src_x >> 16) / 2 - eba;
+}
 
-        if (ipu_plane->enabled && fb->pitches[0] != ipu_plane->stride[0]) {
-                DRM_DEBUG_KMS("pitches must not change while plane is enabled.\n");
-                return -EINVAL;
-        }
+static void ipu_plane_atomic_set_base(struct ipu_plane *ipu_plane,
+                                      struct drm_plane_state *old_state)
+{
+        struct drm_plane *plane = &ipu_plane->base;
+        struct drm_plane_state *state = plane->state;
+        struct drm_framebuffer *fb = state->fb;
+        unsigned long eba, ubo, vbo;
+        int active;
 
-        ipu_plane->stride[0] = fb->pitches[0];
+        eba = drm_plane_state_to_eba(state);
 
         switch (fb->pixel_format) {
         case DRM_FORMAT_YUV420:
         case DRM_FORMAT_YVU420:
+                if (old_state->fb)
+                        break;
+
                 /*
                  * Multiplanar formats have to meet the following restrictions:
                  * - The (up to) three plane addresses are EBA, EBA+UBO, EBA+VBO
@@ -117,59 +128,28 @@ int ipu_plane_set_base(struct ipu_plane *ipu_plane, struct drm_framebuffer *fb,
                  * - Only EBA may be changed while scanout is active
                  * - The strides of U and V planes must be identical.
                  */
-                ubo = cma_obj[1]->paddr + fb->offsets[1] +
-                      fb->pitches[1] * y / 2 + x / 2 - eba;
-                vbo = cma_obj[2]->paddr + fb->offsets[2] +
-                      fb->pitches[2] * y / 2 + x / 2 - eba;
+                ubo = drm_plane_state_to_ubo(state);
+                vbo = drm_plane_state_to_vbo(state);
 
-                if ((ubo & 0x7) || (vbo & 0x7)) {
-                        DRM_DEBUG_KMS("U/V buffer offsets must be a multiple of 8.\n");
-                        return -EINVAL;
-                }
-
-                if ((ubo > 0xfffff8) || (vbo > 0xfffff8)) {
-                        DRM_DEBUG_KMS("U/V buffer offsets must be positive and not larger than 0xfffff8.\n");
-                        return -EINVAL;
-                }
-
-                if (ipu_plane->enabled && ((ipu_plane->u_offset != ubo) ||
-                                           (ipu_plane->v_offset != vbo))) {
-                        DRM_DEBUG_KMS("U/V buffer offsets must not change while plane is enabled.\n");
-                        return -EINVAL;
-                }
-
-                if (fb->pitches[1] != fb->pitches[2]) {
-                        DRM_DEBUG_KMS("U/V pitches must be identical.\n");
-                        return -EINVAL;
-                }
-
-                if (fb->pitches[1] < 1 || fb->pitches[1] > 16384) {
-                        DRM_DEBUG_KMS("U/V pitches out of range.\n");
-                        return -EINVAL;
-                }
-
-                if (ipu_plane->enabled &&
-                    (ipu_plane->stride[1] != fb->pitches[1])) {
-                        DRM_DEBUG_KMS("U/V pitches must not change while plane is enabled.\n");
-                        return -EINVAL;
-                }
-
-                ipu_plane->u_offset = ubo;
-                ipu_plane->v_offset = vbo;
-                ipu_plane->stride[1] = fb->pitches[1];
+                if (fb->pixel_format == DRM_FORMAT_YUV420)
+                        ipu_cpmem_set_yuv_planar_full(ipu_plane->ipu_ch,
+                                                      fb->pitches[1], ubo, vbo);
+                else
+                        ipu_cpmem_set_yuv_planar_full(ipu_plane->ipu_ch,
+                                                      fb->pitches[1], vbo, ubo);
 
                 dev_dbg(ipu_plane->base.dev->dev,
-                        "phys = %pad %pad %pad, x = %d, y = %d",
-                        &cma_obj[0]->paddr, &cma_obj[1]->paddr,
-                        &cma_obj[2]->paddr, x, y);
+                        "phy = %lu %lu %lu, x = %d, y = %d", eba, ubo, vbo,
+                        state->src_x >> 16, state->src_y >> 16);
                 break;
         default:
-                dev_dbg(ipu_plane->base.dev->dev, "phys = %pad, x = %d, y = %d",
-                        &cma_obj[0]->paddr, x, y);
+                dev_dbg(ipu_plane->base.dev->dev, "phys = %lu, x = %d, y = %d",
+                        eba, state->src_x >> 16, state->src_y >> 16);
+
                 break;
         }
 
-        if (ipu_plane->enabled) {
+        if (old_state->fb) {
                 active = ipu_idmac_get_current_buffer(ipu_plane->ipu_ch);
                 ipu_cpmem_set_buffer(ipu_plane->ipu_ch, !active, eba);
                 ipu_idmac_select_buffer(ipu_plane->ipu_ch, !active);
@@ -177,155 +157,6 @@ int ipu_plane_set_base(struct ipu_plane *ipu_plane, struct drm_framebuffer *fb,
                 ipu_cpmem_set_buffer(ipu_plane->ipu_ch, 0, eba);
                 ipu_cpmem_set_buffer(ipu_plane->ipu_ch, 1, eba);
         }
-
-        /* cache offsets for subsequent pageflips */
-        ipu_plane->x = x;
-        ipu_plane->y = y;
-
-        return 0;
-}
-
-int ipu_plane_mode_set(struct ipu_plane *ipu_plane, struct drm_crtc *crtc,
-                       struct drm_display_mode *mode,
-                       struct drm_framebuffer *fb, int crtc_x, int crtc_y,
-                       unsigned int crtc_w, unsigned int crtc_h,
-                       uint32_t src_x, uint32_t src_y,
-                       uint32_t src_w, uint32_t src_h, bool interlaced)
-{
-        struct device *dev = ipu_plane->base.dev->dev;
-        int ret;
-
-        /* no scaling */
-        if (src_w != crtc_w || src_h != crtc_h)
-                return -EINVAL;
-
-        /* clip to crtc bounds */
-        if (crtc_x < 0) {
-                if (-crtc_x > crtc_w)
-                        return -EINVAL;
-                src_x += -crtc_x;
-                src_w -= -crtc_x;
-                crtc_w -= -crtc_x;
-                crtc_x = 0;
-        }
-        if (crtc_y < 0) {
-                if (-crtc_y > crtc_h)
-                        return -EINVAL;
-                src_y += -crtc_y;
-                src_h -= -crtc_y;
-                crtc_h -= -crtc_y;
-                crtc_y = 0;
-        }
-        if (crtc_x + crtc_w > mode->hdisplay) {
-                if (crtc_x > mode->hdisplay)
-                        return -EINVAL;
-                crtc_w = mode->hdisplay - crtc_x;
-                src_w = crtc_w;
-        }
-        if (crtc_y + crtc_h > mode->vdisplay) {
-                if (crtc_y > mode->vdisplay)
-                        return -EINVAL;
-                crtc_h = mode->vdisplay - crtc_y;
-                src_h = crtc_h;
-        }
-        /* full plane minimum width is 13 pixels */
-        if (crtc_w < 13 && (ipu_plane->dp_flow != IPU_DP_FLOW_SYNC_FG))
-                return -EINVAL;
-        if (crtc_h < 2)
-                return -EINVAL;
-
-        /*
-         * since we cannot touch active IDMAC channels, we do not support
-         * resizing the enabled plane or changing its format
-         */
-        if (ipu_plane->enabled) {
-                if (src_w != ipu_plane->w || src_h != ipu_plane->h ||
-                    fb->pixel_format != ipu_plane->base.fb->pixel_format)
-                        return -EINVAL;
-
-                return ipu_plane_set_base(ipu_plane, fb, src_x, src_y);
-        }
-
-        switch (ipu_plane->dp_flow) {
-        case IPU_DP_FLOW_SYNC_BG:
-                ret = ipu_dp_setup_channel(ipu_plane->dp,
-                                IPUV3_COLORSPACE_RGB,
-                                IPUV3_COLORSPACE_RGB);
-                if (ret) {
-                        dev_err(dev,
-                                "initializing display processor failed with %d\n",
-                                ret);
-                        return ret;
-                }
-                ipu_dp_set_global_alpha(ipu_plane->dp, true, 0, true);
-                break;
-        case IPU_DP_FLOW_SYNC_FG:
-                ipu_dp_setup_channel(ipu_plane->dp,
-                                ipu_drm_fourcc_to_colorspace(fb->pixel_format),
-                                IPUV3_COLORSPACE_UNKNOWN);
-                ipu_dp_set_window_pos(ipu_plane->dp, crtc_x, crtc_y);
-                /* Enable local alpha on partial plane */
-                switch (fb->pixel_format) {
-                case DRM_FORMAT_ARGB1555:
-                case DRM_FORMAT_ABGR1555:
-                case DRM_FORMAT_RGBA5551:
-                case DRM_FORMAT_BGRA5551:
-                case DRM_FORMAT_ARGB4444:
-                case DRM_FORMAT_ARGB8888:
-                case DRM_FORMAT_ABGR8888:
-                case DRM_FORMAT_RGBA8888:
-                case DRM_FORMAT_BGRA8888:
-                        ipu_dp_set_global_alpha(ipu_plane->dp, false, 0, false);
-                        break;
-                default:
-                        break;
-                }
-        }
-
-        ret = ipu_dmfc_alloc_bandwidth(ipu_plane->dmfc,
-                        calc_bandwidth(crtc_w, crtc_h,
-                                       calc_vref(mode)), 64);
-        if (ret) {
-                dev_err(dev, "allocating dmfc bandwidth failed with %d\n", ret);
-                return ret;
-        }
-
-        ipu_dmfc_config_wait4eot(ipu_plane->dmfc, crtc_w);
-
-        ipu_cpmem_zero(ipu_plane->ipu_ch);
-        ipu_cpmem_set_resolution(ipu_plane->ipu_ch, src_w, src_h);
-        ret = ipu_cpmem_set_fmt(ipu_plane->ipu_ch, fb->pixel_format);
-        if (ret < 0) {
-                dev_err(dev, "unsupported pixel format 0x%08x\n",
-                        fb->pixel_format);
-                return ret;
-        }
-        ipu_cpmem_set_high_priority(ipu_plane->ipu_ch);
-        ipu_idmac_set_double_buffer(ipu_plane->ipu_ch, 1);
-        ipu_cpmem_set_stride(ipu_plane->ipu_ch, fb->pitches[0]);
-
-        ret = ipu_plane_set_base(ipu_plane, fb, src_x, src_y);
-        if (ret < 0)
-                return ret;
-        if (interlaced)
-                ipu_cpmem_interlaced_scan(ipu_plane->ipu_ch, fb->pitches[0]);
-
-        if (fb->pixel_format == DRM_FORMAT_YUV420) {
-                ipu_cpmem_set_yuv_planar_full(ipu_plane->ipu_ch,
-                                              ipu_plane->stride[1],
-                                              ipu_plane->u_offset,
-                                              ipu_plane->v_offset);
-        } else if (fb->pixel_format == DRM_FORMAT_YVU420) {
-                ipu_cpmem_set_yuv_planar_full(ipu_plane->ipu_ch,
-                                              ipu_plane->stride[1],
-                                              ipu_plane->v_offset,
-                                              ipu_plane->u_offset);
-        }
-
-        ipu_plane->w = src_w;
-        ipu_plane->h = src_h;
-
-        return 0;
 }
 
 void ipu_plane_put_resources(struct ipu_plane *ipu_plane)
@@ -372,7 +203,7 @@ err_out:
         return ret;
 }
 
-void ipu_plane_enable(struct ipu_plane *ipu_plane)
+static void ipu_plane_enable(struct ipu_plane *ipu_plane)
 {
         if (ipu_plane->dp)
                 ipu_dp_enable(ipu_plane->ipu);
@@ -380,14 +211,10 @@ void ipu_plane_enable(struct ipu_plane *ipu_plane)
         ipu_idmac_enable_channel(ipu_plane->ipu_ch);
         if (ipu_plane->dp)
                 ipu_dp_enable_channel(ipu_plane->dp);
-
-        ipu_plane->enabled = true;
 }
 
-void ipu_plane_disable(struct ipu_plane *ipu_plane)
+static void ipu_plane_disable(struct ipu_plane *ipu_plane)
 {
-        ipu_plane->enabled = false;
-
         ipu_idmac_wait_busy(ipu_plane->ipu_ch, 50);
 
         if (ipu_plane->dp)
@@ -398,74 +225,225 @@ void ipu_plane_disable(struct ipu_plane *ipu_plane)
                 ipu_dp_disable(ipu_plane->ipu);
 }
 
-/*
- * drm_plane API
- */
-
-static int ipu_update_plane(struct drm_plane *plane, struct drm_crtc *crtc,
-                            struct drm_framebuffer *fb, int crtc_x, int crtc_y,
-                            unsigned int crtc_w, unsigned int crtc_h,
-                            uint32_t src_x, uint32_t src_y,
-                            uint32_t src_w, uint32_t src_h)
+static int ipu_disable_plane(struct drm_plane *plane)
 {
         struct ipu_plane *ipu_plane = to_ipu_plane(plane);
-        int ret = 0;
-
-        DRM_DEBUG_KMS("plane - %p\n", plane);
-
-        if (!ipu_plane->enabled)
-                ret = ipu_plane_get_resources(ipu_plane);
-        if (ret < 0)
-                return ret;
-
-        ret = ipu_plane_mode_set(ipu_plane, crtc, &crtc->hwmode, fb,
-                        crtc_x, crtc_y, crtc_w, crtc_h,
-                        src_x >> 16, src_y >> 16, src_w >> 16, src_h >> 16,
-                        false);
-        if (ret < 0) {
-                ipu_plane_put_resources(ipu_plane);
-                return ret;
-        }
 
-        if (crtc != plane->crtc)
-                dev_dbg(plane->dev->dev, "crtc change: %p -> %p\n",
-                                plane->crtc, crtc);
+        DRM_DEBUG_KMS("[%d] %s\n", __LINE__, __func__);
 
-        if (!ipu_plane->enabled)
-                ipu_plane_enable(ipu_plane);
+        ipu_plane_disable(ipu_plane);
 
         return 0;
 }
 
-static int ipu_disable_plane(struct drm_plane *plane)
+static void ipu_plane_destroy(struct drm_plane *plane)
 {
         struct ipu_plane *ipu_plane = to_ipu_plane(plane);
 
         DRM_DEBUG_KMS("[%d] %s\n", __LINE__, __func__);
 
-        if (ipu_plane->enabled)
-                ipu_plane_disable(ipu_plane);
+        ipu_disable_plane(plane);
+        drm_plane_cleanup(plane);
+        kfree(ipu_plane);
+}
 
-        ipu_plane_put_resources(ipu_plane);
+static const struct drm_plane_funcs ipu_plane_funcs = {
+        .update_plane   = drm_atomic_helper_update_plane,
+        .disable_plane  = drm_atomic_helper_disable_plane,
+        .destroy        = ipu_plane_destroy,
+        .reset          = drm_atomic_helper_plane_reset,
+        .atomic_duplicate_state = drm_atomic_helper_plane_duplicate_state,
+        .atomic_destroy_state   = drm_atomic_helper_plane_destroy_state,
+};
+
+static int ipu_plane_atomic_check(struct drm_plane *plane,
+                                  struct drm_plane_state *state)
+{
+        struct drm_plane_state *old_state = plane->state;
+        struct drm_crtc_state *crtc_state;
+        struct device *dev = plane->dev->dev;
+        struct drm_framebuffer *fb = state->fb;
+        struct drm_framebuffer *old_fb = old_state->fb;
+        unsigned long eba, ubo, vbo, old_ubo, old_vbo;
+
+        /* Ok to disable */
+        if (!fb)
+                return 0;
+
+        if (!state->crtc)
+                return -EINVAL;
+
+        crtc_state =
+                drm_atomic_get_existing_crtc_state(state->state, state->crtc);
+        if (WARN_ON(!crtc_state))
+                return -EINVAL;
+
+        /* CRTC should be enabled */
+        if (!crtc_state->enable)
+                return -EINVAL;
+
+        /* no scaling */
+        if (state->src_w >> 16 != state->crtc_w ||
+            state->src_h >> 16 != state->crtc_h)
+                return -EINVAL;
+
+        switch (plane->type) {
+        case DRM_PLANE_TYPE_PRIMARY:
+                /* full plane doesn't support partial off screen */
+                if (state->crtc_x || state->crtc_y ||
+                    state->crtc_w != crtc_state->adjusted_mode.hdisplay ||
+                    state->crtc_h != crtc_state->adjusted_mode.vdisplay)
+                        return -EINVAL;
+
+                /* full plane minimum width is 13 pixels */
+                if (state->crtc_w < 13)
+                        return -EINVAL;
+                break;
+        case DRM_PLANE_TYPE_OVERLAY:
+                if (state->crtc_x < 0 || state->crtc_y < 0)
+                        return -EINVAL;
+
+                if (state->crtc_x + state->crtc_w >
+                    crtc_state->adjusted_mode.hdisplay)
+                        return -EINVAL;
+                if (state->crtc_y + state->crtc_h >
+                    crtc_state->adjusted_mode.vdisplay)
+                        return -EINVAL;
+                break;
+        default:
+                dev_warn(dev, "Unsupported plane type\n");
+                return -EINVAL;
+        }
+
+        if (state->crtc_h < 2)
+                return -EINVAL;
+
+        /*
+         * since we cannot touch active IDMAC channels, we do not support
+         * resizing the enabled plane or changing its format
+         */
+        if (old_fb && (state->src_w != old_state->src_w ||
+                              state->src_h != old_state->src_h ||
+                              fb->pixel_format != old_fb->pixel_format))
+                return -EINVAL;
+
+        eba = drm_plane_state_to_eba(state);
+
+        if (eba & 0x7)
+                return -EINVAL;
+
+        if (fb->pitches[0] < 1 || fb->pitches[0] > 16384)
+                return -EINVAL;
+
+        if (old_fb && fb->pitches[0] != old_fb->pitches[0])
+                return -EINVAL;
+
+        switch (fb->pixel_format) {
+        case DRM_FORMAT_YUV420:
+        case DRM_FORMAT_YVU420:
+                /*
+                 * Multiplanar formats have to meet the following restrictions:
+                 * - The (up to) three plane addresses are EBA, EBA+UBO, EBA+VBO
+                 * - EBA, UBO and VBO are a multiple of 8
+                 * - UBO and VBO are unsigned and not larger than 0xfffff8
+                 * - Only EBA may be changed while scanout is active
+                 * - The strides of U and V planes must be identical.
+                 */
+                ubo = drm_plane_state_to_ubo(state);
+                vbo = drm_plane_state_to_vbo(state);
+
+                if ((ubo & 0x7) || (vbo & 0x7))
+                        return -EINVAL;
+
+                if ((ubo > 0xfffff8) || (vbo > 0xfffff8))
+                        return -EINVAL;
+
+                if (old_fb) {
+                        old_ubo = drm_plane_state_to_ubo(old_state);
+                        old_vbo = drm_plane_state_to_vbo(old_state);
+                        if (ubo != old_ubo || vbo != old_vbo)
+                                return -EINVAL;
+                }
+
+                if (fb->pitches[1] != fb->pitches[2])
+                        return -EINVAL;
+
+                if (fb->pitches[1] < 1 || fb->pitches[1] > 16384)
+                        return -EINVAL;
+
+                if (old_fb && old_fb->pitches[1] != fb->pitches[1])
+                        return -EINVAL;
+        }
 
         return 0;
 }
 
-static void ipu_plane_destroy(struct drm_plane *plane)
+static void ipu_plane_atomic_disable(struct drm_plane *plane,
+                                     struct drm_plane_state *old_state)
+{
+        ipu_disable_plane(plane);
+}
+
+static void ipu_plane_atomic_update(struct drm_plane *plane,
+                                    struct drm_plane_state *old_state)
 {
         struct ipu_plane *ipu_plane = to_ipu_plane(plane);
+        struct drm_plane_state *state = plane->state;
+        enum ipu_color_space ics;
 
-        DRM_DEBUG_KMS("[%d] %s\n", __LINE__, __func__);
+        if (old_state->fb) {
+                ipu_plane_atomic_set_base(ipu_plane, old_state);
+                return;
+        }
 
-        ipu_disable_plane(plane);
-        drm_plane_cleanup(plane);
-        kfree(ipu_plane);
+        switch (ipu_plane->dp_flow) {
+        case IPU_DP_FLOW_SYNC_BG:
+                ipu_dp_setup_channel(ipu_plane->dp,
+                                        IPUV3_COLORSPACE_RGB,
+                                        IPUV3_COLORSPACE_RGB);
+                ipu_dp_set_global_alpha(ipu_plane->dp, true, 0, true);
+                break;
+        case IPU_DP_FLOW_SYNC_FG:
+                ics = ipu_drm_fourcc_to_colorspace(state->fb->pixel_format);
+                ipu_dp_setup_channel(ipu_plane->dp, ics,
+                                        IPUV3_COLORSPACE_UNKNOWN);
+                ipu_dp_set_window_pos(ipu_plane->dp, state->crtc_x,
+                                        state->crtc_y);
+                /* Enable local alpha on partial plane */
+                switch (state->fb->pixel_format) {
+                case DRM_FORMAT_ARGB1555:
+                case DRM_FORMAT_ABGR1555:
+                case DRM_FORMAT_RGBA5551:
+                case DRM_FORMAT_BGRA5551:
+                case DRM_FORMAT_ARGB4444:
+                case DRM_FORMAT_ARGB8888:
+                case DRM_FORMAT_ABGR8888:
+                case DRM_FORMAT_RGBA8888:
+                case DRM_FORMAT_BGRA8888:
+                        ipu_dp_set_global_alpha(ipu_plane->dp, false, 0, false);
+                        break;
+                default:
+                        break;
+                }
+        }
+
+        ipu_dmfc_config_wait4eot(ipu_plane->dmfc, state->crtc_w);
+
+        ipu_cpmem_zero(ipu_plane->ipu_ch);
+        ipu_cpmem_set_resolution(ipu_plane->ipu_ch, state->src_w >> 16,
+                                        state->src_h >> 16);
+        ipu_cpmem_set_fmt(ipu_plane->ipu_ch, state->fb->pixel_format);
+        ipu_cpmem_set_high_priority(ipu_plane->ipu_ch);
+        ipu_idmac_set_double_buffer(ipu_plane->ipu_ch, 1);
+        ipu_cpmem_set_stride(ipu_plane->ipu_ch, state->fb->pitches[0]);
+        ipu_plane_atomic_set_base(ipu_plane, old_state);
+        ipu_plane_enable(ipu_plane);
 }
 
-static const struct drm_plane_funcs ipu_plane_funcs = {
-        .update_plane   = ipu_update_plane,
-        .disable_plane  = ipu_disable_plane,
-        .destroy        = ipu_plane_destroy,
+static const struct drm_plane_helper_funcs ipu_plane_helper_funcs = {
+        .atomic_check = ipu_plane_atomic_check,
+        .atomic_disable = ipu_plane_atomic_disable,
+        .atomic_update = ipu_plane_atomic_update,
 };
 
 struct ipu_plane *ipu_plane_init(struct drm_device *dev, struct ipu_soc *ipu,
@@ -498,5 +476,7 @@ struct ipu_plane *ipu_plane_init(struct drm_device *dev, struct ipu_soc *ipu,
                 return ERR_PTR(ret);
         }
 
+        drm_plane_helper_add(&ipu_plane->base, &ipu_plane_helper_funcs);
+
         return ipu_plane;
 }
diff --git a/drivers/gpu/drm/imx/ipuv3-plane.h b/drivers/gpu/drm/imx/ipuv3-plane.h
index 4448fd4ad4eb..338b88a74eb6 100644
--- a/drivers/gpu/drm/imx/ipuv3-plane.h
+++ b/drivers/gpu/drm/imx/ipuv3-plane.h
@@ -23,17 +23,6 @@ struct ipu_plane {
 
         int                     dma;
         int                     dp_flow;
-
-        int                     x;
-        int                     y;
-        int                     w;
-        int                     h;
-
-        unsigned int            u_offset;
-        unsigned int            v_offset;
-        unsigned int            stride[2];
-
-        bool                    enabled;
 };
 
 struct ipu_plane *ipu_plane_init(struct drm_device *dev, struct ipu_soc *ipu,
@@ -48,11 +37,6 @@ int ipu_plane_mode_set(struct ipu_plane *plane, struct drm_crtc *crtc,
                        uint32_t src_x, uint32_t src_y, uint32_t src_w,
                        uint32_t src_h, bool interlaced);
 
-void ipu_plane_enable(struct ipu_plane *plane);
-void ipu_plane_disable(struct ipu_plane *plane);
-int ipu_plane_set_base(struct ipu_plane *plane, struct drm_framebuffer *fb,
-                       int x, int y);
-
 int ipu_plane_get_resources(struct ipu_plane *plane);
 void ipu_plane_put_resources(struct ipu_plane *plane);
 
diff --git a/drivers/gpu/drm/imx/parallel-display.c b/drivers/gpu/drm/imx/parallel-display.c
index 2d1fd02cd3d6..1dad297b01fd 100644
--- a/drivers/gpu/drm/imx/parallel-display.c
+++ b/drivers/gpu/drm/imx/parallel-display.c
@@ -16,6 +16,7 @@
 #include <linux/component.h>
 #include <linux/module.h>
 #include <drm/drmP.h>
+#include <drm/drm_atomic_helper.h>
 #include <drm/drm_fb_helper.h>
 #include <drm/drm_crtc_helper.h>
 #include <drm/drm_panel.h>
@@ -25,9 +26,6 @@
 
 #include "imx-drm.h"
 
-#define con_to_imxpd(x) container_of(x, struct imx_parallel_display, connector)
-#define enc_to_imxpd(x) container_of(x, struct imx_parallel_display, encoder)
-
 struct imx_parallel_display {
         struct drm_connector connector;
         struct drm_encoder encoder;
@@ -37,8 +35,19 @@ struct imx_parallel_display {
         u32 bus_format;
         struct drm_display_mode mode;
         struct drm_panel *panel;
+        struct drm_bridge *bridge;
 };
 
+static inline struct imx_parallel_display *con_to_imxpd(struct drm_connector *c)
+{
+        return container_of(c, struct imx_parallel_display, connector);
+}
+
+static inline struct imx_parallel_display *enc_to_imxpd(struct drm_encoder *e)
+{
+        return container_of(e, struct imx_parallel_display, encoder);
+}
+
 static enum drm_connector_status imx_pd_connector_detect(
                 struct drm_connector *connector, bool force)
 {
@@ -53,11 +62,7 @@ static int imx_pd_connector_get_modes(struct drm_connector *connector)
 
         if (imxpd->panel && imxpd->panel->funcs &&
             imxpd->panel->funcs->get_modes) {
-                struct drm_display_info *di = &connector->display_info;
-
                 num_modes = imxpd->panel->funcs->get_modes(imxpd->panel);
-                if (!imxpd->bus_format && di->num_bus_formats)
-                        imxpd->bus_format = di->bus_formats[0];
                 if (num_modes > 0)
                         return num_modes;
         }
@@ -69,10 +74,16 @@ static int imx_pd_connector_get_modes(struct drm_connector *connector)
 
         if (np) {
                 struct drm_display_mode *mode = drm_mode_create(connector->dev);
+                int ret;
 
                 if (!mode)
                         return -EINVAL;
-                of_get_drm_display_mode(np, &imxpd->mode, OF_USE_NATIVE_MODE);
+
+                ret = of_get_drm_display_mode(np, &imxpd->mode,
+                                              OF_USE_NATIVE_MODE);
+                if (ret)
+                        return ret;
+
                 drm_mode_copy(mode, &imxpd->mode);
                 mode->type |= DRM_MODE_TYPE_DRIVER | DRM_MODE_TYPE_PREFERRED,
                 drm_mode_probed_add(connector, mode);
@@ -90,24 +101,7 @@ static struct drm_encoder *imx_pd_connector_best_encoder(
         return &imxpd->encoder;
 }
 
-static void imx_pd_encoder_dpms(struct drm_encoder *encoder, int mode)
-{
-        struct imx_parallel_display *imxpd = enc_to_imxpd(encoder);
-
-        if (mode != DRM_MODE_DPMS_ON)
-                drm_panel_disable(imxpd->panel);
-        else
-                drm_panel_enable(imxpd->panel);
-}
-
-static void imx_pd_encoder_prepare(struct drm_encoder *encoder)
-{
-        struct imx_parallel_display *imxpd = enc_to_imxpd(encoder);
-        imx_drm_set_bus_config(encoder, imxpd->bus_format, 2, 3,
-                               imxpd->connector.display_info.bus_flags);
-}
-
-static void imx_pd_encoder_commit(struct drm_encoder *encoder)
+static void imx_pd_encoder_enable(struct drm_encoder *encoder)
 {
         struct imx_parallel_display *imxpd = enc_to_imxpd(encoder);
 
@@ -115,12 +109,6 @@ static void imx_pd_encoder_commit(struct drm_encoder *encoder)
         drm_panel_enable(imxpd->panel);
 }
 
-static void imx_pd_encoder_mode_set(struct drm_encoder *encoder,
-                         struct drm_display_mode *orig_mode,
-                         struct drm_display_mode *mode)
-{
-}
-
 static void imx_pd_encoder_disable(struct drm_encoder *encoder)
 {
         struct imx_parallel_display *imxpd = enc_to_imxpd(encoder);
@@ -129,11 +117,33 @@ static void imx_pd_encoder_disable(struct drm_encoder *encoder)
         drm_panel_unprepare(imxpd->panel);
 }
 
+static int imx_pd_encoder_atomic_check(struct drm_encoder *encoder,
+                                       struct drm_crtc_state *crtc_state,
+                                       struct drm_connector_state *conn_state)
+{
+        struct imx_crtc_state *imx_crtc_state = to_imx_crtc_state(crtc_state);
+        struct drm_display_info *di = &conn_state->connector->display_info;
+        struct imx_parallel_display *imxpd = enc_to_imxpd(encoder);
+
+        imx_crtc_state->bus_flags = di->bus_flags;
+        if (!imxpd->bus_format && di->num_bus_formats)
+                imx_crtc_state->bus_format = di->bus_formats[0];
+        else
+                imx_crtc_state->bus_format = imxpd->bus_format;
+        imx_crtc_state->di_hsync_pin = 2;
+        imx_crtc_state->di_vsync_pin = 3;
+
+        return 0;
+}
+
 static const struct drm_connector_funcs imx_pd_connector_funcs = {
-        .dpms = drm_helper_connector_dpms,
+        .dpms = drm_atomic_helper_connector_dpms,
         .fill_modes = drm_helper_probe_single_connector_modes,
         .detect = imx_pd_connector_detect,
         .destroy = imx_drm_connector_destroy,
+        .reset = drm_atomic_helper_connector_reset,
+        .atomic_duplicate_state = drm_atomic_helper_connector_duplicate_state,
+        .atomic_destroy_state = drm_atomic_helper_connector_destroy_state,
 };
 
 static const struct drm_connector_helper_funcs imx_pd_connector_helper_funcs = {
@@ -146,20 +156,18 @@ static const struct drm_encoder_funcs imx_pd_encoder_funcs = {
 };
 
 static const struct drm_encoder_helper_funcs imx_pd_encoder_helper_funcs = {
-        .dpms = imx_pd_encoder_dpms,
-        .prepare = imx_pd_encoder_prepare,
-        .commit = imx_pd_encoder_commit,
-        .mode_set = imx_pd_encoder_mode_set,
+        .enable = imx_pd_encoder_enable,
         .disable = imx_pd_encoder_disable,
+        .atomic_check = imx_pd_encoder_atomic_check,
 };
 
 static int imx_pd_register(struct drm_device *drm,
         struct imx_parallel_display *imxpd)
 {
+        struct drm_encoder *encoder = &imxpd->encoder;
         int ret;
 
-        ret = imx_drm_encoder_parse_of(drm, &imxpd->encoder,
-                                       imxpd->dev->of_node);
+        ret = imx_drm_encoder_parse_of(drm, encoder, imxpd->dev->of_node);
         if (ret)
                 return ret;
 
@@ -170,19 +178,33 @@ static int imx_pd_register(struct drm_device *drm,
          */
         imxpd->connector.dpms = DRM_MODE_DPMS_OFF;
 
-        drm_encoder_helper_add(&imxpd->encoder, &imx_pd_encoder_helper_funcs);
-        drm_encoder_init(drm, &imxpd->encoder, &imx_pd_encoder_funcs,
+        drm_encoder_helper_add(encoder, &imx_pd_encoder_helper_funcs);
+        drm_encoder_init(drm, encoder, &imx_pd_encoder_funcs,
                          DRM_MODE_ENCODER_NONE, NULL);
 
-        drm_connector_helper_add(&imxpd->connector,
-                        &imx_pd_connector_helper_funcs);
-        drm_connector_init(drm, &imxpd->connector, &imx_pd_connector_funcs,
-                           DRM_MODE_CONNECTOR_VGA);
+        if (!imxpd->bridge) {
+                drm_connector_helper_add(&imxpd->connector,
+                                &imx_pd_connector_helper_funcs);
+                drm_connector_init(drm, &imxpd->connector,
+                                   &imx_pd_connector_funcs,
+                                   DRM_MODE_CONNECTOR_VGA);
+        }
 
         if (imxpd->panel)
                 drm_panel_attach(imxpd->panel, &imxpd->connector);
 
-        drm_mode_connector_attach_encoder(&imxpd->connector, &imxpd->encoder);
+        if (imxpd->bridge) {
+                imxpd->bridge->encoder = encoder;
+                encoder->bridge = imxpd->bridge;
+                ret = drm_bridge_attach(drm, imxpd->bridge);
+                if (ret < 0) {
+                        dev_err(imxpd->dev, "failed to attach bridge: %d\n",
+                                ret);
+                        return ret;
+                }
+        } else {
+                drm_mode_connector_attach_encoder(&imxpd->connector, encoder);
+        }
 
         return 0;
 }
@@ -195,6 +217,7 @@ static int imx_pd_bind(struct device *dev, struct device *master, void *data)
         const u8 *edidp;
         struct imx_parallel_display *imxpd;
         int ret;
+        u32 bus_format = 0;
         const char *fmt;
 
         imxpd = devm_kzalloc(dev, sizeof(*imxpd), GFP_KERNEL);
@@ -208,14 +231,15 @@ static int imx_pd_bind(struct device *dev, struct device *master, void *data)
         ret = of_property_read_string(np, "interface-pix-fmt", &fmt);
         if (!ret) {
                 if (!strcmp(fmt, "rgb24"))
-                        imxpd->bus_format = MEDIA_BUS_FMT_RGB888_1X24;
+                        bus_format = MEDIA_BUS_FMT_RGB888_1X24;
                 else if (!strcmp(fmt, "rgb565"))
-                        imxpd->bus_format = MEDIA_BUS_FMT_RGB565_1X16;
+                        bus_format = MEDIA_BUS_FMT_RGB565_1X16;
                 else if (!strcmp(fmt, "bgr666"))
-                        imxpd->bus_format = MEDIA_BUS_FMT_RGB666_1X18;
+                        bus_format = MEDIA_BUS_FMT_RGB666_1X18;
                 else if (!strcmp(fmt, "lvds666"))
-                        imxpd->bus_format = MEDIA_BUS_FMT_RGB666_1X24_CPADHI;
+                        bus_format = MEDIA_BUS_FMT_RGB666_1X24_CPADHI;
         }
+        imxpd->bus_format = bus_format;
 
         /* port@1 is the output port */
         ep = of_graph_get_endpoint_by_regs(np, 1, -1);
@@ -223,13 +247,30 @@ static int imx_pd_bind(struct device *dev, struct device *master, void *data)
                 struct device_node *remote;
 
                 remote = of_graph_get_remote_port_parent(ep);
+                if (!remote) {
+                        dev_warn(dev, "endpoint %s not connected\n",
+                                 ep->full_name);
+                        of_node_put(ep);
+                        return -ENODEV;
+                }
                 of_node_put(ep);
-                if (remote) {
-                        imxpd->panel = of_drm_find_panel(remote);
-                        of_node_put(remote);
+
+                imxpd->panel = of_drm_find_panel(remote);
+                if (imxpd->panel) {
+                        dev_dbg(dev, "found panel %s\n", remote->full_name);
+                } else {
+                        imxpd->bridge = of_drm_find_bridge(remote);
+                        if (imxpd->bridge)
+                                dev_dbg(dev, "found bridge %s\n",
+                                        remote->full_name);
                 }
-                if (!imxpd->panel)
+                if (!imxpd->panel && !imxpd->bridge) {
+                        dev_dbg(dev, "waiting for panel or bridge %s\n",
+                                remote->full_name);
+                        of_node_put(remote);
                         return -EPROBE_DEFER;
+                }
+                of_node_put(remote);
         }
 
         imxpd->dev = dev;
diff --git a/drivers/gpu/drm/mediatek/Kconfig b/drivers/gpu/drm/mediatek/Kconfig
index eeefc971801a..23ac8041c562 100644
--- a/drivers/gpu/drm/mediatek/Kconfig
+++ b/drivers/gpu/drm/mediatek/Kconfig
@@ -6,7 +6,6 @@ config DRM_MEDIATEK
         select DRM_KMS_HELPER
         select DRM_MIPI_DSI
         select DRM_PANEL
-        select IOMMU_DMA
         select MEMORY
         select MTK_SMI
         help
@@ -14,3 +13,11 @@ config DRM_MEDIATEK
           The module will be called mediatek-drm
           This driver provides kernel mode setting and
           buffer management to userspace.
+
+config DRM_MEDIATEK_HDMI
+        tristate "DRM HDMI Support for Mediatek SoCs"
+        depends on DRM_MEDIATEK
+        select SND_SOC_HDMI_CODEC if SND_SOC
+        select GENERIC_PHY
+        help
+          DRM/KMS HDMI driver for Mediatek SoCs
diff --git a/drivers/gpu/drm/mediatek/Makefile b/drivers/gpu/drm/mediatek/Makefile
index 5fcf58e87786..bf2e5be1ab30 100644
--- a/drivers/gpu/drm/mediatek/Makefile
+++ b/drivers/gpu/drm/mediatek/Makefile
@@ -12,3 +12,10 @@ mediatek-drm-y := mtk_disp_ovl.o \
                   mtk_dpi.o
 
 obj-$(CONFIG_DRM_MEDIATEK) += mediatek-drm.o
+
+mediatek-drm-hdmi-objs := mtk_cec.o \
+                          mtk_hdmi.o \
+                          mtk_hdmi_ddc.o \
+                          mtk_mt8173_hdmi_phy.o
+
+obj-$(CONFIG_DRM_MEDIATEK_HDMI) += mediatek-drm-hdmi.o
diff --git a/drivers/gpu/drm/mediatek/mtk_cec.c b/drivers/gpu/drm/mediatek/mtk_cec.c
new file mode 100644
index 000000000000..7a3eb8c17ef9
--- /dev/null
+++ b/drivers/gpu/drm/mediatek/mtk_cec.c
@@ -0,0 +1,265 @@
+/*
+ * Copyright (c) 2014 MediaTek Inc.
+ * Author: Jie Qiu <jie.qiu@mediatek.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+#include <linux/clk.h>
+#include <linux/delay.h>
+#include <linux/io.h>
+#include <linux/interrupt.h>
+#include <linux/platform_device.h>
+
+#include "mtk_cec.h"
+
+#define TR_CONFIG               0x00
+#define CLEAR_CEC_IRQ                   BIT(15)
+
+#define CEC_CKGEN               0x04
+#define CEC_32K_PDN                     BIT(19)
+#define PDN                             BIT(16)
+
+#define RX_EVENT                0x54
+#define HDMI_PORD                       BIT(25)
+#define HDMI_HTPLG                      BIT(24)
+#define HDMI_PORD_INT_EN                BIT(9)
+#define HDMI_HTPLG_INT_EN               BIT(8)
+
+#define RX_GEN_WD               0x58
+#define HDMI_PORD_INT_32K_STATUS        BIT(26)
+#define RX_RISC_INT_32K_STATUS          BIT(25)
+#define HDMI_HTPLG_INT_32K_STATUS       BIT(24)
+#define HDMI_PORD_INT_32K_CLR           BIT(18)
+#define RX_INT_32K_CLR                  BIT(17)
+#define HDMI_HTPLG_INT_32K_CLR          BIT(16)
+#define HDMI_PORD_INT_32K_STA_MASK      BIT(10)
+#define RX_RISC_INT_32K_STA_MASK        BIT(9)
+#define HDMI_HTPLG_INT_32K_STA_MASK     BIT(8)
+#define HDMI_PORD_INT_32K_EN            BIT(2)
+#define RX_INT_32K_EN                   BIT(1)
+#define HDMI_HTPLG_INT_32K_EN           BIT(0)
+
+#define NORMAL_INT_CTRL         0x5C
+#define HDMI_HTPLG_INT_STA              BIT(0)
+#define HDMI_PORD_INT_STA               BIT(1)
+#define HDMI_HTPLG_INT_CLR              BIT(16)
+#define HDMI_PORD_INT_CLR               BIT(17)
+#define HDMI_FULL_INT_CLR               BIT(20)
+
+struct mtk_cec {
+        void __iomem *regs;
+        struct clk *clk;
+        int irq;
+        bool hpd;
+        void (*hpd_event)(bool hpd, struct device *dev);
+        struct device *hdmi_dev;
+        spinlock_t lock;
+};
+
+static void mtk_cec_clear_bits(struct mtk_cec *cec, unsigned int offset,
+                               unsigned int bits)
+{
+        void __iomem *reg = cec->regs + offset;
+        u32 tmp;
+
+        tmp = readl(reg);
+        tmp &= ~bits;
+        writel(tmp, reg);
+}
+
+static void mtk_cec_set_bits(struct mtk_cec *cec, unsigned int offset,
+                             unsigned int bits)
+{
+        void __iomem *reg = cec->regs + offset;
+        u32 tmp;
+
+        tmp = readl(reg);
+        tmp |= bits;
+        writel(tmp, reg);
+}
+
+static void mtk_cec_mask(struct mtk_cec *cec, unsigned int offset,
+                         unsigned int val, unsigned int mask)
+{
+        u32 tmp = readl(cec->regs + offset) & ~mask;
+
+        tmp |= val & mask;
+        writel(val, cec->regs + offset);
+}
+
+void mtk_cec_set_hpd_event(struct device *dev,
+                           void (*hpd_event)(bool hpd, struct device *dev),
+                           struct device *hdmi_dev)
+{
+        struct mtk_cec *cec = dev_get_drvdata(dev);
+        unsigned long flags;
+
+        spin_lock_irqsave(&cec->lock, flags);
+        cec->hdmi_dev = hdmi_dev;
+        cec->hpd_event = hpd_event;
+        spin_unlock_irqrestore(&cec->lock, flags);
+}
+
+bool mtk_cec_hpd_high(struct device *dev)
+{
+        struct mtk_cec *cec = dev_get_drvdata(dev);
+        unsigned int status;
+
+        status = readl(cec->regs + RX_EVENT);
+
+        return (status & (HDMI_PORD | HDMI_HTPLG)) == (HDMI_PORD | HDMI_HTPLG);
+}
+
+static void mtk_cec_htplg_irq_init(struct mtk_cec *cec)
+{
+        mtk_cec_mask(cec, CEC_CKGEN, 0 | CEC_32K_PDN, PDN | CEC_32K_PDN);
+        mtk_cec_set_bits(cec, RX_GEN_WD, HDMI_PORD_INT_32K_CLR |
+                         RX_INT_32K_CLR | HDMI_HTPLG_INT_32K_CLR);
+        mtk_cec_mask(cec, RX_GEN_WD, 0, HDMI_PORD_INT_32K_CLR | RX_INT_32K_CLR |
+                     HDMI_HTPLG_INT_32K_CLR | HDMI_PORD_INT_32K_EN |
+                     RX_INT_32K_EN | HDMI_HTPLG_INT_32K_EN);
+}
+
+static void mtk_cec_htplg_irq_enable(struct mtk_cec *cec)
+{
+        mtk_cec_set_bits(cec, RX_EVENT, HDMI_PORD_INT_EN | HDMI_HTPLG_INT_EN);
+}
+
+static void mtk_cec_htplg_irq_disable(struct mtk_cec *cec)
+{
+        mtk_cec_clear_bits(cec, RX_EVENT, HDMI_PORD_INT_EN | HDMI_HTPLG_INT_EN);
+}
+
+static void mtk_cec_clear_htplg_irq(struct mtk_cec *cec)
+{
+        mtk_cec_set_bits(cec, TR_CONFIG, CLEAR_CEC_IRQ);
+        mtk_cec_set_bits(cec, NORMAL_INT_CTRL, HDMI_HTPLG_INT_CLR |
+                         HDMI_PORD_INT_CLR | HDMI_FULL_INT_CLR);
+        mtk_cec_set_bits(cec, RX_GEN_WD, HDMI_PORD_INT_32K_CLR |
+                         RX_INT_32K_CLR | HDMI_HTPLG_INT_32K_CLR);
+        usleep_range(5, 10);
+        mtk_cec_clear_bits(cec, NORMAL_INT_CTRL, HDMI_HTPLG_INT_CLR |
+                           HDMI_PORD_INT_CLR | HDMI_FULL_INT_CLR);
+        mtk_cec_clear_bits(cec, TR_CONFIG, CLEAR_CEC_IRQ);
+        mtk_cec_clear_bits(cec, RX_GEN_WD, HDMI_PORD_INT_32K_CLR |
+                           RX_INT_32K_CLR | HDMI_HTPLG_INT_32K_CLR);
+}
+
+static void mtk_cec_hpd_event(struct mtk_cec *cec, bool hpd)
+{
+        void (*hpd_event)(bool hpd, struct device *dev);
+        struct device *hdmi_dev;
+        unsigned long flags;
+
+        spin_lock_irqsave(&cec->lock, flags);
+        hpd_event = cec->hpd_event;
+        hdmi_dev = cec->hdmi_dev;
+        spin_unlock_irqrestore(&cec->lock, flags);
+
+        if (hpd_event)
+                hpd_event(hpd, hdmi_dev);
+}
+
+static irqreturn_t mtk_cec_htplg_isr_thread(int irq, void *arg)
+{
+        struct device *dev = arg;
+        struct mtk_cec *cec = dev_get_drvdata(dev);
+        bool hpd;
+
+        mtk_cec_clear_htplg_irq(cec);
+        hpd = mtk_cec_hpd_high(dev);
+
+        if (cec->hpd != hpd) {
+                dev_dbg(dev, "hotplug event! cur hpd = %d, hpd = %d\n",
+                        cec->hpd, hpd);
+                cec->hpd = hpd;
+                mtk_cec_hpd_event(cec, hpd);
+        }
+        return IRQ_HANDLED;
+}
+
+static int mtk_cec_probe(struct platform_device *pdev)
+{
+        struct device *dev = &pdev->dev;
+        struct mtk_cec *cec;
+        struct resource *res;
+        int ret;
+
+        cec = devm_kzalloc(dev, sizeof(*cec), GFP_KERNEL);
+        if (!cec)
+                return -ENOMEM;
+
+        platform_set_drvdata(pdev, cec);
+        spin_lock_init(&cec->lock);
+
+        res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+        cec->regs = devm_ioremap_resource(dev, res);
+        if (IS_ERR(cec->regs)) {
+                ret = PTR_ERR(cec->regs);
+                dev_err(dev, "Failed to ioremap cec: %d\n", ret);
+                return ret;
+        }
+
+        cec->clk = devm_clk_get(dev, NULL);
+        if (IS_ERR(cec->clk)) {
+                ret = PTR_ERR(cec->clk);
+                dev_err(dev, "Failed to get cec clock: %d\n", ret);
+                return ret;
+        }
+
+        cec->irq = platform_get_irq(pdev, 0);
+        if (cec->irq < 0) {
+                dev_err(dev, "Failed to get cec irq: %d\n", cec->irq);
+                return cec->irq;
+        }
+
+        ret = devm_request_threaded_irq(dev, cec->irq, NULL,
+                                        mtk_cec_htplg_isr_thread,
+                                        IRQF_SHARED | IRQF_TRIGGER_LOW |
+                                        IRQF_ONESHOT, "hdmi hpd", dev);
+        if (ret) {
+                dev_err(dev, "Failed to register cec irq: %d\n", ret);
+                return ret;
+        }
+
+        ret = clk_prepare_enable(cec->clk);
+        if (ret) {
+                dev_err(dev, "Failed to enable cec clock: %d\n", ret);
+                return ret;
+        }
+
+        mtk_cec_htplg_irq_init(cec);
+        mtk_cec_htplg_irq_enable(cec);
+
+        return 0;
+}
+
+static int mtk_cec_remove(struct platform_device *pdev)
+{
+        struct mtk_cec *cec = platform_get_drvdata(pdev);
+
+        mtk_cec_htplg_irq_disable(cec);
+        clk_disable_unprepare(cec->clk);
+        return 0;
+}
+
+static const struct of_device_id mtk_cec_of_ids[] = {
+        { .compatible = "mediatek,mt8173-cec", },
+        {}
+};
+
+struct platform_driver mtk_cec_driver = {
+        .probe = mtk_cec_probe,
+        .remove = mtk_cec_remove,
+        .driver = {
+                .name = "mediatek-cec",
+                .of_match_table = mtk_cec_of_ids,
+        },
+};
diff --git a/drivers/gpu/drm/mediatek/mtk_cec.h b/drivers/gpu/drm/mediatek/mtk_cec.h
new file mode 100644
index 000000000000..10057b7eabec
--- /dev/null
+++ b/drivers/gpu/drm/mediatek/mtk_cec.h
@@ -0,0 +1,26 @@
+/*
+ * Copyright (c) 2014 MediaTek Inc.
+ * Author: Jie Qiu <jie.qiu@mediatek.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+#ifndef _MTK_CEC_H
+#define _MTK_CEC_H
+
+#include <linux/types.h>
+
+struct device;
+
+void mtk_cec_set_hpd_event(struct device *dev,
+                           void (*hotplug_event)(bool hpd, struct device *dev),
+                           struct device *hdmi_dev);
+bool mtk_cec_hpd_high(struct device *dev);
+
+#endif /* _MTK_CEC_H */
diff --git a/drivers/gpu/drm/mediatek/mtk_drm_drv.c b/drivers/gpu/drm/mediatek/mtk_drm_drv.c
index b1223d54d0ab..eebb7d881c2b 100644
--- a/drivers/gpu/drm/mediatek/mtk_drm_drv.c
+++ b/drivers/gpu/drm/mediatek/mtk_drm_drv.c
@@ -91,7 +91,7 @@ static int mtk_atomic_commit(struct drm_device *drm,
         mutex_lock(&private->commit.lock);
         flush_work(&private->commit.work);
 
-        drm_atomic_helper_swap_state(drm, state);
+        drm_atomic_helper_swap_state(state, true);
 
         if (async)
                 mtk_atomic_schedule(private, state);
@@ -243,7 +243,7 @@ static struct drm_driver mtk_drm_driver = {
         .enable_vblank = mtk_drm_crtc_enable_vblank,
         .disable_vblank = mtk_drm_crtc_disable_vblank,
 
-        .gem_free_object = mtk_drm_gem_free_object,
+        .gem_free_object_unlocked = mtk_drm_gem_free_object,
         .gem_vm_ops = &drm_gem_cma_vm_ops,
         .dumb_create = mtk_drm_gem_dumb_create,
         .dumb_map_offset = mtk_drm_gem_dumb_map_offset,
@@ -280,8 +280,6 @@ static int mtk_drm_bind(struct device *dev)
         if (!drm)
                 return -ENOMEM;
 
-        drm_dev_set_unique(drm, dev_name(dev));
-
         drm->dev_private = private;
         private->drm = drm;
 
@@ -293,14 +291,8 @@ static int mtk_drm_bind(struct device *dev)
         if (ret < 0)
                 goto err_deinit;
 
-        ret = drm_connector_register_all(drm);
-        if (ret < 0)
-                goto err_unregister;
-
         return 0;
 
-err_unregister:
-        drm_dev_unregister(drm);
 err_deinit:
         mtk_drm_kms_deinit(drm);
 err_free:
@@ -455,7 +447,6 @@ static int mtk_drm_remove(struct platform_device *pdev)
         struct drm_device *drm = private->drm;
         int i;
 
-        drm_connector_unregister_all(drm);
         drm_dev_unregister(drm);
         mtk_drm_kms_deinit(drm);
         drm_dev_unref(drm);
diff --git a/drivers/gpu/drm/mediatek/mtk_drm_plane.c b/drivers/gpu/drm/mediatek/mtk_drm_plane.c
index 51bc8988fc26..3995765a90dc 100644
--- a/drivers/gpu/drm/mediatek/mtk_drm_plane.c
+++ b/drivers/gpu/drm/mediatek/mtk_drm_plane.c
@@ -170,6 +170,7 @@ static int mtk_plane_atomic_check(struct drm_plane *plane,
 
         return drm_plane_helper_check_update(plane, state->crtc, fb,
                                              &src, &dest, &clip,
+                                             state->rotation,
                                              DRM_PLANE_HELPER_NO_SCALING,
                                              DRM_PLANE_HELPER_NO_SCALING,
                                              true, true, &visible);
diff --git a/drivers/gpu/drm/mediatek/mtk_dsi.c b/drivers/gpu/drm/mediatek/mtk_dsi.c
index 769559124562..28b2044ed9f2 100644
--- a/drivers/gpu/drm/mediatek/mtk_dsi.c
+++ b/drivers/gpu/drm/mediatek/mtk_dsi.c
@@ -575,14 +575,6 @@ static int mtk_dsi_connector_get_modes(struct drm_connector *connector)
         return drm_panel_get_modes(dsi->panel);
 }
 
-static struct drm_encoder *mtk_dsi_connector_best_encoder(
-                struct drm_connector *connector)
-{
-        struct mtk_dsi *dsi = connector_to_dsi(connector);
-
-        return &dsi->encoder;
-}
-
 static const struct drm_encoder_helper_funcs mtk_dsi_encoder_helper_funcs = {
         .mode_fixup = mtk_dsi_encoder_mode_fixup,
         .mode_set = mtk_dsi_encoder_mode_set,
@@ -603,7 +595,6 @@ static const struct drm_connector_funcs mtk_dsi_connector_funcs = {
 static const struct drm_connector_helper_funcs
         mtk_dsi_connector_helper_funcs = {
         .get_modes = mtk_dsi_connector_get_modes,
-        .best_encoder = mtk_dsi_connector_best_encoder,
 };
 
 static int mtk_drm_attach_bridge(struct drm_bridge *bridge,
diff --git a/drivers/gpu/drm/mediatek/mtk_hdmi.c b/drivers/gpu/drm/mediatek/mtk_hdmi.c
new file mode 100644
index 000000000000..334562d06731
--- /dev/null
+++ b/drivers/gpu/drm/mediatek/mtk_hdmi.c
@@ -0,0 +1,1828 @@
+/*
+ * Copyright (c) 2014 MediaTek Inc.
+ * Author: Jie Qiu <jie.qiu@mediatek.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+#include <drm/drmP.h>
+#include <drm/drm_atomic_helper.h>
+#include <drm/drm_crtc.h>
+#include <drm/drm_crtc_helper.h>
+#include <drm/drm_edid.h>
+#include <linux/arm-smccc.h>
+#include <linux/clk.h>
+#include <linux/delay.h>
+#include <linux/hdmi.h>
+#include <linux/i2c.h>
+#include <linux/io.h>
+#include <linux/kernel.h>
+#include <linux/mfd/syscon.h>
+#include <linux/of_platform.h>
+#include <linux/of.h>
+#include <linux/of_gpio.h>
+#include <linux/of_graph.h>
+#include <linux/phy/phy.h>
+#include <linux/platform_device.h>
+#include <linux/regmap.h>
+#include <sound/hdmi-codec.h>
+#include "mtk_cec.h"
+#include "mtk_hdmi.h"
+#include "mtk_hdmi_regs.h"
+
+#define NCTS_BYTES      7
+
+enum mtk_hdmi_clk_id {
+        MTK_HDMI_CLK_HDMI_PIXEL,
+        MTK_HDMI_CLK_HDMI_PLL,
+        MTK_HDMI_CLK_AUD_BCLK,
+        MTK_HDMI_CLK_AUD_SPDIF,
+        MTK_HDMI_CLK_COUNT
+};
+
+enum hdmi_aud_input_type {
+        HDMI_AUD_INPUT_I2S = 0,
+        HDMI_AUD_INPUT_SPDIF,
+};
+
+enum hdmi_aud_i2s_fmt {
+        HDMI_I2S_MODE_RJT_24BIT = 0,
+        HDMI_I2S_MODE_RJT_16BIT,
+        HDMI_I2S_MODE_LJT_24BIT,
+        HDMI_I2S_MODE_LJT_16BIT,
+        HDMI_I2S_MODE_I2S_24BIT,
+        HDMI_I2S_MODE_I2S_16BIT
+};
+
+enum hdmi_aud_mclk {
+        HDMI_AUD_MCLK_128FS,
+        HDMI_AUD_MCLK_192FS,
+        HDMI_AUD_MCLK_256FS,
+        HDMI_AUD_MCLK_384FS,
+        HDMI_AUD_MCLK_512FS,
+        HDMI_AUD_MCLK_768FS,
+        HDMI_AUD_MCLK_1152FS,
+};
+
+enum hdmi_aud_channel_type {
+        HDMI_AUD_CHAN_TYPE_1_0 = 0,
+        HDMI_AUD_CHAN_TYPE_1_1,
+        HDMI_AUD_CHAN_TYPE_2_0,
+        HDMI_AUD_CHAN_TYPE_2_1,
+        HDMI_AUD_CHAN_TYPE_3_0,
+        HDMI_AUD_CHAN_TYPE_3_1,
+        HDMI_AUD_CHAN_TYPE_4_0,
+        HDMI_AUD_CHAN_TYPE_4_1,
+        HDMI_AUD_CHAN_TYPE_5_0,
+        HDMI_AUD_CHAN_TYPE_5_1,
+        HDMI_AUD_CHAN_TYPE_6_0,
+        HDMI_AUD_CHAN_TYPE_6_1,
+        HDMI_AUD_CHAN_TYPE_7_0,
+        HDMI_AUD_CHAN_TYPE_7_1,
+        HDMI_AUD_CHAN_TYPE_3_0_LRS,
+        HDMI_AUD_CHAN_TYPE_3_1_LRS,
+        HDMI_AUD_CHAN_TYPE_4_0_CLRS,
+        HDMI_AUD_CHAN_TYPE_4_1_CLRS,
+        HDMI_AUD_CHAN_TYPE_6_1_CS,
+        HDMI_AUD_CHAN_TYPE_6_1_CH,
+        HDMI_AUD_CHAN_TYPE_6_1_OH,
+        HDMI_AUD_CHAN_TYPE_6_1_CHR,
+        HDMI_AUD_CHAN_TYPE_7_1_LH_RH,
+        HDMI_AUD_CHAN_TYPE_7_1_LSR_RSR,
+        HDMI_AUD_CHAN_TYPE_7_1_LC_RC,
+        HDMI_AUD_CHAN_TYPE_7_1_LW_RW,
+        HDMI_AUD_CHAN_TYPE_7_1_LSD_RSD,
+        HDMI_AUD_CHAN_TYPE_7_1_LSS_RSS,
+        HDMI_AUD_CHAN_TYPE_7_1_LHS_RHS,
+        HDMI_AUD_CHAN_TYPE_7_1_CS_CH,
+        HDMI_AUD_CHAN_TYPE_7_1_CS_OH,
+        HDMI_AUD_CHAN_TYPE_7_1_CS_CHR,
+        HDMI_AUD_CHAN_TYPE_7_1_CH_OH,
+        HDMI_AUD_CHAN_TYPE_7_1_CH_CHR,
+        HDMI_AUD_CHAN_TYPE_7_1_OH_CHR,
+        HDMI_AUD_CHAN_TYPE_7_1_LSS_RSS_LSR_RSR,
+        HDMI_AUD_CHAN_TYPE_6_0_CS,
+        HDMI_AUD_CHAN_TYPE_6_0_CH,
+        HDMI_AUD_CHAN_TYPE_6_0_OH,
+        HDMI_AUD_CHAN_TYPE_6_0_CHR,
+        HDMI_AUD_CHAN_TYPE_7_0_LH_RH,
+        HDMI_AUD_CHAN_TYPE_7_0_LSR_RSR,
+        HDMI_AUD_CHAN_TYPE_7_0_LC_RC,
+        HDMI_AUD_CHAN_TYPE_7_0_LW_RW,
+        HDMI_AUD_CHAN_TYPE_7_0_LSD_RSD,
+        HDMI_AUD_CHAN_TYPE_7_0_LSS_RSS,
+        HDMI_AUD_CHAN_TYPE_7_0_LHS_RHS,
+        HDMI_AUD_CHAN_TYPE_7_0_CS_CH,
+        HDMI_AUD_CHAN_TYPE_7_0_CS_OH,
+        HDMI_AUD_CHAN_TYPE_7_0_CS_CHR,
+        HDMI_AUD_CHAN_TYPE_7_0_CH_OH,
+        HDMI_AUD_CHAN_TYPE_7_0_CH_CHR,
+        HDMI_AUD_CHAN_TYPE_7_0_OH_CHR,
+        HDMI_AUD_CHAN_TYPE_7_0_LSS_RSS_LSR_RSR,
+        HDMI_AUD_CHAN_TYPE_8_0_LH_RH_CS,
+        HDMI_AUD_CHAN_TYPE_UNKNOWN = 0xFF
+};
+
+enum hdmi_aud_channel_swap_type {
+        HDMI_AUD_SWAP_LR,
+        HDMI_AUD_SWAP_LFE_CC,
+        HDMI_AUD_SWAP_LSRS,
+        HDMI_AUD_SWAP_RLS_RRS,
+        HDMI_AUD_SWAP_LR_STATUS,
+};
+
+struct hdmi_audio_param {
+        enum hdmi_audio_coding_type aud_codec;
+        enum hdmi_audio_sample_size aud_sampe_size;
+        enum hdmi_aud_input_type aud_input_type;
+        enum hdmi_aud_i2s_fmt aud_i2s_fmt;
+        enum hdmi_aud_mclk aud_mclk;
+        enum hdmi_aud_channel_type aud_input_chan_type;
+        struct hdmi_codec_params codec_params;
+};
+
+struct mtk_hdmi {
+        struct drm_bridge bridge;
+        struct drm_connector conn;
+        struct device *dev;
+        struct phy *phy;
+        struct device *cec_dev;
+        struct i2c_adapter *ddc_adpt;
+        struct clk *clk[MTK_HDMI_CLK_COUNT];
+        struct drm_display_mode mode;
+        bool dvi_mode;
+        u32 min_clock;
+        u32 max_clock;
+        u32 max_hdisplay;
+        u32 max_vdisplay;
+        u32 ibias;
+        u32 ibias_up;
+        struct regmap *sys_regmap;
+        unsigned int sys_offset;
+        void __iomem *regs;
+        enum hdmi_colorspace csp;
+        struct hdmi_audio_param aud_param;
+        bool audio_enable;
+        bool powered;
+        bool enabled;
+};
+
+static inline struct mtk_hdmi *hdmi_ctx_from_bridge(struct drm_bridge *b)
+{
+        return container_of(b, struct mtk_hdmi, bridge);
+}
+
+static inline struct mtk_hdmi *hdmi_ctx_from_conn(struct drm_connector *c)
+{
+        return container_of(c, struct mtk_hdmi, conn);
+}
+
+static u32 mtk_hdmi_read(struct mtk_hdmi *hdmi, u32 offset)
+{
+        return readl(hdmi->regs + offset);
+}
+
+static void mtk_hdmi_write(struct mtk_hdmi *hdmi, u32 offset, u32 val)
+{
+        writel(val, hdmi->regs + offset);
+}
+
+static void mtk_hdmi_clear_bits(struct mtk_hdmi *hdmi, u32 offset, u32 bits)
+{
+        void __iomem *reg = hdmi->regs + offset;
+        u32 tmp;
+
+        tmp = readl(reg);
+        tmp &= ~bits;
+        writel(tmp, reg);
+}
+
+static void mtk_hdmi_set_bits(struct mtk_hdmi *hdmi, u32 offset, u32 bits)
+{
+        void __iomem *reg = hdmi->regs + offset;
+        u32 tmp;
+
+        tmp = readl(reg);
+        tmp |= bits;
+        writel(tmp, reg);
+}
+
+static void mtk_hdmi_mask(struct mtk_hdmi *hdmi, u32 offset, u32 val, u32 mask)
+{
+        void __iomem *reg = hdmi->regs + offset;
+        u32 tmp;
+
+        tmp = readl(reg);
+        tmp = (tmp & ~mask) | (val & mask);
+        writel(tmp, reg);
+}
+
+static void mtk_hdmi_hw_vid_black(struct mtk_hdmi *hdmi, bool black)
+{
+        mtk_hdmi_mask(hdmi, VIDEO_CFG_4, black ? GEN_RGB : NORMAL_PATH,
+                      VIDEO_SOURCE_SEL);
+}
+
+static void mtk_hdmi_hw_make_reg_writable(struct mtk_hdmi *hdmi, bool enable)
+{
+        struct arm_smccc_res res;
+
+        /*
+         * MT8173 HDMI hardware has an output control bit to enable/disable HDMI
+         * output. This bit can only be controlled in ARM supervisor mode.
+         * The ARM trusted firmware provides an API for the HDMI driver to set
+         * this control bit to enable HDMI output in supervisor mode.
+         */
+        arm_smccc_smc(MTK_SIP_SET_AUTHORIZED_SECURE_REG, 0x14000904, 0x80000000,
+                      0, 0, 0, 0, 0, &res);
+
+        regmap_update_bits(hdmi->sys_regmap, hdmi->sys_offset + HDMI_SYS_CFG20,
+                           HDMI_PCLK_FREE_RUN, enable ? HDMI_PCLK_FREE_RUN : 0);
+        regmap_update_bits(hdmi->sys_regmap, hdmi->sys_offset + HDMI_SYS_CFG1C,
+                           HDMI_ON | ANLG_ON, enable ? (HDMI_ON | ANLG_ON) : 0);
+}
+
+static void mtk_hdmi_hw_1p4_version_enable(struct mtk_hdmi *hdmi, bool enable)
+{
+        regmap_update_bits(hdmi->sys_regmap, hdmi->sys_offset + HDMI_SYS_CFG20,
+                           HDMI2P0_EN, enable ? 0 : HDMI2P0_EN);
+}
+
+static void mtk_hdmi_hw_aud_mute(struct mtk_hdmi *hdmi)
+{
+        mtk_hdmi_set_bits(hdmi, GRL_AUDIO_CFG, AUDIO_ZERO);
+}
+
+static void mtk_hdmi_hw_aud_unmute(struct mtk_hdmi *hdmi)
+{
+        mtk_hdmi_clear_bits(hdmi, GRL_AUDIO_CFG, AUDIO_ZERO);
+}
+
+static void mtk_hdmi_hw_reset(struct mtk_hdmi *hdmi)
+{
+        regmap_update_bits(hdmi->sys_regmap, hdmi->sys_offset + HDMI_SYS_CFG1C,
+                           HDMI_RST, HDMI_RST);
+        regmap_update_bits(hdmi->sys_regmap, hdmi->sys_offset + HDMI_SYS_CFG1C,
+                           HDMI_RST, 0);
+        mtk_hdmi_clear_bits(hdmi, GRL_CFG3, CFG3_CONTROL_PACKET_DELAY);
+        regmap_update_bits(hdmi->sys_regmap, hdmi->sys_offset + HDMI_SYS_CFG1C,
+                           ANLG_ON, ANLG_ON);
+}
+
+static void mtk_hdmi_hw_enable_notice(struct mtk_hdmi *hdmi, bool enable_notice)
+{
+        mtk_hdmi_mask(hdmi, GRL_CFG2, enable_notice ? CFG2_NOTICE_EN : 0,
+                      CFG2_NOTICE_EN);
+}
+
+static void mtk_hdmi_hw_write_int_mask(struct mtk_hdmi *hdmi, u32 int_mask)
+{
+        mtk_hdmi_write(hdmi, GRL_INT_MASK, int_mask);
+}
+
+static void mtk_hdmi_hw_enable_dvi_mode(struct mtk_hdmi *hdmi, bool enable)
+{
+        mtk_hdmi_mask(hdmi, GRL_CFG1, enable ? CFG1_DVI : 0, CFG1_DVI);
+}
+
+static void mtk_hdmi_hw_send_info_frame(struct mtk_hdmi *hdmi, u8 *buffer,
+                                        u8 len)
+{
+        u32 ctrl_reg = GRL_CTRL;
+        int i;
+        u8 *frame_data;
+        enum hdmi_infoframe_type frame_type;
+        u8 frame_ver;
+        u8 frame_len;
+        u8 checksum;
+        int ctrl_frame_en = 0;
+
+        frame_type = *buffer;
+        buffer += 1;
+        frame_ver = *buffer;
+        buffer += 1;
+        frame_len = *buffer;
+        buffer += 1;
+        checksum = *buffer;
+        buffer += 1;
+        frame_data = buffer;
+
+        dev_dbg(hdmi->dev,
+                "frame_type:0x%x,frame_ver:0x%x,frame_len:0x%x,checksum:0x%x\n",
+                frame_type, frame_ver, frame_len, checksum);
+
+        switch (frame_type) {
+        case HDMI_INFOFRAME_TYPE_AVI:
+                ctrl_frame_en = CTRL_AVI_EN;
+                ctrl_reg = GRL_CTRL;
+                break;
+        case HDMI_INFOFRAME_TYPE_SPD:
+                ctrl_frame_en = CTRL_SPD_EN;
+                ctrl_reg = GRL_CTRL;
+                break;
+        case HDMI_INFOFRAME_TYPE_AUDIO:
+                ctrl_frame_en = CTRL_AUDIO_EN;
+                ctrl_reg = GRL_CTRL;
+                break;
+        case HDMI_INFOFRAME_TYPE_VENDOR:
+                ctrl_frame_en = VS_EN;
+                ctrl_reg = GRL_ACP_ISRC_CTRL;
+                break;
+        }
+        mtk_hdmi_clear_bits(hdmi, ctrl_reg, ctrl_frame_en);
+        mtk_hdmi_write(hdmi, GRL_INFOFRM_TYPE, frame_type);
+        mtk_hdmi_write(hdmi, GRL_INFOFRM_VER, frame_ver);
+        mtk_hdmi_write(hdmi, GRL_INFOFRM_LNG, frame_len);
+
+        mtk_hdmi_write(hdmi, GRL_IFM_PORT, checksum);
+        for (i = 0; i < frame_len; i++)
+                mtk_hdmi_write(hdmi, GRL_IFM_PORT, frame_data[i]);
+
+        mtk_hdmi_set_bits(hdmi, ctrl_reg, ctrl_frame_en);
+}
+
+static void mtk_hdmi_hw_send_aud_packet(struct mtk_hdmi *hdmi, bool enable)
+{
+        mtk_hdmi_mask(hdmi, GRL_SHIFT_R2, enable ? 0 : AUDIO_PACKET_OFF,
+                      AUDIO_PACKET_OFF);
+}
+
+static void mtk_hdmi_hw_config_sys(struct mtk_hdmi *hdmi)
+{
+        regmap_update_bits(hdmi->sys_regmap, hdmi->sys_offset + HDMI_SYS_CFG20,
+                           HDMI_OUT_FIFO_EN | MHL_MODE_ON, 0);
+        usleep_range(2000, 4000);
+        regmap_update_bits(hdmi->sys_regmap, hdmi->sys_offset + HDMI_SYS_CFG20,
+                           HDMI_OUT_FIFO_EN | MHL_MODE_ON, HDMI_OUT_FIFO_EN);
+}
+
+static void mtk_hdmi_hw_set_deep_color_mode(struct mtk_hdmi *hdmi)
+{
+        regmap_update_bits(hdmi->sys_regmap, hdmi->sys_offset + HDMI_SYS_CFG20,
+                           DEEP_COLOR_MODE_MASK | DEEP_COLOR_EN,
+                           COLOR_8BIT_MODE);
+}
+
+static void mtk_hdmi_hw_send_av_mute(struct mtk_hdmi *hdmi)
+{
+        mtk_hdmi_clear_bits(hdmi, GRL_CFG4, CTRL_AVMUTE);
+        usleep_range(2000, 4000);
+        mtk_hdmi_set_bits(hdmi, GRL_CFG4, CTRL_AVMUTE);
+}
+
+static void mtk_hdmi_hw_send_av_unmute(struct mtk_hdmi *hdmi)
+{
+        mtk_hdmi_mask(hdmi, GRL_CFG4, CFG4_AV_UNMUTE_EN,
+                      CFG4_AV_UNMUTE_EN | CFG4_AV_UNMUTE_SET);
+        usleep_range(2000, 4000);
+        mtk_hdmi_mask(hdmi, GRL_CFG4, CFG4_AV_UNMUTE_SET,
+                      CFG4_AV_UNMUTE_EN | CFG4_AV_UNMUTE_SET);
+}
+
+static void mtk_hdmi_hw_ncts_enable(struct mtk_hdmi *hdmi, bool on)
+{
+        mtk_hdmi_mask(hdmi, GRL_CTS_CTRL, on ? 0 : CTS_CTRL_SOFT,
+                      CTS_CTRL_SOFT);
+}
+
+static void mtk_hdmi_hw_ncts_auto_write_enable(struct mtk_hdmi *hdmi,
+                                               bool enable)
+{
+        mtk_hdmi_mask(hdmi, GRL_CTS_CTRL, enable ? NCTS_WRI_ANYTIME : 0,
+                      NCTS_WRI_ANYTIME);
+}
+
+static void mtk_hdmi_hw_msic_setting(struct mtk_hdmi *hdmi,
+                                     struct drm_display_mode *mode)
+{
+        mtk_hdmi_clear_bits(hdmi, GRL_CFG4, CFG4_MHL_MODE);
+
+        if (mode->flags & DRM_MODE_FLAG_INTERLACE &&
+            mode->clock == 74250 &&
+            mode->vdisplay == 1080)
+                mtk_hdmi_clear_bits(hdmi, GRL_CFG2, CFG2_MHL_DE_SEL);
+        else
+                mtk_hdmi_set_bits(hdmi, GRL_CFG2, CFG2_MHL_DE_SEL);
+}
+
+static void mtk_hdmi_hw_aud_set_channel_swap(struct mtk_hdmi *hdmi,
+                                        enum hdmi_aud_channel_swap_type swap)
+{
+        u8 swap_bit;
+
+        switch (swap) {
+        case HDMI_AUD_SWAP_LR:
+                swap_bit = LR_SWAP;
+                break;
+        case HDMI_AUD_SWAP_LFE_CC:
+                swap_bit = LFE_CC_SWAP;
+                break;
+        case HDMI_AUD_SWAP_LSRS:
+                swap_bit = LSRS_SWAP;
+                break;
+        case HDMI_AUD_SWAP_RLS_RRS:
+                swap_bit = RLS_RRS_SWAP;
+                break;
+        case HDMI_AUD_SWAP_LR_STATUS:
+                swap_bit = LR_STATUS_SWAP;
+                break;
+        default:
+                swap_bit = LFE_CC_SWAP;
+                break;
+        }
+        mtk_hdmi_mask(hdmi, GRL_CH_SWAP, swap_bit, 0xff);
+}
+
+static void mtk_hdmi_hw_aud_set_bit_num(struct mtk_hdmi *hdmi,
+                                        enum hdmi_audio_sample_size bit_num)
+{
+        u32 val;
+
+        switch (bit_num) {
+        case HDMI_AUDIO_SAMPLE_SIZE_16:
+                val = AOUT_16BIT;
+                break;
+        case HDMI_AUDIO_SAMPLE_SIZE_20:
+                val = AOUT_20BIT;
+                break;
+        case HDMI_AUDIO_SAMPLE_SIZE_24:
+        case HDMI_AUDIO_SAMPLE_SIZE_STREAM:
+                val = AOUT_24BIT;
+                break;
+        }
+
+        mtk_hdmi_mask(hdmi, GRL_AOUT_CFG, val, AOUT_BNUM_SEL_MASK);
+}
+
+static void mtk_hdmi_hw_aud_set_i2s_fmt(struct mtk_hdmi *hdmi,
+                                        enum hdmi_aud_i2s_fmt i2s_fmt)
+{
+        u32 val;
+
+        val = mtk_hdmi_read(hdmi, GRL_CFG0);
+        val &= ~(CFG0_W_LENGTH_MASK | CFG0_I2S_MODE_MASK);
+
+        switch (i2s_fmt) {
+        case HDMI_I2S_MODE_RJT_24BIT:
+                val |= CFG0_I2S_MODE_RTJ | CFG0_W_LENGTH_24BIT;
+                break;
+        case HDMI_I2S_MODE_RJT_16BIT:
+                val |= CFG0_I2S_MODE_RTJ | CFG0_W_LENGTH_16BIT;
+                break;
+        case HDMI_I2S_MODE_LJT_24BIT:
+        default:
+                val |= CFG0_I2S_MODE_LTJ | CFG0_W_LENGTH_24BIT;
+                break;
+        case HDMI_I2S_MODE_LJT_16BIT:
+                val |= CFG0_I2S_MODE_LTJ | CFG0_W_LENGTH_16BIT;
+                break;
+        case HDMI_I2S_MODE_I2S_24BIT:
+                val |= CFG0_I2S_MODE_I2S | CFG0_W_LENGTH_24BIT;
+                break;
+        case HDMI_I2S_MODE_I2S_16BIT:
+                val |= CFG0_I2S_MODE_I2S | CFG0_W_LENGTH_16BIT;
+                break;
+        }
+        mtk_hdmi_write(hdmi, GRL_CFG0, val);
+}
+
+static void mtk_hdmi_hw_audio_config(struct mtk_hdmi *hdmi, bool dst)
+{
+        const u8 mask = HIGH_BIT_RATE | DST_NORMAL_DOUBLE | SACD_DST | DSD_SEL;
+        u8 val;
+
+        /* Disable high bitrate, set DST packet normal/double */
+        mtk_hdmi_clear_bits(hdmi, GRL_AOUT_CFG, HIGH_BIT_RATE_PACKET_ALIGN);
+
+        if (dst)
+                val = DST_NORMAL_DOUBLE | SACD_DST;
+        else
+                val = 0;
+
+        mtk_hdmi_mask(hdmi, GRL_AUDIO_CFG, val, mask);
+}
+
+static void mtk_hdmi_hw_aud_set_i2s_chan_num(struct mtk_hdmi *hdmi,
+                                        enum hdmi_aud_channel_type channel_type,
+                                        u8 channel_count)
+{
+        unsigned int ch_switch;
+        u8 i2s_uv;
+
+        ch_switch = CH_SWITCH(7, 7) | CH_SWITCH(6, 6) |
+                    CH_SWITCH(5, 5) | CH_SWITCH(4, 4) |
+                    CH_SWITCH(3, 3) | CH_SWITCH(1, 2) |
+                    CH_SWITCH(2, 1) | CH_SWITCH(0, 0);
+
+        if (channel_count == 2) {
+                i2s_uv = I2S_UV_CH_EN(0);
+        } else if (channel_count == 3 || channel_count == 4) {
+                if (channel_count == 4 &&
+                    (channel_type == HDMI_AUD_CHAN_TYPE_3_0_LRS ||
+                    channel_type == HDMI_AUD_CHAN_TYPE_4_0))
+                        i2s_uv = I2S_UV_CH_EN(2) | I2S_UV_CH_EN(0);
+                else
+                        i2s_uv = I2S_UV_CH_EN(3) | I2S_UV_CH_EN(2);
+        } else if (channel_count == 6 || channel_count == 5) {
+                if (channel_count == 6 &&
+                    channel_type != HDMI_AUD_CHAN_TYPE_5_1 &&
+                    channel_type != HDMI_AUD_CHAN_TYPE_4_1_CLRS) {
+                        i2s_uv = I2S_UV_CH_EN(3) | I2S_UV_CH_EN(2) |
+                                 I2S_UV_CH_EN(1) | I2S_UV_CH_EN(0);
+                } else {
+                        i2s_uv = I2S_UV_CH_EN(2) | I2S_UV_CH_EN(1) |
+                                 I2S_UV_CH_EN(0);
+                }
+        } else if (channel_count == 8 || channel_count == 7) {
+                i2s_uv = I2S_UV_CH_EN(3) | I2S_UV_CH_EN(2) |
+                         I2S_UV_CH_EN(1) | I2S_UV_CH_EN(0);
+        } else {
+                i2s_uv = I2S_UV_CH_EN(0);
+        }
+
+        mtk_hdmi_write(hdmi, GRL_CH_SW0, ch_switch & 0xff);
+        mtk_hdmi_write(hdmi, GRL_CH_SW1, (ch_switch >> 8) & 0xff);
+        mtk_hdmi_write(hdmi, GRL_CH_SW2, (ch_switch >> 16) & 0xff);
+        mtk_hdmi_write(hdmi, GRL_I2S_UV, i2s_uv);
+}
+
+static void mtk_hdmi_hw_aud_set_input_type(struct mtk_hdmi *hdmi,
+                                           enum hdmi_aud_input_type input_type)
+{
+        u32 val;
+
+        val = mtk_hdmi_read(hdmi, GRL_CFG1);
+        if (input_type == HDMI_AUD_INPUT_I2S &&
+            (val & CFG1_SPDIF) == CFG1_SPDIF) {
+                val &= ~CFG1_SPDIF;
+        } else if (input_type == HDMI_AUD_INPUT_SPDIF &&
+                (val & CFG1_SPDIF) == 0) {
+                val |= CFG1_SPDIF;
+        }
+        mtk_hdmi_write(hdmi, GRL_CFG1, val);
+}
+
+static void mtk_hdmi_hw_aud_set_channel_status(struct mtk_hdmi *hdmi,
+                                               u8 *channel_status)
+{
+        int i;
+
+        for (i = 0; i < 5; i++) {
+                mtk_hdmi_write(hdmi, GRL_I2S_C_STA0 + i * 4, channel_status[i]);
+                mtk_hdmi_write(hdmi, GRL_L_STATUS_0 + i * 4, channel_status[i]);
+                mtk_hdmi_write(hdmi, GRL_R_STATUS_0 + i * 4, channel_status[i]);
+        }
+        for (; i < 24; i++) {
+                mtk_hdmi_write(hdmi, GRL_L_STATUS_0 + i * 4, 0);
+                mtk_hdmi_write(hdmi, GRL_R_STATUS_0 + i * 4, 0);
+        }
+}
+
+static void mtk_hdmi_hw_aud_src_reenable(struct mtk_hdmi *hdmi)
+{
+        u32 val;
+
+        val = mtk_hdmi_read(hdmi, GRL_MIX_CTRL);
+        if (val & MIX_CTRL_SRC_EN) {
+                val &= ~MIX_CTRL_SRC_EN;
+                mtk_hdmi_write(hdmi, GRL_MIX_CTRL, val);
+                usleep_range(255, 512);
+                val |= MIX_CTRL_SRC_EN;
+                mtk_hdmi_write(hdmi, GRL_MIX_CTRL, val);
+        }
+}
+
+static void mtk_hdmi_hw_aud_src_disable(struct mtk_hdmi *hdmi)
+{
+        u32 val;
+
+        val = mtk_hdmi_read(hdmi, GRL_MIX_CTRL);
+        val &= ~MIX_CTRL_SRC_EN;
+        mtk_hdmi_write(hdmi, GRL_MIX_CTRL, val);
+        mtk_hdmi_write(hdmi, GRL_SHIFT_L1, 0x00);
+}
+
+static void mtk_hdmi_hw_aud_set_mclk(struct mtk_hdmi *hdmi,
+                                     enum hdmi_aud_mclk mclk)
+{
+        u32 val;
+
+        val = mtk_hdmi_read(hdmi, GRL_CFG5);
+        val &= CFG5_CD_RATIO_MASK;
+
+        switch (mclk) {
+        case HDMI_AUD_MCLK_128FS:
+                val |= CFG5_FS128;
+                break;
+        case HDMI_AUD_MCLK_256FS:
+                val |= CFG5_FS256;
+                break;
+        case HDMI_AUD_MCLK_384FS:
+                val |= CFG5_FS384;
+                break;
+        case HDMI_AUD_MCLK_512FS:
+                val |= CFG5_FS512;
+                break;
+        case HDMI_AUD_MCLK_768FS:
+                val |= CFG5_FS768;
+                break;
+        default:
+                val |= CFG5_FS256;
+                break;
+        }
+        mtk_hdmi_write(hdmi, GRL_CFG5, val);
+}
+
+struct hdmi_acr_n {
+        unsigned int clock;
+        unsigned int n[3];
+};
+
+/* Recommended N values from HDMI specification, tables 7-1 to 7-3 */
+static const struct hdmi_acr_n hdmi_rec_n_table[] = {
+        /* Clock, N: 32kHz 44.1kHz 48kHz */
+        {  25175, {  4576,  7007,  6864 } },
+        {  74176, { 11648, 17836, 11648 } },
+        { 148352, { 11648,  8918,  5824 } },
+        { 296703, {  5824,  4459,  5824 } },
+        { 297000, {  3072,  4704,  5120 } },
+        {      0, {  4096,  6272,  6144 } }, /* all other TMDS clocks */
+};
+
+/**
+ * hdmi_recommended_n() - Return N value recommended by HDMI specification
+ * @freq: audio sample rate in Hz
+ * @clock: rounded TMDS clock in kHz
+ */
+static unsigned int hdmi_recommended_n(unsigned int freq, unsigned int clock)
+{
+        const struct hdmi_acr_n *recommended;
+        unsigned int i;
+
+        for (i = 0; i < ARRAY_SIZE(hdmi_rec_n_table) - 1; i++) {
+                if (clock == hdmi_rec_n_table[i].clock)
+                        break;
+        }
+        recommended = hdmi_rec_n_table + i;
+
+        switch (freq) {
+        case 32000:
+                return recommended->n[0];
+        case 44100:
+                return recommended->n[1];
+        case 48000:
+                return recommended->n[2];
+        case 88200:
+                return recommended->n[1] * 2;
+        case 96000:
+                return recommended->n[2] * 2;
+        case 176400:
+                return recommended->n[1] * 4;
+        case 192000:
+                return recommended->n[2] * 4;
+        default:
+                return (128 * freq) / 1000;
+        }
+}
+
+static unsigned int hdmi_mode_clock_to_hz(unsigned int clock)
+{
+        switch (clock) {
+        case 25175:
+                return 25174825;        /* 25.2/1.001 MHz */
+        case 74176:
+                return 74175824;        /* 74.25/1.001 MHz */
+        case 148352:
+                return 148351648;       /* 148.5/1.001 MHz */
+        case 296703:
+                return 296703297;       /* 297/1.001 MHz */
+        default:
+                return clock * 1000;
+        }
+}
+
+static unsigned int hdmi_expected_cts(unsigned int audio_sample_rate,
+                                      unsigned int tmds_clock, unsigned int n)
+{
+        return DIV_ROUND_CLOSEST_ULL((u64)hdmi_mode_clock_to_hz(tmds_clock) * n,
+                                     128 * audio_sample_rate);
+}
+
+static void do_hdmi_hw_aud_set_ncts(struct mtk_hdmi *hdmi, unsigned int n,
+                                    unsigned int cts)
+{
+        unsigned char val[NCTS_BYTES];
+        int i;
+
+        mtk_hdmi_write(hdmi, GRL_NCTS, 0);
+        mtk_hdmi_write(hdmi, GRL_NCTS, 0);
+        mtk_hdmi_write(hdmi, GRL_NCTS, 0);
+        memset(val, 0, sizeof(val));
+
+        val[0] = (cts >> 24) & 0xff;
+        val[1] = (cts >> 16) & 0xff;
+        val[2] = (cts >> 8) & 0xff;
+        val[3] = cts & 0xff;
+
+        val[4] = (n >> 16) & 0xff;
+        val[5] = (n >> 8) & 0xff;
+        val[6] = n & 0xff;
+
+        for (i = 0; i < NCTS_BYTES; i++)
+                mtk_hdmi_write(hdmi, GRL_NCTS, val[i]);
+}
+
+static void mtk_hdmi_hw_aud_set_ncts(struct mtk_hdmi *hdmi,
+                                     unsigned int sample_rate,
+                                     unsigned int clock)
+{
+        unsigned int n, cts;
+
+        n = hdmi_recommended_n(sample_rate, clock);
+        cts = hdmi_expected_cts(sample_rate, clock, n);
+
+        dev_dbg(hdmi->dev, "%s: sample_rate=%u, clock=%d, cts=%u, n=%u\n",
+                __func__, sample_rate, clock, n, cts);
+
+        mtk_hdmi_mask(hdmi, DUMMY_304, AUDIO_I2S_NCTS_SEL_64,
+                      AUDIO_I2S_NCTS_SEL);
+        do_hdmi_hw_aud_set_ncts(hdmi, n, cts);
+}
+
+static u8 mtk_hdmi_aud_get_chnl_count(enum hdmi_aud_channel_type channel_type)
+{
+        switch (channel_type) {
+        case HDMI_AUD_CHAN_TYPE_1_0:
+        case HDMI_AUD_CHAN_TYPE_1_1:
+        case HDMI_AUD_CHAN_TYPE_2_0:
+                return 2;
+        case HDMI_AUD_CHAN_TYPE_2_1:
+        case HDMI_AUD_CHAN_TYPE_3_0:
+                return 3;
+        case HDMI_AUD_CHAN_TYPE_3_1:
+        case HDMI_AUD_CHAN_TYPE_4_0:
+        case HDMI_AUD_CHAN_TYPE_3_0_LRS:
+                return 4;
+        case HDMI_AUD_CHAN_TYPE_4_1:
+        case HDMI_AUD_CHAN_TYPE_5_0:
+        case HDMI_AUD_CHAN_TYPE_3_1_LRS:
+        case HDMI_AUD_CHAN_TYPE_4_0_CLRS:
+                return 5;
+        case HDMI_AUD_CHAN_TYPE_5_1:
+        case HDMI_AUD_CHAN_TYPE_6_0:
+        case HDMI_AUD_CHAN_TYPE_4_1_CLRS:
+        case HDMI_AUD_CHAN_TYPE_6_0_CS:
+        case HDMI_AUD_CHAN_TYPE_6_0_CH:
+        case HDMI_AUD_CHAN_TYPE_6_0_OH:
+        case HDMI_AUD_CHAN_TYPE_6_0_CHR:
+                return 6;
+        case HDMI_AUD_CHAN_TYPE_6_1:
+        case HDMI_AUD_CHAN_TYPE_6_1_CS:
+        case HDMI_AUD_CHAN_TYPE_6_1_CH:
+        case HDMI_AUD_CHAN_TYPE_6_1_OH:
+        case HDMI_AUD_CHAN_TYPE_6_1_CHR:
+        case HDMI_AUD_CHAN_TYPE_7_0:
+        case HDMI_AUD_CHAN_TYPE_7_0_LH_RH:
+        case HDMI_AUD_CHAN_TYPE_7_0_LSR_RSR:
+        case HDMI_AUD_CHAN_TYPE_7_0_LC_RC:
+        case HDMI_AUD_CHAN_TYPE_7_0_LW_RW:
+        case HDMI_AUD_CHAN_TYPE_7_0_LSD_RSD:
+        case HDMI_AUD_CHAN_TYPE_7_0_LSS_RSS:
+        case HDMI_AUD_CHAN_TYPE_7_0_LHS_RHS:
+        case HDMI_AUD_CHAN_TYPE_7_0_CS_CH:
+        case HDMI_AUD_CHAN_TYPE_7_0_CS_OH:
+        case HDMI_AUD_CHAN_TYPE_7_0_CS_CHR:
+        case HDMI_AUD_CHAN_TYPE_7_0_CH_OH:
+        case HDMI_AUD_CHAN_TYPE_7_0_CH_CHR:
+        case HDMI_AUD_CHAN_TYPE_7_0_OH_CHR:
+        case HDMI_AUD_CHAN_TYPE_7_0_LSS_RSS_LSR_RSR:
+        case HDMI_AUD_CHAN_TYPE_8_0_LH_RH_CS:
+                return 7;
+        case HDMI_AUD_CHAN_TYPE_7_1:
+        case HDMI_AUD_CHAN_TYPE_7_1_LH_RH:
+        case HDMI_AUD_CHAN_TYPE_7_1_LSR_RSR:
+        case HDMI_AUD_CHAN_TYPE_7_1_LC_RC:
+        case HDMI_AUD_CHAN_TYPE_7_1_LW_RW:
+        case HDMI_AUD_CHAN_TYPE_7_1_LSD_RSD:
+        case HDMI_AUD_CHAN_TYPE_7_1_LSS_RSS:
+        case HDMI_AUD_CHAN_TYPE_7_1_LHS_RHS:
+        case HDMI_AUD_CHAN_TYPE_7_1_CS_CH:
+        case HDMI_AUD_CHAN_TYPE_7_1_CS_OH:
+        case HDMI_AUD_CHAN_TYPE_7_1_CS_CHR:
+        case HDMI_AUD_CHAN_TYPE_7_1_CH_OH:
+        case HDMI_AUD_CHAN_TYPE_7_1_CH_CHR:
+        case HDMI_AUD_CHAN_TYPE_7_1_OH_CHR:
+        case HDMI_AUD_CHAN_TYPE_7_1_LSS_RSS_LSR_RSR:
+                return 8;
+        default:
+                return 2;
+        }
+}
+
+static int mtk_hdmi_video_change_vpll(struct mtk_hdmi *hdmi, u32 clock)
+{
+        unsigned long rate;
+        int ret;
+
+        /* The DPI driver already should have set TVDPLL to the correct rate */
+        ret = clk_set_rate(hdmi->clk[MTK_HDMI_CLK_HDMI_PLL], clock);
+        if (ret) {
+                dev_err(hdmi->dev, "Failed to set PLL to %u Hz: %d\n", clock,
+                        ret);
+                return ret;
+        }
+
+        rate = clk_get_rate(hdmi->clk[MTK_HDMI_CLK_HDMI_PLL]);
+
+        if (DIV_ROUND_CLOSEST(rate, 1000) != DIV_ROUND_CLOSEST(clock, 1000))
+                dev_warn(hdmi->dev, "Want PLL %u Hz, got %lu Hz\n", clock,
+                         rate);
+        else
+                dev_dbg(hdmi->dev, "Want PLL %u Hz, got %lu Hz\n", clock, rate);
+
+        mtk_hdmi_hw_config_sys(hdmi);
+        mtk_hdmi_hw_set_deep_color_mode(hdmi);
+        return 0;
+}
+
+static void mtk_hdmi_video_set_display_mode(struct mtk_hdmi *hdmi,
+                                            struct drm_display_mode *mode)
+{
+        mtk_hdmi_hw_reset(hdmi);
+        mtk_hdmi_hw_enable_notice(hdmi, true);
+        mtk_hdmi_hw_write_int_mask(hdmi, 0xff);
+        mtk_hdmi_hw_enable_dvi_mode(hdmi, hdmi->dvi_mode);
+        mtk_hdmi_hw_ncts_auto_write_enable(hdmi, true);
+
+        mtk_hdmi_hw_msic_setting(hdmi, mode);
+}
+
+static int mtk_hdmi_aud_enable_packet(struct mtk_hdmi *hdmi, bool enable)
+{
+        mtk_hdmi_hw_send_aud_packet(hdmi, enable);
+        return 0;
+}
+
+static int mtk_hdmi_aud_on_off_hw_ncts(struct mtk_hdmi *hdmi, bool on)
+{
+        mtk_hdmi_hw_ncts_enable(hdmi, on);
+        return 0;
+}
+
+static int mtk_hdmi_aud_set_input(struct mtk_hdmi *hdmi)
+{
+        enum hdmi_aud_channel_type chan_type;
+        u8 chan_count;
+        bool dst;
+
+        mtk_hdmi_hw_aud_set_channel_swap(hdmi, HDMI_AUD_SWAP_LFE_CC);
+        mtk_hdmi_set_bits(hdmi, GRL_MIX_CTRL, MIX_CTRL_FLAT);
+
+        if (hdmi->aud_param.aud_input_type == HDMI_AUD_INPUT_SPDIF &&
+            hdmi->aud_param.aud_codec == HDMI_AUDIO_CODING_TYPE_DST) {
+                mtk_hdmi_hw_aud_set_bit_num(hdmi, HDMI_AUDIO_SAMPLE_SIZE_24);
+        } else if (hdmi->aud_param.aud_i2s_fmt == HDMI_I2S_MODE_LJT_24BIT) {
+                hdmi->aud_param.aud_i2s_fmt = HDMI_I2S_MODE_LJT_16BIT;
+        }
+
+        mtk_hdmi_hw_aud_set_i2s_fmt(hdmi, hdmi->aud_param.aud_i2s_fmt);
+        mtk_hdmi_hw_aud_set_bit_num(hdmi, HDMI_AUDIO_SAMPLE_SIZE_24);
+
+        dst = ((hdmi->aud_param.aud_input_type == HDMI_AUD_INPUT_SPDIF) &&
+               (hdmi->aud_param.aud_codec == HDMI_AUDIO_CODING_TYPE_DST));
+        mtk_hdmi_hw_audio_config(hdmi, dst);
+
+        if (hdmi->aud_param.aud_input_type == HDMI_AUD_INPUT_SPDIF)
+                chan_type = HDMI_AUD_CHAN_TYPE_2_0;
+        else
+                chan_type = hdmi->aud_param.aud_input_chan_type;
+        chan_count = mtk_hdmi_aud_get_chnl_count(chan_type);
+        mtk_hdmi_hw_aud_set_i2s_chan_num(hdmi, chan_type, chan_count);
+        mtk_hdmi_hw_aud_set_input_type(hdmi, hdmi->aud_param.aud_input_type);
+
+        return 0;
+}
+
+static int mtk_hdmi_aud_set_src(struct mtk_hdmi *hdmi,
+                                struct drm_display_mode *display_mode)
+{
+        unsigned int sample_rate = hdmi->aud_param.codec_params.sample_rate;
+
+        mtk_hdmi_aud_on_off_hw_ncts(hdmi, false);
+        mtk_hdmi_hw_aud_src_disable(hdmi);
+        mtk_hdmi_clear_bits(hdmi, GRL_CFG2, CFG2_ACLK_INV);
+
+        if (hdmi->aud_param.aud_input_type == HDMI_AUD_INPUT_I2S) {
+                switch (sample_rate) {
+                case 32000:
+                case 44100:
+                case 48000:
+                case 88200:
+                case 96000:
+                        break;
+                default:
+                        return -EINVAL;
+                }
+                mtk_hdmi_hw_aud_set_mclk(hdmi, hdmi->aud_param.aud_mclk);
+        } else {
+                switch (sample_rate) {
+                case 32000:
+                case 44100:
+                case 48000:
+                        break;
+                default:
+                        return -EINVAL;
+                }
+                mtk_hdmi_hw_aud_set_mclk(hdmi, HDMI_AUD_MCLK_128FS);
+        }
+
+        mtk_hdmi_hw_aud_set_ncts(hdmi, sample_rate, display_mode->clock);
+
+        mtk_hdmi_hw_aud_src_reenable(hdmi);
+        return 0;
+}
+
+static int mtk_hdmi_aud_output_config(struct mtk_hdmi *hdmi,
+                                      struct drm_display_mode *display_mode)
+{
+        mtk_hdmi_hw_aud_mute(hdmi);
+        mtk_hdmi_aud_enable_packet(hdmi, false);
+
+        mtk_hdmi_aud_set_input(hdmi);
+        mtk_hdmi_aud_set_src(hdmi, display_mode);
+        mtk_hdmi_hw_aud_set_channel_status(hdmi,
+                        hdmi->aud_param.codec_params.iec.status);
+
+        usleep_range(50, 100);
+
+        mtk_hdmi_aud_on_off_hw_ncts(hdmi, true);
+        mtk_hdmi_aud_enable_packet(hdmi, true);
+        mtk_hdmi_hw_aud_unmute(hdmi);
+        return 0;
+}
+
+static int mtk_hdmi_setup_avi_infoframe(struct mtk_hdmi *hdmi,
+                                        struct drm_display_mode *mode)
+{
+        struct hdmi_avi_infoframe frame;
+        u8 buffer[17];
+        ssize_t err;
+
+        err = drm_hdmi_avi_infoframe_from_display_mode(&frame, mode);
+        if (err < 0) {
+                dev_err(hdmi->dev,
+                        "Failed to get AVI infoframe from mode: %zd\n", err);
+                return err;
+        }
+
+        err = hdmi_avi_infoframe_pack(&frame, buffer, sizeof(buffer));
+        if (err < 0) {
+                dev_err(hdmi->dev, "Failed to pack AVI infoframe: %zd\n", err);
+                return err;
+        }
+
+        mtk_hdmi_hw_send_info_frame(hdmi, buffer, sizeof(buffer));
+        return 0;
+}
+
+static int mtk_hdmi_setup_spd_infoframe(struct mtk_hdmi *hdmi,
+                                        const char *vendor,
+                                        const char *product)
+{
+        struct hdmi_spd_infoframe frame;
+        u8 buffer[29];
+        ssize_t err;
+
+        err = hdmi_spd_infoframe_init(&frame, vendor, product);
+        if (err < 0) {
+                dev_err(hdmi->dev, "Failed to initialize SPD infoframe: %zd\n",
+                        err);
+                return err;
+        }
+
+        err = hdmi_spd_infoframe_pack(&frame, buffer, sizeof(buffer));
+        if (err < 0) {
+                dev_err(hdmi->dev, "Failed to pack SDP infoframe: %zd\n", err);
+                return err;
+        }
+
+        mtk_hdmi_hw_send_info_frame(hdmi, buffer, sizeof(buffer));
+        return 0;
+}
+
+static int mtk_hdmi_setup_audio_infoframe(struct mtk_hdmi *hdmi)
+{
+        struct hdmi_audio_infoframe frame;
+        u8 buffer[14];
+        ssize_t err;
+
+        err = hdmi_audio_infoframe_init(&frame);
+        if (err < 0) {
+                dev_err(hdmi->dev, "Failed to setup audio infoframe: %zd\n",
+                        err);
+                return err;
+        }
+
+        frame.coding_type = HDMI_AUDIO_CODING_TYPE_STREAM;
+        frame.sample_frequency = HDMI_AUDIO_SAMPLE_FREQUENCY_STREAM;
+        frame.sample_size = HDMI_AUDIO_SAMPLE_SIZE_STREAM;
+        frame.channels = mtk_hdmi_aud_get_chnl_count(
+                                        hdmi->aud_param.aud_input_chan_type);
+
+        err = hdmi_audio_infoframe_pack(&frame, buffer, sizeof(buffer));
+        if (err < 0) {
+                dev_err(hdmi->dev, "Failed to pack audio infoframe: %zd\n",
+                        err);
+                return err;
+        }
+
+        mtk_hdmi_hw_send_info_frame(hdmi, buffer, sizeof(buffer));
+        return 0;
+}
+
+static int mtk_hdmi_setup_vendor_specific_infoframe(struct mtk_hdmi *hdmi,
+                                                struct drm_display_mode *mode)
+{
+        struct hdmi_vendor_infoframe frame;
+        u8 buffer[10];
+        ssize_t err;
+
+        err = drm_hdmi_vendor_infoframe_from_display_mode(&frame, mode);
+        if (err) {
+                dev_err(hdmi->dev,
+                        "Failed to get vendor infoframe from mode: %zd\n", err);
+                return err;
+        }
+
+        err = hdmi_vendor_infoframe_pack(&frame, buffer, sizeof(buffer));
+        if (err) {
+                dev_err(hdmi->dev, "Failed to pack vendor infoframe: %zd\n",
+                        err);
+                return err;
+        }
+
+        mtk_hdmi_hw_send_info_frame(hdmi, buffer, sizeof(buffer));
+        return 0;
+}
+
+static int mtk_hdmi_output_init(struct mtk_hdmi *hdmi)
+{
+        struct hdmi_audio_param *aud_param = &hdmi->aud_param;
+
+        hdmi->csp = HDMI_COLORSPACE_RGB;
+        aud_param->aud_codec = HDMI_AUDIO_CODING_TYPE_PCM;
+        aud_param->aud_sampe_size = HDMI_AUDIO_SAMPLE_SIZE_16;
+        aud_param->aud_input_type = HDMI_AUD_INPUT_I2S;
+        aud_param->aud_i2s_fmt = HDMI_I2S_MODE_I2S_24BIT;
+        aud_param->aud_mclk = HDMI_AUD_MCLK_128FS;
+        aud_param->aud_input_chan_type = HDMI_AUD_CHAN_TYPE_2_0;
+
+        return 0;
+}
+
+void mtk_hdmi_audio_enable(struct mtk_hdmi *hdmi)
+{
+        mtk_hdmi_aud_enable_packet(hdmi, true);
+        hdmi->audio_enable = true;
+}
+
+void mtk_hdmi_audio_disable(struct mtk_hdmi *hdmi)
+{
+        mtk_hdmi_aud_enable_packet(hdmi, false);
+        hdmi->audio_enable = false;
+}
+
+int mtk_hdmi_audio_set_param(struct mtk_hdmi *hdmi,
+                             struct hdmi_audio_param *param)
+{
+        if (!hdmi->audio_enable) {
+                dev_err(hdmi->dev, "hdmi audio is in disable state!\n");
+                return -EINVAL;
+        }
+        dev_dbg(hdmi->dev, "codec:%d, input:%d, channel:%d, fs:%d\n",
+                param->aud_codec, param->aud_input_type,
+                param->aud_input_chan_type, param->codec_params.sample_rate);
+        memcpy(&hdmi->aud_param, param, sizeof(*param));
+        return mtk_hdmi_aud_output_config(hdmi, &hdmi->mode);
+}
+
+static int mtk_hdmi_output_set_display_mode(struct mtk_hdmi *hdmi,
+                                            struct drm_display_mode *mode)
+{
+        int ret;
+
+        mtk_hdmi_hw_vid_black(hdmi, true);
+        mtk_hdmi_hw_aud_mute(hdmi);
+        mtk_hdmi_hw_send_av_mute(hdmi);
+        phy_power_off(hdmi->phy);
+
+        ret = mtk_hdmi_video_change_vpll(hdmi,
+                                         mode->clock * 1000);
+        if (ret) {
+                dev_err(hdmi->dev, "Failed to set vpll: %d\n", ret);
+                return ret;
+        }
+        mtk_hdmi_video_set_display_mode(hdmi, mode);
+
+        phy_power_on(hdmi->phy);
+        mtk_hdmi_aud_output_config(hdmi, mode);
+
+        mtk_hdmi_setup_audio_infoframe(hdmi);
+        mtk_hdmi_setup_avi_infoframe(hdmi, mode);
+        mtk_hdmi_setup_spd_infoframe(hdmi, "mediatek", "On-chip HDMI");
+        if (mode->flags & DRM_MODE_FLAG_3D_MASK)
+                mtk_hdmi_setup_vendor_specific_infoframe(hdmi, mode);
+
+        mtk_hdmi_hw_vid_black(hdmi, false);
+        mtk_hdmi_hw_aud_unmute(hdmi);
+        mtk_hdmi_hw_send_av_unmute(hdmi);
+
+        return 0;
+}
+
+static const char * const mtk_hdmi_clk_names[MTK_HDMI_CLK_COUNT] = {
+        [MTK_HDMI_CLK_HDMI_PIXEL] = "pixel",
+        [MTK_HDMI_CLK_HDMI_PLL] = "pll",
+        [MTK_HDMI_CLK_AUD_BCLK] = "bclk",
+        [MTK_HDMI_CLK_AUD_SPDIF] = "spdif",
+};
+
+static int mtk_hdmi_get_all_clk(struct mtk_hdmi *hdmi,
+                                struct device_node *np)
+{
+        int i;
+
+        for (i = 0; i < ARRAY_SIZE(mtk_hdmi_clk_names); i++) {
+                hdmi->clk[i] = of_clk_get_by_name(np,
+                                                  mtk_hdmi_clk_names[i]);
+                if (IS_ERR(hdmi->clk[i]))
+                        return PTR_ERR(hdmi->clk[i]);
+        }
+        return 0;
+}
+
+static int mtk_hdmi_clk_enable_audio(struct mtk_hdmi *hdmi)
+{
+        int ret;
+
+        ret = clk_prepare_enable(hdmi->clk[MTK_HDMI_CLK_AUD_BCLK]);
+        if (ret)
+                return ret;
+
+        ret = clk_prepare_enable(hdmi->clk[MTK_HDMI_CLK_AUD_SPDIF]);
+        if (ret)
+                goto err;
+
+        return 0;
+err:
+        clk_disable_unprepare(hdmi->clk[MTK_HDMI_CLK_AUD_BCLK]);
+        return ret;
+}
+
+static void mtk_hdmi_clk_disable_audio(struct mtk_hdmi *hdmi)
+{
+        clk_disable_unprepare(hdmi->clk[MTK_HDMI_CLK_AUD_BCLK]);
+        clk_disable_unprepare(hdmi->clk[MTK_HDMI_CLK_AUD_SPDIF]);
+}
+
+static enum drm_connector_status hdmi_conn_detect(struct drm_connector *conn,
+                                                  bool force)
+{
+        struct mtk_hdmi *hdmi = hdmi_ctx_from_conn(conn);
+
+        return mtk_cec_hpd_high(hdmi->cec_dev) ?
+               connector_status_connected : connector_status_disconnected;
+}
+
+static void hdmi_conn_destroy(struct drm_connector *conn)
+{
+        struct mtk_hdmi *hdmi = hdmi_ctx_from_conn(conn);
+
+        mtk_cec_set_hpd_event(hdmi->cec_dev, NULL, NULL);
+
+        drm_connector_cleanup(conn);
+}
+
+static int mtk_hdmi_conn_get_modes(struct drm_connector *conn)
+{
+        struct mtk_hdmi *hdmi = hdmi_ctx_from_conn(conn);
+        struct edid *edid;
+        int ret;
+
+        if (!hdmi->ddc_adpt)
+                return -ENODEV;
+
+        edid = drm_get_edid(conn, hdmi->ddc_adpt);
+        if (!edid)
+                return -ENODEV;
+
+        hdmi->dvi_mode = !drm_detect_monitor_audio(edid);
+
+        drm_mode_connector_update_edid_property(conn, edid);
+
+        ret = drm_add_edid_modes(conn, edid);
+        drm_edid_to_eld(conn, edid);
+        kfree(edid);
+        return ret;
+}
+
+static int mtk_hdmi_conn_mode_valid(struct drm_connector *conn,
+                                    struct drm_display_mode *mode)
+{
+        struct mtk_hdmi *hdmi = hdmi_ctx_from_conn(conn);
+
+        dev_dbg(hdmi->dev, "xres=%d, yres=%d, refresh=%d, intl=%d clock=%d\n",
+                mode->hdisplay, mode->vdisplay, mode->vrefresh,
+                !!(mode->flags & DRM_MODE_FLAG_INTERLACE), mode->clock * 1000);
+
+        if (hdmi->bridge.next) {
+                struct drm_display_mode adjusted_mode;
+
+                drm_mode_copy(&adjusted_mode, mode);
+                if (!drm_bridge_mode_fixup(hdmi->bridge.next, mode,
+                                           &adjusted_mode))
+                        return MODE_BAD;
+        }
+
+        if (mode->clock < 27000)
+                return MODE_CLOCK_LOW;
+        if (mode->clock > 297000)
+                return MODE_CLOCK_HIGH;
+
+        return drm_mode_validate_size(mode, 0x1fff, 0x1fff);
+}
+
+static struct drm_encoder *mtk_hdmi_conn_best_enc(struct drm_connector *conn)
+{
+        struct mtk_hdmi *hdmi = hdmi_ctx_from_conn(conn);
+
+        return hdmi->bridge.encoder;
+}
+
+static const struct drm_connector_funcs mtk_hdmi_connector_funcs = {
+        .dpms = drm_atomic_helper_connector_dpms,
+        .detect = hdmi_conn_detect,
+        .fill_modes = drm_helper_probe_single_connector_modes,
+        .destroy = hdmi_conn_destroy,
+        .reset = drm_atomic_helper_connector_reset,
+        .atomic_duplicate_state = drm_atomic_helper_connector_duplicate_state,
+        .atomic_destroy_state = drm_atomic_helper_connector_destroy_state,
+};
+
+static const struct drm_connector_helper_funcs
+                mtk_hdmi_connector_helper_funcs = {
+        .get_modes = mtk_hdmi_conn_get_modes,
+        .mode_valid = mtk_hdmi_conn_mode_valid,
+        .best_encoder = mtk_hdmi_conn_best_enc,
+};
+
+static void mtk_hdmi_hpd_event(bool hpd, struct device *dev)
+{
+        struct mtk_hdmi *hdmi = dev_get_drvdata(dev);
+
+        if (hdmi && hdmi->bridge.encoder && hdmi->bridge.encoder->dev)
+                drm_helper_hpd_irq_event(hdmi->bridge.encoder->dev);
+}
+
+/*
+ * Bridge callbacks
+ */
+
+static int mtk_hdmi_bridge_attach(struct drm_bridge *bridge)
+{
+        struct mtk_hdmi *hdmi = hdmi_ctx_from_bridge(bridge);
+        int ret;
+
+        ret = drm_connector_init(bridge->encoder->dev, &hdmi->conn,
+                                 &mtk_hdmi_connector_funcs,
+                                 DRM_MODE_CONNECTOR_HDMIA);
+        if (ret) {
+                dev_err(hdmi->dev, "Failed to initialize connector: %d\n", ret);
+                return ret;
+        }
+        drm_connector_helper_add(&hdmi->conn, &mtk_hdmi_connector_helper_funcs);
+
+        hdmi->conn.polled = DRM_CONNECTOR_POLL_HPD;
+        hdmi->conn.interlace_allowed = true;
+        hdmi->conn.doublescan_allowed = false;
+
+        ret = drm_mode_connector_attach_encoder(&hdmi->conn,
+                                                bridge->encoder);
+        if (ret) {
+                dev_err(hdmi->dev,
+                        "Failed to attach connector to encoder: %d\n", ret);
+                return ret;
+        }
+
+        if (bridge->next) {
+                bridge->next->encoder = bridge->encoder;
+                ret = drm_bridge_attach(bridge->encoder->dev, bridge->next);
+                if (ret) {
+                        dev_err(hdmi->dev,
+                                "Failed to attach external bridge: %d\n", ret);
+                        return ret;
+                }
+        }
+
+        mtk_cec_set_hpd_event(hdmi->cec_dev, mtk_hdmi_hpd_event, hdmi->dev);
+
+        return 0;
+}
+
+static bool mtk_hdmi_bridge_mode_fixup(struct drm_bridge *bridge,
+                                       const struct drm_display_mode *mode,
+                                       struct drm_display_mode *adjusted_mode)
+{
+        return true;
+}
+
+static void mtk_hdmi_bridge_disable(struct drm_bridge *bridge)
+{
+        struct mtk_hdmi *hdmi = hdmi_ctx_from_bridge(bridge);
+
+        if (!hdmi->enabled)
+                return;
+
+        phy_power_off(hdmi->phy);
+        clk_disable_unprepare(hdmi->clk[MTK_HDMI_CLK_HDMI_PIXEL]);
+        clk_disable_unprepare(hdmi->clk[MTK_HDMI_CLK_HDMI_PLL]);
+
+        hdmi->enabled = false;
+}
+
+static void mtk_hdmi_bridge_post_disable(struct drm_bridge *bridge)
+{
+        struct mtk_hdmi *hdmi = hdmi_ctx_from_bridge(bridge);
+
+        if (!hdmi->powered)
+                return;
+
+        mtk_hdmi_hw_1p4_version_enable(hdmi, true);
+        mtk_hdmi_hw_make_reg_writable(hdmi, false);
+
+        hdmi->powered = false;
+}
+
+static void mtk_hdmi_bridge_mode_set(struct drm_bridge *bridge,
+                                     struct drm_display_mode *mode,
+                                     struct drm_display_mode *adjusted_mode)
+{
+        struct mtk_hdmi *hdmi = hdmi_ctx_from_bridge(bridge);
+
+        dev_dbg(hdmi->dev, "cur info: name:%s, hdisplay:%d\n",
+                adjusted_mode->name, adjusted_mode->hdisplay);
+        dev_dbg(hdmi->dev, "hsync_start:%d,hsync_end:%d, htotal:%d",
+                adjusted_mode->hsync_start, adjusted_mode->hsync_end,
+                adjusted_mode->htotal);
+        dev_dbg(hdmi->dev, "hskew:%d, vdisplay:%d\n",
+                adjusted_mode->hskew, adjusted_mode->vdisplay);
+        dev_dbg(hdmi->dev, "vsync_start:%d, vsync_end:%d, vtotal:%d",
+                adjusted_mode->vsync_start, adjusted_mode->vsync_end,
+                adjusted_mode->vtotal);
+        dev_dbg(hdmi->dev, "vscan:%d, flag:%d\n",
+                adjusted_mode->vscan, adjusted_mode->flags);
+
+        drm_mode_copy(&hdmi->mode, adjusted_mode);
+}
+
+static void mtk_hdmi_bridge_pre_enable(struct drm_bridge *bridge)
+{
+        struct mtk_hdmi *hdmi = hdmi_ctx_from_bridge(bridge);
+
+        mtk_hdmi_hw_make_reg_writable(hdmi, true);
+        mtk_hdmi_hw_1p4_version_enable(hdmi, true);
+
+        hdmi->powered = true;
+}
+
+static void mtk_hdmi_bridge_enable(struct drm_bridge *bridge)
+{
+        struct mtk_hdmi *hdmi = hdmi_ctx_from_bridge(bridge);
+
+        mtk_hdmi_output_set_display_mode(hdmi, &hdmi->mode);
+        clk_prepare_enable(hdmi->clk[MTK_HDMI_CLK_HDMI_PLL]);
+        clk_prepare_enable(hdmi->clk[MTK_HDMI_CLK_HDMI_PIXEL]);
+        phy_power_on(hdmi->phy);
+
+        hdmi->enabled = true;
+}
+
+static const struct drm_bridge_funcs mtk_hdmi_bridge_funcs = {
+        .attach = mtk_hdmi_bridge_attach,
+        .mode_fixup = mtk_hdmi_bridge_mode_fixup,
+        .disable = mtk_hdmi_bridge_disable,
+        .post_disable = mtk_hdmi_bridge_post_disable,
+        .mode_set = mtk_hdmi_bridge_mode_set,
+        .pre_enable = mtk_hdmi_bridge_pre_enable,
+        .enable = mtk_hdmi_bridge_enable,
+};
+
+static int mtk_hdmi_dt_parse_pdata(struct mtk_hdmi *hdmi,
+                                   struct platform_device *pdev)
+{
+        struct device *dev = &pdev->dev;
+        struct device_node *np = dev->of_node;
+        struct device_node *cec_np, *port, *ep, *remote, *i2c_np;
+        struct platform_device *cec_pdev;
+        struct regmap *regmap;
+        struct resource *mem;
+        int ret;
+
+        ret = mtk_hdmi_get_all_clk(hdmi, np);
+        if (ret) {
+                dev_err(dev, "Failed to get clocks: %d\n", ret);
+                return ret;
+        }
+
+        /* The CEC module handles HDMI hotplug detection */
+        cec_np = of_find_compatible_node(np->parent, NULL,
+                                         "mediatek,mt8173-cec");
+        if (!cec_np) {
+                dev_err(dev, "Failed to find CEC node\n");
+                return -EINVAL;
+        }
+
+        cec_pdev = of_find_device_by_node(cec_np);
+        if (!cec_pdev) {
+                dev_err(hdmi->dev, "Waiting for CEC device %s\n",
+                        cec_np->full_name);
+                return -EPROBE_DEFER;
+        }
+        hdmi->cec_dev = &cec_pdev->dev;
+
+        /*
+         * The mediatek,syscon-hdmi property contains a phandle link to the
+         * MMSYS_CONFIG device and the register offset of the HDMI_SYS_CFG
+         * registers it contains.
+         */
+        regmap = syscon_regmap_lookup_by_phandle(np, "mediatek,syscon-hdmi");
+        ret = of_property_read_u32_index(np, "mediatek,syscon-hdmi", 1,
+                                         &hdmi->sys_offset);
+        if (IS_ERR(regmap))
+                ret = PTR_ERR(regmap);
+        if (ret) {
+                ret = PTR_ERR(regmap);
+                dev_err(dev,
+                        "Failed to get system configuration registers: %d\n",
+                        ret);
+                return ret;
+        }
+        hdmi->sys_regmap = regmap;
+
+        mem = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+        hdmi->regs = devm_ioremap_resource(dev, mem);
+        if (IS_ERR(hdmi->regs))
+                return PTR_ERR(hdmi->regs);
+
+        port = of_graph_get_port_by_id(np, 1);
+        if (!port) {
+                dev_err(dev, "Missing output port node\n");
+                return -EINVAL;
+        }
+
+        ep = of_get_child_by_name(port, "endpoint");
+        if (!ep) {
+                dev_err(dev, "Missing endpoint node in port %s\n",
+                        port->full_name);
+                of_node_put(port);
+                return -EINVAL;
+        }
+        of_node_put(port);
+
+        remote = of_graph_get_remote_port_parent(ep);
+        if (!remote) {
+                dev_err(dev, "Missing connector/bridge node for endpoint %s\n",
+                        ep->full_name);
+                of_node_put(ep);
+                return -EINVAL;
+        }
+        of_node_put(ep);
+
+        if (!of_device_is_compatible(remote, "hdmi-connector")) {
+                hdmi->bridge.next = of_drm_find_bridge(remote);
+                if (!hdmi->bridge.next) {
+                        dev_err(dev, "Waiting for external bridge\n");
+                        of_node_put(remote);
+                        return -EPROBE_DEFER;
+                }
+        }
+
+        i2c_np = of_parse_phandle(remote, "ddc-i2c-bus", 0);
+        if (!i2c_np) {
+                dev_err(dev, "Failed to find ddc-i2c-bus node in %s\n",
+                        remote->full_name);
+                of_node_put(remote);
+                return -EINVAL;
+        }
+        of_node_put(remote);
+
+        hdmi->ddc_adpt = of_find_i2c_adapter_by_node(i2c_np);
+        if (!hdmi->ddc_adpt) {
+                dev_err(dev, "Failed to get ddc i2c adapter by node\n");
+                return -EINVAL;
+        }
+
+        return 0;
+}
+
+/*
+ * HDMI audio codec callbacks
+ */
+
+static int mtk_hdmi_audio_hw_params(struct device *dev, void *data,
+                                    struct hdmi_codec_daifmt *daifmt,
+                                    struct hdmi_codec_params *params)
+{
+        struct mtk_hdmi *hdmi = dev_get_drvdata(dev);
+        struct hdmi_audio_param hdmi_params;
+        unsigned int chan = params->cea.channels;
+
+        dev_dbg(hdmi->dev, "%s: %u Hz, %d bit, %d channels\n", __func__,
+                params->sample_rate, params->sample_width, chan);
+
+        if (!hdmi->bridge.encoder)
+                return -ENODEV;
+
+        switch (chan) {
+        case 2:
+                hdmi_params.aud_input_chan_type = HDMI_AUD_CHAN_TYPE_2_0;
+                break;
+        case 4:
+                hdmi_params.aud_input_chan_type = HDMI_AUD_CHAN_TYPE_4_0;
+                break;
+        case 6:
+                hdmi_params.aud_input_chan_type = HDMI_AUD_CHAN_TYPE_5_1;
+                break;
+        case 8:
+                hdmi_params.aud_input_chan_type = HDMI_AUD_CHAN_TYPE_7_1;
+                break;
+        default:
+                dev_err(hdmi->dev, "channel[%d] not supported!\n", chan);
+                return -EINVAL;
+        }
+
+        switch (params->sample_rate) {
+        case 32000:
+        case 44100:
+        case 48000:
+        case 88200:
+        case 96000:
+        case 176400:
+        case 192000:
+                break;
+        default:
+                dev_err(hdmi->dev, "rate[%d] not supported!\n",
+                        params->sample_rate);
+                return -EINVAL;
+        }
+
+        switch (daifmt->fmt) {
+        case HDMI_I2S:
+                hdmi_params.aud_codec = HDMI_AUDIO_CODING_TYPE_PCM;
+                hdmi_params.aud_sampe_size = HDMI_AUDIO_SAMPLE_SIZE_16;
+                hdmi_params.aud_input_type = HDMI_AUD_INPUT_I2S;
+                hdmi_params.aud_i2s_fmt = HDMI_I2S_MODE_I2S_24BIT;
+                hdmi_params.aud_mclk = HDMI_AUD_MCLK_128FS;
+                break;
+        default:
+                dev_err(hdmi->dev, "%s: Invalid DAI format %d\n", __func__,
+                        daifmt->fmt);
+                return -EINVAL;
+        }
+
+        memcpy(&hdmi_params.codec_params, params,
+               sizeof(hdmi_params.codec_params));
+
+        mtk_hdmi_audio_set_param(hdmi, &hdmi_params);
+
+        return 0;
+}
+
+static int mtk_hdmi_audio_startup(struct device *dev, void *data)
+{
+        struct mtk_hdmi *hdmi = dev_get_drvdata(dev);
+
+        dev_dbg(dev, "%s\n", __func__);
+
+        mtk_hdmi_audio_enable(hdmi);
+
+        return 0;
+}
+
+static void mtk_hdmi_audio_shutdown(struct device *dev, void *data)
+{
+        struct mtk_hdmi *hdmi = dev_get_drvdata(dev);
+
+        dev_dbg(dev, "%s\n", __func__);
+
+        mtk_hdmi_audio_disable(hdmi);
+}
+
+int mtk_hdmi_audio_digital_mute(struct device *dev, void *data, bool enable)
+{
+        struct mtk_hdmi *hdmi = dev_get_drvdata(dev);
+
+        dev_dbg(dev, "%s(%d)\n", __func__, enable);
+
+        if (enable)
+                mtk_hdmi_hw_aud_mute(hdmi);
+        else
+                mtk_hdmi_hw_aud_unmute(hdmi);
+
+        return 0;
+}
+
+static int mtk_hdmi_audio_get_eld(struct device *dev, void *data, uint8_t *buf, size_t len)
+{
+        struct mtk_hdmi *hdmi = dev_get_drvdata(dev);
+
+        dev_dbg(dev, "%s\n", __func__);
+
+        memcpy(buf, hdmi->conn.eld, min(sizeof(hdmi->conn.eld), len));
+
+        return 0;
+}
+
+static const struct hdmi_codec_ops mtk_hdmi_audio_codec_ops = {
+        .hw_params = mtk_hdmi_audio_hw_params,
+        .audio_startup = mtk_hdmi_audio_startup,
+        .audio_shutdown = mtk_hdmi_audio_shutdown,
+        .digital_mute = mtk_hdmi_audio_digital_mute,
+        .get_eld = mtk_hdmi_audio_get_eld,
+};
+
+static void mtk_hdmi_register_audio_driver(struct device *dev)
+{
+        struct hdmi_codec_pdata codec_data = {
+                .ops = &mtk_hdmi_audio_codec_ops,
+                .max_i2s_channels = 2,
+                .i2s = 1,
+        };
+        struct platform_device *pdev;
+
+        pdev = platform_device_register_data(dev, HDMI_CODEC_DRV_NAME,
+                                             PLATFORM_DEVID_AUTO, &codec_data,
+                                             sizeof(codec_data));
+        if (IS_ERR(pdev))
+                return;
+
+        DRM_INFO("%s driver bound to HDMI\n", HDMI_CODEC_DRV_NAME);
+}
+
+static int mtk_drm_hdmi_probe(struct platform_device *pdev)
+{
+        struct mtk_hdmi *hdmi;
+        struct device *dev = &pdev->dev;
+        int ret;
+
+        hdmi = devm_kzalloc(dev, sizeof(*hdmi), GFP_KERNEL);
+        if (!hdmi)
+                return -ENOMEM;
+
+        hdmi->dev = dev;
+
+        ret = mtk_hdmi_dt_parse_pdata(hdmi, pdev);
+        if (ret)
+                return ret;
+
+        hdmi->phy = devm_phy_get(dev, "hdmi");
+        if (IS_ERR(hdmi->phy)) {
+                ret = PTR_ERR(hdmi->phy);
+                dev_err(dev, "Failed to get HDMI PHY: %d\n", ret);
+                return ret;
+        }
+
+        platform_set_drvdata(pdev, hdmi);
+
+        ret = mtk_hdmi_output_init(hdmi);
+        if (ret) {
+                dev_err(dev, "Failed to initialize hdmi output\n");
+                return ret;
+        }
+
+        mtk_hdmi_register_audio_driver(dev);
+
+        hdmi->bridge.funcs = &mtk_hdmi_bridge_funcs;
+        hdmi->bridge.of_node = pdev->dev.of_node;
+        ret = drm_bridge_add(&hdmi->bridge);
+        if (ret) {
+                dev_err(dev, "failed to add bridge, ret = %d\n", ret);
+                return ret;
+        }
+
+        ret = mtk_hdmi_clk_enable_audio(hdmi);
+        if (ret) {
+                dev_err(dev, "Failed to enable audio clocks: %d\n", ret);
+                goto err_bridge_remove;
+        }
+
+        dev_dbg(dev, "mediatek hdmi probe success\n");
+        return 0;
+
+err_bridge_remove:
+        drm_bridge_remove(&hdmi->bridge);
+        return ret;
+}
+
+static int mtk_drm_hdmi_remove(struct platform_device *pdev)
+{
+        struct mtk_hdmi *hdmi = platform_get_drvdata(pdev);
+
+        drm_bridge_remove(&hdmi->bridge);
+        mtk_hdmi_clk_disable_audio(hdmi);
+        return 0;
+}
+
+#ifdef CONFIG_PM_SLEEP
+static int mtk_hdmi_suspend(struct device *dev)
+{
+        struct mtk_hdmi *hdmi = dev_get_drvdata(dev);
+
+        mtk_hdmi_clk_disable_audio(hdmi);
+        dev_dbg(dev, "hdmi suspend success!\n");
+        return 0;
+}
+
+static int mtk_hdmi_resume(struct device *dev)
+{
+        struct mtk_hdmi *hdmi = dev_get_drvdata(dev);
+        int ret = 0;
+
+        ret = mtk_hdmi_clk_enable_audio(hdmi);
+        if (ret) {
+                dev_err(dev, "hdmi resume failed!\n");
+                return ret;
+        }
+
+        dev_dbg(dev, "hdmi resume success!\n");
+        return 0;
+}
+#endif
+static SIMPLE_DEV_PM_OPS(mtk_hdmi_pm_ops,
+                         mtk_hdmi_suspend, mtk_hdmi_resume);
+
+static const struct of_device_id mtk_drm_hdmi_of_ids[] = {
+        { .compatible = "mediatek,mt8173-hdmi", },
+        {}
+};
+
+static struct platform_driver mtk_hdmi_driver = {
+        .probe = mtk_drm_hdmi_probe,
+        .remove = mtk_drm_hdmi_remove,
+        .driver = {
+                .name = "mediatek-drm-hdmi",
+                .of_match_table = mtk_drm_hdmi_of_ids,
+                .pm = &mtk_hdmi_pm_ops,
+        },
+};
+
+static struct platform_driver * const mtk_hdmi_drivers[] = {
+        &mtk_hdmi_phy_driver,
+        &mtk_hdmi_ddc_driver,
+        &mtk_cec_driver,
+        &mtk_hdmi_driver,
+};
+
+static int __init mtk_hdmitx_init(void)
+{
+        int ret;
+        int i;
+
+        for (i = 0; i < ARRAY_SIZE(mtk_hdmi_drivers); i++) {
+                ret = platform_driver_register(mtk_hdmi_drivers[i]);
+                if (ret < 0) {
+                        pr_err("Failed to register %s driver: %d\n",
+                               mtk_hdmi_drivers[i]->driver.name, ret);
+                        goto err;
+                }
+        }
+
+        return 0;
+
+err:
+        while (--i >= 0)
+                platform_driver_unregister(mtk_hdmi_drivers[i]);
+
+        return ret;
+}
+
+static void __exit mtk_hdmitx_exit(void)
+{
+        int i;
+
+        for (i = ARRAY_SIZE(mtk_hdmi_drivers) - 1; i >= 0; i--)
+                platform_driver_unregister(mtk_hdmi_drivers[i]);
+}
+
+module_init(mtk_hdmitx_init);
+module_exit(mtk_hdmitx_exit);
+
+MODULE_AUTHOR("Jie Qiu <jie.qiu@mediatek.com>");
+MODULE_DESCRIPTION("MediaTek HDMI Driver");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/gpu/drm/mediatek/mtk_hdmi.h b/drivers/gpu/drm/mediatek/mtk_hdmi.h
new file mode 100644
index 000000000000..6371b3de1ff6
--- /dev/null
+++ b/drivers/gpu/drm/mediatek/mtk_hdmi.h
@@ -0,0 +1,23 @@
+/*
+ * Copyright (c) 2014 MediaTek Inc.
+ * Author: Jie Qiu <jie.qiu@mediatek.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+#ifndef _MTK_HDMI_CTRL_H
+#define _MTK_HDMI_CTRL_H
+
+struct platform_driver;
+
+extern struct platform_driver mtk_cec_driver;
+extern struct platform_driver mtk_hdmi_ddc_driver;
+extern struct platform_driver mtk_hdmi_phy_driver;
+
+#endif /* _MTK_HDMI_CTRL_H */
diff --git a/drivers/gpu/drm/mediatek/mtk_hdmi_ddc.c b/drivers/gpu/drm/mediatek/mtk_hdmi_ddc.c
new file mode 100644
index 000000000000..33c9e1bdb114
--- /dev/null
+++ b/drivers/gpu/drm/mediatek/mtk_hdmi_ddc.c
@@ -0,0 +1,358 @@
+/*
+ * Copyright (c) 2014 MediaTek Inc.
+ * Author: Jie Qiu <jie.qiu@mediatek.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/i2c.h>
+#include <linux/time.h>
+#include <linux/delay.h>
+#include <linux/errno.h>
+#include <linux/err.h>
+#include <linux/platform_device.h>
+#include <linux/clk.h>
+#include <linux/slab.h>
+#include <linux/io.h>
+#include <linux/iopoll.h>
+#include <linux/of_address.h>
+#include <linux/of_irq.h>
+#include <linux/of_platform.h>
+
+#define SIF1_CLOK               (288)
+#define DDC_DDCMCTL0            (0x0)
+#define DDCM_ODRAIN                     BIT(31)
+#define DDCM_CLK_DIV_OFFSET             (16)
+#define DDCM_CLK_DIV_MASK               (0xfff << 16)
+#define DDCM_CS_STATUS                  BIT(4)
+#define DDCM_SCL_STATE                  BIT(3)
+#define DDCM_SDA_STATE                  BIT(2)
+#define DDCM_SM0EN                      BIT(1)
+#define DDCM_SCL_STRECH                 BIT(0)
+#define DDC_DDCMCTL1            (0x4)
+#define DDCM_ACK_OFFSET                 (16)
+#define DDCM_ACK_MASK                   (0xff << 16)
+#define DDCM_PGLEN_OFFSET               (8)
+#define DDCM_PGLEN_MASK                 (0x7 << 8)
+#define DDCM_SIF_MODE_OFFSET            (4)
+#define DDCM_SIF_MODE_MASK              (0x7 << 4)
+#define DDCM_START                      (0x1)
+#define DDCM_WRITE_DATA                 (0x2)
+#define DDCM_STOP                       (0x3)
+#define DDCM_READ_DATA_NO_ACK           (0x4)
+#define DDCM_READ_DATA_ACK              (0x5)
+#define DDCM_TRI                        BIT(0)
+#define DDC_DDCMD0              (0x8)
+#define DDCM_DATA3                      (0xff << 24)
+#define DDCM_DATA2                      (0xff << 16)
+#define DDCM_DATA1                      (0xff << 8)
+#define DDCM_DATA0                      (0xff << 0)
+#define DDC_DDCMD1              (0xc)
+#define DDCM_DATA7                      (0xff << 24)
+#define DDCM_DATA6                      (0xff << 16)
+#define DDCM_DATA5                      (0xff << 8)
+#define DDCM_DATA4                      (0xff << 0)
+
+struct mtk_hdmi_ddc {
+        struct i2c_adapter adap;
+        struct clk *clk;
+        void __iomem *regs;
+};
+
+static inline void sif_set_bit(struct mtk_hdmi_ddc *ddc, unsigned int offset,
+                               unsigned int val)
+{
+        writel(readl(ddc->regs + offset) | val, ddc->regs + offset);
+}
+
+static inline void sif_clr_bit(struct mtk_hdmi_ddc *ddc, unsigned int offset,
+                               unsigned int val)
+{
+        writel(readl(ddc->regs + offset) & ~val, ddc->regs + offset);
+}
+
+static inline bool sif_bit_is_set(struct mtk_hdmi_ddc *ddc, unsigned int offset,
+                                  unsigned int val)
+{
+        return (readl(ddc->regs + offset) & val) == val;
+}
+
+static inline void sif_write_mask(struct mtk_hdmi_ddc *ddc, unsigned int offset,
+                                  unsigned int mask, unsigned int shift,
+                                  unsigned int val)
+{
+        unsigned int tmp;
+
+        tmp = readl(ddc->regs + offset);
+        tmp &= ~mask;
+        tmp |= (val << shift) & mask;
+        writel(tmp, ddc->regs + offset);
+}
+
+static inline unsigned int sif_read_mask(struct mtk_hdmi_ddc *ddc,
+                                         unsigned int offset, unsigned int mask,
+                                         unsigned int shift)
+{
+        return (readl(ddc->regs + offset) & mask) >> shift;
+}
+
+static void ddcm_trigger_mode(struct mtk_hdmi_ddc *ddc, int mode)
+{
+        u32 val;
+
+        sif_write_mask(ddc, DDC_DDCMCTL1, DDCM_SIF_MODE_MASK,
+                       DDCM_SIF_MODE_OFFSET, mode);
+        sif_set_bit(ddc, DDC_DDCMCTL1, DDCM_TRI);
+        readl_poll_timeout(ddc->regs + DDC_DDCMCTL1, val,
+                           (val & DDCM_TRI) != DDCM_TRI, 4, 20000);
+}
+
+static int mtk_hdmi_ddc_read_msg(struct mtk_hdmi_ddc *ddc, struct i2c_msg *msg)
+{
+        struct device *dev = ddc->adap.dev.parent;
+        u32 remain_count, ack_count, ack_final, read_count, temp_count;
+        u32 index = 0;
+        u32 ack;
+        int i;
+
+        ddcm_trigger_mode(ddc, DDCM_START);
+        sif_write_mask(ddc, DDC_DDCMD0, 0xff, 0, (msg->addr << 1) | 0x01);
+        sif_write_mask(ddc, DDC_DDCMCTL1, DDCM_PGLEN_MASK, DDCM_PGLEN_OFFSET,
+                       0x00);
+        ddcm_trigger_mode(ddc, DDCM_WRITE_DATA);
+        ack = sif_read_mask(ddc, DDC_DDCMCTL1, DDCM_ACK_MASK, DDCM_ACK_OFFSET);
+        dev_dbg(dev, "ack = 0x%x\n", ack);
+        if (ack != 0x01) {
+                dev_err(dev, "i2c ack err!\n");
+                return -ENXIO;
+        }
+
+        remain_count = msg->len;
+        ack_count = (msg->len - 1) / 8;
+        ack_final = 0;
+
+        while (remain_count > 0) {
+                if (ack_count > 0) {
+                        read_count = 8;
+                        ack_final = 0;
+                        ack_count--;
+                } else {
+                        read_count = remain_count;
+                        ack_final = 1;
+                }
+
+                sif_write_mask(ddc, DDC_DDCMCTL1, DDCM_PGLEN_MASK,
+                               DDCM_PGLEN_OFFSET, read_count - 1);
+                ddcm_trigger_mode(ddc, (ack_final == 1) ?
+                                  DDCM_READ_DATA_NO_ACK :
+                                  DDCM_READ_DATA_ACK);
+
+                ack = sif_read_mask(ddc, DDC_DDCMCTL1, DDCM_ACK_MASK,
+                                    DDCM_ACK_OFFSET);
+                temp_count = 0;
+                while (((ack & (1 << temp_count)) != 0) && (temp_count < 8))
+                        temp_count++;
+                if (((ack_final == 1) && (temp_count != (read_count - 1))) ||
+                    ((ack_final == 0) && (temp_count != read_count))) {
+                        dev_err(dev, "Address NACK! ACK(0x%x)\n", ack);
+                        break;
+                }
+
+                for (i = read_count; i >= 1; i--) {
+                        int shift;
+                        int offset;
+
+                        if (i > 4) {
+                                offset = DDC_DDCMD1;
+                                shift = (i - 5) * 8;
+                        } else {
+                                offset = DDC_DDCMD0;
+                                shift = (i - 1) * 8;
+                        }
+
+                        msg->buf[index + i - 1] = sif_read_mask(ddc, offset,
+                                                                0xff << shift,
+                                                                shift);
+                }
+
+                remain_count -= read_count;
+                index += read_count;
+        }
+
+        return 0;
+}
+
+static int mtk_hdmi_ddc_write_msg(struct mtk_hdmi_ddc *ddc, struct i2c_msg *msg)
+{
+        struct device *dev = ddc->adap.dev.parent;
+        u32 ack;
+
+        ddcm_trigger_mode(ddc, DDCM_START);
+        sif_write_mask(ddc, DDC_DDCMD0, DDCM_DATA0, 0, msg->addr << 1);
+        sif_write_mask(ddc, DDC_DDCMD0, DDCM_DATA1, 8, msg->buf[0]);
+        sif_write_mask(ddc, DDC_DDCMCTL1, DDCM_PGLEN_MASK, DDCM_PGLEN_OFFSET,
+                       0x1);
+        ddcm_trigger_mode(ddc, DDCM_WRITE_DATA);
+
+        ack = sif_read_mask(ddc, DDC_DDCMCTL1, DDCM_ACK_MASK, DDCM_ACK_OFFSET);
+        dev_dbg(dev, "ack = %d\n", ack);
+
+        if (ack != 0x03) {
+                dev_err(dev, "i2c ack err!\n");
+                return -EIO;
+        }
+
+        return 0;
+}
+
+static int mtk_hdmi_ddc_xfer(struct i2c_adapter *adapter,
+                             struct i2c_msg *msgs, int num)
+{
+        struct mtk_hdmi_ddc *ddc = adapter->algo_data;
+        struct device *dev = adapter->dev.parent;
+        int ret;
+        int i;
+
+        if (!ddc) {
+                dev_err(dev, "invalid arguments\n");
+                return -EINVAL;
+        }
+
+        sif_set_bit(ddc, DDC_DDCMCTL0, DDCM_SCL_STRECH);
+        sif_set_bit(ddc, DDC_DDCMCTL0, DDCM_SM0EN);
+        sif_clr_bit(ddc, DDC_DDCMCTL0, DDCM_ODRAIN);
+
+        if (sif_bit_is_set(ddc, DDC_DDCMCTL1, DDCM_TRI)) {
+                dev_err(dev, "ddc line is busy!\n");
+                return -EBUSY;
+        }
+
+        sif_write_mask(ddc, DDC_DDCMCTL0, DDCM_CLK_DIV_MASK,
+                       DDCM_CLK_DIV_OFFSET, SIF1_CLOK);
+
+        for (i = 0; i < num; i++) {
+                struct i2c_msg *msg = &msgs[i];
+
+                dev_dbg(dev, "i2c msg, adr:0x%x, flags:%d, len :0x%x\n",
+                        msg->addr, msg->flags, msg->len);
+
+                if (msg->flags & I2C_M_RD)
+                        ret = mtk_hdmi_ddc_read_msg(ddc, msg);
+                else
+                        ret = mtk_hdmi_ddc_write_msg(ddc, msg);
+                if (ret < 0)
+                        goto xfer_end;
+        }
+
+        ddcm_trigger_mode(ddc, DDCM_STOP);
+
+        return i;
+
+xfer_end:
+        ddcm_trigger_mode(ddc, DDCM_STOP);
+        dev_err(dev, "ddc failed!\n");
+        return ret;
+}
+
+static u32 mtk_hdmi_ddc_func(struct i2c_adapter *adapter)
+{
+        return I2C_FUNC_I2C | I2C_FUNC_SMBUS_EMUL;
+}
+
+static const struct i2c_algorithm mtk_hdmi_ddc_algorithm = {
+        .master_xfer = mtk_hdmi_ddc_xfer,
+        .functionality = mtk_hdmi_ddc_func,
+};
+
+static int mtk_hdmi_ddc_probe(struct platform_device *pdev)
+{
+        struct device *dev = &pdev->dev;
+        struct mtk_hdmi_ddc *ddc;
+        struct resource *mem;
+        int ret;
+
+        ddc = devm_kzalloc(dev, sizeof(struct mtk_hdmi_ddc), GFP_KERNEL);
+        if (!ddc)
+                return -ENOMEM;
+
+        ddc->clk = devm_clk_get(dev, "ddc-i2c");
+        if (IS_ERR(ddc->clk)) {
+                dev_err(dev, "get ddc_clk failed: %p ,\n", ddc->clk);
+                return PTR_ERR(ddc->clk);
+        }
+
+        mem = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+        ddc->regs = devm_ioremap_resource(&pdev->dev, mem);
+        if (IS_ERR(ddc->regs))
+                return PTR_ERR(ddc->regs);
+
+        ret = clk_prepare_enable(ddc->clk);
+        if (ret) {
+                dev_err(dev, "enable ddc clk failed!\n");
+                return ret;
+        }
+
+        strlcpy(ddc->adap.name, "mediatek-hdmi-ddc", sizeof(ddc->adap.name));
+        ddc->adap.owner = THIS_MODULE;
+        ddc->adap.class = I2C_CLASS_DDC;
+        ddc->adap.algo = &mtk_hdmi_ddc_algorithm;
+        ddc->adap.retries = 3;
+        ddc->adap.dev.of_node = dev->of_node;
+        ddc->adap.algo_data = ddc;
+        ddc->adap.dev.parent = &pdev->dev;
+
+        ret = i2c_add_adapter(&ddc->adap);
+        if (ret < 0) {
+                dev_err(dev, "failed to add bus to i2c core\n");
+                goto err_clk_disable;
+        }
+
+        platform_set_drvdata(pdev, ddc);
+
+        dev_dbg(dev, "ddc->adap: %p\n", &ddc->adap);
+        dev_dbg(dev, "ddc->clk: %p\n", ddc->clk);
+        dev_dbg(dev, "physical adr: %pa, end: %pa\n", &mem->start,
+                &mem->end);
+
+        return 0;
+
+err_clk_disable:
+        clk_disable_unprepare(ddc->clk);
+        return ret;
+}
+
+static int mtk_hdmi_ddc_remove(struct platform_device *pdev)
+{
+        struct mtk_hdmi_ddc *ddc = platform_get_drvdata(pdev);
+
+        i2c_del_adapter(&ddc->adap);
+        clk_disable_unprepare(ddc->clk);
+
+        return 0;
+}
+
+static const struct of_device_id mtk_hdmi_ddc_match[] = {
+        { .compatible = "mediatek,mt8173-hdmi-ddc", },
+        {},
+};
+
+struct platform_driver mtk_hdmi_ddc_driver = {
+        .probe = mtk_hdmi_ddc_probe,
+        .remove = mtk_hdmi_ddc_remove,
+        .driver = {
+                .name = "mediatek-hdmi-ddc",
+                .of_match_table = mtk_hdmi_ddc_match,
+        },
+};
+
+MODULE_AUTHOR("Jie Qiu <jie.qiu@mediatek.com>");
+MODULE_DESCRIPTION("MediaTek HDMI DDC Driver");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/gpu/drm/mediatek/mtk_hdmi_regs.h b/drivers/gpu/drm/mediatek/mtk_hdmi_regs.h
new file mode 100644
index 000000000000..a5cb07d12c9c
--- /dev/null
+++ b/drivers/gpu/drm/mediatek/mtk_hdmi_regs.h
@@ -0,0 +1,238 @@
+/*
+ * Copyright (c) 2014 MediaTek Inc.
+ * Author: Jie Qiu <jie.qiu@mediatek.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+#ifndef _MTK_HDMI_REGS_H
+#define _MTK_HDMI_REGS_H
+
+#define GRL_INT_MASK            0x18
+#define GRL_IFM_PORT            0x188
+#define GRL_CH_SWAP             0x198
+#define LR_SWAP                         BIT(0)
+#define LFE_CC_SWAP                     BIT(1)
+#define LSRS_SWAP                       BIT(2)
+#define RLS_RRS_SWAP                    BIT(3)
+#define LR_STATUS_SWAP                  BIT(4)
+#define GRL_I2S_C_STA0          0x140
+#define GRL_I2S_C_STA1          0x144
+#define GRL_I2S_C_STA2          0x148
+#define GRL_I2S_C_STA3          0x14C
+#define GRL_I2S_C_STA4          0x150
+#define GRL_I2S_UV              0x154
+#define I2S_UV_V                        BIT(0)
+#define I2S_UV_U                        BIT(1)
+#define I2S_UV_CH_EN_MASK               0x3c
+#define I2S_UV_CH_EN(x)                 BIT((x) + 2)
+#define I2S_UV_TMDS_DEBUG               BIT(6)
+#define I2S_UV_NORMAL_INFO_INV          BIT(7)
+#define GRL_ACP_ISRC_CTRL       0x158
+#define VS_EN                           BIT(0)
+#define ACP_EN                          BIT(1)
+#define ISRC1_EN                        BIT(2)
+#define ISRC2_EN                        BIT(3)
+#define GAMUT_EN                        BIT(4)
+#define GRL_CTS_CTRL            0x160
+#define CTS_CTRL_SOFT                   BIT(0)
+#define GRL_INT                 0x14
+#define INT_MDI                         BIT(0)
+#define INT_HDCP                        BIT(1)
+#define INT_FIFO_O                      BIT(2)
+#define INT_FIFO_U                      BIT(3)
+#define INT_IFM_ERR                     BIT(4)
+#define INT_INF_DONE                    BIT(5)
+#define INT_NCTS_DONE                   BIT(6)
+#define INT_CTRL_PKT_DONE               BIT(7)
+#define GRL_INT_MASK            0x18
+#define GRL_CTRL                0x1C
+#define CTRL_GEN_EN                     BIT(2)
+#define CTRL_SPD_EN                     BIT(3)
+#define CTRL_MPEG_EN                    BIT(4)
+#define CTRL_AUDIO_EN                   BIT(5)
+#define CTRL_AVI_EN                     BIT(6)
+#define CTRL_AVMUTE                     BIT(7)
+#define GRL_STATUS              0x20
+#define STATUS_HTPLG                    BIT(0)
+#define STATUS_PORD                     BIT(1)
+#define GRL_DIVN                0x170
+#define NCTS_WRI_ANYTIME                BIT(6)
+#define GRL_AUDIO_CFG           0x17C
+#define AUDIO_ZERO                      BIT(0)
+#define HIGH_BIT_RATE                   BIT(1)
+#define SACD_DST                        BIT(2)
+#define DST_NORMAL_DOUBLE               BIT(3)
+#define DSD_INV                         BIT(4)
+#define LR_INV                          BIT(5)
+#define LR_MIX                          BIT(6)
+#define DSD_SEL                         BIT(7)
+#define GRL_NCTS                0x184
+#define GRL_CH_SW0              0x18C
+#define GRL_CH_SW1              0x190
+#define GRL_CH_SW2              0x194
+#define CH_SWITCH(from, to)             ((from) << ((to) * 3))
+#define GRL_INFOFRM_VER         0x19C
+#define GRL_INFOFRM_TYPE        0x1A0
+#define GRL_INFOFRM_LNG         0x1A4
+#define GRL_MIX_CTRL            0x1B4
+#define MIX_CTRL_SRC_EN                 BIT(0)
+#define BYPASS_VOLUME                   BIT(1)
+#define MIX_CTRL_FLAT                   BIT(7)
+#define GRL_AOUT_CFG            0x1C4
+#define AOUT_BNUM_SEL_MASK              0x03
+#define AOUT_24BIT                      0x00
+#define AOUT_20BIT                      0x02
+#define AOUT_16BIT                      0x03
+#define AOUT_FIFO_ADAP_CTRL             BIT(6)
+#define AOUT_BURST_PREAMBLE_EN          BIT(7)
+#define HIGH_BIT_RATE_PACKET_ALIGN      (AOUT_BURST_PREAMBLE_EN | \
+                                         AOUT_FIFO_ADAP_CTRL)
+#define GRL_SHIFT_L1            0x1C0
+#define GRL_SHIFT_R2            0x1B0
+#define AUDIO_PACKET_OFF                BIT(6)
+#define GRL_CFG0                0x24
+#define CFG0_I2S_MODE_MASK              0x3
+#define CFG0_I2S_MODE_RTJ               0x1
+#define CFG0_I2S_MODE_LTJ               0x0
+#define CFG0_I2S_MODE_I2S               0x2
+#define CFG0_W_LENGTH_MASK              0x30
+#define CFG0_W_LENGTH_24BIT             0x00
+#define CFG0_W_LENGTH_16BIT             0x10
+#define GRL_CFG1                0x28
+#define CFG1_EDG_SEL                    BIT(0)
+#define CFG1_SPDIF                      BIT(1)
+#define CFG1_DVI                        BIT(2)
+#define CFG1_HDCP_DEBUG                 BIT(3)
+#define GRL_CFG2                0x2c
+#define CFG2_MHL_DE_SEL                 BIT(3)
+#define CFG2_MHL_FAKE_DE_SEL            BIT(4)
+#define CFG2_MHL_DATA_REMAP             BIT(5)
+#define CFG2_NOTICE_EN                  BIT(6)
+#define CFG2_ACLK_INV                   BIT(7)
+#define GRL_CFG3                0x30
+#define CFG3_AES_KEY_INDEX_MASK         0x3f
+#define CFG3_CONTROL_PACKET_DELAY       BIT(6)
+#define CFG3_KSV_LOAD_START             BIT(7)
+#define GRL_CFG4                0x34
+#define CFG4_AES_KEY_LOAD               BIT(4)
+#define CFG4_AV_UNMUTE_EN               BIT(5)
+#define CFG4_AV_UNMUTE_SET              BIT(6)
+#define CFG4_MHL_MODE                   BIT(7)
+#define GRL_CFG5                0x38
+#define CFG5_CD_RATIO_MASK      0x8F
+#define CFG5_FS128                      (0x1 << 4)
+#define CFG5_FS256                      (0x2 << 4)
+#define CFG5_FS384                      (0x3 << 4)
+#define CFG5_FS512                      (0x4 << 4)
+#define CFG5_FS768                      (0x6 << 4)
+#define DUMMY_304               0x304
+#define CHMO_SEL                        (0x3 << 2)
+#define CHM1_SEL                        (0x3 << 4)
+#define CHM2_SEL                        (0x3 << 6)
+#define AUDIO_I2S_NCTS_SEL              BIT(1)
+#define AUDIO_I2S_NCTS_SEL_64           (1 << 1)
+#define AUDIO_I2S_NCTS_SEL_128          (0 << 1)
+#define NEW_GCP_CTRL                    BIT(0)
+#define NEW_GCP_CTRL_MERGE              BIT(0)
+#define GRL_L_STATUS_0          0x200
+#define GRL_L_STATUS_1          0x204
+#define GRL_L_STATUS_2          0x208
+#define GRL_L_STATUS_3          0x20c
+#define GRL_L_STATUS_4          0x210
+#define GRL_L_STATUS_5          0x214
+#define GRL_L_STATUS_6          0x218
+#define GRL_L_STATUS_7          0x21c
+#define GRL_L_STATUS_8          0x220
+#define GRL_L_STATUS_9          0x224
+#define GRL_L_STATUS_10         0x228
+#define GRL_L_STATUS_11         0x22c
+#define GRL_L_STATUS_12         0x230
+#define GRL_L_STATUS_13         0x234
+#define GRL_L_STATUS_14         0x238
+#define GRL_L_STATUS_15         0x23c
+#define GRL_L_STATUS_16         0x240
+#define GRL_L_STATUS_17         0x244
+#define GRL_L_STATUS_18         0x248
+#define GRL_L_STATUS_19         0x24c
+#define GRL_L_STATUS_20         0x250
+#define GRL_L_STATUS_21         0x254
+#define GRL_L_STATUS_22         0x258
+#define GRL_L_STATUS_23         0x25c
+#define GRL_R_STATUS_0          0x260
+#define GRL_R_STATUS_1          0x264
+#define GRL_R_STATUS_2          0x268
+#define GRL_R_STATUS_3          0x26c
+#define GRL_R_STATUS_4          0x270
+#define GRL_R_STATUS_5          0x274
+#define GRL_R_STATUS_6          0x278
+#define GRL_R_STATUS_7          0x27c
+#define GRL_R_STATUS_8          0x280
+#define GRL_R_STATUS_9          0x284
+#define GRL_R_STATUS_10         0x288
+#define GRL_R_STATUS_11         0x28c
+#define GRL_R_STATUS_12         0x290
+#define GRL_R_STATUS_13         0x294
+#define GRL_R_STATUS_14         0x298
+#define GRL_R_STATUS_15         0x29c
+#define GRL_R_STATUS_16         0x2a0
+#define GRL_R_STATUS_17         0x2a4
+#define GRL_R_STATUS_18         0x2a8
+#define GRL_R_STATUS_19         0x2ac
+#define GRL_R_STATUS_20         0x2b0
+#define GRL_R_STATUS_21         0x2b4
+#define GRL_R_STATUS_22         0x2b8
+#define GRL_R_STATUS_23         0x2bc
+#define GRL_ABIST_CTRL0         0x2D4
+#define GRL_ABIST_CTRL1         0x2D8
+#define ABIST_EN                        BIT(7)
+#define ABIST_DATA_FMT                  (0x7 << 0)
+#define VIDEO_CFG_0             0x380
+#define VIDEO_CFG_1             0x384
+#define VIDEO_CFG_2             0x388
+#define VIDEO_CFG_3             0x38c
+#define VIDEO_CFG_4             0x390
+#define VIDEO_SOURCE_SEL                BIT(7)
+#define NORMAL_PATH                     (1 << 7)
+#define GEN_RGB                         (0 << 7)
+
+#define HDMI_SYS_CFG1C          0x000
+#define HDMI_ON                         BIT(0)
+#define HDMI_RST                        BIT(1)
+#define ANLG_ON                         BIT(2)
+#define CFG10_DVI                       BIT(3)
+#define HDMI_TST                        BIT(3)
+#define SYS_KEYMASK1                    (0xff << 8)
+#define SYS_KEYMASK2                    (0xff << 16)
+#define AUD_OUTSYNC_EN                  BIT(24)
+#define AUD_OUTSYNC_PRE_EN              BIT(25)
+#define I2CM_ON                         BIT(26)
+#define E2PROM_TYPE_8BIT                BIT(27)
+#define MCM_E2PROM_ON                   BIT(28)
+#define EXT_E2PROM_ON                   BIT(29)
+#define HTPLG_PIN_SEL_OFF               BIT(30)
+#define AES_EFUSE_ENABLE                BIT(31)
+#define HDMI_SYS_CFG20          0x004
+#define DEEP_COLOR_MODE_MASK            (3 << 1)
+#define COLOR_8BIT_MODE                 (0 << 1)
+#define COLOR_10BIT_MODE                (1 << 1)
+#define COLOR_12BIT_MODE                (2 << 1)
+#define COLOR_16BIT_MODE                (3 << 1)
+#define DEEP_COLOR_EN                   BIT(0)
+#define HDMI_AUDIO_TEST_SEL             BIT(8)
+#define HDMI2P0_EN                      BIT(11)
+#define HDMI_OUT_FIFO_EN                BIT(16)
+#define HDMI_OUT_FIFO_CLK_INV           BIT(17)
+#define MHL_MODE_ON                     BIT(28)
+#define MHL_PP_MODE                     BIT(29)
+#define MHL_SYNC_AUTO_EN                BIT(30)
+#define HDMI_PCLK_FREE_RUN              BIT(31)
+
+#define MTK_SIP_SET_AUTHORIZED_SECURE_REG 0x82000001
+#endif
diff --git a/drivers/gpu/drm/mediatek/mtk_mipi_tx.c b/drivers/gpu/drm/mediatek/mtk_mipi_tx.c
index cf8f38d39e10..1c366f8cb2d0 100644
--- a/drivers/gpu/drm/mediatek/mtk_mipi_tx.c
+++ b/drivers/gpu/drm/mediatek/mtk_mipi_tx.c
@@ -431,7 +431,7 @@ static int mtk_mipi_tx_probe(struct platform_device *pdev)
         phy_set_drvdata(phy, mipi_tx);
 
         phy_provider = devm_of_phy_provider_register(dev, of_phy_simple_xlate);
-        if (IS_ERR(phy)) {
+        if (IS_ERR(phy_provider)) {
                 ret = PTR_ERR(phy_provider);
                 return ret;
         }
diff --git a/drivers/gpu/drm/mediatek/mtk_mt8173_hdmi_phy.c b/drivers/gpu/drm/mediatek/mtk_mt8173_hdmi_phy.c
new file mode 100644
index 000000000000..8a24754b440f
--- /dev/null
+++ b/drivers/gpu/drm/mediatek/mtk_mt8173_hdmi_phy.c
@@ -0,0 +1,515 @@
+/*
+ * Copyright (c) 2014 MediaTek Inc.
+ * Author: Jie Qiu <jie.qiu@mediatek.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ */
+
+#include <linux/clk.h>
+#include <linux/clk-provider.h>
+#include <linux/delay.h>
+#include <linux/io.h>
+#include <linux/mfd/syscon.h>
+#include <linux/module.h>
+#include <linux/phy/phy.h>
+#include <linux/platform_device.h>
+#include <linux/types.h>
+
+#define HDMI_CON0               0x00
+#define RG_HDMITX_PLL_EN                BIT(31)
+#define RG_HDMITX_PLL_FBKDIV            (0x7f << 24)
+#define PLL_FBKDIV_SHIFT                24
+#define RG_HDMITX_PLL_FBKSEL            (0x3 << 22)
+#define PLL_FBKSEL_SHIFT                22
+#define RG_HDMITX_PLL_PREDIV            (0x3 << 20)
+#define PREDIV_SHIFT                    20
+#define RG_HDMITX_PLL_POSDIV            (0x3 << 18)
+#define POSDIV_SHIFT                    18
+#define RG_HDMITX_PLL_RST_DLY           (0x3 << 16)
+#define RG_HDMITX_PLL_IR                (0xf << 12)
+#define PLL_IR_SHIFT                    12
+#define RG_HDMITX_PLL_IC                (0xf << 8)
+#define PLL_IC_SHIFT                    8
+#define RG_HDMITX_PLL_BP                (0xf << 4)
+#define PLL_BP_SHIFT                    4
+#define RG_HDMITX_PLL_BR                (0x3 << 2)
+#define PLL_BR_SHIFT                    2
+#define RG_HDMITX_PLL_BC                (0x3 << 0)
+#define PLL_BC_SHIFT                    0
+#define HDMI_CON1               0x04
+#define RG_HDMITX_PLL_DIVEN             (0x7 << 29)
+#define PLL_DIVEN_SHIFT                 29
+#define RG_HDMITX_PLL_AUTOK_EN          BIT(28)
+#define RG_HDMITX_PLL_AUTOK_KF          (0x3 << 26)
+#define RG_HDMITX_PLL_AUTOK_KS          (0x3 << 24)
+#define RG_HDMITX_PLL_AUTOK_LOAD        BIT(23)
+#define RG_HDMITX_PLL_BAND              (0x3f << 16)
+#define RG_HDMITX_PLL_REF_SEL           BIT(15)
+#define RG_HDMITX_PLL_BIAS_EN           BIT(14)
+#define RG_HDMITX_PLL_BIAS_LPF_EN       BIT(13)
+#define RG_HDMITX_PLL_TXDIV_EN          BIT(12)
+#define RG_HDMITX_PLL_TXDIV             (0x3 << 10)
+#define PLL_TXDIV_SHIFT                 10
+#define RG_HDMITX_PLL_LVROD_EN          BIT(9)
+#define RG_HDMITX_PLL_MONVC_EN          BIT(8)
+#define RG_HDMITX_PLL_MONCK_EN          BIT(7)
+#define RG_HDMITX_PLL_MONREF_EN         BIT(6)
+#define RG_HDMITX_PLL_TST_EN            BIT(5)
+#define RG_HDMITX_PLL_TST_CK_EN         BIT(4)
+#define RG_HDMITX_PLL_TST_SEL           (0xf << 0)
+#define HDMI_CON2               0x08
+#define RGS_HDMITX_PLL_AUTOK_BAND       (0x7f << 8)
+#define RGS_HDMITX_PLL_AUTOK_FAIL       BIT(1)
+#define RG_HDMITX_EN_TX_CKLDO           BIT(0)
+#define HDMI_CON3               0x0c
+#define RG_HDMITX_SER_EN                (0xf << 28)
+#define RG_HDMITX_PRD_EN                (0xf << 24)
+#define RG_HDMITX_PRD_IMP_EN            (0xf << 20)
+#define RG_HDMITX_DRV_EN                (0xf << 16)
+#define RG_HDMITX_DRV_IMP_EN            (0xf << 12)
+#define DRV_IMP_EN_SHIFT                12
+#define RG_HDMITX_MHLCK_FORCE           BIT(10)
+#define RG_HDMITX_MHLCK_PPIX_EN         BIT(9)
+#define RG_HDMITX_MHLCK_EN              BIT(8)
+#define RG_HDMITX_SER_DIN_SEL           (0xf << 4)
+#define RG_HDMITX_SER_5T1_BIST_EN       BIT(3)
+#define RG_HDMITX_SER_BIST_TOG          BIT(2)
+#define RG_HDMITX_SER_DIN_TOG           BIT(1)
+#define RG_HDMITX_SER_CLKDIG_INV        BIT(0)
+#define HDMI_CON4               0x10
+#define RG_HDMITX_PRD_IBIAS_CLK         (0xf << 24)
+#define RG_HDMITX_PRD_IBIAS_D2          (0xf << 16)
+#define RG_HDMITX_PRD_IBIAS_D1          (0xf << 8)
+#define RG_HDMITX_PRD_IBIAS_D0          (0xf << 0)
+#define PRD_IBIAS_CLK_SHIFT             24
+#define PRD_IBIAS_D2_SHIFT              16
+#define PRD_IBIAS_D1_SHIFT              8
+#define PRD_IBIAS_D0_SHIFT              0
+#define HDMI_CON5               0x14
+#define RG_HDMITX_DRV_IBIAS_CLK         (0x3f << 24)
+#define RG_HDMITX_DRV_IBIAS_D2          (0x3f << 16)
+#define RG_HDMITX_DRV_IBIAS_D1          (0x3f << 8)
+#define RG_HDMITX_DRV_IBIAS_D0          (0x3f << 0)
+#define DRV_IBIAS_CLK_SHIFT             24
+#define DRV_IBIAS_D2_SHIFT              16
+#define DRV_IBIAS_D1_SHIFT              8
+#define DRV_IBIAS_D0_SHIFT              0
+#define HDMI_CON6               0x18
+#define RG_HDMITX_DRV_IMP_CLK           (0x3f << 24)
+#define RG_HDMITX_DRV_IMP_D2            (0x3f << 16)
+#define RG_HDMITX_DRV_IMP_D1            (0x3f << 8)
+#define RG_HDMITX_DRV_IMP_D0            (0x3f << 0)
+#define DRV_IMP_CLK_SHIFT               24
+#define DRV_IMP_D2_SHIFT                16
+#define DRV_IMP_D1_SHIFT                8
+#define DRV_IMP_D0_SHIFT                0
+#define HDMI_CON7               0x1c
+#define RG_HDMITX_MHLCK_DRV_IBIAS       (0x1f << 27)
+#define RG_HDMITX_SER_DIN               (0x3ff << 16)
+#define RG_HDMITX_CHLDC_TST             (0xf << 12)
+#define RG_HDMITX_CHLCK_TST             (0xf << 8)
+#define RG_HDMITX_RESERVE               (0xff << 0)
+#define HDMI_CON8               0x20
+#define RGS_HDMITX_2T1_LEV              (0xf << 16)
+#define RGS_HDMITX_2T1_EDG              (0xf << 12)
+#define RGS_HDMITX_5T1_LEV              (0xf << 8)
+#define RGS_HDMITX_5T1_EDG              (0xf << 4)
+#define RGS_HDMITX_PLUG_TST             BIT(0)
+
+struct mtk_hdmi_phy {
+        void __iomem *regs;
+        struct device *dev;
+        struct clk *pll;
+        struct clk_hw pll_hw;
+        unsigned long pll_rate;
+        u8 drv_imp_clk;
+        u8 drv_imp_d2;
+        u8 drv_imp_d1;
+        u8 drv_imp_d0;
+        u32 ibias;
+        u32 ibias_up;
+};
+
+static const u8 PREDIV[3][4] = {
+        {0x0, 0x0, 0x0, 0x0},   /* 27Mhz */
+        {0x1, 0x1, 0x1, 0x1},   /* 74Mhz */
+        {0x1, 0x1, 0x1, 0x1}    /* 148Mhz */
+};
+
+static const u8 TXDIV[3][4] = {
+        {0x3, 0x3, 0x3, 0x2},   /* 27Mhz */
+        {0x2, 0x1, 0x1, 0x1},   /* 74Mhz */
+        {0x1, 0x0, 0x0, 0x0}    /* 148Mhz */
+};
+
+static const u8 FBKSEL[3][4] = {
+        {0x1, 0x1, 0x1, 0x1},   /* 27Mhz */
+        {0x1, 0x0, 0x1, 0x1},   /* 74Mhz */
+        {0x1, 0x0, 0x1, 0x1}    /* 148Mhz */
+};
+
+static const u8 FBKDIV[3][4] = {
+        {19, 24, 29, 19},       /* 27Mhz */
+        {19, 24, 14, 19},       /* 74Mhz */
+        {19, 24, 14, 19}        /* 148Mhz */
+};
+
+static const u8 DIVEN[3][4] = {
+        {0x2, 0x1, 0x1, 0x2},   /* 27Mhz */
+        {0x2, 0x2, 0x2, 0x2},   /* 74Mhz */
+        {0x2, 0x2, 0x2, 0x2}    /* 148Mhz */
+};
+
+static const u8 HTPLLBP[3][4] = {
+        {0xc, 0xc, 0x8, 0xc},   /* 27Mhz */
+        {0xc, 0xf, 0xf, 0xc},   /* 74Mhz */
+        {0xc, 0xf, 0xf, 0xc}    /* 148Mhz */
+};
+
+static const u8 HTPLLBC[3][4] = {
+        {0x2, 0x3, 0x3, 0x2},   /* 27Mhz */
+        {0x2, 0x3, 0x3, 0x2},   /* 74Mhz */
+        {0x2, 0x3, 0x3, 0x2}    /* 148Mhz */
+};
+
+static const u8 HTPLLBR[3][4] = {
+        {0x1, 0x1, 0x0, 0x1},   /* 27Mhz */
+        {0x1, 0x2, 0x2, 0x1},   /* 74Mhz */
+        {0x1, 0x2, 0x2, 0x1}    /* 148Mhz */
+};
+
+static void mtk_hdmi_phy_clear_bits(struct mtk_hdmi_phy *hdmi_phy, u32 offset,
+                                    u32 bits)
+{
+        void __iomem *reg = hdmi_phy->regs + offset;
+        u32 tmp;
+
+        tmp = readl(reg);
+        tmp &= ~bits;
+        writel(tmp, reg);
+}
+
+static void mtk_hdmi_phy_set_bits(struct mtk_hdmi_phy *hdmi_phy, u32 offset,
+                                  u32 bits)
+{
+        void __iomem *reg = hdmi_phy->regs + offset;
+        u32 tmp;
+
+        tmp = readl(reg);
+        tmp |= bits;
+        writel(tmp, reg);
+}
+
+static void mtk_hdmi_phy_mask(struct mtk_hdmi_phy *hdmi_phy, u32 offset,
+                              u32 val, u32 mask)
+{
+        void __iomem *reg = hdmi_phy->regs + offset;
+        u32 tmp;
+
+        tmp = readl(reg);
+        tmp = (tmp & ~mask) | (val & mask);
+        writel(tmp, reg);
+}
+
+static inline struct mtk_hdmi_phy *to_mtk_hdmi_phy(struct clk_hw *hw)
+{
+        return container_of(hw, struct mtk_hdmi_phy, pll_hw);
+}
+
+static int mtk_hdmi_pll_prepare(struct clk_hw *hw)
+{
+        struct mtk_hdmi_phy *hdmi_phy = to_mtk_hdmi_phy(hw);
+
+        dev_dbg(hdmi_phy->dev, "%s\n", __func__);
+
+        mtk_hdmi_phy_set_bits(hdmi_phy, HDMI_CON1, RG_HDMITX_PLL_AUTOK_EN);
+        mtk_hdmi_phy_set_bits(hdmi_phy, HDMI_CON0, RG_HDMITX_PLL_POSDIV);
+        mtk_hdmi_phy_clear_bits(hdmi_phy, HDMI_CON3, RG_HDMITX_MHLCK_EN);
+        mtk_hdmi_phy_set_bits(hdmi_phy, HDMI_CON1, RG_HDMITX_PLL_BIAS_EN);
+        usleep_range(100, 150);
+        mtk_hdmi_phy_set_bits(hdmi_phy, HDMI_CON0, RG_HDMITX_PLL_EN);
+        usleep_range(100, 150);
+        mtk_hdmi_phy_set_bits(hdmi_phy, HDMI_CON1, RG_HDMITX_PLL_BIAS_LPF_EN);
+        mtk_hdmi_phy_set_bits(hdmi_phy, HDMI_CON1, RG_HDMITX_PLL_TXDIV_EN);
+
+        return 0;
+}
+
+static void mtk_hdmi_pll_unprepare(struct clk_hw *hw)
+{
+        struct mtk_hdmi_phy *hdmi_phy = to_mtk_hdmi_phy(hw);
+
+        dev_dbg(hdmi_phy->dev, "%s\n", __func__);
+
+        mtk_hdmi_phy_clear_bits(hdmi_phy, HDMI_CON1, RG_HDMITX_PLL_TXDIV_EN);
+        mtk_hdmi_phy_clear_bits(hdmi_phy, HDMI_CON1, RG_HDMITX_PLL_BIAS_LPF_EN);
+        usleep_range(100, 150);
+        mtk_hdmi_phy_clear_bits(hdmi_phy, HDMI_CON0, RG_HDMITX_PLL_EN);
+        usleep_range(100, 150);
+        mtk_hdmi_phy_clear_bits(hdmi_phy, HDMI_CON1, RG_HDMITX_PLL_BIAS_EN);
+        mtk_hdmi_phy_clear_bits(hdmi_phy, HDMI_CON0, RG_HDMITX_PLL_POSDIV);
+        mtk_hdmi_phy_clear_bits(hdmi_phy, HDMI_CON1, RG_HDMITX_PLL_AUTOK_EN);
+        usleep_range(100, 150);
+}
+
+static int mtk_hdmi_pll_set_rate(struct clk_hw *hw, unsigned long rate,
+                                 unsigned long parent_rate)
+{
+        struct mtk_hdmi_phy *hdmi_phy = to_mtk_hdmi_phy(hw);
+        unsigned int pre_div;
+        unsigned int div;
+
+        dev_dbg(hdmi_phy->dev, "%s: %lu Hz, parent: %lu Hz\n", __func__,
+                rate, parent_rate);
+
+        if (rate <= 27000000) {
+                pre_div = 0;
+                div = 3;
+        } else if (rate <= 74250000) {
+                pre_div = 1;
+                div = 2;
+        } else {
+                pre_div = 1;
+                div = 1;
+        }
+
+        mtk_hdmi_phy_mask(hdmi_phy, HDMI_CON0,
+                          (pre_div << PREDIV_SHIFT), RG_HDMITX_PLL_PREDIV);
+        mtk_hdmi_phy_set_bits(hdmi_phy, HDMI_CON0, RG_HDMITX_PLL_POSDIV);
+        mtk_hdmi_phy_mask(hdmi_phy, HDMI_CON0,
+                          (0x1 << PLL_IC_SHIFT) | (0x1 << PLL_IR_SHIFT),
+                          RG_HDMITX_PLL_IC | RG_HDMITX_PLL_IR);
+        mtk_hdmi_phy_mask(hdmi_phy, HDMI_CON1,
+                          (div << PLL_TXDIV_SHIFT), RG_HDMITX_PLL_TXDIV);
+        mtk_hdmi_phy_mask(hdmi_phy, HDMI_CON0,
+                          (0x1 << PLL_FBKSEL_SHIFT) | (19 << PLL_FBKDIV_SHIFT),
+                          RG_HDMITX_PLL_FBKSEL | RG_HDMITX_PLL_FBKDIV);
+        mtk_hdmi_phy_mask(hdmi_phy, HDMI_CON1,
+                          (0x2 << PLL_DIVEN_SHIFT), RG_HDMITX_PLL_DIVEN);
+        mtk_hdmi_phy_mask(hdmi_phy, HDMI_CON0,
+                          (0xc << PLL_BP_SHIFT) | (0x2 << PLL_BC_SHIFT) |
+                          (0x1 << PLL_BR_SHIFT),
+                          RG_HDMITX_PLL_BP | RG_HDMITX_PLL_BC |
+                          RG_HDMITX_PLL_BR);
+        mtk_hdmi_phy_clear_bits(hdmi_phy, HDMI_CON3, RG_HDMITX_PRD_IMP_EN);
+        mtk_hdmi_phy_mask(hdmi_phy, HDMI_CON4,
+                          (0x3 << PRD_IBIAS_CLK_SHIFT) |
+                          (0x3 << PRD_IBIAS_D2_SHIFT) |
+                          (0x3 << PRD_IBIAS_D1_SHIFT) |
+                          (0x3 << PRD_IBIAS_D0_SHIFT),
+                          RG_HDMITX_PRD_IBIAS_CLK |
+                          RG_HDMITX_PRD_IBIAS_D2 |
+                          RG_HDMITX_PRD_IBIAS_D1 |
+                          RG_HDMITX_PRD_IBIAS_D0);
+        mtk_hdmi_phy_mask(hdmi_phy, HDMI_CON3,
+                          (0x0 << DRV_IMP_EN_SHIFT), RG_HDMITX_DRV_IMP_EN);
+        mtk_hdmi_phy_mask(hdmi_phy, HDMI_CON6,
+                          (hdmi_phy->drv_imp_clk << DRV_IMP_CLK_SHIFT) |
+                          (hdmi_phy->drv_imp_d2 << DRV_IMP_D2_SHIFT) |
+                          (hdmi_phy->drv_imp_d1 << DRV_IMP_D1_SHIFT) |
+                          (hdmi_phy->drv_imp_d0 << DRV_IMP_D0_SHIFT),
+                          RG_HDMITX_DRV_IMP_CLK | RG_HDMITX_DRV_IMP_D2 |
+                          RG_HDMITX_DRV_IMP_D1 | RG_HDMITX_DRV_IMP_D0);
+        mtk_hdmi_phy_mask(hdmi_phy, HDMI_CON5,
+                          (hdmi_phy->ibias << DRV_IBIAS_CLK_SHIFT) |
+                          (hdmi_phy->ibias << DRV_IBIAS_D2_SHIFT) |
+                          (hdmi_phy->ibias << DRV_IBIAS_D1_SHIFT) |
+                          (hdmi_phy->ibias << DRV_IBIAS_D0_SHIFT),
+                          RG_HDMITX_DRV_IBIAS_CLK | RG_HDMITX_DRV_IBIAS_D2 |
+                          RG_HDMITX_DRV_IBIAS_D1 | RG_HDMITX_DRV_IBIAS_D0);
+        return 0;
+}
+
+static long mtk_hdmi_pll_round_rate(struct clk_hw *hw, unsigned long rate,
+                                    unsigned long *parent_rate)
+{
+        struct mtk_hdmi_phy *hdmi_phy = to_mtk_hdmi_phy(hw);
+
+        hdmi_phy->pll_rate = rate;
+        if (rate <= 74250000)
+                *parent_rate = rate;
+        else
+                *parent_rate = rate / 2;
+
+        return rate;
+}
+
+static unsigned long mtk_hdmi_pll_recalc_rate(struct clk_hw *hw,
+                                              unsigned long parent_rate)
+{
+        struct mtk_hdmi_phy *hdmi_phy = to_mtk_hdmi_phy(hw);
+
+        return hdmi_phy->pll_rate;
+}
+
+static const struct clk_ops mtk_hdmi_pll_ops = {
+        .prepare = mtk_hdmi_pll_prepare,
+        .unprepare = mtk_hdmi_pll_unprepare,
+        .set_rate = mtk_hdmi_pll_set_rate,
+        .round_rate = mtk_hdmi_pll_round_rate,
+        .recalc_rate = mtk_hdmi_pll_recalc_rate,
+};
+
+static void mtk_hdmi_phy_enable_tmds(struct mtk_hdmi_phy *hdmi_phy)
+{
+        mtk_hdmi_phy_set_bits(hdmi_phy, HDMI_CON3,
+                              RG_HDMITX_SER_EN | RG_HDMITX_PRD_EN |
+                              RG_HDMITX_DRV_EN);
+        usleep_range(100, 150);
+}
+
+static void mtk_hdmi_phy_disable_tmds(struct mtk_hdmi_phy *hdmi_phy)
+{
+        mtk_hdmi_phy_clear_bits(hdmi_phy, HDMI_CON3,
+                                RG_HDMITX_DRV_EN | RG_HDMITX_PRD_EN |
+                                RG_HDMITX_SER_EN);
+}
+
+static int mtk_hdmi_phy_power_on(struct phy *phy)
+{
+        struct mtk_hdmi_phy *hdmi_phy = phy_get_drvdata(phy);
+        int ret;
+
+        ret = clk_prepare_enable(hdmi_phy->pll);
+        if (ret < 0)
+                return ret;
+
+        mtk_hdmi_phy_enable_tmds(hdmi_phy);
+
+        return 0;
+}
+
+static int mtk_hdmi_phy_power_off(struct phy *phy)
+{
+        struct mtk_hdmi_phy *hdmi_phy = phy_get_drvdata(phy);
+
+        mtk_hdmi_phy_disable_tmds(hdmi_phy);
+        clk_disable_unprepare(hdmi_phy->pll);
+
+        return 0;
+}
+
+static const struct phy_ops mtk_hdmi_phy_ops = {
+        .power_on = mtk_hdmi_phy_power_on,
+        .power_off = mtk_hdmi_phy_power_off,
+        .owner = THIS_MODULE,
+};
+
+static int mtk_hdmi_phy_probe(struct platform_device *pdev)
+{
+        struct device *dev = &pdev->dev;
+        struct mtk_hdmi_phy *hdmi_phy;
+        struct resource *mem;
+        struct clk *ref_clk;
+        const char *ref_clk_name;
+        struct clk_init_data clk_init = {
+                .ops = &mtk_hdmi_pll_ops,
+                .num_parents = 1,
+                .parent_names = (const char * const *)&ref_clk_name,
+                .flags = CLK_SET_RATE_PARENT | CLK_SET_RATE_GATE,
+        };
+        struct phy *phy;
+        struct phy_provider *phy_provider;
+        int ret;
+
+        hdmi_phy = devm_kzalloc(dev, sizeof(*hdmi_phy), GFP_KERNEL);
+        if (!hdmi_phy)
+                return -ENOMEM;
+
+        mem = platform_get_resource(pdev, IORESOURCE_MEM, 0);
+        hdmi_phy->regs = devm_ioremap_resource(dev, mem);
+        if (IS_ERR(hdmi_phy->regs)) {
+                ret = PTR_ERR(hdmi_phy->regs);
+                dev_err(dev, "Failed to get memory resource: %d\n", ret);
+                return ret;
+        }
+
+        ref_clk = devm_clk_get(dev, "pll_ref");
+        if (IS_ERR(ref_clk)) {
+                ret = PTR_ERR(ref_clk);
+                dev_err(&pdev->dev, "Failed to get PLL reference clock: %d\n",
+                        ret);
+                return ret;
+        }
+        ref_clk_name = __clk_get_name(ref_clk);
+
+        ret = of_property_read_string(dev->of_node, "clock-output-names",
+                                      &clk_init.name);
+        if (ret < 0) {
+                dev_err(dev, "Failed to read clock-output-names: %d\n", ret);
+                return ret;
+        }
+
+        hdmi_phy->pll_hw.init = &clk_init;
+        hdmi_phy->pll = devm_clk_register(dev, &hdmi_phy->pll_hw);
+        if (IS_ERR(hdmi_phy->pll)) {
+                ret = PTR_ERR(hdmi_phy->pll);
+                dev_err(dev, "Failed to register PLL: %d\n", ret);
+                return ret;
+        }
+
+        ret = of_property_read_u32(dev->of_node, "mediatek,ibias",
+                                   &hdmi_phy->ibias);
+        if (ret < 0) {
+                dev_err(&pdev->dev, "Failed to get ibias: %d\n", ret);
+                return ret;
+        }
+
+        ret = of_property_read_u32(dev->of_node, "mediatek,ibias_up",
+                                   &hdmi_phy->ibias_up);
+        if (ret < 0) {
+                dev_err(&pdev->dev, "Failed to get ibias up: %d\n", ret);
+                return ret;
+        }
+
+        dev_info(dev, "Using default TX DRV impedance: 4.2k/36\n");
+        hdmi_phy->drv_imp_clk = 0x30;
+        hdmi_phy->drv_imp_d2 = 0x30;
+        hdmi_phy->drv_imp_d1 = 0x30;
+        hdmi_phy->drv_imp_d0 = 0x30;
+
+        phy = devm_phy_create(dev, NULL, &mtk_hdmi_phy_ops);
+        if (IS_ERR(phy)) {
+                dev_err(dev, "Failed to create HDMI PHY\n");
+                return PTR_ERR(phy);
+        }
+        phy_set_drvdata(phy, hdmi_phy);
+
+        phy_provider = devm_of_phy_provider_register(dev, of_phy_simple_xlate);
+        if (IS_ERR(phy_provider))
+                return PTR_ERR(phy_provider);
+
+        hdmi_phy->dev = dev;
+        return of_clk_add_provider(dev->of_node, of_clk_src_simple_get,
+                                   hdmi_phy->pll);
+}
+
+static int mtk_hdmi_phy_remove(struct platform_device *pdev)
+{
+        return 0;
+}
+
+static const struct of_device_id mtk_hdmi_phy_match[] = {
+        { .compatible = "mediatek,mt8173-hdmi-phy", },
+        {},
+};
+
+struct platform_driver mtk_hdmi_phy_driver = {
+        .probe = mtk_hdmi_phy_probe,
+        .remove = mtk_hdmi_phy_remove,
+        .driver = {
+                .name = "mediatek-hdmi-phy",
+                .of_match_table = mtk_hdmi_phy_match,
+        },
+};
+
+MODULE_AUTHOR("Jie Qiu <jie.qiu@mediatek.com>");
+MODULE_DESCRIPTION("MediaTek MT8173 HDMI PHY Driver");
+MODULE_LICENSE("GPL v2");
diff --git a/drivers/gpu/drm/mgag200/Kconfig b/drivers/gpu/drm/mgag200/Kconfig
index 3a1c5fbae54a..520e5e668d6c 100644
--- a/drivers/gpu/drm/mgag200/Kconfig
+++ b/drivers/gpu/drm/mgag200/Kconfig
@@ -1,11 +1,7 @@
 config DRM_MGAG200
         tristate "Kernel modesetting driver for MGA G200 server engines"
         depends on DRM && PCI
-        select FB_SYS_FILLRECT
-        select FB_SYS_COPYAREA
-        select FB_SYS_IMAGEBLIT
         select DRM_KMS_HELPER
-        select DRM_KMS_FB_HELPER
         select DRM_TTM
         help
          This is a KMS driver for the MGA G200 server chips, it
diff --git a/drivers/gpu/drm/mgag200/mgag200_drv.c b/drivers/gpu/drm/mgag200/mgag200_drv.c
index ebb470ff7200..2b4b125eebc3 100644
--- a/drivers/gpu/drm/mgag200/mgag200_drv.c
+++ b/drivers/gpu/drm/mgag200/mgag200_drv.c
@@ -101,7 +101,7 @@ static struct drm_driver driver = {
         .minor = DRIVER_MINOR,
         .patchlevel = DRIVER_PATCHLEVEL,
 
-        .gem_free_object = mgag200_gem_free_object,
+        .gem_free_object_unlocked = mgag200_gem_free_object,
         .dumb_create = mgag200_dumb_create,
         .dumb_map_offset = mgag200_dumb_mmap_offset,
         .dumb_destroy = drm_gem_dumb_destroy,
diff --git a/drivers/gpu/drm/mgag200/mgag200_main.c b/drivers/gpu/drm/mgag200/mgag200_main.c
index 615cbb08ba29..13798b3e6beb 100644
--- a/drivers/gpu/drm/mgag200/mgag200_main.c
+++ b/drivers/gpu/drm/mgag200/mgag200_main.c
@@ -17,8 +17,8 @@
 static void mga_user_framebuffer_destroy(struct drm_framebuffer *fb)
 {
         struct mga_framebuffer *mga_fb = to_mga_framebuffer(fb);
-        if (mga_fb->obj)
-                drm_gem_object_unreference_unlocked(mga_fb->obj);
+
+        drm_gem_object_unreference_unlocked(mga_fb->obj);
         drm_framebuffer_cleanup(fb);
         kfree(fb);
 }
diff --git a/drivers/gpu/drm/mgag200/mgag200_mode.c b/drivers/gpu/drm/mgag200/mgag200_mode.c
index d347dca17267..6b21cb27e1cc 100644
--- a/drivers/gpu/drm/mgag200/mgag200_mode.c
+++ b/drivers/gpu/drm/mgag200/mgag200_mode.c
@@ -1352,19 +1352,20 @@ static void mga_crtc_commit(struct drm_crtc *crtc)
  * use this for 8-bit mode so can't perform smooth fades on deeper modes,
  * but it's a requirement that we provide the function
  */
-static void mga_crtc_gamma_set(struct drm_crtc *crtc, u16 *red, u16 *green,
-                                  u16 *blue, uint32_t start, uint32_t size)
+static int mga_crtc_gamma_set(struct drm_crtc *crtc, u16 *red, u16 *green,
+                              u16 *blue, uint32_t size)
 {
         struct mga_crtc *mga_crtc = to_mga_crtc(crtc);
-        int end = (start + size > MGAG200_LUT_SIZE) ? MGAG200_LUT_SIZE : start + size;
         int i;
 
-        for (i = start; i < end; i++) {
+        for (i = 0; i < size; i++) {
                 mga_crtc->lut_r[i] = red[i] >> 8;
                 mga_crtc->lut_g[i] = green[i] >> 8;
                 mga_crtc->lut_b[i] = blue[i] >> 8;
         }
         mga_crtc_load_lut(crtc);
+
+        return 0;
 }
 
 /* Simple cleanup function */
diff --git a/drivers/gpu/drm/mgag200/mgag200_ttm.c b/drivers/gpu/drm/mgag200/mgag200_ttm.c
index 9d5083d0f1ee..68268e55d595 100644
--- a/drivers/gpu/drm/mgag200/mgag200_ttm.c
+++ b/drivers/gpu/drm/mgag200/mgag200_ttm.c
@@ -186,17 +186,6 @@ static void mgag200_ttm_io_mem_free(struct ttm_bo_device *bdev, struct ttm_mem_r
 {
 }
 
-static int mgag200_bo_move(struct ttm_buffer_object *bo,
-                       bool evict, bool interruptible,
-                       bool no_wait_gpu,
-                       struct ttm_mem_reg *new_mem)
-{
-        int r;
-        r = ttm_bo_move_memcpy(bo, evict, no_wait_gpu, new_mem);
-        return r;
-}
-
-
 static void mgag200_ttm_backend_destroy(struct ttm_tt *tt)
 {
         ttm_tt_fini(tt);
@@ -241,7 +230,7 @@ struct ttm_bo_driver mgag200_bo_driver = {
         .ttm_tt_unpopulate = mgag200_ttm_tt_unpopulate,
         .init_mem_type = mgag200_bo_init_mem_type,
         .evict_flags = mgag200_bo_evict_flags,
-        .move = mgag200_bo_move,
+        .move = NULL,
         .verify_access = mgag200_bo_verify_access,
         .io_mem_reserve = &mgag200_ttm_io_mem_reserve,
         .io_mem_free = &mgag200_ttm_io_mem_free,
diff --git a/drivers/gpu/drm/msm/Kconfig b/drivers/gpu/drm/msm/Kconfig
index 167a4971f47c..7c7a0314a756 100644
--- a/drivers/gpu/drm/msm/Kconfig
+++ b/drivers/gpu/drm/msm/Kconfig
@@ -10,6 +10,7 @@ config DRM_MSM
         select SHMEM
         select TMPFS
         select QCOM_SCM
+        select SND_SOC_HDMI_CODEC if SND_SOC
         default y
         help
           DRM/KMS driver for MSM/snapdragon.
diff --git a/drivers/gpu/drm/msm/Makefile b/drivers/gpu/drm/msm/Makefile
index 60cb02624dc0..4e2806cf778c 100644
--- a/drivers/gpu/drm/msm/Makefile
+++ b/drivers/gpu/drm/msm/Makefile
@@ -35,6 +35,7 @@ msm-y := \
         mdp/mdp5/mdp5_crtc.o \
         mdp/mdp5/mdp5_encoder.o \
         mdp/mdp5/mdp5_irq.o \
+        mdp/mdp5/mdp5_mdss.o \
         mdp/mdp5/mdp5_kms.o \
         mdp/mdp5/mdp5_plane.o \
         mdp/mdp5/mdp5_smp.o \
@@ -45,6 +46,7 @@ msm-y := \
         msm_fence.o \
         msm_gem.o \
         msm_gem_prime.o \
+        msm_gem_shrinker.o \
         msm_gem_submit.o \
         msm_gpu.o \
         msm_iommu.o \
diff --git a/drivers/gpu/drm/msm/adreno/adreno_gpu.c b/drivers/gpu/drm/msm/adreno/adreno_gpu.c
index 2aec27dbb5bb..f386f463278d 100644
--- a/drivers/gpu/drm/msm/adreno/adreno_gpu.c
+++ b/drivers/gpu/drm/msm/adreno/adreno_gpu.c
@@ -139,7 +139,7 @@ void adreno_submit(struct msm_gpu *gpu, struct msm_gem_submit *submit,
         struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu);
         struct msm_drm_private *priv = gpu->dev->dev_private;
         struct msm_ringbuffer *ring = gpu->rb;
-        unsigned i, ibs = 0;
+        unsigned i;
 
         for (i = 0; i < submit->nr_cmds; i++) {
                 switch (submit->cmd[i].type) {
@@ -155,18 +155,11 @@ void adreno_submit(struct msm_gpu *gpu, struct msm_gem_submit *submit,
                                 CP_INDIRECT_BUFFER_PFE : CP_INDIRECT_BUFFER_PFD, 2);
                         OUT_RING(ring, submit->cmd[i].iova);
                         OUT_RING(ring, submit->cmd[i].size);
-                        ibs++;
+                        OUT_PKT2(ring);
                         break;
                 }
         }
 
-        /* on a320, at least, we seem to need to pad things out to an
-         * even number of qwords to avoid issue w/ CP hanging on wrap-
-         * around:
-         */
-        if (ibs % 2)
-                OUT_PKT2(ring);
-
         OUT_PKT0(ring, REG_AXXX_CP_SCRATCH_REG2, 1);
         OUT_RING(ring, submit->fence->seqno);
 
@@ -407,7 +400,7 @@ int adreno_gpu_init(struct drm_device *drm, struct platform_device *pdev,
                 return ret;
         }
 
-        adreno_gpu->memptrs = msm_gem_vaddr(adreno_gpu->memptrs_bo);
+        adreno_gpu->memptrs = msm_gem_get_vaddr(adreno_gpu->memptrs_bo);
         if (IS_ERR(adreno_gpu->memptrs)) {
                 dev_err(drm->dev, "could not vmap memptrs\n");
                 return -ENOMEM;
@@ -426,8 +419,12 @@ int adreno_gpu_init(struct drm_device *drm, struct platform_device *pdev,
 void adreno_gpu_cleanup(struct adreno_gpu *gpu)
 {
         if (gpu->memptrs_bo) {
+                if (gpu->memptrs)
+                        msm_gem_put_vaddr(gpu->memptrs_bo);
+
                 if (gpu->memptrs_iova)
                         msm_gem_put_iova(gpu->memptrs_bo, gpu->base.id);
+
                 drm_gem_object_unreference_unlocked(gpu->memptrs_bo);
         }
         release_firmware(gpu->pm4);
diff --git a/drivers/gpu/drm/msm/dsi/dsi.c b/drivers/gpu/drm/msm/dsi/dsi.c
index 6edcd6f57e70..ec572f8389ed 100644
--- a/drivers/gpu/drm/msm/dsi/dsi.c
+++ b/drivers/gpu/drm/msm/dsi/dsi.c
@@ -29,7 +29,7 @@ static int dsi_get_phy(struct msm_dsi *msm_dsi)
         struct platform_device *phy_pdev;
         struct device_node *phy_node;
 
-        phy_node = of_parse_phandle(pdev->dev.of_node, "qcom,dsi-phy", 0);
+        phy_node = of_parse_phandle(pdev->dev.of_node, "phys", 0);
         if (!phy_node) {
                 dev_err(&pdev->dev, "cannot find phy device\n");
                 return -ENXIO;
diff --git a/drivers/gpu/drm/msm/dsi/dsi_cfg.c b/drivers/gpu/drm/msm/dsi/dsi_cfg.c
index 93c1ee094eac..63436d8ee470 100644
--- a/drivers/gpu/drm/msm/dsi/dsi_cfg.c
+++ b/drivers/gpu/drm/msm/dsi/dsi_cfg.c
@@ -29,6 +29,8 @@ static const struct msm_dsi_config apq8064_dsi_cfg = {
         },
         .bus_clk_names = dsi_v2_bus_clk_names,
         .num_bus_clks = ARRAY_SIZE(dsi_v2_bus_clk_names),
+        .io_start = { 0x4700000, 0x5800000 },
+        .num_dsi = 2,
 };
 
 static const char * const dsi_6g_bus_clk_names[] = {
@@ -48,6 +50,8 @@ static const struct msm_dsi_config msm8974_apq8084_dsi_cfg = {
         },
         .bus_clk_names = dsi_6g_bus_clk_names,
         .num_bus_clks = ARRAY_SIZE(dsi_6g_bus_clk_names),
+        .io_start = { 0xfd922800, 0xfd922b00 },
+        .num_dsi = 2,
 };
 
 static const char * const dsi_8916_bus_clk_names[] = {
@@ -66,6 +70,8 @@ static const struct msm_dsi_config msm8916_dsi_cfg = {
         },
         .bus_clk_names = dsi_8916_bus_clk_names,
         .num_bus_clks = ARRAY_SIZE(dsi_8916_bus_clk_names),
+        .io_start = { 0x1a98000 },
+        .num_dsi = 1,
 };
 
 static const struct msm_dsi_config msm8994_dsi_cfg = {
@@ -84,6 +90,8 @@ static const struct msm_dsi_config msm8994_dsi_cfg = {
         },
         .bus_clk_names = dsi_6g_bus_clk_names,
         .num_bus_clks = ARRAY_SIZE(dsi_6g_bus_clk_names),
+        .io_start = { 0xfd998000, 0xfd9a0000 },
+        .num_dsi = 2,
 };
 
 static const struct msm_dsi_cfg_handler dsi_cfg_handlers[] = {
diff --git a/drivers/gpu/drm/msm/dsi/dsi_cfg.h b/drivers/gpu/drm/msm/dsi/dsi_cfg.h
index a68c836744a3..eeacc3232494 100644
--- a/drivers/gpu/drm/msm/dsi/dsi_cfg.h
+++ b/drivers/gpu/drm/msm/dsi/dsi_cfg.h
@@ -34,6 +34,8 @@ struct msm_dsi_config {
         struct dsi_reg_config reg_cfg;
         const char * const *bus_clk_names;
         const int num_bus_clks;
+        const resource_size_t io_start[DSI_MAX];
+        const int num_dsi;
 };
 
 struct msm_dsi_cfg_handler {
diff --git a/drivers/gpu/drm/msm/dsi/dsi_host.c b/drivers/gpu/drm/msm/dsi/dsi_host.c
index a3e47ad83eb3..f05ed0e1f3d6 100644
--- a/drivers/gpu/drm/msm/dsi/dsi_host.c
+++ b/drivers/gpu/drm/msm/dsi/dsi_host.c
@@ -1066,7 +1066,7 @@ static int dsi_cmd_dma_add(struct msm_dsi_host *msm_host,
         }
 
         if (cfg_hnd->major == MSM_DSI_VER_MAJOR_6G) {
-                data = msm_gem_vaddr(msm_host->tx_gem_obj);
+                data = msm_gem_get_vaddr(msm_host->tx_gem_obj);
                 if (IS_ERR(data)) {
                         ret = PTR_ERR(data);
                         pr_err("%s: get vaddr failed, %d\n", __func__, ret);
@@ -1094,6 +1094,9 @@ static int dsi_cmd_dma_add(struct msm_dsi_host *msm_host,
         if (packet.size < len)
                 memset(data + packet.size, 0xff, len - packet.size);
 
+        if (cfg_hnd->major == MSM_DSI_VER_MAJOR_6G)
+                msm_gem_put_vaddr(msm_host->tx_gem_obj);
+
         return len;
 }
 
@@ -1543,7 +1546,7 @@ static int dsi_host_parse_lane_data(struct msm_dsi_host *msm_host,
         u32 lane_map[4];
         int ret, i, len, num_lanes;
 
-        prop = of_find_property(ep, "qcom,data-lane-map", &len);
+        prop = of_find_property(ep, "data-lanes", &len);
         if (!prop) {
                 dev_dbg(dev, "failed to find data lane mapping\n");
                 return -EINVAL;
@@ -1558,7 +1561,7 @@ static int dsi_host_parse_lane_data(struct msm_dsi_host *msm_host,
 
         msm_host->num_data_lanes = num_lanes;
 
-        ret = of_property_read_u32_array(ep, "qcom,data-lane-map", lane_map,
+        ret = of_property_read_u32_array(ep, "data-lanes", lane_map,
                                          num_lanes);
         if (ret) {
                 dev_err(dev, "failed to read lane data\n");
@@ -1573,8 +1576,19 @@ static int dsi_host_parse_lane_data(struct msm_dsi_host *msm_host,
                 const int *swap = supported_data_lane_swaps[i];
                 int j;
 
+                /*
+                 * the data-lanes array we get from DT has a logical->physical
+                 * mapping. The "data lane swap" register field represents
+                 * supported configurations in a physical->logical mapping.
+                 * Translate the DT mapping to what we understand and find a
+                 * configuration that works.
+                 */
                 for (j = 0; j < num_lanes; j++) {
-                        if (swap[j] != lane_map[j])
+                        if (lane_map[j] < 0 || lane_map[j] > 3)
+                                dev_err(dev, "bad physical lane entry %u\n",
+                                        lane_map[j]);
+
+                        if (swap[lane_map[j]] != j)
                                 break;
                 }
 
@@ -1594,20 +1608,13 @@ static int dsi_host_parse_dt(struct msm_dsi_host *msm_host)
         struct device_node *endpoint, *device_node;
         int ret;
 
-        ret = of_property_read_u32(np, "qcom,dsi-host-index", &msm_host->id);
-        if (ret) {
-                dev_err(dev, "%s: host index not specified, ret=%d\n",
-                        __func__, ret);
-                return ret;
-        }
-
         /*
-         * Get the first endpoint node. In our case, dsi has one output port
-         * to which the panel is connected. Don't return an error if a port
-         * isn't defined. It's possible that there is nothing connected to
-         * the dsi output.
+         * Get the endpoint of the output port of the DSI host. In our case,
+         * this is mapped to port number with reg = 1. Don't return an error if
+         * the remote endpoint isn't defined. It's possible that there is
+         * nothing connected to the dsi output.
          */
-        endpoint = of_graph_get_next_endpoint(np, NULL);
+        endpoint = of_graph_get_endpoint_by_regs(np, 1, -1);
         if (!endpoint) {
                 dev_dbg(dev, "%s: no endpoint\n", __func__);
                 return 0;
@@ -1648,6 +1655,25 @@ err:
         return ret;
 }
 
+static int dsi_host_get_id(struct msm_dsi_host *msm_host)
+{
+        struct platform_device *pdev = msm_host->pdev;
+        const struct msm_dsi_config *cfg = msm_host->cfg_hnd->cfg;
+        struct resource *res;
+        int i;
+
+        res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "dsi_ctrl");
+        if (!res)
+                return -EINVAL;
+
+        for (i = 0; i < cfg->num_dsi; i++) {
+                if (cfg->io_start[i] == res->start)
+                        return i;
+        }
+
+        return -EINVAL;
+}
+
 int msm_dsi_host_init(struct msm_dsi *msm_dsi)
 {
         struct msm_dsi_host *msm_host = NULL;
@@ -1684,6 +1710,13 @@ int msm_dsi_host_init(struct msm_dsi *msm_dsi)
                 goto fail;
         }
 
+        msm_host->id = dsi_host_get_id(msm_host);
+        if (msm_host->id < 0) {
+                ret = msm_host->id;
+                pr_err("%s: unable to identify DSI host index\n", __func__);
+                goto fail;
+        }
+
         /* fixup base address by io offset */
         msm_host->ctrl_base += msm_host->cfg_hnd->cfg->io_offset;
 
@@ -2245,9 +2278,9 @@ int msm_dsi_host_set_display_mode(struct mipi_dsi_host *host,
         }
 
         msm_host->mode = drm_mode_duplicate(msm_host->dev, mode);
-        if (IS_ERR(msm_host->mode)) {
+        if (!msm_host->mode) {
                 pr_err("%s: cannot duplicate mode\n", __func__);
-                return PTR_ERR(msm_host->mode);
+                return -ENOMEM;
         }
 
         return 0;
diff --git a/drivers/gpu/drm/msm/dsi/phy/dsi_phy.c b/drivers/gpu/drm/msm/dsi/phy/dsi_phy.c
index e2f42d8ea294..f39386ed75e4 100644
--- a/drivers/gpu/drm/msm/dsi/phy/dsi_phy.c
+++ b/drivers/gpu/drm/msm/dsi/phy/dsi_phy.c
@@ -271,6 +271,30 @@ static const struct of_device_id dsi_phy_dt_match[] = {
         {}
 };
 
+/*
+ * Currently, we only support one SoC for each PHY type. When we have multiple
+ * SoCs for the same PHY, we can try to make the index searching a bit more
+ * clever.
+ */
+static int dsi_phy_get_id(struct msm_dsi_phy *phy)
+{
+        struct platform_device *pdev = phy->pdev;
+        const struct msm_dsi_phy_cfg *cfg = phy->cfg;
+        struct resource *res;
+        int i;
+
+        res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "dsi_phy");
+        if (!res)
+                return -EINVAL;
+
+        for (i = 0; i < cfg->num_dsi_phy; i++) {
+                if (cfg->io_start[i] == res->start)
+                        return i;
+        }
+
+        return -EINVAL;
+}
+
 static int dsi_phy_driver_probe(struct platform_device *pdev)
 {
         struct msm_dsi_phy *phy;
@@ -289,10 +313,10 @@ static int dsi_phy_driver_probe(struct platform_device *pdev)
         phy->cfg = match->data;
         phy->pdev = pdev;
 
-        ret = of_property_read_u32(dev->of_node,
-                                "qcom,dsi-phy-index", &phy->id);
-        if (ret) {
-                dev_err(dev, "%s: PHY index not specified, %d\n",
+        phy->id = dsi_phy_get_id(phy);
+        if (phy->id < 0) {
+                ret = phy->id;
+                dev_err(dev, "%s: couldn't identify PHY index, %d\n",
                         __func__, ret);
                 goto fail;
         }
diff --git a/drivers/gpu/drm/msm/dsi/phy/dsi_phy.h b/drivers/gpu/drm/msm/dsi/phy/dsi_phy.h
index 0d54ed00386d..f24a85439b94 100644
--- a/drivers/gpu/drm/msm/dsi/phy/dsi_phy.h
+++ b/drivers/gpu/drm/msm/dsi/phy/dsi_phy.h
@@ -38,6 +38,8 @@ struct msm_dsi_phy_cfg {
          * Fill default H/W values in illegal cells, eg. cell {0, 1}.
          */
         bool src_pll_truthtable[DSI_MAX][DSI_MAX];
+        const resource_size_t io_start[DSI_MAX];
+        const int num_dsi_phy;
 };
 
 extern const struct msm_dsi_phy_cfg dsi_phy_28nm_hpm_cfgs;
diff --git a/drivers/gpu/drm/msm/dsi/phy/dsi_phy_20nm.c b/drivers/gpu/drm/msm/dsi/phy/dsi_phy_20nm.c
index f4bc11af849a..c757e2070cac 100644
--- a/drivers/gpu/drm/msm/dsi/phy/dsi_phy_20nm.c
+++ b/drivers/gpu/drm/msm/dsi/phy/dsi_phy_20nm.c
@@ -145,6 +145,8 @@ const struct msm_dsi_phy_cfg dsi_phy_20nm_cfgs = {
         .ops = {
                 .enable = dsi_20nm_phy_enable,
                 .disable = dsi_20nm_phy_disable,
-        }
+        },
+        .io_start = { 0xfd998300, 0xfd9a0300 },
+        .num_dsi_phy = 2,
 };
 
diff --git a/drivers/gpu/drm/msm/dsi/phy/dsi_phy_28nm.c b/drivers/gpu/drm/msm/dsi/phy/dsi_phy_28nm.c
index 96d1852af418..63d7fba31380 100644
--- a/drivers/gpu/drm/msm/dsi/phy/dsi_phy_28nm.c
+++ b/drivers/gpu/drm/msm/dsi/phy/dsi_phy_28nm.c
@@ -145,6 +145,8 @@ const struct msm_dsi_phy_cfg dsi_phy_28nm_hpm_cfgs = {
                 .enable = dsi_28nm_phy_enable,
                 .disable = dsi_28nm_phy_disable,
         },
+        .io_start = { 0xfd922b00, 0xfd923100 },
+        .num_dsi_phy = 2,
 };
 
 const struct msm_dsi_phy_cfg dsi_phy_28nm_lp_cfgs = {
@@ -160,5 +162,7 @@ const struct msm_dsi_phy_cfg dsi_phy_28nm_lp_cfgs = {
                 .enable = dsi_28nm_phy_enable,
                 .disable = dsi_28nm_phy_disable,
         },
+        .io_start = { 0x1a98500 },
+        .num_dsi_phy = 1,
 };
 
diff --git a/drivers/gpu/drm/msm/dsi/phy/dsi_phy_28nm_8960.c b/drivers/gpu/drm/msm/dsi/phy/dsi_phy_28nm_8960.c
index 213355a3e767..7bdb9de54968 100644
--- a/drivers/gpu/drm/msm/dsi/phy/dsi_phy_28nm_8960.c
+++ b/drivers/gpu/drm/msm/dsi/phy/dsi_phy_28nm_8960.c
@@ -192,4 +192,6 @@ const struct msm_dsi_phy_cfg dsi_phy_28nm_8960_cfgs = {
                 .enable = dsi_28nm_phy_enable,
                 .disable = dsi_28nm_phy_disable,
         },
+        .io_start = { 0x4700300, 0x5800300 },
+        .num_dsi_phy = 2,
 };
diff --git a/drivers/gpu/drm/msm/edp/edp_connector.c b/drivers/gpu/drm/msm/edp/edp_connector.c
index 72360cd038c0..5960628ceb93 100644
--- a/drivers/gpu/drm/msm/edp/edp_connector.c
+++ b/drivers/gpu/drm/msm/edp/edp_connector.c
@@ -91,15 +91,6 @@ static int edp_connector_mode_valid(struct drm_connector *connector,
         return MODE_OK;
 }
 
-static struct drm_encoder *
-edp_connector_best_encoder(struct drm_connector *connector)
-{
-        struct edp_connector *edp_connector = to_edp_connector(connector);
-
-        DBG("");
-        return edp_connector->edp->encoder;
-}
-
 static const struct drm_connector_funcs edp_connector_funcs = {
         .dpms = drm_atomic_helper_connector_dpms,
         .detect = edp_connector_detect,
@@ -113,7 +104,6 @@ static const struct drm_connector_funcs edp_connector_funcs = {
 static const struct drm_connector_helper_funcs edp_connector_helper_funcs = {
         .get_modes = edp_connector_get_modes,
         .mode_valid = edp_connector_mode_valid,
-        .best_encoder = edp_connector_best_encoder,
 };
 
 /* initialize connector */
diff --git a/drivers/gpu/drm/msm/hdmi/hdmi.c b/drivers/gpu/drm/msm/hdmi/hdmi.c
index 51b9ea552f97..973720792236 100644
--- a/drivers/gpu/drm/msm/hdmi/hdmi.c
+++ b/drivers/gpu/drm/msm/hdmi/hdmi.c
@@ -19,6 +19,7 @@
 #include <linux/of_irq.h>
 #include <linux/of_gpio.h>
 
+#include <sound/hdmi-codec.h>
 #include "hdmi.h"
 
 void msm_hdmi_set_mode(struct hdmi *hdmi, bool power_on)
@@ -434,6 +435,111 @@ static int msm_hdmi_get_gpio(struct device_node *of_node, const char *name)
         return gpio;
 }
 
+/*
+ * HDMI audio codec callbacks
+ */
+static int msm_hdmi_audio_hw_params(struct device *dev, void *data,
+                                    struct hdmi_codec_daifmt *daifmt,
+                                    struct hdmi_codec_params *params)
+{
+        struct hdmi *hdmi = dev_get_drvdata(dev);
+        unsigned int chan;
+        unsigned int channel_allocation = 0;
+        unsigned int rate;
+        unsigned int level_shift  = 0; /* 0dB */
+        bool down_mix = false;
+
+        dev_dbg(dev, "%u Hz, %d bit, %d channels\n", params->sample_rate,
+                 params->sample_width, params->cea.channels);
+
+        switch (params->cea.channels) {
+        case 2:
+                /* FR and FL speakers */
+                channel_allocation  = 0;
+                chan = MSM_HDMI_AUDIO_CHANNEL_2;
+                break;
+        case 4:
+                /* FC, LFE, FR and FL speakers */
+                channel_allocation  = 0x3;
+                chan = MSM_HDMI_AUDIO_CHANNEL_4;
+                break;
+        case 6:
+                /* RR, RL, FC, LFE, FR and FL speakers */
+                channel_allocation  = 0x0B;
+                chan = MSM_HDMI_AUDIO_CHANNEL_6;
+                break;
+        case 8:
+                /* FRC, FLC, RR, RL, FC, LFE, FR and FL speakers */
+                channel_allocation  = 0x1F;
+                chan = MSM_HDMI_AUDIO_CHANNEL_8;
+                break;
+        default:
+                return -EINVAL;
+        }
+
+        switch (params->sample_rate) {
+        case 32000:
+                rate = HDMI_SAMPLE_RATE_32KHZ;
+                break;
+        case 44100:
+                rate = HDMI_SAMPLE_RATE_44_1KHZ;
+                break;
+        case 48000:
+                rate = HDMI_SAMPLE_RATE_48KHZ;
+                break;
+        case 88200:
+                rate = HDMI_SAMPLE_RATE_88_2KHZ;
+                break;
+        case 96000:
+                rate = HDMI_SAMPLE_RATE_96KHZ;
+                break;
+        case 176400:
+                rate = HDMI_SAMPLE_RATE_176_4KHZ;
+                break;
+        case 192000:
+                rate = HDMI_SAMPLE_RATE_192KHZ;
+                break;
+        default:
+                dev_err(dev, "rate[%d] not supported!\n",
+                        params->sample_rate);
+                return -EINVAL;
+        }
+
+        msm_hdmi_audio_set_sample_rate(hdmi, rate);
+        msm_hdmi_audio_info_setup(hdmi, 1, chan, channel_allocation,
+                              level_shift, down_mix);
+
+        return 0;
+}
+
+static void msm_hdmi_audio_shutdown(struct device *dev, void *data)
+{
+        struct hdmi *hdmi = dev_get_drvdata(dev);
+
+        msm_hdmi_audio_info_setup(hdmi, 0, 0, 0, 0, 0);
+}
+
+static const struct hdmi_codec_ops msm_hdmi_audio_codec_ops = {
+        .hw_params = msm_hdmi_audio_hw_params,
+        .audio_shutdown = msm_hdmi_audio_shutdown,
+};
+
+static struct hdmi_codec_pdata codec_data = {
+        .ops = &msm_hdmi_audio_codec_ops,
+        .max_i2s_channels = 8,
+        .i2s = 1,
+};
+
+static int msm_hdmi_register_audio_driver(struct hdmi *hdmi, struct device *dev)
+{
+        hdmi->audio_pdev = platform_device_register_data(dev,
+                                                         HDMI_CODEC_DRV_NAME,
+                                                         PLATFORM_DEVID_AUTO,
+                                                         &codec_data,
+                                                         sizeof(codec_data));
+        return PTR_ERR_OR_ZERO(hdmi->audio_pdev);
+}
+
 static int msm_hdmi_bind(struct device *dev, struct device *master, void *data)
 {
         struct drm_device *drm = dev_get_drvdata(master);
@@ -441,7 +547,7 @@ static int msm_hdmi_bind(struct device *dev, struct device *master, void *data)
         static struct hdmi_platform_config *hdmi_cfg;
         struct hdmi *hdmi;
         struct device_node *of_node = dev->of_node;
-        int i;
+        int i, err;
 
         hdmi_cfg = (struct hdmi_platform_config *)
                         of_device_get_match_data(dev);
@@ -468,6 +574,12 @@ static int msm_hdmi_bind(struct device *dev, struct device *master, void *data)
                 return PTR_ERR(hdmi);
         priv->hdmi = hdmi;
 
+        err = msm_hdmi_register_audio_driver(hdmi, dev);
+        if (err) {
+                DRM_ERROR("Failed to attach an audio codec %d\n", err);
+                hdmi->audio_pdev = NULL;
+        }
+
         return 0;
 }
 
@@ -477,6 +589,9 @@ static void msm_hdmi_unbind(struct device *dev, struct device *master,
         struct drm_device *drm = dev_get_drvdata(master);
         struct msm_drm_private *priv = drm->dev_private;
         if (priv->hdmi) {
+                if (priv->hdmi->audio_pdev)
+                        platform_device_unregister(priv->hdmi->audio_pdev);
+
                 msm_hdmi_destroy(priv->hdmi);
                 priv->hdmi = NULL;
         }
diff --git a/drivers/gpu/drm/msm/hdmi/hdmi.h b/drivers/gpu/drm/msm/hdmi/hdmi.h
index bc7ba0bdee07..accc9a61611d 100644
--- a/drivers/gpu/drm/msm/hdmi/hdmi.h
+++ b/drivers/gpu/drm/msm/hdmi/hdmi.h
@@ -50,6 +50,7 @@ struct hdmi_hdcp_ctrl;
 struct hdmi {
         struct drm_device *dev;
         struct platform_device *pdev;
+        struct platform_device *audio_pdev;
 
         const struct hdmi_platform_config *config;
 
@@ -210,6 +211,19 @@ static inline int msm_hdmi_pll_8996_init(struct platform_device *pdev)
 /*
  * audio:
  */
+/* Supported HDMI Audio channels and rates */
+#define MSM_HDMI_AUDIO_CHANNEL_2        0
+#define MSM_HDMI_AUDIO_CHANNEL_4        1
+#define MSM_HDMI_AUDIO_CHANNEL_6        2
+#define MSM_HDMI_AUDIO_CHANNEL_8        3
+
+#define HDMI_SAMPLE_RATE_32KHZ          0
+#define HDMI_SAMPLE_RATE_44_1KHZ        1
+#define HDMI_SAMPLE_RATE_48KHZ          2
+#define HDMI_SAMPLE_RATE_88_2KHZ        3
+#define HDMI_SAMPLE_RATE_96KHZ          4
+#define HDMI_SAMPLE_RATE_176_4KHZ       5
+#define HDMI_SAMPLE_RATE_192KHZ         6
 
 int msm_hdmi_audio_update(struct hdmi *hdmi);
 int msm_hdmi_audio_info_setup(struct hdmi *hdmi, bool enabled,
diff --git a/drivers/gpu/drm/msm/hdmi/hdmi_connector.c b/drivers/gpu/drm/msm/hdmi/hdmi_connector.c
index b15d72683112..a2515b466ce5 100644
--- a/drivers/gpu/drm/msm/hdmi/hdmi_connector.c
+++ b/drivers/gpu/drm/msm/hdmi/hdmi_connector.c
@@ -406,13 +406,6 @@ static int msm_hdmi_connector_mode_valid(struct drm_connector *connector,
         return 0;
 }
 
-static struct drm_encoder *
-msm_hdmi_connector_best_encoder(struct drm_connector *connector)
-{
-        struct hdmi_connector *hdmi_connector = to_hdmi_connector(connector);
-        return hdmi_connector->hdmi->encoder;
-}
-
 static const struct drm_connector_funcs hdmi_connector_funcs = {
         .dpms = drm_atomic_helper_connector_dpms,
         .detect = hdmi_connector_detect,
@@ -426,7 +419,6 @@ static const struct drm_connector_funcs hdmi_connector_funcs = {
 static const struct drm_connector_helper_funcs msm_hdmi_connector_helper_funcs = {
         .get_modes = msm_hdmi_connector_get_modes,
         .mode_valid = msm_hdmi_connector_mode_valid,
-        .best_encoder = msm_hdmi_connector_best_encoder,
 };
 
 /* initialize connector */
diff --git a/drivers/gpu/drm/msm/hdmi/hdmi_hdcp.c b/drivers/gpu/drm/msm/hdmi/hdmi_hdcp.c
index 0baaaaabd002..6e767979aab3 100644
--- a/drivers/gpu/drm/msm/hdmi/hdmi_hdcp.c
+++ b/drivers/gpu/drm/msm/hdmi/hdmi_hdcp.c
@@ -1430,7 +1430,7 @@ struct hdmi_hdcp_ctrl *msm_hdmi_hdcp_init(struct hdmi *hdmi)
 
 void msm_hdmi_hdcp_destroy(struct hdmi *hdmi)
 {
-        if (hdmi && hdmi->hdcp_ctrl) {
+        if (hdmi) {
                 kfree(hdmi->hdcp_ctrl);
                 hdmi->hdcp_ctrl = NULL;
         }
diff --git a/drivers/gpu/drm/msm/mdp/mdp4/mdp4_dtv_encoder.c b/drivers/gpu/drm/msm/mdp/mdp4/mdp4_dtv_encoder.c
index 35ad78a1dc1c..24258e3025e3 100644
--- a/drivers/gpu/drm/msm/mdp/mdp4/mdp4_dtv_encoder.c
+++ b/drivers/gpu/drm/msm/mdp/mdp4/mdp4_dtv_encoder.c
@@ -23,7 +23,6 @@
 
 struct mdp4_dtv_encoder {
         struct drm_encoder base;
-        struct clk *src_clk;
         struct clk *hdmi_clk;
         struct clk *mdp_clk;
         unsigned long int pixclock;
@@ -179,7 +178,6 @@ static void mdp4_dtv_encoder_disable(struct drm_encoder *encoder)
          */
         mdp_irq_wait(&mdp4_kms->base, MDP4_IRQ_EXTERNAL_VSYNC);
 
-        clk_disable_unprepare(mdp4_dtv_encoder->src_clk);
         clk_disable_unprepare(mdp4_dtv_encoder->hdmi_clk);
         clk_disable_unprepare(mdp4_dtv_encoder->mdp_clk);
 
@@ -208,19 +206,21 @@ static void mdp4_dtv_encoder_enable(struct drm_encoder *encoder)
 
         bs_set(mdp4_dtv_encoder, 1);
 
-        DBG("setting src_clk=%lu", pc);
+        DBG("setting mdp_clk=%lu", pc);
 
-        ret = clk_set_rate(mdp4_dtv_encoder->src_clk, pc);
+        ret = clk_set_rate(mdp4_dtv_encoder->mdp_clk, pc);
         if (ret)
-                dev_err(dev->dev, "failed to set src_clk to %lu: %d\n", pc, ret);
-        clk_prepare_enable(mdp4_dtv_encoder->src_clk);
-        ret = clk_prepare_enable(mdp4_dtv_encoder->hdmi_clk);
-        if (ret)
-                dev_err(dev->dev, "failed to enable hdmi_clk: %d\n", ret);
+                dev_err(dev->dev, "failed to set mdp_clk to %lu: %d\n",
+                        pc, ret);
+
         ret = clk_prepare_enable(mdp4_dtv_encoder->mdp_clk);
         if (ret)
                 dev_err(dev->dev, "failed to enabled mdp_clk: %d\n", ret);
 
+        ret = clk_prepare_enable(mdp4_dtv_encoder->hdmi_clk);
+        if (ret)
+                dev_err(dev->dev, "failed to enable hdmi_clk: %d\n", ret);
+
         mdp4_write(mdp4_kms, REG_MDP4_DTV_ENABLE, 1);
 
         mdp4_dtv_encoder->enabled = true;
@@ -235,7 +235,7 @@ static const struct drm_encoder_helper_funcs mdp4_dtv_encoder_helper_funcs = {
 long mdp4_dtv_round_pixclk(struct drm_encoder *encoder, unsigned long rate)
 {
         struct mdp4_dtv_encoder *mdp4_dtv_encoder = to_mdp4_dtv_encoder(encoder);
-        return clk_round_rate(mdp4_dtv_encoder->src_clk, rate);
+        return clk_round_rate(mdp4_dtv_encoder->mdp_clk, rate);
 }
 
 /* initialize encoder */
@@ -257,13 +257,6 @@ struct drm_encoder *mdp4_dtv_encoder_init(struct drm_device *dev)
                          DRM_MODE_ENCODER_TMDS, NULL);
         drm_encoder_helper_add(encoder, &mdp4_dtv_encoder_helper_funcs);
 
-        mdp4_dtv_encoder->src_clk = devm_clk_get(dev->dev, "src_clk");
-        if (IS_ERR(mdp4_dtv_encoder->src_clk)) {
-                dev_err(dev->dev, "failed to get src_clk\n");
-                ret = PTR_ERR(mdp4_dtv_encoder->src_clk);
-                goto fail;
-        }
-
         mdp4_dtv_encoder->hdmi_clk = devm_clk_get(dev->dev, "hdmi_clk");
         if (IS_ERR(mdp4_dtv_encoder->hdmi_clk)) {
                 dev_err(dev->dev, "failed to get hdmi_clk\n");
@@ -271,9 +264,9 @@ struct drm_encoder *mdp4_dtv_encoder_init(struct drm_device *dev)
                 goto fail;
         }
 
-        mdp4_dtv_encoder->mdp_clk = devm_clk_get(dev->dev, "mdp_clk");
+        mdp4_dtv_encoder->mdp_clk = devm_clk_get(dev->dev, "tv_clk");
         if (IS_ERR(mdp4_dtv_encoder->mdp_clk)) {
-                dev_err(dev->dev, "failed to get mdp_clk\n");
+                dev_err(dev->dev, "failed to get tv_clk\n");
                 ret = PTR_ERR(mdp4_dtv_encoder->mdp_clk);
                 goto fail;
         }
diff --git a/drivers/gpu/drm/msm/mdp/mdp4/mdp4_kms.c b/drivers/gpu/drm/msm/mdp/mdp4/mdp4_kms.c
index 67442d50a6c2..7b39e89fbc2b 100644
--- a/drivers/gpu/drm/msm/mdp/mdp4/mdp4_kms.c
+++ b/drivers/gpu/drm/msm/mdp/mdp4/mdp4_kms.c
@@ -106,31 +106,27 @@ out:
 static void mdp4_prepare_commit(struct msm_kms *kms, struct drm_atomic_state *state)
 {
         struct mdp4_kms *mdp4_kms = to_mdp4_kms(to_mdp_kms(kms));
-        int i, ncrtcs = state->dev->mode_config.num_crtc;
+        int i;
+        struct drm_crtc *crtc;
+        struct drm_crtc_state *crtc_state;
 
         mdp4_enable(mdp4_kms);
 
         /* see 119ecb7fd */
-        for (i = 0; i < ncrtcs; i++) {
-                struct drm_crtc *crtc = state->crtcs[i];
-                if (!crtc)
-                        continue;
+        for_each_crtc_in_state(state, crtc, crtc_state, i)
                 drm_crtc_vblank_get(crtc);
-        }
 }
 
 static void mdp4_complete_commit(struct msm_kms *kms, struct drm_atomic_state *state)
 {
         struct mdp4_kms *mdp4_kms = to_mdp4_kms(to_mdp_kms(kms));
-        int i, ncrtcs = state->dev->mode_config.num_crtc;
+        int i;
+        struct drm_crtc *crtc;
+        struct drm_crtc_state *crtc_state;
 
         /* see 119ecb7fd */
-        for (i = 0; i < ncrtcs; i++) {
-                struct drm_crtc *crtc = state->crtcs[i];
-                if (!crtc)
-                        continue;
+        for_each_crtc_in_state(state, crtc, crtc_state, i)
                 drm_crtc_vblank_put(crtc);
-        }
 
         mdp4_disable(mdp4_kms);
 }
@@ -162,6 +158,7 @@ static const char * const iommu_ports[] = {
 static void mdp4_destroy(struct msm_kms *kms)
 {
         struct mdp4_kms *mdp4_kms = to_mdp4_kms(to_mdp_kms(kms));
+        struct device *dev = mdp4_kms->dev->dev;
         struct msm_mmu *mmu = mdp4_kms->mmu;
 
         if (mmu) {
@@ -171,8 +168,11 @@ static void mdp4_destroy(struct msm_kms *kms)
 
         if (mdp4_kms->blank_cursor_iova)
                 msm_gem_put_iova(mdp4_kms->blank_cursor_bo, mdp4_kms->id);
-        if (mdp4_kms->blank_cursor_bo)
-                drm_gem_object_unreference_unlocked(mdp4_kms->blank_cursor_bo);
+        drm_gem_object_unreference_unlocked(mdp4_kms->blank_cursor_bo);
+
+        if (mdp4_kms->rpm_enabled)
+                pm_runtime_disable(dev);
+
         kfree(mdp4_kms);
 }
 
@@ -440,7 +440,7 @@ struct msm_kms *mdp4_kms_init(struct drm_device *dev)
         struct mdp4_kms *mdp4_kms;
         struct msm_kms *kms = NULL;
         struct msm_mmu *mmu;
-        int ret;
+        int irq, ret;
 
         mdp4_kms = kzalloc(sizeof(*mdp4_kms), GFP_KERNEL);
         if (!mdp4_kms) {
@@ -461,6 +461,15 @@ struct msm_kms *mdp4_kms_init(struct drm_device *dev)
                 goto fail;
         }
 
+        irq = platform_get_irq(pdev, 0);
+        if (irq < 0) {
+                ret = irq;
+                dev_err(dev->dev, "failed to get irq: %d\n", ret);
+                goto fail;
+        }
+
+        kms->irq = irq;
+
         /* NOTE: driver for this regulator still missing upstream.. use
          * _get_exclusive() and ignore the error if it does not exist
          * (and hope that the bootloader left it on for us)
@@ -496,7 +505,7 @@ struct msm_kms *mdp4_kms_init(struct drm_device *dev)
                 goto fail;
         }
 
-        mdp4_kms->axi_clk = devm_clk_get(&pdev->dev, "mdp_axi_clk");
+        mdp4_kms->axi_clk = devm_clk_get(&pdev->dev, "bus_clk");
         if (IS_ERR(mdp4_kms->axi_clk)) {
                 dev_err(dev->dev, "failed to get axi_clk\n");
                 ret = PTR_ERR(mdp4_kms->axi_clk);
@@ -506,6 +515,9 @@ struct msm_kms *mdp4_kms_init(struct drm_device *dev)
         clk_set_rate(mdp4_kms->clk, config->max_clk);
         clk_set_rate(mdp4_kms->lut_clk, config->max_clk);
 
+        pm_runtime_enable(dev->dev);
+        mdp4_kms->rpm_enabled = true;
+
         /* make sure things are off before attaching iommu (bootloader could
          * have left things on, in which case we'll start getting faults if
          * we don't disable):
diff --git a/drivers/gpu/drm/msm/mdp/mdp4/mdp4_kms.h b/drivers/gpu/drm/msm/mdp/mdp4/mdp4_kms.h
index c5d045d5680d..25fb83997119 100644
--- a/drivers/gpu/drm/msm/mdp/mdp4/mdp4_kms.h
+++ b/drivers/gpu/drm/msm/mdp/mdp4/mdp4_kms.h
@@ -47,6 +47,8 @@ struct mdp4_kms {
 
         struct mdp_irq error_handler;
 
+        bool rpm_enabled;
+
         /* empty/blank cursor bo to use when cursor is "disabled" */
         struct drm_gem_object *blank_cursor_bo;
         uint32_t blank_cursor_iova;
diff --git a/drivers/gpu/drm/msm/mdp/mdp4/mdp4_lvds_connector.c b/drivers/gpu/drm/msm/mdp/mdp4/mdp4_lvds_connector.c
index 2648cd7631ef..353429b05733 100644
--- a/drivers/gpu/drm/msm/mdp/mdp4/mdp4_lvds_connector.c
+++ b/drivers/gpu/drm/msm/mdp/mdp4/mdp4_lvds_connector.c
@@ -90,14 +90,6 @@ static int mdp4_lvds_connector_mode_valid(struct drm_connector *connector,
         return MODE_OK;
 }
 
-static struct drm_encoder *
-mdp4_lvds_connector_best_encoder(struct drm_connector *connector)
-{
-        struct mdp4_lvds_connector *mdp4_lvds_connector =
-                        to_mdp4_lvds_connector(connector);
-        return mdp4_lvds_connector->encoder;
-}
-
 static const struct drm_connector_funcs mdp4_lvds_connector_funcs = {
         .dpms = drm_atomic_helper_connector_dpms,
         .detect = mdp4_lvds_connector_detect,
@@ -111,7 +103,6 @@ static const struct drm_connector_funcs mdp4_lvds_connector_funcs = {
 static const struct drm_connector_helper_funcs mdp4_lvds_connector_helper_funcs = {
         .get_modes = mdp4_lvds_connector_get_modes,
         .mode_valid = mdp4_lvds_connector_mode_valid,
-        .best_encoder = mdp4_lvds_connector_best_encoder,
 };
 
 /* initialize connector */
diff --git a/drivers/gpu/drm/msm/mdp/mdp5/mdp5.xml.h b/drivers/gpu/drm/msm/mdp/mdp5/mdp5.xml.h
index b275ce11b24b..ca6ca30650a0 100644
--- a/drivers/gpu/drm/msm/mdp/mdp5/mdp5.xml.h
+++ b/drivers/gpu/drm/msm/mdp/mdp5/mdp5.xml.h
@@ -8,19 +8,11 @@ http://github.com/freedreno/envytools/
 git clone https://github.com/freedreno/envytools.git
 
 The rules-ng-ng source files this header was generated from are:
-- /home/robclark/src/freedreno/envytools/rnndb/msm.xml                 (    676 bytes, from 2015-05-20 20:03:14)
-- /home/robclark/src/freedreno/envytools/rnndb/freedreno_copyright.xml (   1572 bytes, from 2016-02-10 17:07:21)
-- /home/robclark/src/freedreno/envytools/rnndb/mdp/mdp4.xml            (  20915 bytes, from 2015-05-20 20:03:14)
-- /home/robclark/src/freedreno/envytools/rnndb/mdp/mdp_common.xml      (   2849 bytes, from 2015-09-18 12:07:28)
-- /home/robclark/src/freedreno/envytools/rnndb/mdp/mdp5.xml            (  37194 bytes, from 2015-09-18 12:07:28)
-- /home/robclark/src/freedreno/envytools/rnndb/dsi/dsi.xml             (  27887 bytes, from 2015-10-22 16:34:52)
-- /home/robclark/src/freedreno/envytools/rnndb/dsi/sfpb.xml            (    602 bytes, from 2015-10-22 16:35:02)
-- /home/robclark/src/freedreno/envytools/rnndb/dsi/mmss_cc.xml         (   1686 bytes, from 2015-05-20 20:03:14)
-- /home/robclark/src/freedreno/envytools/rnndb/hdmi/qfprom.xml         (    600 bytes, from 2015-05-20 20:03:07)
-- /home/robclark/src/freedreno/envytools/rnndb/hdmi/hdmi.xml           (  41472 bytes, from 2016-01-22 18:18:18)
-- /home/robclark/src/freedreno/envytools/rnndb/edp/edp.xml             (  10416 bytes, from 2015-05-20 20:03:14)
-
-Copyright (C) 2013-2015 by the following authors:
+- /local/mnt/workspace/source_trees/envytools/rnndb/../rnndb/mdp/mdp5.xml   (  36965 bytes, from 2016-05-10 05:06:30)
+- /local/mnt/workspace/source_trees/envytools/rnndb/freedreno_copyright.xml (   1572 bytes, from 2016-05-09 06:32:54)
+- /local/mnt/workspace/source_trees/envytools/rnndb/mdp/mdp_common.xml      (   2849 bytes, from 2016-01-07 08:45:55)
+
+Copyright (C) 2013-2016 by the following authors:
 - Rob Clark <robdclark@gmail.com> (robclark)
 - Ilia Mirkin <imirkin@alum.mit.edu> (imirkin)
 
@@ -198,118 +190,109 @@ static inline uint32_t MDSS_HW_VERSION_MAJOR(uint32_t val)
 #define MDSS_HW_INTR_STATUS_INTR_HDMI                           0x00000100
 #define MDSS_HW_INTR_STATUS_INTR_EDP                            0x00001000
 
-static inline uint32_t __offset_MDP(uint32_t idx)
-{
-        switch (idx) {
-                case 0: return (mdp5_cfg->mdp.base[0]);
-                default: return INVALID_IDX(idx);
-        }
-}
-static inline uint32_t REG_MDP5_MDP(uint32_t i0) { return 0x00000000 + __offset_MDP(i0); }
-
-static inline uint32_t REG_MDP5_MDP_HW_VERSION(uint32_t i0) { return 0x00000000 + __offset_MDP(i0); }
-#define MDP5_MDP_HW_VERSION_STEP__MASK                          0x0000ffff
-#define MDP5_MDP_HW_VERSION_STEP__SHIFT                         0
-static inline uint32_t MDP5_MDP_HW_VERSION_STEP(uint32_t val)
+#define REG_MDP5_HW_VERSION                                     0x00000000
+#define MDP5_HW_VERSION_STEP__MASK                              0x0000ffff
+#define MDP5_HW_VERSION_STEP__SHIFT                             0
+static inline uint32_t MDP5_HW_VERSION_STEP(uint32_t val)
 {
-        return ((val) << MDP5_MDP_HW_VERSION_STEP__SHIFT) & MDP5_MDP_HW_VERSION_STEP__MASK;
+        return ((val) << MDP5_HW_VERSION_STEP__SHIFT) & MDP5_HW_VERSION_STEP__MASK;
 }
-#define MDP5_MDP_HW_VERSION_MINOR__MASK                         0x0fff0000
-#define MDP5_MDP_HW_VERSION_MINOR__SHIFT                        16
-static inline uint32_t MDP5_MDP_HW_VERSION_MINOR(uint32_t val)
+#define MDP5_HW_VERSION_MINOR__MASK                             0x0fff0000
+#define MDP5_HW_VERSION_MINOR__SHIFT                            16
+static inline uint32_t MDP5_HW_VERSION_MINOR(uint32_t val)
 {
-        return ((val) << MDP5_MDP_HW_VERSION_MINOR__SHIFT) & MDP5_MDP_HW_VERSION_MINOR__MASK;
+        return ((val) << MDP5_HW_VERSION_MINOR__SHIFT) & MDP5_HW_VERSION_MINOR__MASK;
 }
-#define MDP5_MDP_HW_VERSION_MAJOR__MASK                         0xf0000000
-#define MDP5_MDP_HW_VERSION_MAJOR__SHIFT                        28
-static inline uint32_t MDP5_MDP_HW_VERSION_MAJOR(uint32_t val)
+#define MDP5_HW_VERSION_MAJOR__MASK                             0xf0000000
+#define MDP5_HW_VERSION_MAJOR__SHIFT                            28
+static inline uint32_t MDP5_HW_VERSION_MAJOR(uint32_t val)
 {
-        return ((val) << MDP5_MDP_HW_VERSION_MAJOR__SHIFT) & MDP5_MDP_HW_VERSION_MAJOR__MASK;
+        return ((val) << MDP5_HW_VERSION_MAJOR__SHIFT) & MDP5_HW_VERSION_MAJOR__MASK;
 }
 
-static inline uint32_t REG_MDP5_MDP_DISP_INTF_SEL(uint32_t i0) { return 0x00000004 + __offset_MDP(i0); }
-#define MDP5_MDP_DISP_INTF_SEL_INTF0__MASK                      0x000000ff
-#define MDP5_MDP_DISP_INTF_SEL_INTF0__SHIFT                     0
-static inline uint32_t MDP5_MDP_DISP_INTF_SEL_INTF0(enum mdp5_intf_type val)
+#define REG_MDP5_DISP_INTF_SEL                                  0x00000004
+#define MDP5_DISP_INTF_SEL_INTF0__MASK                          0x000000ff
+#define MDP5_DISP_INTF_SEL_INTF0__SHIFT                         0
+static inline uint32_t MDP5_DISP_INTF_SEL_INTF0(enum mdp5_intf_type val)
 {
-        return ((val) << MDP5_MDP_DISP_INTF_SEL_INTF0__SHIFT) & MDP5_MDP_DISP_INTF_SEL_INTF0__MASK;
+        return ((val) << MDP5_DISP_INTF_SEL_INTF0__SHIFT) & MDP5_DISP_INTF_SEL_INTF0__MASK;
 }
-#define MDP5_MDP_DISP_INTF_SEL_INTF1__MASK                      0x0000ff00
-#define MDP5_MDP_DISP_INTF_SEL_INTF1__SHIFT                     8
-static inline uint32_t MDP5_MDP_DISP_INTF_SEL_INTF1(enum mdp5_intf_type val)
+#define MDP5_DISP_INTF_SEL_INTF1__MASK                          0x0000ff00
+#define MDP5_DISP_INTF_SEL_INTF1__SHIFT                         8
+static inline uint32_t MDP5_DISP_INTF_SEL_INTF1(enum mdp5_intf_type val)
 {
-        return ((val) << MDP5_MDP_DISP_INTF_SEL_INTF1__SHIFT) & MDP5_MDP_DISP_INTF_SEL_INTF1__MASK;
+        return ((val) << MDP5_DISP_INTF_SEL_INTF1__SHIFT) & MDP5_DISP_INTF_SEL_INTF1__MASK;
 }
-#define MDP5_MDP_DISP_INTF_SEL_INTF2__MASK                      0x00ff0000
-#define MDP5_MDP_DISP_INTF_SEL_INTF2__SHIFT                     16
-static inline uint32_t MDP5_MDP_DISP_INTF_SEL_INTF2(enum mdp5_intf_type val)
+#define MDP5_DISP_INTF_SEL_INTF2__MASK                          0x00ff0000
+#define MDP5_DISP_INTF_SEL_INTF2__SHIFT                         16
+static inline uint32_t MDP5_DISP_INTF_SEL_INTF2(enum mdp5_intf_type val)
 {
-        return ((val) << MDP5_MDP_DISP_INTF_SEL_INTF2__SHIFT) & MDP5_MDP_DISP_INTF_SEL_INTF2__MASK;
+        return ((val) << MDP5_DISP_INTF_SEL_INTF2__SHIFT) & MDP5_DISP_INTF_SEL_INTF2__MASK;
 }
-#define MDP5_MDP_DISP_INTF_SEL_INTF3__MASK                      0xff000000
-#define MDP5_MDP_DISP_INTF_SEL_INTF3__SHIFT                     24
-static inline uint32_t MDP5_MDP_DISP_INTF_SEL_INTF3(enum mdp5_intf_type val)
+#define MDP5_DISP_INTF_SEL_INTF3__MASK                          0xff000000
+#define MDP5_DISP_INTF_SEL_INTF3__SHIFT                         24
+static inline uint32_t MDP5_DISP_INTF_SEL_INTF3(enum mdp5_intf_type val)
 {
-        return ((val) << MDP5_MDP_DISP_INTF_SEL_INTF3__SHIFT) & MDP5_MDP_DISP_INTF_SEL_INTF3__MASK;
+        return ((val) << MDP5_DISP_INTF_SEL_INTF3__SHIFT) & MDP5_DISP_INTF_SEL_INTF3__MASK;
 }
 
-static inline uint32_t REG_MDP5_MDP_INTR_EN(uint32_t i0) { return 0x00000010 + __offset_MDP(i0); }
+#define REG_MDP5_INTR_EN                                        0x00000010
 
-static inline uint32_t REG_MDP5_MDP_INTR_STATUS(uint32_t i0) { return 0x00000014 + __offset_MDP(i0); }
+#define REG_MDP5_INTR_STATUS                                    0x00000014
 
-static inline uint32_t REG_MDP5_MDP_INTR_CLEAR(uint32_t i0) { return 0x00000018 + __offset_MDP(i0); }
+#define REG_MDP5_INTR_CLEAR                                     0x00000018
 
-static inline uint32_t REG_MDP5_MDP_HIST_INTR_EN(uint32_t i0) { return 0x0000001c + __offset_MDP(i0); }
+#define REG_MDP5_HIST_INTR_EN                                   0x0000001c
 
-static inline uint32_t REG_MDP5_MDP_HIST_INTR_STATUS(uint32_t i0) { return 0x00000020 + __offset_MDP(i0); }
+#define REG_MDP5_HIST_INTR_STATUS                               0x00000020
 
-static inline uint32_t REG_MDP5_MDP_HIST_INTR_CLEAR(uint32_t i0) { return 0x00000024 + __offset_MDP(i0); }
+#define REG_MDP5_HIST_INTR_CLEAR                                0x00000024
 
-static inline uint32_t REG_MDP5_MDP_SPARE_0(uint32_t i0) { return 0x00000028 + __offset_MDP(i0); }
-#define MDP5_MDP_SPARE_0_SPLIT_DPL_SINGLE_FLUSH_EN              0x00000001
+#define REG_MDP5_SPARE_0                                        0x00000028
+#define MDP5_SPARE_0_SPLIT_DPL_SINGLE_FLUSH_EN                  0x00000001
 
-static inline uint32_t REG_MDP5_MDP_SMP_ALLOC_W(uint32_t i0, uint32_t i1) { return 0x00000080 + __offset_MDP(i0) + 0x4*i1; }
+static inline uint32_t REG_MDP5_SMP_ALLOC_W(uint32_t i0) { return 0x00000080 + 0x4*i0; }
 
-static inline uint32_t REG_MDP5_MDP_SMP_ALLOC_W_REG(uint32_t i0, uint32_t i1) { return 0x00000080 + __offset_MDP(i0) + 0x4*i1; }
-#define MDP5_MDP_SMP_ALLOC_W_REG_CLIENT0__MASK                  0x000000ff
-#define MDP5_MDP_SMP_ALLOC_W_REG_CLIENT0__SHIFT                 0
-static inline uint32_t MDP5_MDP_SMP_ALLOC_W_REG_CLIENT0(uint32_t val)
+static inline uint32_t REG_MDP5_SMP_ALLOC_W_REG(uint32_t i0) { return 0x00000080 + 0x4*i0; }
+#define MDP5_SMP_ALLOC_W_REG_CLIENT0__MASK                      0x000000ff
+#define MDP5_SMP_ALLOC_W_REG_CLIENT0__SHIFT                     0
+static inline uint32_t MDP5_SMP_ALLOC_W_REG_CLIENT0(uint32_t val)
 {
-        return ((val) << MDP5_MDP_SMP_ALLOC_W_REG_CLIENT0__SHIFT) & MDP5_MDP_SMP_ALLOC_W_REG_CLIENT0__MASK;
+        return ((val) << MDP5_SMP_ALLOC_W_REG_CLIENT0__SHIFT) & MDP5_SMP_ALLOC_W_REG_CLIENT0__MASK;
 }
-#define MDP5_MDP_SMP_ALLOC_W_REG_CLIENT1__MASK                  0x0000ff00
-#define MDP5_MDP_SMP_ALLOC_W_REG_CLIENT1__SHIFT                 8
-static inline uint32_t MDP5_MDP_SMP_ALLOC_W_REG_CLIENT1(uint32_t val)
+#define MDP5_SMP_ALLOC_W_REG_CLIENT1__MASK                      0x0000ff00
+#define MDP5_SMP_ALLOC_W_REG_CLIENT1__SHIFT                     8
+static inline uint32_t MDP5_SMP_ALLOC_W_REG_CLIENT1(uint32_t val)
 {
-        return ((val) << MDP5_MDP_SMP_ALLOC_W_REG_CLIENT1__SHIFT) & MDP5_MDP_SMP_ALLOC_W_REG_CLIENT1__MASK;
+        return ((val) << MDP5_SMP_ALLOC_W_REG_CLIENT1__SHIFT) & MDP5_SMP_ALLOC_W_REG_CLIENT1__MASK;
 }
-#define MDP5_MDP_SMP_ALLOC_W_REG_CLIENT2__MASK                  0x00ff0000
-#define MDP5_MDP_SMP_ALLOC_W_REG_CLIENT2__SHIFT                 16
-static inline uint32_t MDP5_MDP_SMP_ALLOC_W_REG_CLIENT2(uint32_t val)
+#define MDP5_SMP_ALLOC_W_REG_CLIENT2__MASK                      0x00ff0000
+#define MDP5_SMP_ALLOC_W_REG_CLIENT2__SHIFT                     16
+static inline uint32_t MDP5_SMP_ALLOC_W_REG_CLIENT2(uint32_t val)
 {
-        return ((val) << MDP5_MDP_SMP_ALLOC_W_REG_CLIENT2__SHIFT) & MDP5_MDP_SMP_ALLOC_W_REG_CLIENT2__MASK;
+        return ((val) << MDP5_SMP_ALLOC_W_REG_CLIENT2__SHIFT) & MDP5_SMP_ALLOC_W_REG_CLIENT2__MASK;
 }
 
-static inline uint32_t REG_MDP5_MDP_SMP_ALLOC_R(uint32_t i0, uint32_t i1) { return 0x00000130 + __offset_MDP(i0) + 0x4*i1; }
+static inline uint32_t REG_MDP5_SMP_ALLOC_R(uint32_t i0) { return 0x00000130 + 0x4*i0; }
 
-static inline uint32_t REG_MDP5_MDP_SMP_ALLOC_R_REG(uint32_t i0, uint32_t i1) { return 0x00000130 + __offset_MDP(i0) + 0x4*i1; }
-#define MDP5_MDP_SMP_ALLOC_R_REG_CLIENT0__MASK                  0x000000ff
-#define MDP5_MDP_SMP_ALLOC_R_REG_CLIENT0__SHIFT                 0
-static inline uint32_t MDP5_MDP_SMP_ALLOC_R_REG_CLIENT0(uint32_t val)
+static inline uint32_t REG_MDP5_SMP_ALLOC_R_REG(uint32_t i0) { return 0x00000130 + 0x4*i0; }
+#define MDP5_SMP_ALLOC_R_REG_CLIENT0__MASK                      0x000000ff
+#define MDP5_SMP_ALLOC_R_REG_CLIENT0__SHIFT                     0
+static inline uint32_t MDP5_SMP_ALLOC_R_REG_CLIENT0(uint32_t val)
 {
-        return ((val) << MDP5_MDP_SMP_ALLOC_R_REG_CLIENT0__SHIFT) & MDP5_MDP_SMP_ALLOC_R_REG_CLIENT0__MASK;
+        return ((val) << MDP5_SMP_ALLOC_R_REG_CLIENT0__SHIFT) & MDP5_SMP_ALLOC_R_REG_CLIENT0__MASK;
 }
-#define MDP5_MDP_SMP_ALLOC_R_REG_CLIENT1__MASK                  0x0000ff00
-#define MDP5_MDP_SMP_ALLOC_R_REG_CLIENT1__SHIFT                 8
-static inline uint32_t MDP5_MDP_SMP_ALLOC_R_REG_CLIENT1(uint32_t val)
+#define MDP5_SMP_ALLOC_R_REG_CLIENT1__MASK                      0x0000ff00
+#define MDP5_SMP_ALLOC_R_REG_CLIENT1__SHIFT                     8
+static inline uint32_t MDP5_SMP_ALLOC_R_REG_CLIENT1(uint32_t val)
 {
-        return ((val) << MDP5_MDP_SMP_ALLOC_R_REG_CLIENT1__SHIFT) & MDP5_MDP_SMP_ALLOC_R_REG_CLIENT1__MASK;
+        return ((val) << MDP5_SMP_ALLOC_R_REG_CLIENT1__SHIFT) & MDP5_SMP_ALLOC_R_REG_CLIENT1__MASK;
 }
-#define MDP5_MDP_SMP_ALLOC_R_REG_CLIENT2__MASK                  0x00ff0000
-#define MDP5_MDP_SMP_ALLOC_R_REG_CLIENT2__SHIFT                 16
-static inline uint32_t MDP5_MDP_SMP_ALLOC_R_REG_CLIENT2(uint32_t val)
+#define MDP5_SMP_ALLOC_R_REG_CLIENT2__MASK                      0x00ff0000
+#define MDP5_SMP_ALLOC_R_REG_CLIENT2__SHIFT                     16
+static inline uint32_t MDP5_SMP_ALLOC_R_REG_CLIENT2(uint32_t val)
 {
-        return ((val) << MDP5_MDP_SMP_ALLOC_R_REG_CLIENT2__SHIFT) & MDP5_MDP_SMP_ALLOC_R_REG_CLIENT2__MASK;
+        return ((val) << MDP5_SMP_ALLOC_R_REG_CLIENT2__SHIFT) & MDP5_SMP_ALLOC_R_REG_CLIENT2__MASK;
 }
 
 static inline uint32_t __offset_IGC(enum mdp5_igc_type idx)
@@ -322,35 +305,35 @@ static inline uint32_t __offset_IGC(enum mdp5_igc_type idx)
                 default: return INVALID_IDX(idx);
         }
 }
-static inline uint32_t REG_MDP5_MDP_IGC(uint32_t i0, enum mdp5_igc_type i1) { return 0x00000000 + __offset_MDP(i0) + __offset_IGC(i1); }
+static inline uint32_t REG_MDP5_IGC(enum mdp5_igc_type i0) { return 0x00000000 + __offset_IGC(i0); }
 
-static inline uint32_t REG_MDP5_MDP_IGC_LUT(uint32_t i0, enum mdp5_igc_type i1, uint32_t i2) { return 0x00000000 + __offset_MDP(i0) + __offset_IGC(i1) + 0x4*i2; }
+static inline uint32_t REG_MDP5_IGC_LUT(enum mdp5_igc_type i0, uint32_t i1) { return 0x00000000 + __offset_IGC(i0) + 0x4*i1; }
 
-static inline uint32_t REG_MDP5_MDP_IGC_LUT_REG(uint32_t i0, enum mdp5_igc_type i1, uint32_t i2) { return 0x00000000 + __offset_MDP(i0) + __offset_IGC(i1) + 0x4*i2; }
-#define MDP5_MDP_IGC_LUT_REG_VAL__MASK                          0x00000fff
-#define MDP5_MDP_IGC_LUT_REG_VAL__SHIFT                         0
-static inline uint32_t MDP5_MDP_IGC_LUT_REG_VAL(uint32_t val)
+static inline uint32_t REG_MDP5_IGC_LUT_REG(enum mdp5_igc_type i0, uint32_t i1) { return 0x00000000 + __offset_IGC(i0) + 0x4*i1; }
+#define MDP5_IGC_LUT_REG_VAL__MASK                              0x00000fff
+#define MDP5_IGC_LUT_REG_VAL__SHIFT                             0
+static inline uint32_t MDP5_IGC_LUT_REG_VAL(uint32_t val)
 {
-        return ((val) << MDP5_MDP_IGC_LUT_REG_VAL__SHIFT) & MDP5_MDP_IGC_LUT_REG_VAL__MASK;
+        return ((val) << MDP5_IGC_LUT_REG_VAL__SHIFT) & MDP5_IGC_LUT_REG_VAL__MASK;
 }
-#define MDP5_MDP_IGC_LUT_REG_INDEX_UPDATE                       0x02000000
-#define MDP5_MDP_IGC_LUT_REG_DISABLE_PIPE_0                     0x10000000
-#define MDP5_MDP_IGC_LUT_REG_DISABLE_PIPE_1                     0x20000000
-#define MDP5_MDP_IGC_LUT_REG_DISABLE_PIPE_2                     0x40000000
+#define MDP5_IGC_LUT_REG_INDEX_UPDATE                           0x02000000
+#define MDP5_IGC_LUT_REG_DISABLE_PIPE_0                         0x10000000
+#define MDP5_IGC_LUT_REG_DISABLE_PIPE_1                         0x20000000
+#define MDP5_IGC_LUT_REG_DISABLE_PIPE_2                         0x40000000
 
-static inline uint32_t REG_MDP5_MDP_SPLIT_DPL_EN(uint32_t i0) { return 0x000002f4 + __offset_MDP(i0); }
+#define REG_MDP5_SPLIT_DPL_EN                                   0x000002f4
 
-static inline uint32_t REG_MDP5_MDP_SPLIT_DPL_UPPER(uint32_t i0) { return 0x000002f8 + __offset_MDP(i0); }
-#define MDP5_MDP_SPLIT_DPL_UPPER_SMART_PANEL                    0x00000002
-#define MDP5_MDP_SPLIT_DPL_UPPER_SMART_PANEL_FREE_RUN           0x00000004
-#define MDP5_MDP_SPLIT_DPL_UPPER_INTF1_SW_TRG_MUX               0x00000010
-#define MDP5_MDP_SPLIT_DPL_UPPER_INTF2_SW_TRG_MUX               0x00000100
+#define REG_MDP5_SPLIT_DPL_UPPER                                0x000002f8
+#define MDP5_SPLIT_DPL_UPPER_SMART_PANEL                        0x00000002
+#define MDP5_SPLIT_DPL_UPPER_SMART_PANEL_FREE_RUN               0x00000004
+#define MDP5_SPLIT_DPL_UPPER_INTF1_SW_TRG_MUX                   0x00000010
+#define MDP5_SPLIT_DPL_UPPER_INTF2_SW_TRG_MUX                   0x00000100
 
-static inline uint32_t REG_MDP5_MDP_SPLIT_DPL_LOWER(uint32_t i0) { return 0x000003f0 + __offset_MDP(i0); }
-#define MDP5_MDP_SPLIT_DPL_LOWER_SMART_PANEL                    0x00000002
-#define MDP5_MDP_SPLIT_DPL_LOWER_SMART_PANEL_FREE_RUN           0x00000004
-#define MDP5_MDP_SPLIT_DPL_LOWER_INTF1_TG_SYNC                  0x00000010
-#define MDP5_MDP_SPLIT_DPL_LOWER_INTF2_TG_SYNC                  0x00000100
+#define REG_MDP5_SPLIT_DPL_LOWER                                0x000003f0
+#define MDP5_SPLIT_DPL_LOWER_SMART_PANEL                        0x00000002
+#define MDP5_SPLIT_DPL_LOWER_SMART_PANEL_FREE_RUN               0x00000004
+#define MDP5_SPLIT_DPL_LOWER_INTF1_TG_SYNC                      0x00000010
+#define MDP5_SPLIT_DPL_LOWER_INTF2_TG_SYNC                      0x00000100
 
 static inline uint32_t __offset_CTL(uint32_t idx)
 {
diff --git a/drivers/gpu/drm/msm/mdp/mdp5/mdp5_cfg.c b/drivers/gpu/drm/msm/mdp/mdp5/mdp5_cfg.c
index 57f73f0c120d..ac9e4cde1380 100644
--- a/drivers/gpu/drm/msm/mdp/mdp5/mdp5_cfg.c
+++ b/drivers/gpu/drm/msm/mdp/mdp5/mdp5_cfg.c
@@ -26,7 +26,6 @@ const struct mdp5_cfg_hw msm8x74v1_config = {
         .name = "msm8x74v1",
         .mdp = {
                 .count = 1,
-                .base = { 0x00100 },
                 .caps = MDP_CAP_SMP |
                         0,
         },
@@ -41,12 +40,12 @@ const struct mdp5_cfg_hw msm8x74v1_config = {
         },
         .ctl = {
                 .count = 5,
-                .base = { 0x00600, 0x00700, 0x00800, 0x00900, 0x00a00 },
+                .base = { 0x00500, 0x00600, 0x00700, 0x00800, 0x00900 },
                 .flush_hw_mask = 0x0003ffff,
         },
         .pipe_vig = {
                 .count = 3,
-                .base = { 0x01200, 0x01600, 0x01a00 },
+                .base = { 0x01100, 0x01500, 0x01900 },
                 .caps = MDP_PIPE_CAP_HFLIP |
                         MDP_PIPE_CAP_VFLIP |
                         MDP_PIPE_CAP_SCALE |
@@ -55,7 +54,7 @@ const struct mdp5_cfg_hw msm8x74v1_config = {
         },
         .pipe_rgb = {
                 .count = 3,
-                .base = { 0x01e00, 0x02200, 0x02600 },
+                .base = { 0x01d00, 0x02100, 0x02500 },
                 .caps = MDP_PIPE_CAP_HFLIP |
                         MDP_PIPE_CAP_VFLIP |
                         MDP_PIPE_CAP_SCALE |
@@ -63,26 +62,26 @@ const struct mdp5_cfg_hw msm8x74v1_config = {
         },
         .pipe_dma = {
                 .count = 2,
-                .base = { 0x02a00, 0x02e00 },
+                .base = { 0x02900, 0x02d00 },
                 .caps = MDP_PIPE_CAP_HFLIP |
                         MDP_PIPE_CAP_VFLIP |
                         0,
         },
         .lm = {
                 .count = 5,
-                .base = { 0x03200, 0x03600, 0x03a00, 0x03e00, 0x04200 },
+                .base = { 0x03100, 0x03500, 0x03900, 0x03d00, 0x04100 },
                 .nb_stages = 5,
         },
         .dspp = {
                 .count = 3,
-                .base = { 0x04600, 0x04a00, 0x04e00 },
+                .base = { 0x04500, 0x04900, 0x04d00 },
         },
         .pp = {
                 .count = 3,
-                .base = { 0x21b00, 0x21c00, 0x21d00 },
+                .base = { 0x21a00, 0x21b00, 0x21c00 },
         },
         .intf = {
-                .base = { 0x21100, 0x21300, 0x21500, 0x21700 },
+                .base = { 0x21000, 0x21200, 0x21400, 0x21600 },
                 .connect = {
                         [0] = INTF_eDP,
                         [1] = INTF_DSI,
@@ -97,7 +96,6 @@ const struct mdp5_cfg_hw msm8x74v2_config = {
         .name = "msm8x74",
         .mdp = {
                 .count = 1,
-                .base = { 0x00100 },
                 .caps = MDP_CAP_SMP |
                         0,
         },
@@ -112,48 +110,48 @@ const struct mdp5_cfg_hw msm8x74v2_config = {
         },
         .ctl = {
                 .count = 5,
-                .base = { 0x00600, 0x00700, 0x00800, 0x00900, 0x00a00 },
+                .base = { 0x00500, 0x00600, 0x00700, 0x00800, 0x00900 },
                 .flush_hw_mask = 0x0003ffff,
         },
         .pipe_vig = {
                 .count = 3,
-                .base = { 0x01200, 0x01600, 0x01a00 },
+                .base = { 0x01100, 0x01500, 0x01900 },
                 .caps = MDP_PIPE_CAP_HFLIP | MDP_PIPE_CAP_VFLIP |
                                 MDP_PIPE_CAP_SCALE | MDP_PIPE_CAP_CSC |
                                 MDP_PIPE_CAP_DECIMATION,
         },
         .pipe_rgb = {
                 .count = 3,
-                .base = { 0x01e00, 0x02200, 0x02600 },
+                .base = { 0x01d00, 0x02100, 0x02500 },
                 .caps = MDP_PIPE_CAP_HFLIP | MDP_PIPE_CAP_VFLIP |
                                 MDP_PIPE_CAP_SCALE | MDP_PIPE_CAP_DECIMATION,
         },
         .pipe_dma = {
                 .count = 2,
-                .base = { 0x02a00, 0x02e00 },
+                .base = { 0x02900, 0x02d00 },
                 .caps = MDP_PIPE_CAP_HFLIP | MDP_PIPE_CAP_VFLIP,
         },
         .lm = {
                 .count = 5,
-                .base = { 0x03200, 0x03600, 0x03a00, 0x03e00, 0x04200 },
+                .base = { 0x03100, 0x03500, 0x03900, 0x03d00, 0x04100 },
                 .nb_stages = 5,
                 .max_width = 2048,
                 .max_height = 0xFFFF,
         },
         .dspp = {
                 .count = 3,
-                .base = { 0x04600, 0x04a00, 0x04e00 },
+                .base = { 0x04500, 0x04900, 0x04d00 },
         },
         .ad = {
                 .count = 2,
-                .base = { 0x13100, 0x13300 },
+                .base = { 0x13000, 0x13200 },
         },
         .pp = {
                 .count = 3,
-                .base = { 0x12d00, 0x12e00, 0x12f00 },
+                .base = { 0x12c00, 0x12d00, 0x12e00 },
         },
         .intf = {
-                .base = { 0x12500, 0x12700, 0x12900, 0x12b00 },
+                .base = { 0x12400, 0x12600, 0x12800, 0x12a00 },
                 .connect = {
                         [0] = INTF_eDP,
                         [1] = INTF_DSI,
@@ -168,7 +166,6 @@ const struct mdp5_cfg_hw apq8084_config = {
         .name = "apq8084",
         .mdp = {
                 .count = 1,
-                .base = { 0x00100 },
                 .caps = MDP_CAP_SMP |
                         0,
         },
@@ -190,49 +187,49 @@ const struct mdp5_cfg_hw apq8084_config = {
         },
         .ctl = {
                 .count = 5,
-                .base = { 0x00600, 0x00700, 0x00800, 0x00900, 0x00a00 },
+                .base = { 0x00500, 0x00600, 0x00700, 0x00800, 0x00900 },
                 .flush_hw_mask = 0x003fffff,
         },
         .pipe_vig = {
                 .count = 4,
-                .base = { 0x01200, 0x01600, 0x01a00, 0x01e00 },
+                .base = { 0x01100, 0x01500, 0x01900, 0x01d00 },
                 .caps = MDP_PIPE_CAP_HFLIP | MDP_PIPE_CAP_VFLIP |
                                 MDP_PIPE_CAP_SCALE | MDP_PIPE_CAP_CSC |
                                 MDP_PIPE_CAP_DECIMATION,
         },
         .pipe_rgb = {
                 .count = 4,
-                .base = { 0x02200, 0x02600, 0x02a00, 0x02e00 },
+                .base = { 0x02100, 0x02500, 0x02900, 0x02d00 },
                 .caps = MDP_PIPE_CAP_HFLIP | MDP_PIPE_CAP_VFLIP |
                                 MDP_PIPE_CAP_SCALE | MDP_PIPE_CAP_DECIMATION,
         },
         .pipe_dma = {
                 .count = 2,
-                .base = { 0x03200, 0x03600 },
+                .base = { 0x03100, 0x03500 },
                 .caps = MDP_PIPE_CAP_HFLIP | MDP_PIPE_CAP_VFLIP,
         },
         .lm = {
                 .count = 6,
-                .base = { 0x03a00, 0x03e00, 0x04200, 0x04600, 0x04a00, 0x04e00 },
+                .base = { 0x03900, 0x03d00, 0x04100, 0x04500, 0x04900, 0x04d00 },
                 .nb_stages = 5,
                 .max_width = 2048,
                 .max_height = 0xFFFF,
         },
         .dspp = {
                 .count = 4,
-                .base = { 0x05200, 0x05600, 0x05a00, 0x05e00 },
+                .base = { 0x05100, 0x05500, 0x05900, 0x05d00 },
 
         },
         .ad = {
                 .count = 3,
-                .base = { 0x13500, 0x13700, 0x13900 },
+                .base = { 0x13400, 0x13600, 0x13800 },
         },
         .pp = {
                 .count = 4,
-                .base = { 0x12f00, 0x13000, 0x13100, 0x13200 },
+                .base = { 0x12e00, 0x12f00, 0x13000, 0x13100 },
         },
         .intf = {
-                .base = { 0x12500, 0x12700, 0x12900, 0x12b00, 0x12d00 },
+                .base = { 0x12400, 0x12600, 0x12800, 0x12a00, 0x12c00 },
                 .connect = {
                         [0] = INTF_eDP,
                         [1] = INTF_DSI,
@@ -247,7 +244,7 @@ const struct mdp5_cfg_hw msm8x16_config = {
         .name = "msm8x16",
         .mdp = {
                 .count = 1,
-                .base = { 0x01000 },
+                .base = { 0x0 },
                 .caps = MDP_CAP_SMP |
                         0,
         },
@@ -261,41 +258,41 @@ const struct mdp5_cfg_hw msm8x16_config = {
         },
         .ctl = {
                 .count = 5,
-                .base = { 0x02000, 0x02200, 0x02400, 0x02600, 0x02800 },
+                .base = { 0x01000, 0x01200, 0x01400, 0x01600, 0x01800 },
                 .flush_hw_mask = 0x4003ffff,
         },
         .pipe_vig = {
                 .count = 1,
-                .base = { 0x05000 },
+                .base = { 0x04000 },
                 .caps = MDP_PIPE_CAP_HFLIP | MDP_PIPE_CAP_VFLIP |
                                 MDP_PIPE_CAP_SCALE | MDP_PIPE_CAP_CSC |
                                 MDP_PIPE_CAP_DECIMATION,
         },
         .pipe_rgb = {
                 .count = 2,
-                .base = { 0x15000, 0x17000 },
+                .base = { 0x14000, 0x16000 },
                 .caps = MDP_PIPE_CAP_HFLIP | MDP_PIPE_CAP_VFLIP |
                                 MDP_PIPE_CAP_SCALE | MDP_PIPE_CAP_DECIMATION,
         },
         .pipe_dma = {
                 .count = 1,
-                .base = { 0x25000 },
+                .base = { 0x24000 },
                 .caps = MDP_PIPE_CAP_HFLIP | MDP_PIPE_CAP_VFLIP,
         },
         .lm = {
                 .count = 2, /* LM0 and LM3 */
-                .base = { 0x45000, 0x48000 },
+                .base = { 0x44000, 0x47000 },
                 .nb_stages = 5,
                 .max_width = 2048,
                 .max_height = 0xFFFF,
         },
         .dspp = {
                 .count = 1,
-                .base = { 0x55000 },
+                .base = { 0x54000 },
 
         },
         .intf = {
-                .base = { 0x00000, 0x6b800 },
+                .base = { 0x00000, 0x6a800 },
                 .connect = {
                         [0] = INTF_DISABLED,
                         [1] = INTF_DSI,
@@ -308,7 +305,6 @@ const struct mdp5_cfg_hw msm8x94_config = {
         .name = "msm8x94",
         .mdp = {
                 .count = 1,
-                .base = { 0x01000 },
                 .caps = MDP_CAP_SMP |
                         0,
         },
@@ -330,49 +326,49 @@ const struct mdp5_cfg_hw msm8x94_config = {
         },
         .ctl = {
                 .count = 5,
-                .base = { 0x02000, 0x02200, 0x02400, 0x02600, 0x02800 },
+                .base = { 0x01000, 0x01200, 0x01400, 0x01600, 0x01800 },
                 .flush_hw_mask = 0xf0ffffff,
         },
         .pipe_vig = {
                 .count = 4,
-                .base = { 0x05000, 0x07000, 0x09000, 0x0b000 },
+                .base = { 0x04000, 0x06000, 0x08000, 0x0a000 },
                 .caps = MDP_PIPE_CAP_HFLIP | MDP_PIPE_CAP_VFLIP |
                                 MDP_PIPE_CAP_SCALE | MDP_PIPE_CAP_CSC |
                                 MDP_PIPE_CAP_DECIMATION,
         },
         .pipe_rgb = {
                 .count = 4,
-                .base = { 0x15000, 0x17000, 0x19000, 0x1b000 },
+                .base = { 0x14000, 0x16000, 0x18000, 0x1a000 },
                 .caps = MDP_PIPE_CAP_HFLIP | MDP_PIPE_CAP_VFLIP |
                                 MDP_PIPE_CAP_SCALE | MDP_PIPE_CAP_DECIMATION,
         },
         .pipe_dma = {
                 .count = 2,
-                .base = { 0x25000, 0x27000 },
+                .base = { 0x24000, 0x26000 },
                 .caps = MDP_PIPE_CAP_HFLIP | MDP_PIPE_CAP_VFLIP,
         },
         .lm = {
                 .count = 6,
-                .base = { 0x45000, 0x46000, 0x47000, 0x48000, 0x49000, 0x4a000 },
+                .base = { 0x44000, 0x45000, 0x46000, 0x47000, 0x48000, 0x49000 },
                 .nb_stages = 8,
                 .max_width = 2048,
                 .max_height = 0xFFFF,
         },
         .dspp = {
                 .count = 4,
-                .base = { 0x55000, 0x57000, 0x59000, 0x5b000 },
+                .base = { 0x54000, 0x56000, 0x58000, 0x5a000 },
 
         },
         .ad = {
                 .count = 3,
-                .base = { 0x79000, 0x79800, 0x7a000 },
+                .base = { 0x78000, 0x78800, 0x79000 },
         },
         .pp = {
                 .count = 4,
-                .base = { 0x71000, 0x71800, 0x72000, 0x72800 },
+                .base = { 0x70000, 0x70800, 0x71000, 0x71800 },
         },
         .intf = {
-                .base = { 0x6b000, 0x6b800, 0x6c000, 0x6c800, 0x6d000 },
+                .base = { 0x6a000, 0x6a800, 0x6b000, 0x6b800, 0x6c000 },
                 .connect = {
                         [0] = INTF_DISABLED,
                         [1] = INTF_DSI,
@@ -387,19 +383,18 @@ const struct mdp5_cfg_hw msm8x96_config = {
         .name = "msm8x96",
         .mdp = {
                 .count = 1,
-                .base = { 0x01000 },
                 .caps = MDP_CAP_DSC |
                         MDP_CAP_CDM |
                         0,
         },
         .ctl = {
                 .count = 5,
-                .base = { 0x02000, 0x02200, 0x02400, 0x02600, 0x02800 },
+                .base = { 0x01000, 0x01200, 0x01400, 0x01600, 0x01800 },
                 .flush_hw_mask = 0xf4ffffff,
         },
         .pipe_vig = {
                 .count = 4,
-                .base = { 0x05000, 0x07000, 0x09000, 0x0b000 },
+                .base = { 0x04000, 0x06000, 0x08000, 0x0a000 },
                 .caps = MDP_PIPE_CAP_HFLIP      |
                         MDP_PIPE_CAP_VFLIP      |
                         MDP_PIPE_CAP_SCALE      |
@@ -410,7 +405,7 @@ const struct mdp5_cfg_hw msm8x96_config = {
         },
         .pipe_rgb = {
                 .count = 4,
-                .base = { 0x15000, 0x17000, 0x19000, 0x1b000 },
+                .base = { 0x14000, 0x16000, 0x18000, 0x1a000 },
                 .caps = MDP_PIPE_CAP_HFLIP      |
                         MDP_PIPE_CAP_VFLIP      |
                         MDP_PIPE_CAP_SCALE      |
@@ -420,7 +415,7 @@ const struct mdp5_cfg_hw msm8x96_config = {
         },
         .pipe_dma = {
                 .count = 2,
-                .base = { 0x25000, 0x27000 },
+                .base = { 0x24000, 0x26000 },
                 .caps = MDP_PIPE_CAP_HFLIP      |
                         MDP_PIPE_CAP_VFLIP      |
                         MDP_PIPE_CAP_SW_PIX_EXT |
@@ -428,33 +423,33 @@ const struct mdp5_cfg_hw msm8x96_config = {
         },
         .lm = {
                 .count = 6,
-                .base = { 0x45000, 0x46000, 0x47000, 0x48000, 0x49000, 0x4a000 },
+                .base = { 0x44000, 0x45000, 0x46000, 0x47000, 0x48000, 0x49000 },
                 .nb_stages = 8,
                 .max_width = 2560,
                 .max_height = 0xFFFF,
         },
         .dspp = {
                 .count = 2,
-                .base = { 0x55000, 0x57000 },
+                .base = { 0x54000, 0x56000 },
         },
         .ad = {
                 .count = 3,
-                .base = { 0x79000, 0x79800, 0x7a000 },
+                .base = { 0x78000, 0x78800, 0x79000 },
         },
         .pp = {
                 .count = 4,
-                .base = { 0x71000, 0x71800, 0x72000, 0x72800 },
+                .base = { 0x70000, 0x70800, 0x71000, 0x71800 },
         },
         .cdm = {
                 .count = 1,
-                .base = { 0x7a200 },
+                .base = { 0x79200 },
         },
         .dsc = {
                 .count = 2,
-                .base = { 0x81000, 0x81400 },
+                .base = { 0x80000, 0x80400 },
         },
         .intf = {
-                .base = { 0x6b000, 0x6b800, 0x6c000, 0x6c800, 0x6d000 },
+                .base = { 0x6a000, 0x6a800, 0x6b000, 0x6b800, 0x6c000 },
                 .connect = {
                         [0] = INTF_DISABLED,
                         [1] = INTF_DSI,
diff --git a/drivers/gpu/drm/msm/mdp/mdp5/mdp5_cmd_encoder.c b/drivers/gpu/drm/msm/mdp/mdp5/mdp5_cmd_encoder.c
index 69094cb28103..c627ab6d0061 100644
--- a/drivers/gpu/drm/msm/mdp/mdp5/mdp5_cmd_encoder.c
+++ b/drivers/gpu/drm/msm/mdp/mdp5/mdp5_cmd_encoder.c
@@ -272,22 +272,22 @@ int mdp5_cmd_encoder_set_split_display(struct drm_encoder *encoder,
          * start signal for the slave encoder
          */
         if (intf_num == 1)
-                data |= MDP5_MDP_SPLIT_DPL_UPPER_INTF2_SW_TRG_MUX;
+                data |= MDP5_SPLIT_DPL_UPPER_INTF2_SW_TRG_MUX;
         else if (intf_num == 2)
-                data |= MDP5_MDP_SPLIT_DPL_UPPER_INTF1_SW_TRG_MUX;
+                data |= MDP5_SPLIT_DPL_UPPER_INTF1_SW_TRG_MUX;
         else
                 return -EINVAL;
 
         /* Smart Panel, Sync mode */
-        data |= MDP5_MDP_SPLIT_DPL_UPPER_SMART_PANEL;
+        data |= MDP5_SPLIT_DPL_UPPER_SMART_PANEL;
 
         /* Make sure clocks are on when connectors calling this function. */
         mdp5_enable(mdp5_kms);
-        mdp5_write(mdp5_kms, REG_MDP5_MDP_SPLIT_DPL_UPPER(0), data);
+        mdp5_write(mdp5_kms, REG_MDP5_SPLIT_DPL_UPPER, data);
 
-        mdp5_write(mdp5_kms, REG_MDP5_MDP_SPLIT_DPL_LOWER(0),
-                        MDP5_MDP_SPLIT_DPL_LOWER_SMART_PANEL);
-        mdp5_write(mdp5_kms, REG_MDP5_MDP_SPLIT_DPL_EN(0), 1);
+        mdp5_write(mdp5_kms, REG_MDP5_SPLIT_DPL_LOWER,
+                   MDP5_SPLIT_DPL_LOWER_SMART_PANEL);
+        mdp5_write(mdp5_kms, REG_MDP5_SPLIT_DPL_EN, 1);
         mdp5_disable(mdp5_kms);
 
         return 0;
diff --git a/drivers/gpu/drm/msm/mdp/mdp5/mdp5_crtc.c b/drivers/gpu/drm/msm/mdp/mdp5/mdp5_crtc.c
index 88fe256c1931..fa2be7ce9468 100644
--- a/drivers/gpu/drm/msm/mdp/mdp5/mdp5_crtc.c
+++ b/drivers/gpu/drm/msm/mdp/mdp5/mdp5_crtc.c
@@ -374,6 +374,7 @@ static int mdp5_crtc_atomic_check(struct drm_crtc *crtc,
         struct drm_device *dev = crtc->dev;
         struct plane_state pstates[STAGE_MAX + 1];
         const struct mdp5_cfg_hw *hw_cfg;
+        const struct drm_plane_state *pstate;
         int cnt = 0, i;
 
         DBG("%s: check", mdp5_crtc->name);
@@ -382,20 +383,13 @@ static int mdp5_crtc_atomic_check(struct drm_crtc *crtc,
          * and that we don't have conflicting mixer stages:
          */
         hw_cfg = mdp5_cfg_get_hw_config(mdp5_kms->cfg);
-        drm_atomic_crtc_state_for_each_plane(plane, state) {
-                struct drm_plane_state *pstate;
+        drm_atomic_crtc_state_for_each_plane_state(plane, pstate, state) {
                 if (cnt >= (hw_cfg->lm.nb_stages)) {
                         dev_err(dev->dev, "too many planes!\n");
                         return -EINVAL;
                 }
 
-                pstate = state->state->plane_states[drm_plane_index(plane)];
 
-                /* plane might not have changed, in which case take
-                 * current state:
-                 */
-                if (!pstate)
-                        pstate = plane->state;
                 pstates[cnt].plane = plane;
                 pstates[cnt].state = to_mdp5_plane_state(pstate);
 
@@ -496,8 +490,7 @@ static int mdp5_crtc_cursor_set(struct drm_crtc *crtc,
         struct mdp5_kms *mdp5_kms = get_kms(crtc);
         struct drm_gem_object *cursor_bo, *old_bo = NULL;
         uint32_t blendcfg, cursor_addr, stride;
-        int ret, bpp, lm;
-        unsigned int depth;
+        int ret, lm;
         enum mdp5_cursor_alpha cur_alpha = CURSOR_ALPHA_PER_PIXEL;
         uint32_t flush_mask = mdp_ctl_flush_mask_cursor(0);
         uint32_t roi_w, roi_h;
@@ -527,8 +520,7 @@ static int mdp5_crtc_cursor_set(struct drm_crtc *crtc,
                 return -EINVAL;
 
         lm = mdp5_crtc->lm;
-        drm_fb_get_bpp_depth(DRM_FORMAT_ARGB8888, &depth, &bpp);
-        stride = width * (bpp >> 3);
+        stride = width * drm_format_plane_cpp(DRM_FORMAT_ARGB8888, 0);
 
         spin_lock_irqsave(&mdp5_crtc->cursor.lock, flags);
         old_bo = mdp5_crtc->cursor.scanout_bo;
diff --git a/drivers/gpu/drm/msm/mdp/mdp5/mdp5_ctl.c b/drivers/gpu/drm/msm/mdp/mdp5/mdp5_ctl.c
index 4e81ca4f964a..d021edc3b307 100644
--- a/drivers/gpu/drm/msm/mdp/mdp5/mdp5_ctl.c
+++ b/drivers/gpu/drm/msm/mdp/mdp5/mdp5_ctl.c
@@ -118,31 +118,31 @@ static void set_display_intf(struct mdp5_kms *mdp5_kms,
         u32 intf_sel;
 
         spin_lock_irqsave(&mdp5_kms->resource_lock, flags);
-        intf_sel = mdp5_read(mdp5_kms, REG_MDP5_MDP_DISP_INTF_SEL(0));
+        intf_sel = mdp5_read(mdp5_kms, REG_MDP5_DISP_INTF_SEL);
 
         switch (intf->num) {
         case 0:
-                intf_sel &= ~MDP5_MDP_DISP_INTF_SEL_INTF0__MASK;
-                intf_sel |= MDP5_MDP_DISP_INTF_SEL_INTF0(intf->type);
+                intf_sel &= ~MDP5_DISP_INTF_SEL_INTF0__MASK;
+                intf_sel |= MDP5_DISP_INTF_SEL_INTF0(intf->type);
                 break;
         case 1:
-                intf_sel &= ~MDP5_MDP_DISP_INTF_SEL_INTF1__MASK;
-                intf_sel |= MDP5_MDP_DISP_INTF_SEL_INTF1(intf->type);
+                intf_sel &= ~MDP5_DISP_INTF_SEL_INTF1__MASK;
+                intf_sel |= MDP5_DISP_INTF_SEL_INTF1(intf->type);
                 break;
         case 2:
-                intf_sel &= ~MDP5_MDP_DISP_INTF_SEL_INTF2__MASK;
-                intf_sel |= MDP5_MDP_DISP_INTF_SEL_INTF2(intf->type);
+                intf_sel &= ~MDP5_DISP_INTF_SEL_INTF2__MASK;
+                intf_sel |= MDP5_DISP_INTF_SEL_INTF2(intf->type);
                 break;
         case 3:
-                intf_sel &= ~MDP5_MDP_DISP_INTF_SEL_INTF3__MASK;
-                intf_sel |= MDP5_MDP_DISP_INTF_SEL_INTF3(intf->type);
+                intf_sel &= ~MDP5_DISP_INTF_SEL_INTF3__MASK;
+                intf_sel |= MDP5_DISP_INTF_SEL_INTF3(intf->type);
                 break;
         default:
                 BUG();
                 break;
         }
 
-        mdp5_write(mdp5_kms, REG_MDP5_MDP_DISP_INTF_SEL(0), intf_sel);
+        mdp5_write(mdp5_kms, REG_MDP5_DISP_INTF_SEL, intf_sel);
         spin_unlock_irqrestore(&mdp5_kms->resource_lock, flags);
 }
 
@@ -557,7 +557,7 @@ int mdp5_ctl_pair(struct mdp5_ctl *ctlx, struct mdp5_ctl *ctly, bool enable)
         if (!enable) {
                 ctlx->pair = NULL;
                 ctly->pair = NULL;
-                mdp5_write(mdp5_kms, REG_MDP5_MDP_SPARE_0(0), 0);
+                mdp5_write(mdp5_kms, REG_MDP5_SPARE_0, 0);
                 return 0;
         } else if ((ctlx->pair != NULL) || (ctly->pair != NULL)) {
                 dev_err(ctl_mgr->dev->dev, "CTLs already paired\n");
@@ -570,8 +570,8 @@ int mdp5_ctl_pair(struct mdp5_ctl *ctlx, struct mdp5_ctl *ctly, bool enable)
         ctlx->pair = ctly;
         ctly->pair = ctlx;
 
-        mdp5_write(mdp5_kms, REG_MDP5_MDP_SPARE_0(0),
-                MDP5_MDP_SPARE_0_SPLIT_DPL_SINGLE_FLUSH_EN);
+        mdp5_write(mdp5_kms, REG_MDP5_SPARE_0,
+                   MDP5_SPARE_0_SPLIT_DPL_SINGLE_FLUSH_EN);
 
         return 0;
 }
diff --git a/drivers/gpu/drm/msm/mdp/mdp5/mdp5_encoder.c b/drivers/gpu/drm/msm/mdp/mdp5/mdp5_encoder.c
index 1d95f9fd9dc7..fe0c22230883 100644
--- a/drivers/gpu/drm/msm/mdp/mdp5/mdp5_encoder.c
+++ b/drivers/gpu/drm/msm/mdp/mdp5/mdp5_encoder.c
@@ -322,18 +322,18 @@ int mdp5_encoder_set_split_display(struct drm_encoder *encoder,
          * to use the master's enable signal for the slave encoder.
          */
         if (intf_num == 1)
-                data |= MDP5_MDP_SPLIT_DPL_LOWER_INTF2_TG_SYNC;
+                data |= MDP5_SPLIT_DPL_LOWER_INTF2_TG_SYNC;
         else if (intf_num == 2)
-                data |= MDP5_MDP_SPLIT_DPL_LOWER_INTF1_TG_SYNC;
+                data |= MDP5_SPLIT_DPL_LOWER_INTF1_TG_SYNC;
         else
                 return -EINVAL;
 
         /* Make sure clocks are on when connectors calling this function. */
         mdp5_enable(mdp5_kms);
         /* Dumb Panel, Sync mode */
-        mdp5_write(mdp5_kms, REG_MDP5_MDP_SPLIT_DPL_UPPER(0), 0);
-        mdp5_write(mdp5_kms, REG_MDP5_MDP_SPLIT_DPL_LOWER(0), data);
-        mdp5_write(mdp5_kms, REG_MDP5_MDP_SPLIT_DPL_EN(0), 1);
+        mdp5_write(mdp5_kms, REG_MDP5_SPLIT_DPL_UPPER, 0);
+        mdp5_write(mdp5_kms, REG_MDP5_SPLIT_DPL_LOWER, data);
+        mdp5_write(mdp5_kms, REG_MDP5_SPLIT_DPL_EN, 1);
 
         mdp5_ctl_pair(mdp5_encoder->ctl, mdp5_slave_enc->ctl, true);
 
diff --git a/drivers/gpu/drm/msm/mdp/mdp5/mdp5_irq.c b/drivers/gpu/drm/msm/mdp/mdp5/mdp5_irq.c
index 73bc3e312fd4..d53e5510fd7c 100644
--- a/drivers/gpu/drm/msm/mdp/mdp5/mdp5_irq.c
+++ b/drivers/gpu/drm/msm/mdp/mdp5/mdp5_irq.c
@@ -15,7 +15,6 @@
  * this program.  If not, see <http://www.gnu.org/licenses/>.
  */
 
-#include <linux/irqdomain.h>
 #include <linux/irq.h>
 
 #include "msm_drv.h"
@@ -24,9 +23,9 @@
 void mdp5_set_irqmask(struct mdp_kms *mdp_kms, uint32_t irqmask,
                 uint32_t old_irqmask)
 {
-        mdp5_write(to_mdp5_kms(mdp_kms), REG_MDP5_MDP_INTR_CLEAR(0),
-                irqmask ^ (irqmask & old_irqmask));
-        mdp5_write(to_mdp5_kms(mdp_kms), REG_MDP5_MDP_INTR_EN(0), irqmask);
+        mdp5_write(to_mdp5_kms(mdp_kms), REG_MDP5_INTR_CLEAR,
+                   irqmask ^ (irqmask & old_irqmask));
+        mdp5_write(to_mdp5_kms(mdp_kms), REG_MDP5_INTR_EN, irqmask);
 }
 
 static void mdp5_irq_error_handler(struct mdp_irq *irq, uint32_t irqstatus)
@@ -38,8 +37,8 @@ void mdp5_irq_preinstall(struct msm_kms *kms)
 {
         struct mdp5_kms *mdp5_kms = to_mdp5_kms(to_mdp_kms(kms));
         mdp5_enable(mdp5_kms);
-        mdp5_write(mdp5_kms, REG_MDP5_MDP_INTR_CLEAR(0), 0xffffffff);
-        mdp5_write(mdp5_kms, REG_MDP5_MDP_INTR_EN(0), 0x00000000);
+        mdp5_write(mdp5_kms, REG_MDP5_INTR_CLEAR, 0xffffffff);
+        mdp5_write(mdp5_kms, REG_MDP5_INTR_EN, 0x00000000);
         mdp5_disable(mdp5_kms);
 }
 
@@ -55,7 +54,9 @@ int mdp5_irq_postinstall(struct msm_kms *kms)
                         MDP5_IRQ_INTF2_UNDER_RUN |
                         MDP5_IRQ_INTF3_UNDER_RUN;
 
+        mdp5_enable(mdp5_kms);
         mdp_irq_register(mdp_kms, error_handler);
+        mdp5_disable(mdp5_kms);
 
         return 0;
 }
@@ -64,21 +65,22 @@ void mdp5_irq_uninstall(struct msm_kms *kms)
 {
         struct mdp5_kms *mdp5_kms = to_mdp5_kms(to_mdp_kms(kms));
         mdp5_enable(mdp5_kms);
-        mdp5_write(mdp5_kms, REG_MDP5_MDP_INTR_EN(0), 0x00000000);
+        mdp5_write(mdp5_kms, REG_MDP5_INTR_EN, 0x00000000);
         mdp5_disable(mdp5_kms);
 }
 
-static void mdp5_irq_mdp(struct mdp_kms *mdp_kms)
+irqreturn_t mdp5_irq(struct msm_kms *kms)
 {
+        struct mdp_kms *mdp_kms = to_mdp_kms(kms);
         struct mdp5_kms *mdp5_kms = to_mdp5_kms(mdp_kms);
         struct drm_device *dev = mdp5_kms->dev;
         struct msm_drm_private *priv = dev->dev_private;
         unsigned int id;
         uint32_t status, enable;
 
-        enable = mdp5_read(mdp5_kms, REG_MDP5_MDP_INTR_EN(0));
-        status = mdp5_read(mdp5_kms, REG_MDP5_MDP_INTR_STATUS(0)) & enable;
-        mdp5_write(mdp5_kms, REG_MDP5_MDP_INTR_CLEAR(0), status);
+        enable = mdp5_read(mdp5_kms, REG_MDP5_INTR_EN);
+        status = mdp5_read(mdp5_kms, REG_MDP5_INTR_STATUS) & enable;
+        mdp5_write(mdp5_kms, REG_MDP5_INTR_CLEAR, status);
 
         VERB("status=%08x", status);
 
@@ -87,29 +89,6 @@ static void mdp5_irq_mdp(struct mdp_kms *mdp_kms)
         for (id = 0; id < priv->num_crtcs; id++)
                 if (status & mdp5_crtc_vblank(priv->crtcs[id]))
                         drm_handle_vblank(dev, id);
-}
-
-irqreturn_t mdp5_irq(struct msm_kms *kms)
-{
-        struct mdp_kms *mdp_kms = to_mdp_kms(kms);
-        struct mdp5_kms *mdp5_kms = to_mdp5_kms(mdp_kms);
-        uint32_t intr;
-
-        intr = mdp5_read(mdp5_kms, REG_MDSS_HW_INTR_STATUS);
-
-        VERB("intr=%08x", intr);
-
-        if (intr & MDSS_HW_INTR_STATUS_INTR_MDP) {
-                mdp5_irq_mdp(mdp_kms);
-                intr &= ~MDSS_HW_INTR_STATUS_INTR_MDP;
-        }
-
-        while (intr) {
-                irq_hw_number_t hwirq = fls(intr) - 1;
-                generic_handle_irq(irq_find_mapping(
-                                mdp5_kms->irqcontroller.domain, hwirq));
-                intr &= ~(1 << hwirq);
-        }
 
         return IRQ_HANDLED;
 }
@@ -135,81 +114,3 @@ void mdp5_disable_vblank(struct msm_kms *kms, struct drm_crtc *crtc)
                         mdp5_crtc_vblank(crtc), false);
         mdp5_disable(mdp5_kms);
 }
-
-/*
- * interrupt-controller implementation, so sub-blocks (hdmi/eDP/dsi/etc)
- * can register to get their irq's delivered
- */
-
-#define VALID_IRQS  (MDSS_HW_INTR_STATUS_INTR_DSI0 | \
-                MDSS_HW_INTR_STATUS_INTR_DSI1 | \
-                MDSS_HW_INTR_STATUS_INTR_HDMI | \
-                MDSS_HW_INTR_STATUS_INTR_EDP)
-
-static void mdp5_hw_mask_irq(struct irq_data *irqd)
-{
-        struct mdp5_kms *mdp5_kms = irq_data_get_irq_chip_data(irqd);
-        smp_mb__before_atomic();
-        clear_bit(irqd->hwirq, &mdp5_kms->irqcontroller.enabled_mask);
-        smp_mb__after_atomic();
-}
-
-static void mdp5_hw_unmask_irq(struct irq_data *irqd)
-{
-        struct mdp5_kms *mdp5_kms = irq_data_get_irq_chip_data(irqd);
-        smp_mb__before_atomic();
-        set_bit(irqd->hwirq, &mdp5_kms->irqcontroller.enabled_mask);
-        smp_mb__after_atomic();
-}
-
-static struct irq_chip mdp5_hw_irq_chip = {
-        .name           = "mdp5",
-        .irq_mask       = mdp5_hw_mask_irq,
-        .irq_unmask     = mdp5_hw_unmask_irq,
-};
-
-static int mdp5_hw_irqdomain_map(struct irq_domain *d,
-                unsigned int irq, irq_hw_number_t hwirq)
-{
-        struct mdp5_kms *mdp5_kms = d->host_data;
-
-        if (!(VALID_IRQS & (1 << hwirq)))
-                return -EPERM;
-
-        irq_set_chip_and_handler(irq, &mdp5_hw_irq_chip, handle_level_irq);
-        irq_set_chip_data(irq, mdp5_kms);
-
-        return 0;
-}
-
-static struct irq_domain_ops mdp5_hw_irqdomain_ops = {
-        .map = mdp5_hw_irqdomain_map,
-        .xlate = irq_domain_xlate_onecell,
-};
-
-
-int mdp5_irq_domain_init(struct mdp5_kms *mdp5_kms)
-{
-        struct device *dev = mdp5_kms->dev->dev;
-        struct irq_domain *d;
-
-        d = irq_domain_add_linear(dev->of_node, 32,
-                        &mdp5_hw_irqdomain_ops, mdp5_kms);
-        if (!d) {
-                dev_err(dev, "mdp5 irq domain add failed\n");
-                return -ENXIO;
-        }
-
-        mdp5_kms->irqcontroller.enabled_mask = 0;
-        mdp5_kms->irqcontroller.domain = d;
-
-        return 0;
-}
-
-void mdp5_irq_domain_fini(struct mdp5_kms *mdp5_kms)
-{
-        if (mdp5_kms->irqcontroller.domain) {
-                irq_domain_remove(mdp5_kms->irqcontroller.domain);
-                mdp5_kms->irqcontroller.domain = NULL;
-        }
-}
diff --git a/drivers/gpu/drm/msm/mdp/mdp5/mdp5_kms.c b/drivers/gpu/drm/msm/mdp/mdp5/mdp5_kms.c
index 484b4d15e71d..ed7143d35b25 100644
--- a/drivers/gpu/drm/msm/mdp/mdp5/mdp5_kms.c
+++ b/drivers/gpu/drm/msm/mdp/mdp5/mdp5_kms.c
@@ -16,6 +16,7 @@
  * this program.  If not, see <http://www.gnu.org/licenses/>.
  */
 
+#include <linux/of_irq.h>
 
 #include "msm_drv.h"
 #include "msm_mmu.h"
@@ -28,10 +29,11 @@ static const char *iommu_ports[] = {
 static int mdp5_hw_init(struct msm_kms *kms)
 {
         struct mdp5_kms *mdp5_kms = to_mdp5_kms(to_mdp_kms(kms));
-        struct drm_device *dev = mdp5_kms->dev;
+        struct platform_device *pdev = mdp5_kms->pdev;
         unsigned long flags;
 
-        pm_runtime_get_sync(dev->dev);
+        pm_runtime_get_sync(&pdev->dev);
+        mdp5_enable(mdp5_kms);
 
         /* Magic unknown register writes:
          *
@@ -58,12 +60,13 @@ static int mdp5_hw_init(struct msm_kms *kms)
          */
 
         spin_lock_irqsave(&mdp5_kms->resource_lock, flags);
-        mdp5_write(mdp5_kms, REG_MDP5_MDP_DISP_INTF_SEL(0), 0);
+        mdp5_write(mdp5_kms, REG_MDP5_DISP_INTF_SEL, 0);
         spin_unlock_irqrestore(&mdp5_kms->resource_lock, flags);
 
         mdp5_ctlm_hw_reset(mdp5_kms->ctlm);
 
-        pm_runtime_put_sync(dev->dev);
+        mdp5_disable(mdp5_kms);
+        pm_runtime_put_sync(&pdev->dev);
 
         return 0;
 }
@@ -78,17 +81,11 @@ static void mdp5_complete_commit(struct msm_kms *kms, struct drm_atomic_state *s
 {
         int i;
         struct mdp5_kms *mdp5_kms = to_mdp5_kms(to_mdp_kms(kms));
-        int nplanes = mdp5_kms->dev->mode_config.num_total_plane;
-
-        for (i = 0; i < nplanes; i++) {
-                struct drm_plane *plane = state->planes[i];
-                struct drm_plane_state *plane_state = state->plane_states[i];
-
-                if (!plane)
-                        continue;
+        struct drm_plane *plane;
+        struct drm_plane_state *plane_state;
 
+        for_each_plane_in_state(state, plane, plane_state, i)
                 mdp5_plane_complete_commit(plane, plane_state);
-        }
 
         mdp5_disable(mdp5_kms);
 }
@@ -117,26 +114,15 @@ static int mdp5_set_split_display(struct msm_kms *kms,
                 return mdp5_encoder_set_split_display(encoder, slave_encoder);
 }
 
-static void mdp5_destroy(struct msm_kms *kms)
+static void mdp5_kms_destroy(struct msm_kms *kms)
 {
         struct mdp5_kms *mdp5_kms = to_mdp5_kms(to_mdp_kms(kms));
         struct msm_mmu *mmu = mdp5_kms->mmu;
 
-        mdp5_irq_domain_fini(mdp5_kms);
-
         if (mmu) {
                 mmu->funcs->detach(mmu, iommu_ports, ARRAY_SIZE(iommu_ports));
                 mmu->funcs->destroy(mmu);
         }
-
-        if (mdp5_kms->ctlm)
-                mdp5_ctlm_destroy(mdp5_kms->ctlm);
-        if (mdp5_kms->smp)
-                mdp5_smp_destroy(mdp5_kms->smp);
-        if (mdp5_kms->cfg)
-                mdp5_cfg_destroy(mdp5_kms->cfg);
-
-        kfree(mdp5_kms);
 }
 
 static const struct mdp_kms_funcs kms_funcs = {
@@ -154,7 +140,7 @@ static const struct mdp_kms_funcs kms_funcs = {
                 .get_format      = mdp_get_format,
                 .round_pixclk    = mdp5_round_pixclk,
                 .set_split_display = mdp5_set_split_display,
-                .destroy         = mdp5_destroy,
+                .destroy         = mdp5_kms_destroy,
         },
         .set_irqmask         = mdp5_set_irqmask,
 };
@@ -351,13 +337,6 @@ static int modeset_init(struct mdp5_kms *mdp5_kms)
 
         hw_cfg = mdp5_cfg_get_hw_config(mdp5_kms->cfg);
 
-        /* register our interrupt-controller for hdmi/eDP/dsi/etc
-         * to use for irqs routed through mdp:
-         */
-        ret = mdp5_irq_domain_init(mdp5_kms);
-        if (ret)
-                goto fail;
-
         /* construct CRTCs and their private planes: */
         for (i = 0; i < hw_cfg->pipe_rgb.count; i++) {
                 struct drm_plane *plane;
@@ -425,17 +404,17 @@ fail:
         return ret;
 }
 
-static void read_hw_revision(struct mdp5_kms *mdp5_kms,
-                uint32_t *major, uint32_t *minor)
+static void read_mdp_hw_revision(struct mdp5_kms *mdp5_kms,
+                                 u32 *major, u32 *minor)
 {
-        uint32_t version;
+        u32 version;
 
         mdp5_enable(mdp5_kms);
-        version = mdp5_read(mdp5_kms, REG_MDSS_HW_VERSION);
+        version = mdp5_read(mdp5_kms, REG_MDP5_HW_VERSION);
         mdp5_disable(mdp5_kms);
 
-        *major = FIELD(version, MDSS_HW_VERSION_MAJOR);
-        *minor = FIELD(version, MDSS_HW_VERSION_MINOR);
+        *major = FIELD(version, MDP5_HW_VERSION_MAJOR);
+        *minor = FIELD(version, MDP5_HW_VERSION_MINOR);
 
         DBG("MDP5 version v%d.%d", *major, *minor);
 }
@@ -580,51 +559,146 @@ static u32 mdp5_get_vblank_counter(struct drm_device *dev, unsigned int pipe)
 
 struct msm_kms *mdp5_kms_init(struct drm_device *dev)
 {
-        struct platform_device *pdev = dev->platformdev;
-        struct mdp5_cfg *config;
+        struct msm_drm_private *priv = dev->dev_private;
+        struct platform_device *pdev;
         struct mdp5_kms *mdp5_kms;
-        struct msm_kms *kms = NULL;
+        struct mdp5_cfg *config;
+        struct msm_kms *kms;
         struct msm_mmu *mmu;
-        uint32_t major, minor;
-        int i, ret;
+        int irq, i, ret;
 
-        mdp5_kms = kzalloc(sizeof(*mdp5_kms), GFP_KERNEL);
-        if (!mdp5_kms) {
-                dev_err(dev->dev, "failed to allocate kms\n");
-                ret = -ENOMEM;
+        /* priv->kms would have been populated by the MDP5 driver */
+        kms = priv->kms;
+        if (!kms)
+                return NULL;
+
+        mdp5_kms = to_mdp5_kms(to_mdp_kms(kms));
+
+        mdp_kms_init(&mdp5_kms->base, &kms_funcs);
+
+        pdev = mdp5_kms->pdev;
+
+        irq = irq_of_parse_and_map(pdev->dev.of_node, 0);
+        if (irq < 0) {
+                ret = irq;
+                dev_err(&pdev->dev, "failed to get irq: %d\n", ret);
                 goto fail;
         }
 
-        spin_lock_init(&mdp5_kms->resource_lock);
+        kms->irq = irq;
 
-        mdp_kms_init(&mdp5_kms->base, &kms_funcs);
+        config = mdp5_cfg_get_config(mdp5_kms->cfg);
 
-        kms = &mdp5_kms->base.base;
+        /* make sure things are off before attaching iommu (bootloader could
+         * have left things on, in which case we'll start getting faults if
+         * we don't disable):
+         */
+        mdp5_enable(mdp5_kms);
+        for (i = 0; i < MDP5_INTF_NUM_MAX; i++) {
+                if (mdp5_cfg_intf_is_virtual(config->hw->intf.connect[i]) ||
+                    !config->hw->intf.base[i])
+                        continue;
+                mdp5_write(mdp5_kms, REG_MDP5_INTF_TIMING_ENGINE_EN(i), 0);
 
-        mdp5_kms->dev = dev;
+                mdp5_write(mdp5_kms, REG_MDP5_INTF_FRAME_LINE_COUNT_EN(i), 0x3);
+        }
+        mdp5_disable(mdp5_kms);
+        mdelay(16);
 
-        /* mdp5_kms->mmio actually represents the MDSS base address */
-        mdp5_kms->mmio = msm_ioremap(pdev, "mdp_phys", "MDP5");
-        if (IS_ERR(mdp5_kms->mmio)) {
-                ret = PTR_ERR(mdp5_kms->mmio);
+        if (config->platform.iommu) {
+                mmu = msm_iommu_new(&pdev->dev, config->platform.iommu);
+                if (IS_ERR(mmu)) {
+                        ret = PTR_ERR(mmu);
+                        dev_err(&pdev->dev, "failed to init iommu: %d\n", ret);
+                        iommu_domain_free(config->platform.iommu);
+                        goto fail;
+                }
+
+                ret = mmu->funcs->attach(mmu, iommu_ports,
+                                ARRAY_SIZE(iommu_ports));
+                if (ret) {
+                        dev_err(&pdev->dev, "failed to attach iommu: %d\n",
+                                ret);
+                        mmu->funcs->destroy(mmu);
+                        goto fail;
+                }
+        } else {
+                dev_info(&pdev->dev,
+                         "no iommu, fallback to phys contig buffers for scanout\n");
+                mmu = NULL;
+        }
+        mdp5_kms->mmu = mmu;
+
+        mdp5_kms->id = msm_register_mmu(dev, mmu);
+        if (mdp5_kms->id < 0) {
+                ret = mdp5_kms->id;
+                dev_err(&pdev->dev, "failed to register mdp5 iommu: %d\n", ret);
                 goto fail;
         }
 
-        mdp5_kms->vbif = msm_ioremap(pdev, "vbif_phys", "VBIF");
-        if (IS_ERR(mdp5_kms->vbif)) {
-                ret = PTR_ERR(mdp5_kms->vbif);
+        ret = modeset_init(mdp5_kms);
+        if (ret) {
+                dev_err(&pdev->dev, "modeset_init failed: %d\n", ret);
                 goto fail;
         }
 
-        mdp5_kms->vdd = devm_regulator_get(&pdev->dev, "vdd");
-        if (IS_ERR(mdp5_kms->vdd)) {
-                ret = PTR_ERR(mdp5_kms->vdd);
+        dev->mode_config.min_width = 0;
+        dev->mode_config.min_height = 0;
+        dev->mode_config.max_width = config->hw->lm.max_width;
+        dev->mode_config.max_height = config->hw->lm.max_height;
+
+        dev->driver->get_vblank_timestamp = mdp5_get_vblank_timestamp;
+        dev->driver->get_scanout_position = mdp5_get_scanoutpos;
+        dev->driver->get_vblank_counter = mdp5_get_vblank_counter;
+        dev->max_vblank_count = 0xffffffff;
+        dev->vblank_disable_immediate = true;
+
+        return kms;
+fail:
+        if (kms)
+                mdp5_kms_destroy(kms);
+        return ERR_PTR(ret);
+}
+
+static void mdp5_destroy(struct platform_device *pdev)
+{
+        struct mdp5_kms *mdp5_kms = platform_get_drvdata(pdev);
+
+        if (mdp5_kms->ctlm)
+                mdp5_ctlm_destroy(mdp5_kms->ctlm);
+        if (mdp5_kms->smp)
+                mdp5_smp_destroy(mdp5_kms->smp);
+        if (mdp5_kms->cfg)
+                mdp5_cfg_destroy(mdp5_kms->cfg);
+
+        if (mdp5_kms->rpm_enabled)
+                pm_runtime_disable(&pdev->dev);
+}
+
+static int mdp5_init(struct platform_device *pdev, struct drm_device *dev)
+{
+        struct msm_drm_private *priv = dev->dev_private;
+        struct mdp5_kms *mdp5_kms;
+        struct mdp5_cfg *config;
+        u32 major, minor;
+        int ret;
+
+        mdp5_kms = devm_kzalloc(&pdev->dev, sizeof(*mdp5_kms), GFP_KERNEL);
+        if (!mdp5_kms) {
+                ret = -ENOMEM;
                 goto fail;
         }
 
-        ret = regulator_enable(mdp5_kms->vdd);
-        if (ret) {
-                dev_err(dev->dev, "failed to enable regulator vdd: %d\n", ret);
+        platform_set_drvdata(pdev, mdp5_kms);
+
+        spin_lock_init(&mdp5_kms->resource_lock);
+
+        mdp5_kms->dev = dev;
+        mdp5_kms->pdev = pdev;
+
+        mdp5_kms->mmio = msm_ioremap(pdev, "mdp_phys", "MDP5");
+        if (IS_ERR(mdp5_kms->mmio)) {
+                ret = PTR_ERR(mdp5_kms->mmio);
                 goto fail;
         }
 
@@ -635,9 +709,6 @@ struct msm_kms *mdp5_kms_init(struct drm_device *dev)
         ret = get_clk(pdev, &mdp5_kms->ahb_clk, "iface_clk", true);
         if (ret)
                 goto fail;
-        ret = get_clk(pdev, &mdp5_kms->src_clk, "core_clk_src", true);
-        if (ret)
-                goto fail;
         ret = get_clk(pdev, &mdp5_kms->core_clk, "core_clk", true);
         if (ret)
                 goto fail;
@@ -652,9 +723,12 @@ struct msm_kms *mdp5_kms_init(struct drm_device *dev)
          * rate first, then figure out hw revision, and then set a
          * more optimal rate:
          */
-        clk_set_rate(mdp5_kms->src_clk, 200000000);
+        clk_set_rate(mdp5_kms->core_clk, 200000000);
 
-        read_hw_revision(mdp5_kms, &major, &minor);
+        pm_runtime_enable(&pdev->dev);
+        mdp5_kms->rpm_enabled = true;
+
+        read_mdp_hw_revision(mdp5_kms, &major, &minor);
 
         mdp5_kms->cfg = mdp5_cfg_init(mdp5_kms, major, minor);
         if (IS_ERR(mdp5_kms->cfg)) {
@@ -667,7 +741,7 @@ struct msm_kms *mdp5_kms_init(struct drm_device *dev)
         mdp5_kms->caps = config->hw->mdp.caps;
 
         /* TODO: compute core clock rate at runtime */
-        clk_set_rate(mdp5_kms->src_clk, config->hw->max_clk);
+        clk_set_rate(mdp5_kms->core_clk, config->hw->max_clk);
 
         /*
          * Some chipsets have a Shared Memory Pool (SMP), while others
@@ -690,73 +764,76 @@ struct msm_kms *mdp5_kms_init(struct drm_device *dev)
                 goto fail;
         }
 
-        /* make sure things are off before attaching iommu (bootloader could
-         * have left things on, in which case we'll start getting faults if
-         * we don't disable):
-         */
-        mdp5_enable(mdp5_kms);
-        for (i = 0; i < MDP5_INTF_NUM_MAX; i++) {
-                if (mdp5_cfg_intf_is_virtual(config->hw->intf.connect[i]) ||
-                                !config->hw->intf.base[i])
-                        continue;
-                mdp5_write(mdp5_kms, REG_MDP5_INTF_TIMING_ENGINE_EN(i), 0);
+        /* set uninit-ed kms */
+        priv->kms = &mdp5_kms->base.base;
 
-                mdp5_write(mdp5_kms, REG_MDP5_INTF_FRAME_LINE_COUNT_EN(i), 0x3);
-        }
-        mdp5_disable(mdp5_kms);
-        mdelay(16);
+        return 0;
+fail:
+        mdp5_destroy(pdev);
+        return ret;
+}
 
-        if (config->platform.iommu) {
-                mmu = msm_iommu_new(&pdev->dev, config->platform.iommu);
-                if (IS_ERR(mmu)) {
-                        ret = PTR_ERR(mmu);
-                        dev_err(dev->dev, "failed to init iommu: %d\n", ret);
-                        iommu_domain_free(config->platform.iommu);
-                        goto fail;
-                }
+static int mdp5_bind(struct device *dev, struct device *master, void *data)
+{
+        struct drm_device *ddev = dev_get_drvdata(master);
+        struct platform_device *pdev = to_platform_device(dev);
 
-                ret = mmu->funcs->attach(mmu, iommu_ports,
-                                ARRAY_SIZE(iommu_ports));
-                if (ret) {
-                        dev_err(dev->dev, "failed to attach iommu: %d\n", ret);
-                        mmu->funcs->destroy(mmu);
-                        goto fail;
-                }
-        } else {
-                dev_info(dev->dev, "no iommu, fallback to phys "
-                                "contig buffers for scanout\n");
-                mmu = NULL;
-        }
-        mdp5_kms->mmu = mmu;
+        DBG("");
 
-        mdp5_kms->id = msm_register_mmu(dev, mmu);
-        if (mdp5_kms->id < 0) {
-                ret = mdp5_kms->id;
-                dev_err(dev->dev, "failed to register mdp5 iommu: %d\n", ret);
-                goto fail;
-        }
+        return mdp5_init(pdev, ddev);
+}
 
-        ret = modeset_init(mdp5_kms);
-        if (ret) {
-                dev_err(dev->dev, "modeset_init failed: %d\n", ret);
-                goto fail;
-        }
+static void mdp5_unbind(struct device *dev, struct device *master,
+                        void *data)
+{
+        struct platform_device *pdev = to_platform_device(dev);
 
-        dev->mode_config.min_width = 0;
-        dev->mode_config.min_height = 0;
-        dev->mode_config.max_width = config->hw->lm.max_width;
-        dev->mode_config.max_height = config->hw->lm.max_height;
+        mdp5_destroy(pdev);
+}
 
-        dev->driver->get_vblank_timestamp = mdp5_get_vblank_timestamp;
-        dev->driver->get_scanout_position = mdp5_get_scanoutpos;
-        dev->driver->get_vblank_counter = mdp5_get_vblank_counter;
-        dev->max_vblank_count = 0xffffffff;
-        dev->vblank_disable_immediate = true;
+static const struct component_ops mdp5_ops = {
+        .bind   = mdp5_bind,
+        .unbind = mdp5_unbind,
+};
 
-        return kms;
+static int mdp5_dev_probe(struct platform_device *pdev)
+{
+        DBG("");
+        return component_add(&pdev->dev, &mdp5_ops);
+}
 
-fail:
-        if (kms)
-                mdp5_destroy(kms);
-        return ERR_PTR(ret);
+static int mdp5_dev_remove(struct platform_device *pdev)
+{
+        DBG("");
+        component_del(&pdev->dev, &mdp5_ops);
+        return 0;
+}
+
+static const struct of_device_id mdp5_dt_match[] = {
+        { .compatible = "qcom,mdp5", },
+        /* to support downstream DT files */
+        { .compatible = "qcom,mdss_mdp", },
+        {}
+};
+MODULE_DEVICE_TABLE(of, mdp5_dt_match);
+
+static struct platform_driver mdp5_driver = {
+        .probe = mdp5_dev_probe,
+        .remove = mdp5_dev_remove,
+        .driver = {
+                .name = "msm_mdp",
+                .of_match_table = mdp5_dt_match,
+        },
+};
+
+void __init msm_mdp_register(void)
+{
+        DBG("");
+        platform_driver_register(&mdp5_driver);
+}
+
+void __exit msm_mdp_unregister(void)
+{
+        DBG("");
+        platform_driver_unregister(&mdp5_driver);
 }
diff --git a/drivers/gpu/drm/msm/mdp/mdp5/mdp5_kms.h b/drivers/gpu/drm/msm/mdp/mdp5/mdp5_kms.h
index 9a25898239d3..03738927be10 100644
--- a/drivers/gpu/drm/msm/mdp/mdp5/mdp5_kms.h
+++ b/drivers/gpu/drm/msm/mdp/mdp5/mdp5_kms.h
@@ -31,6 +31,8 @@ struct mdp5_kms {
 
         struct drm_device *dev;
 
+        struct platform_device *pdev;
+
         struct mdp5_cfg_handler *cfg;
         uint32_t caps;  /* MDP capabilities (MDP_CAP_XXX bits) */
 
@@ -43,29 +45,23 @@ struct mdp5_kms {
         struct mdp5_ctl_manager *ctlm;
 
         /* io/register spaces: */
-        void __iomem *mmio, *vbif;
-
-        struct regulator *vdd;
+        void __iomem *mmio;
 
         struct clk *axi_clk;
         struct clk *ahb_clk;
-        struct clk *src_clk;
         struct clk *core_clk;
         struct clk *lut_clk;
         struct clk *vsync_clk;
 
         /*
          * lock to protect access to global resources: ie., following register:
-         *      - REG_MDP5_MDP_DISP_INTF_SEL
+         *      - REG_MDP5_DISP_INTF_SEL
          */
         spinlock_t resource_lock;
 
-        struct mdp_irq error_handler;
+        bool rpm_enabled;
 
-        struct {
-                volatile unsigned long enabled_mask;
-                struct irq_domain *domain;
-        } irqcontroller;
+        struct mdp_irq error_handler;
 };
 #define to_mdp5_kms(x) container_of(x, struct mdp5_kms, base)
 
diff --git a/drivers/gpu/drm/msm/mdp/mdp5/mdp5_mdss.c b/drivers/gpu/drm/msm/mdp/mdp5/mdp5_mdss.c
new file mode 100644
index 000000000000..d444a6901fff
--- /dev/null
+++ b/drivers/gpu/drm/msm/mdp/mdp5/mdp5_mdss.c
@@ -0,0 +1,235 @@
+/*
+ * Copyright (c) 2016, The Linux Foundation. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 as published by
+ * the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include <linux/irqdomain.h>
+#include <linux/irq.h>
+
+#include "msm_drv.h"
+#include "mdp5_kms.h"
+
+/*
+ * If needed, this can become more specific: something like struct mdp5_mdss,
+ * which contains a 'struct msm_mdss base' member.
+ */
+struct msm_mdss {
+        struct drm_device *dev;
+
+        void __iomem *mmio, *vbif;
+
+        struct regulator *vdd;
+
+        struct {
+                volatile unsigned long enabled_mask;
+                struct irq_domain *domain;
+        } irqcontroller;
+};
+
+static inline void mdss_write(struct msm_mdss *mdss, u32 reg, u32 data)
+{
+        msm_writel(data, mdss->mmio + reg);
+}
+
+static inline u32 mdss_read(struct msm_mdss *mdss, u32 reg)
+{
+        return msm_readl(mdss->mmio + reg);
+}
+
+static irqreturn_t mdss_irq(int irq, void *arg)
+{
+        struct msm_mdss *mdss = arg;
+        u32 intr;
+
+        intr = mdss_read(mdss, REG_MDSS_HW_INTR_STATUS);
+
+        VERB("intr=%08x", intr);
+
+        while (intr) {
+                irq_hw_number_t hwirq = fls(intr) - 1;
+
+                generic_handle_irq(irq_find_mapping(
+                                mdss->irqcontroller.domain, hwirq));
+                intr &= ~(1 << hwirq);
+        }
+
+        return IRQ_HANDLED;
+}
+
+/*
+ * interrupt-controller implementation, so sub-blocks (MDP/HDMI/eDP/DSI/etc)
+ * can register to get their irq's delivered
+ */
+
+#define VALID_IRQS  (MDSS_HW_INTR_STATUS_INTR_MDP | \
+                MDSS_HW_INTR_STATUS_INTR_DSI0 | \
+                MDSS_HW_INTR_STATUS_INTR_DSI1 | \
+                MDSS_HW_INTR_STATUS_INTR_HDMI | \
+                MDSS_HW_INTR_STATUS_INTR_EDP)
+
+static void mdss_hw_mask_irq(struct irq_data *irqd)
+{
+        struct msm_mdss *mdss = irq_data_get_irq_chip_data(irqd);
+
+        smp_mb__before_atomic();
+        clear_bit(irqd->hwirq, &mdss->irqcontroller.enabled_mask);
+        smp_mb__after_atomic();
+}
+
+static void mdss_hw_unmask_irq(struct irq_data *irqd)
+{
+        struct msm_mdss *mdss = irq_data_get_irq_chip_data(irqd);
+
+        smp_mb__before_atomic();
+        set_bit(irqd->hwirq, &mdss->irqcontroller.enabled_mask);
+        smp_mb__after_atomic();
+}
+
+static struct irq_chip mdss_hw_irq_chip = {
+        .name           = "mdss",
+        .irq_mask       = mdss_hw_mask_irq,
+        .irq_unmask     = mdss_hw_unmask_irq,
+};
+
+static int mdss_hw_irqdomain_map(struct irq_domain *d, unsigned int irq,
+                                 irq_hw_number_t hwirq)
+{
+        struct msm_mdss *mdss = d->host_data;
+
+        if (!(VALID_IRQS & (1 << hwirq)))
+                return -EPERM;
+
+        irq_set_chip_and_handler(irq, &mdss_hw_irq_chip, handle_level_irq);
+        irq_set_chip_data(irq, mdss);
+
+        return 0;
+}
+
+static struct irq_domain_ops mdss_hw_irqdomain_ops = {
+        .map = mdss_hw_irqdomain_map,
+        .xlate = irq_domain_xlate_onecell,
+};
+
+
+static int mdss_irq_domain_init(struct msm_mdss *mdss)
+{
+        struct device *dev = mdss->dev->dev;
+        struct irq_domain *d;
+
+        d = irq_domain_add_linear(dev->of_node, 32, &mdss_hw_irqdomain_ops,
+                                  mdss);
+        if (!d) {
+                dev_err(dev, "mdss irq domain add failed\n");
+                return -ENXIO;
+        }
+
+        mdss->irqcontroller.enabled_mask = 0;
+        mdss->irqcontroller.domain = d;
+
+        return 0;
+}
+
+void msm_mdss_destroy(struct drm_device *dev)
+{
+        struct msm_drm_private *priv = dev->dev_private;
+        struct msm_mdss *mdss = priv->mdss;
+
+        if (!mdss)
+                return;
+
+        irq_domain_remove(mdss->irqcontroller.domain);
+        mdss->irqcontroller.domain = NULL;
+
+        regulator_disable(mdss->vdd);
+
+        pm_runtime_put_sync(dev->dev);
+
+        pm_runtime_disable(dev->dev);
+}
+
+int msm_mdss_init(struct drm_device *dev)
+{
+        struct platform_device *pdev = dev->platformdev;
+        struct msm_drm_private *priv = dev->dev_private;
+        struct msm_mdss *mdss;
+        int ret;
+
+        DBG("");
+
+        if (!of_device_is_compatible(dev->dev->of_node, "qcom,mdss"))
+                return 0;
+
+        mdss = devm_kzalloc(dev->dev, sizeof(*mdss), GFP_KERNEL);
+        if (!mdss) {
+                ret = -ENOMEM;
+                goto fail;
+        }
+
+        mdss->dev = dev;
+
+        mdss->mmio = msm_ioremap(pdev, "mdss_phys", "MDSS");
+        if (IS_ERR(mdss->mmio)) {
+                ret = PTR_ERR(mdss->mmio);
+                goto fail;
+        }
+
+        mdss->vbif = msm_ioremap(pdev, "vbif_phys", "VBIF");
+        if (IS_ERR(mdss->vbif)) {
+                ret = PTR_ERR(mdss->vbif);
+                goto fail;
+        }
+
+        /* Regulator to enable GDSCs in downstream kernels */
+        mdss->vdd = devm_regulator_get(dev->dev, "vdd");
+        if (IS_ERR(mdss->vdd)) {
+                ret = PTR_ERR(mdss->vdd);
+                goto fail;
+        }
+
+        ret = regulator_enable(mdss->vdd);
+        if (ret) {
+                dev_err(dev->dev, "failed to enable regulator vdd: %d\n",
+                        ret);
+                goto fail;
+        }
+
+        ret = devm_request_irq(dev->dev, platform_get_irq(pdev, 0),
+                               mdss_irq, 0, "mdss_isr", mdss);
+        if (ret) {
+                dev_err(dev->dev, "failed to init irq: %d\n", ret);
+                goto fail_irq;
+        }
+
+        ret = mdss_irq_domain_init(mdss);
+        if (ret) {
+                dev_err(dev->dev, "failed to init sub-block irqs: %d\n", ret);
+                goto fail_irq;
+        }
+
+        priv->mdss = mdss;
+
+        pm_runtime_enable(dev->dev);
+
+        /*
+         * TODO: This is needed as the MDSS GDSC is only tied to MDSS's power
+         * domain. Remove this once runtime PM is adapted for all the devices.
+         */
+        pm_runtime_get_sync(dev->dev);
+
+        return 0;
+fail_irq:
+        regulator_disable(mdss->vdd);
+fail:
+        return ret;
+}
diff --git a/drivers/gpu/drm/msm/mdp/mdp5/mdp5_smp.c b/drivers/gpu/drm/msm/mdp/mdp5/mdp5_smp.c
index 6f425c25d9fe..27d7b55b52c9 100644
--- a/drivers/gpu/drm/msm/mdp/mdp5/mdp5_smp.c
+++ b/drivers/gpu/drm/msm/mdp/mdp5/mdp5_smp.c
@@ -42,7 +42,7 @@
  *
  *     configured:
  *     The block is allocated to some client, and assigned to that
- *     client in MDP5_MDP_SMP_ALLOC registers.
+ *     client in MDP5_SMP_ALLOC registers.
  *
  *     inuse:
  *     The block is being actively used by a client.
@@ -59,7 +59,7 @@
  *     mdp5_smp_commit.
  *
  *  2) mdp5_smp_configure():
- *     As hw is programmed, before FLUSH, MDP5_MDP_SMP_ALLOC registers
+ *     As hw is programmed, before FLUSH, MDP5_SMP_ALLOC registers
  *     are configured for the union(pending, inuse)
  *     Current pending is copied to configured.
  *     It is assumed that mdp5_smp_request and mdp5_smp_configure not run
@@ -311,25 +311,25 @@ static void update_smp_state(struct mdp5_smp *smp,
                 int idx = blk / 3;
                 int fld = blk % 3;
 
-                val = mdp5_read(mdp5_kms, REG_MDP5_MDP_SMP_ALLOC_W_REG(0, idx));
+                val = mdp5_read(mdp5_kms, REG_MDP5_SMP_ALLOC_W_REG(idx));
 
                 switch (fld) {
                 case 0:
-                        val &= ~MDP5_MDP_SMP_ALLOC_W_REG_CLIENT0__MASK;
-                        val |= MDP5_MDP_SMP_ALLOC_W_REG_CLIENT0(cid);
+                        val &= ~MDP5_SMP_ALLOC_W_REG_CLIENT0__MASK;
+                        val |= MDP5_SMP_ALLOC_W_REG_CLIENT0(cid);
                         break;
                 case 1:
-                        val &= ~MDP5_MDP_SMP_ALLOC_W_REG_CLIENT1__MASK;
-                        val |= MDP5_MDP_SMP_ALLOC_W_REG_CLIENT1(cid);
+                        val &= ~MDP5_SMP_ALLOC_W_REG_CLIENT1__MASK;
+                        val |= MDP5_SMP_ALLOC_W_REG_CLIENT1(cid);
                         break;
                 case 2:
-                        val &= ~MDP5_MDP_SMP_ALLOC_W_REG_CLIENT2__MASK;
-                        val |= MDP5_MDP_SMP_ALLOC_W_REG_CLIENT2(cid);
+                        val &= ~MDP5_SMP_ALLOC_W_REG_CLIENT2__MASK;
+                        val |= MDP5_SMP_ALLOC_W_REG_CLIENT2(cid);
                         break;
                 }
 
-                mdp5_write(mdp5_kms, REG_MDP5_MDP_SMP_ALLOC_W_REG(0, idx), val);
-                mdp5_write(mdp5_kms, REG_MDP5_MDP_SMP_ALLOC_R_REG(0, idx), val);
+                mdp5_write(mdp5_kms, REG_MDP5_SMP_ALLOC_W_REG(idx), val);
+                mdp5_write(mdp5_kms, REG_MDP5_SMP_ALLOC_R_REG(idx), val);
         }
 }
 
diff --git a/drivers/gpu/drm/msm/msm_atomic.c b/drivers/gpu/drm/msm/msm_atomic.c
index e3892c263f27..4a8a6f1f1151 100644
--- a/drivers/gpu/drm/msm/msm_atomic.c
+++ b/drivers/gpu/drm/msm/msm_atomic.c
@@ -84,17 +84,12 @@ static void msm_atomic_wait_for_commit_done(struct drm_device *dev,
                 struct drm_atomic_state *old_state)
 {
         struct drm_crtc *crtc;
+        struct drm_crtc_state *crtc_state;
         struct msm_drm_private *priv = old_state->dev->dev_private;
         struct msm_kms *kms = priv->kms;
-        int ncrtcs = old_state->dev->mode_config.num_crtc;
         int i;
 
-        for (i = 0; i < ncrtcs; i++) {
-                crtc = old_state->crtcs[i];
-
-                if (!crtc)
-                        continue;
-
+        for_each_crtc_in_state(old_state, crtc, crtc_state, i) {
                 if (!crtc->state->enable)
                         continue;
 
@@ -192,9 +187,11 @@ int msm_atomic_commit(struct drm_device *dev,
                 struct drm_atomic_state *state, bool nonblock)
 {
         struct msm_drm_private *priv = dev->dev_private;
-        int nplanes = dev->mode_config.num_total_plane;
-        int ncrtcs = dev->mode_config.num_crtc;
         struct msm_commit *c;
+        struct drm_crtc *crtc;
+        struct drm_crtc_state *crtc_state;
+        struct drm_plane *plane;
+        struct drm_plane_state *plane_state;
         int i, ret;
 
         ret = drm_atomic_helper_prepare_planes(dev, state);
@@ -210,28 +207,18 @@ int msm_atomic_commit(struct drm_device *dev,
         /*
          * Figure out what crtcs we have:
          */
-        for (i = 0; i < ncrtcs; i++) {
-                struct drm_crtc *crtc = state->crtcs[i];
-                if (!crtc)
-                        continue;
-                c->crtc_mask |= (1 << drm_crtc_index(crtc));
-        }
+        for_each_crtc_in_state(state, crtc, crtc_state, i)
+                c->crtc_mask |= drm_crtc_mask(crtc);
 
         /*
          * Figure out what fence to wait for:
          */
-        for (i = 0; i < nplanes; i++) {
-                struct drm_plane *plane = state->planes[i];
-                struct drm_plane_state *new_state = state->plane_states[i];
-
-                if (!plane)
-                        continue;
-
-                if ((plane->state->fb != new_state->fb) && new_state->fb) {
-                        struct drm_gem_object *obj = msm_framebuffer_bo(new_state->fb, 0);
+        for_each_plane_in_state(state, plane, plane_state, i) {
+                if ((plane->state->fb != plane_state->fb) && plane_state->fb) {
+                        struct drm_gem_object *obj = msm_framebuffer_bo(plane_state->fb, 0);
                         struct msm_gem_object *msm_obj = to_msm_bo(obj);
 
-                        new_state->fence = reservation_object_get_excl_rcu(msm_obj->resv);
+                        plane_state->fence = reservation_object_get_excl_rcu(msm_obj->resv);
                 }
         }
 
@@ -251,7 +238,7 @@ int msm_atomic_commit(struct drm_device *dev,
          * the software side now.
          */
 
-        drm_atomic_helper_swap_state(dev, state);
+        drm_atomic_helper_swap_state(state, true);
 
         /*
          * Everything below can be run asynchronously without the need to grab
diff --git a/drivers/gpu/drm/msm/msm_drv.c b/drivers/gpu/drm/msm/msm_drv.c
index 9c654092ef78..26f859ec24b3 100644
--- a/drivers/gpu/drm/msm/msm_drv.c
+++ b/drivers/gpu/drm/msm/msm_drv.c
@@ -21,6 +21,16 @@
 #include "msm_gpu.h"
 #include "msm_kms.h"
 
+
+/*
+ * MSM driver version:
+ * - 1.0.0 - initial interface
+ * - 1.1.0 - adds madvise, and support for submits with > 4 cmd buffers
+ */
+#define MSM_VERSION_MAJOR       1
+#define MSM_VERSION_MINOR       1
+#define MSM_VERSION_PATCHLEVEL  0
+
 static void msm_fb_output_poll_changed(struct drm_device *dev)
 {
         struct msm_drm_private *priv = dev->dev_private;
@@ -195,9 +205,9 @@ static int msm_drm_uninit(struct device *dev)
                 kfree(vbl_ev);
         }
 
-        drm_kms_helper_poll_fini(ddev);
+        msm_gem_shrinker_cleanup(ddev);
 
-        drm_connector_unregister_all(ddev);
+        drm_kms_helper_poll_fini(ddev);
 
         drm_dev_unregister(ddev);
 
@@ -217,10 +227,8 @@ static int msm_drm_uninit(struct device *dev)
         flush_workqueue(priv->atomic_wq);
         destroy_workqueue(priv->atomic_wq);
 
-        if (kms) {
-                pm_runtime_disable(dev);
+        if (kms)
                 kms->funcs->destroy(kms);
-        }
 
         if (gpu) {
                 mutex_lock(&ddev->struct_mutex);
@@ -239,6 +247,8 @@ static int msm_drm_uninit(struct device *dev)
 
         component_unbind_all(dev, ddev);
 
+        msm_mdss_destroy(ddev);
+
         ddev->dev_private = NULL;
         drm_dev_unref(ddev);
 
@@ -284,6 +294,7 @@ static int msm_init_vram(struct drm_device *dev)
         if (node) {
                 struct resource r;
                 ret = of_address_to_resource(node, 0, &r);
+                of_node_put(node);
                 if (ret)
                         return ret;
                 size = r.end - r.start;
@@ -352,6 +363,14 @@ static int msm_drm_init(struct device *dev, struct drm_driver *drv)
         }
 
         ddev->dev_private = priv;
+        priv->dev = ddev;
+
+        ret = msm_mdss_init(ddev);
+        if (ret) {
+                kfree(priv);
+                drm_dev_unref(ddev);
+                return ret;
+        }
 
         priv->wq = alloc_ordered_workqueue("msm", 0);
         priv->atomic_wq = alloc_ordered_workqueue("msm:atomic", 0);
@@ -367,6 +386,7 @@ static int msm_drm_init(struct device *dev, struct drm_driver *drv)
         /* Bind all our sub-components: */
         ret = component_bind_all(dev, ddev);
         if (ret) {
+                msm_mdss_destroy(ddev);
                 kfree(priv);
                 drm_dev_unref(ddev);
                 return ret;
@@ -376,9 +396,12 @@ static int msm_drm_init(struct device *dev, struct drm_driver *drv)
         if (ret)
                 goto fail;
 
+        msm_gem_shrinker_init(ddev);
+
         switch (get_mdp_ver(pdev)) {
         case 4:
                 kms = mdp4_kms_init(ddev);
+                priv->kms = kms;
                 break;
         case 5:
                 kms = mdp5_kms_init(ddev);
@@ -400,10 +423,7 @@ static int msm_drm_init(struct device *dev, struct drm_driver *drv)
                 goto fail;
         }
 
-        priv->kms = kms;
-
         if (kms) {
-                pm_runtime_enable(dev);
                 ret = kms->funcs->hw_init(kms);
                 if (ret) {
                         dev_err(dev, "kms hw init failed: %d\n", ret);
@@ -419,24 +439,20 @@ static int msm_drm_init(struct device *dev, struct drm_driver *drv)
                 goto fail;
         }
 
-        pm_runtime_get_sync(dev);
-        ret = drm_irq_install(ddev, platform_get_irq(pdev, 0));
-        pm_runtime_put_sync(dev);
-        if (ret < 0) {
-                dev_err(dev, "failed to install IRQ handler\n");
-                goto fail;
+        if (kms) {
+                pm_runtime_get_sync(dev);
+                ret = drm_irq_install(ddev, kms->irq);
+                pm_runtime_put_sync(dev);
+                if (ret < 0) {
+                        dev_err(dev, "failed to install IRQ handler\n");
+                        goto fail;
+                }
         }
 
         ret = drm_dev_register(ddev, 0);
         if (ret)
                 goto fail;
 
-        ret = drm_connector_register_all(ddev);
-        if (ret) {
-                dev_err(dev, "failed to register connectors\n");
-                goto fail;
-        }
-
         drm_mode_config_reset(ddev);
 
 #ifdef CONFIG_DRM_FBDEV_EMULATION
@@ -690,6 +706,44 @@ static int msm_ioctl_wait_fence(struct drm_device *dev, void *data,
         return msm_wait_fence(priv->gpu->fctx, args->fence, &timeout, true);
 }
 
+static int msm_ioctl_gem_madvise(struct drm_device *dev, void *data,
+                struct drm_file *file)
+{
+        struct drm_msm_gem_madvise *args = data;
+        struct drm_gem_object *obj;
+        int ret;
+
+        switch (args->madv) {
+        case MSM_MADV_DONTNEED:
+        case MSM_MADV_WILLNEED:
+                break;
+        default:
+                return -EINVAL;
+        }
+
+        ret = mutex_lock_interruptible(&dev->struct_mutex);
+        if (ret)
+                return ret;
+
+        obj = drm_gem_object_lookup(file, args->handle);
+        if (!obj) {
+                ret = -ENOENT;
+                goto unlock;
+        }
+
+        ret = msm_gem_madvise(obj, args->madv);
+        if (ret >= 0) {
+                args->retained = ret;
+                ret = 0;
+        }
+
+        drm_gem_object_unreference(obj);
+
+unlock:
+        mutex_unlock(&dev->struct_mutex);
+        return ret;
+}
+
 static const struct drm_ioctl_desc msm_ioctls[] = {
         DRM_IOCTL_DEF_DRV(MSM_GET_PARAM,    msm_ioctl_get_param,    DRM_AUTH|DRM_RENDER_ALLOW),
         DRM_IOCTL_DEF_DRV(MSM_GEM_NEW,      msm_ioctl_gem_new,      DRM_AUTH|DRM_RENDER_ALLOW),
@@ -698,6 +752,7 @@ static const struct drm_ioctl_desc msm_ioctls[] = {
         DRM_IOCTL_DEF_DRV(MSM_GEM_CPU_FINI, msm_ioctl_gem_cpu_fini, DRM_AUTH|DRM_RENDER_ALLOW),
         DRM_IOCTL_DEF_DRV(MSM_GEM_SUBMIT,   msm_ioctl_gem_submit,   DRM_AUTH|DRM_RENDER_ALLOW),
         DRM_IOCTL_DEF_DRV(MSM_WAIT_FENCE,   msm_ioctl_wait_fence,   DRM_AUTH|DRM_RENDER_ALLOW),
+        DRM_IOCTL_DEF_DRV(MSM_GEM_MADVISE,  msm_ioctl_gem_madvise,  DRM_AUTH|DRM_RENDER_ALLOW),
 };
 
 static const struct vm_operations_struct vm_ops = {
@@ -730,7 +785,6 @@ static struct drm_driver msm_driver = {
         .open               = msm_open,
         .preclose           = msm_preclose,
         .lastclose          = msm_lastclose,
-        .set_busid          = drm_platform_set_busid,
         .irq_handler        = msm_irq,
         .irq_preinstall     = msm_irq_preinstall,
         .irq_postinstall    = msm_irq_postinstall,
@@ -764,8 +818,9 @@ static struct drm_driver msm_driver = {
         .name               = "msm",
         .desc               = "MSM Snapdragon DRM",
         .date               = "20130625",
-        .major              = 1,
-        .minor              = 0,
+        .major              = MSM_VERSION_MAJOR,
+        .minor              = MSM_VERSION_MINOR,
+        .patchlevel         = MSM_VERSION_PATCHLEVEL,
 };
 
 #ifdef CONFIG_PM_SLEEP
@@ -805,22 +860,146 @@ static int compare_of(struct device *dev, void *data)
         return dev->of_node == data;
 }
 
-static int add_components(struct device *dev, struct component_match **matchptr,
-                const char *name)
+/*
+ * Identify what components need to be added by parsing what remote-endpoints
+ * our MDP output ports are connected to. In the case of LVDS on MDP4, there
+ * is no external component that we need to add since LVDS is within MDP4
+ * itself.
+ */
+static int add_components_mdp(struct device *mdp_dev,
+                              struct component_match **matchptr)
 {
-        struct device_node *np = dev->of_node;
-        unsigned i;
+        struct device_node *np = mdp_dev->of_node;
+        struct device_node *ep_node;
+        struct device *master_dev;
+
+        /*
+         * on MDP4 based platforms, the MDP platform device is the component
+         * master that adds other display interface components to itself.
+         *
+         * on MDP5 based platforms, the MDSS platform device is the component
+         * master that adds MDP5 and other display interface components to
+         * itself.
+         */
+        if (of_device_is_compatible(np, "qcom,mdp4"))
+                master_dev = mdp_dev;
+        else
+                master_dev = mdp_dev->parent;
 
-        for (i = 0; ; i++) {
-                struct device_node *node;
+        for_each_endpoint_of_node(np, ep_node) {
+                struct device_node *intf;
+                struct of_endpoint ep;
+                int ret;
 
-                node = of_parse_phandle(np, name, i);
-                if (!node)
-                        break;
+                ret = of_graph_parse_endpoint(ep_node, &ep);
+                if (ret) {
+                        dev_err(mdp_dev, "unable to parse port endpoint\n");
+                        of_node_put(ep_node);
+                        return ret;
+                }
+
+                /*
+                 * The LCDC/LVDS port on MDP4 is a speacial case where the
+                 * remote-endpoint isn't a component that we need to add
+                 */
+                if (of_device_is_compatible(np, "qcom,mdp4") &&
+                    ep.port == 0) {
+                        of_node_put(ep_node);
+                        continue;
+                }
+
+                /*
+                 * It's okay if some of the ports don't have a remote endpoint
+                 * specified. It just means that the port isn't connected to
+                 * any external interface.
+                 */
+                intf = of_graph_get_remote_port_parent(ep_node);
+                if (!intf) {
+                        of_node_put(ep_node);
+                        continue;
+                }
+
+                component_match_add(master_dev, matchptr, compare_of, intf);
+
+                of_node_put(intf);
+                of_node_put(ep_node);
+        }
+
+        return 0;
+}
+
+static int compare_name_mdp(struct device *dev, void *data)
+{
+        return (strstr(dev_name(dev), "mdp") != NULL);
+}
+
+static int add_display_components(struct device *dev,
+                                  struct component_match **matchptr)
+{
+        struct device *mdp_dev;
+        int ret;
+
+        /*
+         * MDP5 based devices don't have a flat hierarchy. There is a top level
+         * parent: MDSS, and children: MDP5, DSI, HDMI, eDP etc. Populate the
+         * children devices, find the MDP5 node, and then add the interfaces
+         * to our components list.
+         */
+        if (of_device_is_compatible(dev->of_node, "qcom,mdss")) {
+                ret = of_platform_populate(dev->of_node, NULL, NULL, dev);
+                if (ret) {
+                        dev_err(dev, "failed to populate children devices\n");
+                        return ret;
+                }
+
+                mdp_dev = device_find_child(dev, NULL, compare_name_mdp);
+                if (!mdp_dev) {
+                        dev_err(dev, "failed to find MDSS MDP node\n");
+                        of_platform_depopulate(dev);
+                        return -ENODEV;
+                }
+
+                put_device(mdp_dev);
 
-                component_match_add(dev, matchptr, compare_of, node);
+                /* add the MDP component itself */
+                component_match_add(dev, matchptr, compare_of,
+                                    mdp_dev->of_node);
+        } else {
+                /* MDP4 */
+                mdp_dev = dev;
         }
 
+        ret = add_components_mdp(mdp_dev, matchptr);
+        if (ret)
+                of_platform_depopulate(dev);
+
+        return ret;
+}
+
+/*
+ * We don't know what's the best binding to link the gpu with the drm device.
+ * Fow now, we just hunt for all the possible gpus that we support, and add them
+ * as components.
+ */
+static const struct of_device_id msm_gpu_match[] = {
+        { .compatible = "qcom,adreno-3xx" },
+        { .compatible = "qcom,kgsl-3d0" },
+        { },
+};
+
+static int add_gpu_components(struct device *dev,
+                              struct component_match **matchptr)
+{
+        struct device_node *np;
+
+        np = of_find_matching_node(NULL, msm_gpu_match);
+        if (!np)
+                return 0;
+
+        component_match_add(dev, matchptr, compare_of, np);
+
+        of_node_put(np);
+
         return 0;
 }
 
@@ -846,9 +1025,15 @@ static const struct component_master_ops msm_drm_ops = {
 static int msm_pdev_probe(struct platform_device *pdev)
 {
         struct component_match *match = NULL;
+        int ret;
 
-        add_components(&pdev->dev, &match, "connectors");
-        add_components(&pdev->dev, &match, "gpus");
+        ret = add_display_components(&pdev->dev, &match);
+        if (ret)
+                return ret;
+
+        ret = add_gpu_components(&pdev->dev, &match);
+        if (ret)
+                return ret;
 
         pdev->dev.coherent_dma_mask = DMA_BIT_MASK(32);
         return component_master_add_with_match(&pdev->dev, &msm_drm_ops, match);
@@ -857,20 +1042,14 @@ static int msm_pdev_probe(struct platform_device *pdev)
 static int msm_pdev_remove(struct platform_device *pdev)
 {
         component_master_del(&pdev->dev, &msm_drm_ops);
+        of_platform_depopulate(&pdev->dev);
 
         return 0;
 }
 
-static const struct platform_device_id msm_id[] = {
-        { "mdp", 0 },
-        { }
-};
-
 static const struct of_device_id dt_match[] = {
-        { .compatible = "qcom,mdp4", .data = (void *) 4 },      /* mdp4 */
-        { .compatible = "qcom,mdp5", .data = (void *) 5 },      /* mdp5 */
-        /* to support downstream DT files */
-        { .compatible = "qcom,mdss_mdp", .data = (void *) 5 },  /* mdp5 */
+        { .compatible = "qcom,mdp4", .data = (void *)4 },       /* MDP4 */
+        { .compatible = "qcom,mdss", .data = (void *)5 },       /* MDP5 MDSS */
         {}
 };
 MODULE_DEVICE_TABLE(of, dt_match);
@@ -883,12 +1062,12 @@ static struct platform_driver msm_platform_driver = {
                 .of_match_table = dt_match,
                 .pm     = &msm_pm_ops,
         },
-        .id_table   = msm_id,
 };
 
 static int __init msm_drm_register(void)
 {
         DBG("init");
+        msm_mdp_register();
         msm_dsi_register();
         msm_edp_register();
         msm_hdmi_register();
@@ -904,6 +1083,7 @@ static void __exit msm_drm_unregister(void)
         adreno_unregister();
         msm_edp_unregister();
         msm_dsi_unregister();
+        msm_mdp_unregister();
 }
 
 module_init(msm_drm_register);
diff --git a/drivers/gpu/drm/msm/msm_drv.h b/drivers/gpu/drm/msm/msm_drv.h
index 5b2963f32291..b4bc7f1ef717 100644
--- a/drivers/gpu/drm/msm/msm_drv.h
+++ b/drivers/gpu/drm/msm/msm_drv.h
@@ -46,6 +46,7 @@
 struct msm_kms;
 struct msm_gpu;
 struct msm_mmu;
+struct msm_mdss;
 struct msm_rd_state;
 struct msm_perf_state;
 struct msm_gem_submit;
@@ -77,11 +78,16 @@ struct msm_vblank_ctrl {
 
 struct msm_drm_private {
 
+        struct drm_device *dev;
+
         struct msm_kms *kms;
 
         /* subordinate devices, if present: */
         struct platform_device *gpu_pdev;
 
+        /* top level MDSS wrapper device (for MDP5 only) */
+        struct msm_mdss *mdss;
+
         /* possibly this should be in the kms component, but it is
          * shared by both mdp4 and mdp5..
          */
@@ -147,6 +153,9 @@ struct msm_drm_private {
                 struct drm_mm mm;
         } vram;
 
+        struct notifier_block vmap_notifier;
+        struct shrinker shrinker;
+
         struct msm_vblank_ctrl vblank_ctrl;
 };
 
@@ -165,6 +174,9 @@ void msm_gem_submit_free(struct msm_gem_submit *submit);
 int msm_ioctl_gem_submit(struct drm_device *dev, void *data,
                 struct drm_file *file);
 
+void msm_gem_shrinker_init(struct drm_device *dev);
+void msm_gem_shrinker_cleanup(struct drm_device *dev);
+
 int msm_gem_mmap_obj(struct drm_gem_object *obj,
                         struct vm_area_struct *vma);
 int msm_gem_mmap(struct file *filp, struct vm_area_struct *vma);
@@ -189,8 +201,13 @@ struct drm_gem_object *msm_gem_prime_import_sg_table(struct drm_device *dev,
                 struct dma_buf_attachment *attach, struct sg_table *sg);
 int msm_gem_prime_pin(struct drm_gem_object *obj);
 void msm_gem_prime_unpin(struct drm_gem_object *obj);
-void *msm_gem_vaddr_locked(struct drm_gem_object *obj);
-void *msm_gem_vaddr(struct drm_gem_object *obj);
+void *msm_gem_get_vaddr_locked(struct drm_gem_object *obj);
+void *msm_gem_get_vaddr(struct drm_gem_object *obj);
+void msm_gem_put_vaddr_locked(struct drm_gem_object *obj);
+void msm_gem_put_vaddr(struct drm_gem_object *obj);
+int msm_gem_madvise(struct drm_gem_object *obj, unsigned madv);
+void msm_gem_purge(struct drm_gem_object *obj);
+void msm_gem_vunmap(struct drm_gem_object *obj);
 int msm_gem_sync_object(struct drm_gem_object *obj,
                 struct msm_fence_context *fctx, bool exclusive);
 void msm_gem_move_to_active(struct drm_gem_object *obj,
@@ -257,6 +274,9 @@ static inline int msm_dsi_modeset_init(struct msm_dsi *msm_dsi,
 }
 #endif
 
+void __init msm_mdp_register(void);
+void __exit msm_mdp_unregister(void);
+
 #ifdef CONFIG_DEBUG_FS
 void msm_gem_describe(struct drm_gem_object *obj, struct seq_file *m);
 void msm_gem_describe_objects(struct list_head *list, struct seq_file *m);
diff --git a/drivers/gpu/drm/msm/msm_fb.c b/drivers/gpu/drm/msm/msm_fb.c
index 461dc8b873f0..95cf8fe72ee5 100644
--- a/drivers/gpu/drm/msm/msm_fb.c
+++ b/drivers/gpu/drm/msm/msm_fb.c
@@ -49,24 +49,16 @@ static void msm_framebuffer_destroy(struct drm_framebuffer *fb)
 
         for (i = 0; i < n; i++) {
                 struct drm_gem_object *bo = msm_fb->planes[i];
-                if (bo)
-                        drm_gem_object_unreference_unlocked(bo);
+
+                drm_gem_object_unreference_unlocked(bo);
         }
 
         kfree(msm_fb);
 }
 
-static int msm_framebuffer_dirty(struct drm_framebuffer *fb,
-                struct drm_file *file_priv, unsigned flags, unsigned color,
-                struct drm_clip_rect *clips, unsigned num_clips)
-{
-        return 0;
-}
-
 static const struct drm_framebuffer_funcs msm_framebuffer_funcs = {
         .create_handle = msm_framebuffer_create_handle,
         .destroy = msm_framebuffer_destroy,
-        .dirty = msm_framebuffer_dirty,
 };
 
 #ifdef CONFIG_DEBUG_FS
diff --git a/drivers/gpu/drm/msm/msm_fbdev.c b/drivers/gpu/drm/msm/msm_fbdev.c
index c6cf837c5193..ffd4a338ca12 100644
--- a/drivers/gpu/drm/msm/msm_fbdev.c
+++ b/drivers/gpu/drm/msm/msm_fbdev.c
@@ -158,7 +158,7 @@ static int msm_fbdev_create(struct drm_fb_helper *helper,
 
         dev->mode_config.fb_base = paddr;
 
-        fbi->screen_base = msm_gem_vaddr_locked(fbdev->bo);
+        fbi->screen_base = msm_gem_get_vaddr_locked(fbdev->bo);
         if (IS_ERR(fbi->screen_base)) {
                 ret = PTR_ERR(fbi->screen_base);
                 goto fail_unlock;
@@ -188,21 +188,7 @@ fail:
         return ret;
 }
 
-static void msm_crtc_fb_gamma_set(struct drm_crtc *crtc,
-                u16 red, u16 green, u16 blue, int regno)
-{
-        DBG("fbdev: set gamma");
-}
-
-static void msm_crtc_fb_gamma_get(struct drm_crtc *crtc,
-                u16 *red, u16 *green, u16 *blue, int regno)
-{
-        DBG("fbdev: get gamma");
-}
-
 static const struct drm_fb_helper_funcs msm_fb_helper_funcs = {
-        .gamma_set = msm_crtc_fb_gamma_set,
-        .gamma_get = msm_crtc_fb_gamma_get,
         .fb_probe = msm_fbdev_create,
 };
 
@@ -265,6 +251,7 @@ void msm_fbdev_free(struct drm_device *dev)
 
         /* this will free the backing object */
         if (fbdev->fb) {
+                msm_gem_put_vaddr(fbdev->bo);
                 drm_framebuffer_unregister_private(fbdev->fb);
                 drm_framebuffer_remove(fbdev->fb);
         }
diff --git a/drivers/gpu/drm/msm/msm_gem.c b/drivers/gpu/drm/msm/msm_gem.c
index 69836f5685b1..6cd4af443139 100644
--- a/drivers/gpu/drm/msm/msm_gem.c
+++ b/drivers/gpu/drm/msm/msm_gem.c
@@ -276,6 +276,26 @@ uint64_t msm_gem_mmap_offset(struct drm_gem_object *obj)
         return offset;
 }
 
+static void
+put_iova(struct drm_gem_object *obj)
+{
+        struct drm_device *dev = obj->dev;
+        struct msm_drm_private *priv = obj->dev->dev_private;
+        struct msm_gem_object *msm_obj = to_msm_bo(obj);
+        int id;
+
+        WARN_ON(!mutex_is_locked(&dev->struct_mutex));
+
+        for (id = 0; id < ARRAY_SIZE(msm_obj->domain); id++) {
+                struct msm_mmu *mmu = priv->mmus[id];
+                if (mmu && msm_obj->domain[id].iova) {
+                        uint32_t offset = msm_obj->domain[id].iova;
+                        mmu->funcs->unmap(mmu, offset, msm_obj->sgt, obj->size);
+                        msm_obj->domain[id].iova = 0;
+                }
+        }
+}
+
 /* should be called under struct_mutex.. although it can be called
  * from atomic context without struct_mutex to acquire an extra
  * iova ref if you know one is already held.
@@ -388,7 +408,7 @@ fail:
         return ret;
 }
 
-void *msm_gem_vaddr_locked(struct drm_gem_object *obj)
+void *msm_gem_get_vaddr_locked(struct drm_gem_object *obj)
 {
         struct msm_gem_object *msm_obj = to_msm_bo(obj);
         WARN_ON(!mutex_is_locked(&obj->dev->struct_mutex));
@@ -401,18 +421,91 @@ void *msm_gem_vaddr_locked(struct drm_gem_object *obj)
                 if (msm_obj->vaddr == NULL)
                         return ERR_PTR(-ENOMEM);
         }
+        msm_obj->vmap_count++;
         return msm_obj->vaddr;
 }
 
-void *msm_gem_vaddr(struct drm_gem_object *obj)
+void *msm_gem_get_vaddr(struct drm_gem_object *obj)
 {
         void *ret;
         mutex_lock(&obj->dev->struct_mutex);
-        ret = msm_gem_vaddr_locked(obj);
+        ret = msm_gem_get_vaddr_locked(obj);
         mutex_unlock(&obj->dev->struct_mutex);
         return ret;
 }
 
+void msm_gem_put_vaddr_locked(struct drm_gem_object *obj)
+{
+        struct msm_gem_object *msm_obj = to_msm_bo(obj);
+        WARN_ON(!mutex_is_locked(&obj->dev->struct_mutex));
+        WARN_ON(msm_obj->vmap_count < 1);
+        msm_obj->vmap_count--;
+}
+
+void msm_gem_put_vaddr(struct drm_gem_object *obj)
+{
+        mutex_lock(&obj->dev->struct_mutex);
+        msm_gem_put_vaddr_locked(obj);
+        mutex_unlock(&obj->dev->struct_mutex);
+}
+
+/* Update madvise status, returns true if not purged, else
+ * false or -errno.
+ */
+int msm_gem_madvise(struct drm_gem_object *obj, unsigned madv)
+{
+        struct msm_gem_object *msm_obj = to_msm_bo(obj);
+
+        WARN_ON(!mutex_is_locked(&obj->dev->struct_mutex));
+
+        if (msm_obj->madv != __MSM_MADV_PURGED)
+                msm_obj->madv = madv;
+
+        return (msm_obj->madv != __MSM_MADV_PURGED);
+}
+
+void msm_gem_purge(struct drm_gem_object *obj)
+{
+        struct drm_device *dev = obj->dev;
+        struct msm_gem_object *msm_obj = to_msm_bo(obj);
+
+        WARN_ON(!mutex_is_locked(&dev->struct_mutex));
+        WARN_ON(!is_purgeable(msm_obj));
+        WARN_ON(obj->import_attach);
+
+        put_iova(obj);
+
+        msm_gem_vunmap(obj);
+
+        put_pages(obj);
+
+        msm_obj->madv = __MSM_MADV_PURGED;
+
+        drm_vma_node_unmap(&obj->vma_node, dev->anon_inode->i_mapping);
+        drm_gem_free_mmap_offset(obj);
+
+        /* Our goal here is to return as much of the memory as
+         * is possible back to the system as we are called from OOM.
+         * To do this we must instruct the shmfs to drop all of its
+         * backing pages, *now*.
+         */
+        shmem_truncate_range(file_inode(obj->filp), 0, (loff_t)-1);
+
+        invalidate_mapping_pages(file_inode(obj->filp)->i_mapping,
+                        0, (loff_t)-1);
+}
+
+void msm_gem_vunmap(struct drm_gem_object *obj)
+{
+        struct msm_gem_object *msm_obj = to_msm_bo(obj);
+
+        if (!msm_obj->vaddr || WARN_ON(!is_vunmapable(msm_obj)))
+                return;
+
+        vunmap(msm_obj->vaddr);
+        msm_obj->vaddr = NULL;
+}
+
 /* must be called before _move_to_active().. */
 int msm_gem_sync_object(struct drm_gem_object *obj,
                 struct msm_fence_context *fctx, bool exclusive)
@@ -464,6 +557,7 @@ void msm_gem_move_to_active(struct drm_gem_object *obj,
                 struct msm_gpu *gpu, bool exclusive, struct fence *fence)
 {
         struct msm_gem_object *msm_obj = to_msm_bo(obj);
+        WARN_ON(msm_obj->madv != MSM_MADV_WILLNEED);
         msm_obj->gpu = gpu;
         if (exclusive)
                 reservation_object_add_excl_fence(msm_obj->resv, fence);
@@ -532,13 +626,27 @@ void msm_gem_describe(struct drm_gem_object *obj, struct seq_file *m)
         struct reservation_object_list *fobj;
         struct fence *fence;
         uint64_t off = drm_vma_node_start(&obj->vma_node);
+        const char *madv;
 
         WARN_ON(!mutex_is_locked(&obj->dev->struct_mutex));
 
-        seq_printf(m, "%08x: %c %2d (%2d) %08llx %p %zu\n",
+        switch (msm_obj->madv) {
+        case __MSM_MADV_PURGED:
+                madv = " purged";
+                break;
+        case MSM_MADV_DONTNEED:
+                madv = " purgeable";
+                break;
+        case MSM_MADV_WILLNEED:
+        default:
+                madv = "";
+                break;
+        }
+
+        seq_printf(m, "%08x: %c %2d (%2d) %08llx %p %zu%s\n",
                         msm_obj->flags, is_active(msm_obj) ? 'A' : 'I',
                         obj->name, obj->refcount.refcount.counter,
-                        off, msm_obj->vaddr, obj->size);
+                        off, msm_obj->vaddr, obj->size, madv);
 
         rcu_read_lock();
         fobj = rcu_dereference(robj->fence);
@@ -578,9 +686,7 @@ void msm_gem_describe_objects(struct list_head *list, struct seq_file *m)
 void msm_gem_free_object(struct drm_gem_object *obj)
 {
         struct drm_device *dev = obj->dev;
-        struct msm_drm_private *priv = obj->dev->dev_private;
         struct msm_gem_object *msm_obj = to_msm_bo(obj);
-        int id;
 
         WARN_ON(!mutex_is_locked(&dev->struct_mutex));
 
@@ -589,13 +695,7 @@ void msm_gem_free_object(struct drm_gem_object *obj)
 
         list_del(&msm_obj->mm_list);
 
-        for (id = 0; id < ARRAY_SIZE(msm_obj->domain); id++) {
-                struct msm_mmu *mmu = priv->mmus[id];
-                if (mmu && msm_obj->domain[id].iova) {
-                        uint32_t offset = msm_obj->domain[id].iova;
-                        mmu->funcs->unmap(mmu, offset, msm_obj->sgt, obj->size);
-                }
-        }
+        put_iova(obj);
 
         if (obj->import_attach) {
                 if (msm_obj->vaddr)
@@ -609,7 +709,7 @@ void msm_gem_free_object(struct drm_gem_object *obj)
 
                 drm_prime_gem_destroy(obj, msm_obj->sgt);
         } else {
-                vunmap(msm_obj->vaddr);
+                msm_gem_vunmap(obj);
                 put_pages(obj);
         }
 
@@ -688,6 +788,7 @@ static int msm_gem_new_impl(struct drm_device *dev,
                 msm_obj->vram_node = (void *)&msm_obj[1];
 
         msm_obj->flags = flags;
+        msm_obj->madv = MSM_MADV_WILLNEED;
 
         if (resv) {
                 msm_obj->resv = resv;
@@ -729,9 +830,7 @@ struct drm_gem_object *msm_gem_new(struct drm_device *dev,
         return obj;
 
 fail:
-        if (obj)
-                drm_gem_object_unreference(obj);
-
+        drm_gem_object_unreference(obj);
         return ERR_PTR(ret);
 }
 
@@ -774,8 +873,6 @@ struct drm_gem_object *msm_gem_import(struct drm_device *dev,
         return obj;
 
 fail:
-        if (obj)
-                drm_gem_object_unreference_unlocked(obj);
-
+        drm_gem_object_unreference_unlocked(obj);
         return ERR_PTR(ret);
 }
diff --git a/drivers/gpu/drm/msm/msm_gem.h b/drivers/gpu/drm/msm/msm_gem.h
index 9facd4b6ffd9..b2f13cfe945e 100644
--- a/drivers/gpu/drm/msm/msm_gem.h
+++ b/drivers/gpu/drm/msm/msm_gem.h
@@ -29,6 +29,16 @@ struct msm_gem_object {
 
         uint32_t flags;
 
+        /**
+         * Advice: are the backing pages purgeable?
+         */
+        uint8_t madv;
+
+        /**
+         * count of active vmap'ing
+         */
+        uint8_t vmap_count;
+
         /* And object is either:
          *  inactive - on priv->inactive_list
          *  active   - on one one of the gpu's active_list..  well, at
@@ -72,7 +82,16 @@ static inline bool is_active(struct msm_gem_object *msm_obj)
         return msm_obj->gpu != NULL;
 }
 
-#define MAX_CMDS 4
+static inline bool is_purgeable(struct msm_gem_object *msm_obj)
+{
+        return (msm_obj->madv == MSM_MADV_DONTNEED) && msm_obj->sgt &&
+                        !msm_obj->base.dma_buf && !msm_obj->base.import_attach;
+}
+
+static inline bool is_vunmapable(struct msm_gem_object *msm_obj)
+{
+        return (msm_obj->vmap_count == 0) && msm_obj->vaddr;
+}
 
 /* Created per submit-ioctl, to track bo's and cmdstream bufs, etc,
  * associated with the cmdstream submission for synchronization (and
@@ -95,7 +114,7 @@ struct msm_gem_submit {
                 uint32_t size;  /* in dwords */
                 uint32_t iova;
                 uint32_t idx;   /* cmdstream buffer idx in bos[] */
-        } cmd[MAX_CMDS];
+        } *cmd;  /* array of size nr_cmds */
         struct {
                 uint32_t flags;
                 struct msm_gem_object *obj;
diff --git a/drivers/gpu/drm/msm/msm_gem_prime.c b/drivers/gpu/drm/msm/msm_gem_prime.c
index 6b90890faffe..60bb290700ce 100644
--- a/drivers/gpu/drm/msm/msm_gem_prime.c
+++ b/drivers/gpu/drm/msm/msm_gem_prime.c
@@ -33,12 +33,12 @@ struct sg_table *msm_gem_prime_get_sg_table(struct drm_gem_object *obj)
 
 void *msm_gem_prime_vmap(struct drm_gem_object *obj)
 {
-        return msm_gem_vaddr(obj);
+        return msm_gem_get_vaddr(obj);
 }
 
 void msm_gem_prime_vunmap(struct drm_gem_object *obj, void *vaddr)
 {
-        /* TODO msm_gem_vunmap() */
+        msm_gem_put_vaddr(obj);
 }
 
 int msm_gem_prime_mmap(struct drm_gem_object *obj, struct vm_area_struct *vma)
diff --git a/drivers/gpu/drm/msm/msm_gem_shrinker.c b/drivers/gpu/drm/msm/msm_gem_shrinker.c
new file mode 100644
index 000000000000..283d2841ba58
--- /dev/null
+++ b/drivers/gpu/drm/msm/msm_gem_shrinker.c
@@ -0,0 +1,168 @@
+/*
+ * Copyright (C) 2016 Red Hat
+ * Author: Rob Clark <robdclark@gmail.com>
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 as published by
+ * the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "msm_drv.h"
+#include "msm_gem.h"
+
+static bool mutex_is_locked_by(struct mutex *mutex, struct task_struct *task)
+{
+        if (!mutex_is_locked(mutex))
+                return false;
+
+#if defined(CONFIG_SMP) || defined(CONFIG_DEBUG_MUTEXES)
+        return mutex->owner == task;
+#else
+        /* Since UP may be pre-empted, we cannot assume that we own the lock */
+        return false;
+#endif
+}
+
+static bool msm_gem_shrinker_lock(struct drm_device *dev, bool *unlock)
+{
+        if (!mutex_trylock(&dev->struct_mutex)) {
+                if (!mutex_is_locked_by(&dev->struct_mutex, current))
+                        return false;
+                *unlock = false;
+        } else {
+                *unlock = true;
+        }
+
+        return true;
+}
+
+
+static unsigned long
+msm_gem_shrinker_count(struct shrinker *shrinker, struct shrink_control *sc)
+{
+        struct msm_drm_private *priv =
+                container_of(shrinker, struct msm_drm_private, shrinker);
+        struct drm_device *dev = priv->dev;
+        struct msm_gem_object *msm_obj;
+        unsigned long count = 0;
+        bool unlock;
+
+        if (!msm_gem_shrinker_lock(dev, &unlock))
+                return 0;
+
+        list_for_each_entry(msm_obj, &priv->inactive_list, mm_list) {
+                if (is_purgeable(msm_obj))
+                        count += msm_obj->base.size >> PAGE_SHIFT;
+        }
+
+        if (unlock)
+                mutex_unlock(&dev->struct_mutex);
+
+        return count;
+}
+
+static unsigned long
+msm_gem_shrinker_scan(struct shrinker *shrinker, struct shrink_control *sc)
+{
+        struct msm_drm_private *priv =
+                container_of(shrinker, struct msm_drm_private, shrinker);
+        struct drm_device *dev = priv->dev;
+        struct msm_gem_object *msm_obj;
+        unsigned long freed = 0;
+        bool unlock;
+
+        if (!msm_gem_shrinker_lock(dev, &unlock))
+                return SHRINK_STOP;
+
+        list_for_each_entry(msm_obj, &priv->inactive_list, mm_list) {
+                if (freed >= sc->nr_to_scan)
+                        break;
+                if (is_purgeable(msm_obj)) {
+                        msm_gem_purge(&msm_obj->base);
+                        freed += msm_obj->base.size >> PAGE_SHIFT;
+                }
+        }
+
+        if (unlock)
+                mutex_unlock(&dev->struct_mutex);
+
+        if (freed > 0)
+                pr_info_ratelimited("Purging %lu bytes\n", freed << PAGE_SHIFT);
+
+        return freed;
+}
+
+static int
+msm_gem_shrinker_vmap(struct notifier_block *nb, unsigned long event, void *ptr)
+{
+        struct msm_drm_private *priv =
+                container_of(nb, struct msm_drm_private, vmap_notifier);
+        struct drm_device *dev = priv->dev;
+        struct msm_gem_object *msm_obj;
+        unsigned unmapped = 0;
+        bool unlock;
+
+        if (!msm_gem_shrinker_lock(dev, &unlock))
+                return NOTIFY_DONE;
+
+        list_for_each_entry(msm_obj, &priv->inactive_list, mm_list) {
+                if (is_vunmapable(msm_obj)) {
+                        msm_gem_vunmap(&msm_obj->base);
+                        /* since we don't know any better, lets bail after a few
+                         * and if necessary the shrinker will be invoked again.
+                         * Seems better than unmapping *everything*
+                         */
+                        if (++unmapped >= 15)
+                                break;
+                }
+        }
+
+        if (unlock)
+                mutex_unlock(&dev->struct_mutex);
+
+        *(unsigned long *)ptr += unmapped;
+
+        if (unmapped > 0)
+                pr_info_ratelimited("Purging %u vmaps\n", unmapped);
+
+        return NOTIFY_DONE;
+}
+
+/**
+ * msm_gem_shrinker_init - Initialize msm shrinker
+ * @dev_priv: msm device
+ *
+ * This function registers and sets up the msm shrinker.
+ */
+void msm_gem_shrinker_init(struct drm_device *dev)
+{
+        struct msm_drm_private *priv = dev->dev_private;
+        priv->shrinker.count_objects = msm_gem_shrinker_count;
+        priv->shrinker.scan_objects = msm_gem_shrinker_scan;
+        priv->shrinker.seeks = DEFAULT_SEEKS;
+        WARN_ON(register_shrinker(&priv->shrinker));
+
+        priv->vmap_notifier.notifier_call = msm_gem_shrinker_vmap;
+        WARN_ON(register_vmap_purge_notifier(&priv->vmap_notifier));
+}
+
+/**
+ * msm_gem_shrinker_cleanup - Clean up msm shrinker
+ * @dev_priv: msm device
+ *
+ * This function unregisters the msm shrinker.
+ */
+void msm_gem_shrinker_cleanup(struct drm_device *dev)
+{
+        struct msm_drm_private *priv = dev->dev_private;
+        WARN_ON(unregister_vmap_purge_notifier(&priv->vmap_notifier));
+        unregister_shrinker(&priv->shrinker);
+}
diff --git a/drivers/gpu/drm/msm/msm_gem_submit.c b/drivers/gpu/drm/msm/msm_gem_submit.c
index eb4bb8b2f3a5..9766f9ae4b7d 100644
--- a/drivers/gpu/drm/msm/msm_gem_submit.c
+++ b/drivers/gpu/drm/msm/msm_gem_submit.c
@@ -29,10 +29,11 @@
 #define BO_PINNED   0x2000
 
 static struct msm_gem_submit *submit_create(struct drm_device *dev,
-                struct msm_gpu *gpu, int nr)
+                struct msm_gpu *gpu, int nr_bos, int nr_cmds)
 {
         struct msm_gem_submit *submit;
-        int sz = sizeof(*submit) + (nr * sizeof(submit->bos[0]));
+        int sz = sizeof(*submit) + (nr_bos * sizeof(submit->bos[0])) +
+                        (nr_cmds * sizeof(*submit->cmd));
 
         submit = kmalloc(sz, GFP_TEMPORARY | __GFP_NOWARN | __GFP_NORETRY);
         if (!submit)
@@ -42,6 +43,7 @@ static struct msm_gem_submit *submit_create(struct drm_device *dev,
         submit->gpu = gpu;
         submit->fence = NULL;
         submit->pid = get_pid(task_pid(current));
+        submit->cmd = (void *)&submit->bos[nr_bos];
 
         /* initially, until copy_from_user() and bo lookup succeeds: */
         submit->nr_bos = 0;
@@ -279,7 +281,7 @@ static int submit_reloc(struct msm_gem_submit *submit, struct msm_gem_object *ob
         /* For now, just map the entire thing.  Eventually we probably
          * to do it page-by-page, w/ kmap() if not vmap()d..
          */
-        ptr = msm_gem_vaddr_locked(&obj->base);
+        ptr = msm_gem_get_vaddr_locked(&obj->base);
 
         if (IS_ERR(ptr)) {
                 ret = PTR_ERR(ptr);
@@ -332,6 +334,8 @@ static int submit_reloc(struct msm_gem_submit *submit, struct msm_gem_object *ob
                 last_offset = off;
         }
 
+        msm_gem_put_vaddr_locked(&obj->base);
+
         return 0;
 }
 
@@ -369,14 +373,15 @@ int msm_ioctl_gem_submit(struct drm_device *dev, void *data,
         if (args->pipe != MSM_PIPE_3D0)
                 return -EINVAL;
 
-        if (args->nr_cmds > MAX_CMDS)
-                return -EINVAL;
-
-        submit = submit_create(dev, gpu, args->nr_bos);
-        if (!submit)
-                return -ENOMEM;
+        ret = mutex_lock_interruptible(&dev->struct_mutex);
+        if (ret)
+                return ret;
 
-        mutex_lock(&dev->struct_mutex);
+        submit = submit_create(dev, gpu, args->nr_bos, args->nr_cmds);
+        if (!submit) {
+                ret = -ENOMEM;
+                goto out_unlock;
+        }
 
         ret = submit_lookup_objects(submit, args, file);
         if (ret)
@@ -462,6 +467,7 @@ out:
         submit_cleanup(submit);
         if (ret)
                 msm_gem_submit_free(submit);
+out_unlock:
         mutex_unlock(&dev->struct_mutex);
         return ret;
 }
diff --git a/drivers/gpu/drm/msm/msm_iommu.c b/drivers/gpu/drm/msm/msm_iommu.c
index a7a0b6d9b057..3a294d0da3a0 100644
--- a/drivers/gpu/drm/msm/msm_iommu.c
+++ b/drivers/gpu/drm/msm/msm_iommu.c
@@ -59,10 +59,10 @@ static int msm_iommu_map(struct msm_mmu *mmu, uint32_t iova,
                 return -EINVAL;
 
         for_each_sg(sgt->sgl, sg, sgt->nents, i) {
-                u32 pa = sg_phys(sg) - sg->offset;
+                dma_addr_t pa = sg_phys(sg) - sg->offset;
                 size_t bytes = sg->length + sg->offset;
 
-                VERB("map[%d]: %08x %08x(%zx)", i, iova, pa, bytes);
+                VERB("map[%d]: %08x %08lx(%zx)", i, da, (unsigned long)pa, bytes);
 
                 ret = iommu_map(domain, da, pa, bytes, prot);
                 if (ret)
@@ -101,7 +101,7 @@ static int msm_iommu_unmap(struct msm_mmu *mmu, uint32_t iova,
                 if (unmapped < bytes)
                         return unmapped;
 
-                VERB("unmap[%d]: %08x(%zx)", i, iova, bytes);
+                VERB("unmap[%d]: %08x(%zx)", i, da, bytes);
 
                 BUG_ON(!PAGE_ALIGNED(bytes));
 
diff --git a/drivers/gpu/drm/msm/msm_kms.h b/drivers/gpu/drm/msm/msm_kms.h
index e32222c3d44f..40e41e5cdbc6 100644
--- a/drivers/gpu/drm/msm/msm_kms.h
+++ b/drivers/gpu/drm/msm/msm_kms.h
@@ -61,10 +61,8 @@ struct msm_kms_funcs {
 struct msm_kms {
         const struct msm_kms_funcs *funcs;
 
-        /* irq handling: */
-        bool in_irq;
-        struct list_head irq_list;    /* list of mdp4_irq */
-        uint32_t vblank_mask;         /* irq bits set for userspace vblank */
+        /* irq number to be passed on to drm_irq_install */
+        int irq;
 };
 
 static inline void msm_kms_init(struct msm_kms *kms,
@@ -75,5 +73,7 @@ static inline void msm_kms_init(struct msm_kms *kms,
 
 struct msm_kms *mdp4_kms_init(struct drm_device *dev);
 struct msm_kms *mdp5_kms_init(struct drm_device *dev);
+int msm_mdss_init(struct drm_device *dev);
+void msm_mdss_destroy(struct drm_device *dev);
 
 #endif /* __MSM_KMS_H__ */
diff --git a/drivers/gpu/drm/msm/msm_perf.c b/drivers/gpu/drm/msm/msm_perf.c
index 830857c47c86..17fe4e53e0d1 100644
--- a/drivers/gpu/drm/msm/msm_perf.c
+++ b/drivers/gpu/drm/msm/msm_perf.c
@@ -132,7 +132,7 @@ static ssize_t perf_read(struct file *file, char __user *buf,
                 size_t sz, loff_t *ppos)
 {
         struct msm_perf_state *perf = file->private_data;
-        int n = 0, ret;
+        int n = 0, ret = 0;
 
         mutex_lock(&perf->read_lock);
 
@@ -143,9 +143,10 @@ static ssize_t perf_read(struct file *file, char __user *buf,
         }
 
         n = min((int)sz, perf->buftot - perf->bufpos);
-        ret = copy_to_user(buf, &perf->buf[perf->bufpos], n);
-        if (ret)
+        if (copy_to_user(buf, &perf->buf[perf->bufpos], n)) {
+                ret = -EFAULT;
                 goto out;
+        }
 
         perf->bufpos += n;
         *ppos += n;
diff --git a/drivers/gpu/drm/msm/msm_rd.c b/drivers/gpu/drm/msm/msm_rd.c
index 0857710c2ff2..3a5fdfcd67ae 100644
--- a/drivers/gpu/drm/msm/msm_rd.c
+++ b/drivers/gpu/drm/msm/msm_rd.c
@@ -27,6 +27,11 @@
  * This bypasses drm_debugfs_create_files() mainly because we need to use
  * our own fops for a bit more control.  In particular, we don't want to
  * do anything if userspace doesn't have the debugfs file open.
+ *
+ * The module-param "rd_full", which defaults to false, enables snapshotting
+ * all (non-written) buffers in the submit, rather than just cmdstream bo's.
+ * This is useful to capture the contents of (for example) vbo's or textures,
+ * or shader programs (if not emitted inline in cmdstream).
  */
 
 #ifdef CONFIG_DEBUG_FS
@@ -40,6 +45,10 @@
 #include "msm_gpu.h"
 #include "msm_gem.h"
 
+static bool rd_full = false;
+MODULE_PARM_DESC(rd_full, "If true, $debugfs/.../rd will snapshot all buffer contents");
+module_param_named(rd_full, rd_full, bool, 0600);
+
 enum rd_sect_type {
         RD_NONE,
         RD_TEST,       /* ascii text */
@@ -140,9 +149,10 @@ static ssize_t rd_read(struct file *file, char __user *buf,
                 goto out;
 
         n = min_t(int, sz, circ_count_to_end(&rd->fifo));
-        ret = copy_to_user(buf, fptr, n);
-        if (ret)
+        if (copy_to_user(buf, fptr, n)) {
+                ret = -EFAULT;
                 goto out;
+        }
 
         fifo->tail = (fifo->tail + n) & (BUF_SZ - 1);
         *ppos += n;
@@ -277,6 +287,31 @@ void msm_rd_debugfs_cleanup(struct drm_minor *minor)
         kfree(rd);
 }
 
+static void snapshot_buf(struct msm_rd_state *rd,
+                struct msm_gem_submit *submit, int idx,
+                uint32_t iova, uint32_t size)
+{
+        struct msm_gem_object *obj = submit->bos[idx].obj;
+        const char *buf;
+
+        buf = msm_gem_get_vaddr_locked(&obj->base);
+        if (IS_ERR(buf))
+                return;
+
+        if (iova) {
+                buf += iova - submit->bos[idx].iova;
+        } else {
+                iova = submit->bos[idx].iova;
+                size = obj->base.size;
+        }
+
+        rd_write_section(rd, RD_GPUADDR,
+                        (uint32_t[2]){ iova, size }, 8);
+        rd_write_section(rd, RD_BUFFER_CONTENTS, buf, size);
+
+        msm_gem_put_vaddr_locked(&obj->base);
+}
+
 /* called under struct_mutex */
 void msm_rd_dump_submit(struct msm_gem_submit *submit)
 {
@@ -300,27 +335,27 @@ void msm_rd_dump_submit(struct msm_gem_submit *submit)
 
         rd_write_section(rd, RD_CMD, msg, ALIGN(n, 4));
 
-        /* could be nice to have an option (module-param?) to snapshot
-         * all the bo's associated with the submit.  Handy to see vtx
-         * buffers, etc.  For now just the cmdstream bo's is enough.
-         */
+        if (rd_full) {
+                for (i = 0; i < submit->nr_bos; i++) {
+                        /* buffers that are written to probably don't start out
+                         * with anything interesting:
+                         */
+                        if (submit->bos[i].flags & MSM_SUBMIT_BO_WRITE)
+                                continue;
+
+                        snapshot_buf(rd, submit, i, 0, 0);
+                }
+        }
 
         for (i = 0; i < submit->nr_cmds; i++) {
-                uint32_t idx  = submit->cmd[i].idx;
                 uint32_t iova = submit->cmd[i].iova;
                 uint32_t szd  = submit->cmd[i].size; /* in dwords */
-                struct msm_gem_object *obj = submit->bos[idx].obj;
-                const char *buf = msm_gem_vaddr_locked(&obj->base);
-
-                if (IS_ERR(buf))
-                        continue;
 
-                buf += iova - submit->bos[idx].iova;
-
-                rd_write_section(rd, RD_GPUADDR,
-                                (uint32_t[2]){ iova, szd * 4 }, 8);
-                rd_write_section(rd, RD_BUFFER_CONTENTS,
-                                buf, szd * 4);
+                /* snapshot cmdstream bo's (if we haven't already): */
+                if (!rd_full) {
+                        snapshot_buf(rd, submit, submit->cmd[i].idx,
+                                        submit->cmd[i].iova, szd * 4);
+                }
 
                 switch (submit->cmd[i].type) {
                 case MSM_SUBMIT_CMD_IB_TARGET_BUF:
diff --git a/drivers/gpu/drm/msm/msm_ringbuffer.c b/drivers/gpu/drm/msm/msm_ringbuffer.c
index 42f5359cf988..f326cf6a32e6 100644
--- a/drivers/gpu/drm/msm/msm_ringbuffer.c
+++ b/drivers/gpu/drm/msm/msm_ringbuffer.c
@@ -39,7 +39,7 @@ struct msm_ringbuffer *msm_ringbuffer_new(struct msm_gpu *gpu, int size)
                 goto fail;
         }
 
-        ring->start = msm_gem_vaddr_locked(ring->bo);
+        ring->start = msm_gem_get_vaddr_locked(ring->bo);
         if (IS_ERR(ring->start)) {
                 ret = PTR_ERR(ring->start);
                 goto fail;
@@ -59,7 +59,9 @@ fail:
 
 void msm_ringbuffer_destroy(struct msm_ringbuffer *ring)
 {
-        if (ring->bo)
+        if (ring->bo) {
+                msm_gem_put_vaddr(ring->bo);
                 drm_gem_object_unreference_unlocked(ring->bo);
+        }
         kfree(ring);
 }
diff --git a/drivers/gpu/drm/nouveau/Kconfig b/drivers/gpu/drm/nouveau/Kconfig
index 5ab13e7939db..2922a82cba8e 100644
--- a/drivers/gpu/drm/nouveau/Kconfig
+++ b/drivers/gpu/drm/nouveau/Kconfig
@@ -3,13 +3,7 @@ config DRM_NOUVEAU
         depends on DRM && PCI
         select FW_LOADER
         select DRM_KMS_HELPER
-        select DRM_KMS_FB_HELPER
         select DRM_TTM
-        select FB_CFB_FILLRECT
-        select FB_CFB_COPYAREA
-        select FB_CFB_IMAGEBLIT
-        select FB
-        select FRAMEBUFFER_CONSOLE if !EXPERT
         select FB_BACKLIGHT if DRM_NOUVEAU_BACKLIGHT
         select ACPI_VIDEO if ACPI && X86 && BACKLIGHT_CLASS_DEVICE && INPUT
         select X86_PLATFORM_DEVICES if ACPI && X86
diff --git a/drivers/gpu/drm/nouveau/dispnv04/crtc.c b/drivers/gpu/drm/nouveau/dispnv04/crtc.c
index 6f318c54da33..0cb7a18cde26 100644
--- a/drivers/gpu/drm/nouveau/dispnv04/crtc.c
+++ b/drivers/gpu/drm/nouveau/dispnv04/crtc.c
@@ -785,14 +785,14 @@ nv_crtc_disable(struct drm_crtc *crtc)
         nouveau_bo_ref(NULL, &disp->image[nv_crtc->index]);
 }
 
-static void
-nv_crtc_gamma_set(struct drm_crtc *crtc, u16 *r, u16 *g, u16 *b, uint32_t start,
+static int
+nv_crtc_gamma_set(struct drm_crtc *crtc, u16 *r, u16 *g, u16 *b,
                   uint32_t size)
 {
-        int end = (start + size > 256) ? 256 : start + size, i;
         struct nouveau_crtc *nv_crtc = nouveau_crtc(crtc);
+        int i;
 
-        for (i = start; i < end; i++) {
+        for (i = 0; i < size; i++) {
                 nv_crtc->lut.r[i] = r[i];
                 nv_crtc->lut.g[i] = g[i];
                 nv_crtc->lut.b[i] = b[i];
@@ -805,10 +805,12 @@ nv_crtc_gamma_set(struct drm_crtc *crtc, u16 *r, u16 *g, u16 *b, uint32_t start,
          */
         if (!nv_crtc->base.primary->fb) {
                 nv_crtc->lut.depth = 0;
-                return;
+                return 0;
         }
 
         nv_crtc_gamma_load(crtc);
+
+        return 0;
 }
 
 static int
diff --git a/drivers/gpu/drm/nouveau/dispnv04/disp.c b/drivers/gpu/drm/nouveau/dispnv04/disp.c
index aea81a547e85..34c0f2f67548 100644
--- a/drivers/gpu/drm/nouveau/dispnv04/disp.c
+++ b/drivers/gpu/drm/nouveau/dispnv04/disp.c
@@ -125,18 +125,8 @@ nv04_display_destroy(struct drm_device *dev)
         struct nv04_display *disp = nv04_display(dev);
         struct nouveau_drm *drm = nouveau_drm(dev);
         struct nouveau_encoder *encoder;
-        struct drm_crtc *crtc;
         struct nouveau_crtc *nv_crtc;
 
-        /* Turn every CRTC off. */
-        list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) {
-                struct drm_mode_set modeset = {
-                        .crtc = crtc,
-                };
-
-                drm_mode_set_config_internal(&modeset);
-        }
-
         /* Restore state */
         list_for_each_entry(encoder, &dev->mode_config.encoder_list, base.base.head)
                 encoder->enc_restore(&encoder->base.base);
diff --git a/drivers/gpu/drm/nouveau/dispnv04/tvnv17.c b/drivers/gpu/drm/nouveau/dispnv04/tvnv17.c
index a665b78b2af5..434d1e29f279 100644
--- a/drivers/gpu/drm/nouveau/dispnv04/tvnv17.c
+++ b/drivers/gpu/drm/nouveau/dispnv04/tvnv17.c
@@ -749,13 +749,8 @@ static int nv17_tv_set_property(struct drm_encoder *encoder,
 
                 /* Disable the crtc to ensure a full modeset is
                  * performed whenever it's turned on again. */
-                if (crtc) {
-                        struct drm_mode_set modeset = {
-                                .crtc = crtc,
-                        };
-
-                        drm_mode_set_config_internal(&modeset);
-                }
+                if (crtc)
+                        drm_crtc_force_disable(crtc);
         }
 
         return 0;
diff --git a/drivers/gpu/drm/nouveau/include/nvif/cl0080.h b/drivers/gpu/drm/nouveau/include/nvif/cl0080.h
index 331620a52afa..287a7d6fa480 100644
--- a/drivers/gpu/drm/nouveau/include/nvif/cl0080.h
+++ b/drivers/gpu/drm/nouveau/include/nvif/cl0080.h
@@ -29,6 +29,7 @@ struct nv_device_info_v0 {
 #define NV_DEVICE_INFO_V0_FERMI                                            0x07
 #define NV_DEVICE_INFO_V0_KEPLER                                           0x08
 #define NV_DEVICE_INFO_V0_MAXWELL                                          0x09
+#define NV_DEVICE_INFO_V0_PASCAL                                           0x0a
         __u8  family;
         __u8  pad06[2];
         __u64 ram_size;
diff --git a/drivers/gpu/drm/nouveau/include/nvif/class.h b/drivers/gpu/drm/nouveau/include/nvif/class.h
index 982aad8fa645..e6e9537537cf 100644
--- a/drivers/gpu/drm/nouveau/include/nvif/class.h
+++ b/drivers/gpu/drm/nouveau/include/nvif/class.h
@@ -39,6 +39,7 @@
 #define KEPLER_CHANNEL_GPFIFO_A                       /* cla06f.h */ 0x0000a06f
 #define KEPLER_CHANNEL_GPFIFO_B                       /* cla06f.h */ 0x0000a16f
 #define MAXWELL_CHANNEL_GPFIFO_A                      /* cla06f.h */ 0x0000b06f
+#define PASCAL_CHANNEL_GPFIFO_A                       /* cla06f.h */ 0x0000c06f
 
 #define NV50_DISP                                     /* cl5070.h */ 0x00005070
 #define G82_DISP                                      /* cl5070.h */ 0x00008270
@@ -50,6 +51,8 @@
 #define GK110_DISP                                    /* cl5070.h */ 0x00009270
 #define GM107_DISP                                    /* cl5070.h */ 0x00009470
 #define GM200_DISP                                    /* cl5070.h */ 0x00009570
+#define GP100_DISP                                    /* cl5070.h */ 0x00009770
+#define GP104_DISP                                    /* cl5070.h */ 0x00009870
 
 #define NV31_MPEG                                                    0x00003174
 #define G82_MPEG                                                     0x00008274
@@ -86,6 +89,8 @@
 #define GK110_DISP_CORE_CHANNEL_DMA                   /* cl507d.h */ 0x0000927d
 #define GM107_DISP_CORE_CHANNEL_DMA                   /* cl507d.h */ 0x0000947d
 #define GM200_DISP_CORE_CHANNEL_DMA                   /* cl507d.h */ 0x0000957d
+#define GP100_DISP_CORE_CHANNEL_DMA                   /* cl507d.h */ 0x0000977d
+#define GP104_DISP_CORE_CHANNEL_DMA                   /* cl507d.h */ 0x0000987d
 
 #define NV50_DISP_OVERLAY_CHANNEL_DMA                 /* cl507e.h */ 0x0000507e
 #define G82_DISP_OVERLAY_CHANNEL_DMA                  /* cl507e.h */ 0x0000827e
@@ -105,6 +110,8 @@
 #define MAXWELL_A                                     /* cl9097.h */ 0x0000b097
 #define MAXWELL_B                                     /* cl9097.h */ 0x0000b197
 
+#define PASCAL_A                                      /* cl9097.h */ 0x0000c097
+
 #define NV74_BSP                                                     0x000074b0
 
 #define GT212_MSVLD                                                  0x000085b1
@@ -128,6 +135,8 @@
 #define FERMI_DMA                                                    0x000090b5
 #define KEPLER_DMA_COPY_A                                            0x0000a0b5
 #define MAXWELL_DMA_COPY_A                                           0x0000b0b5
+#define PASCAL_DMA_COPY_A                                            0x0000c0b5
+#define PASCAL_DMA_COPY_B                                            0x0000c1b5
 
 #define FERMI_DECOMPRESS                                             0x000090b8
 
@@ -137,6 +146,7 @@
 #define KEPLER_COMPUTE_B                                             0x0000a1c0
 #define MAXWELL_COMPUTE_A                                            0x0000b0c0
 #define MAXWELL_COMPUTE_B                                            0x0000b1c0
+#define PASCAL_COMPUTE_A                                             0x0000c0c0
 
 #define NV74_CIPHER                                                  0x000074c1
 #endif
diff --git a/drivers/gpu/drm/nouveau/include/nvkm/core/device.h b/drivers/gpu/drm/nouveau/include/nvkm/core/device.h
index 126a85cc81bc..7ea8aa7ca408 100644
--- a/drivers/gpu/drm/nouveau/include/nvkm/core/device.h
+++ b/drivers/gpu/drm/nouveau/include/nvkm/core/device.h
@@ -33,7 +33,10 @@ enum nvkm_devidx {
         NVKM_ENGINE_CE0,
         NVKM_ENGINE_CE1,
         NVKM_ENGINE_CE2,
-        NVKM_ENGINE_CE_LAST = NVKM_ENGINE_CE2,
+        NVKM_ENGINE_CE3,
+        NVKM_ENGINE_CE4,
+        NVKM_ENGINE_CE5,
+        NVKM_ENGINE_CE_LAST = NVKM_ENGINE_CE5,
 
         NVKM_ENGINE_CIPHER,
         NVKM_ENGINE_DISP,
@@ -50,7 +53,8 @@ enum nvkm_devidx {
 
         NVKM_ENGINE_NVENC0,
         NVKM_ENGINE_NVENC1,
-        NVKM_ENGINE_NVENC_LAST = NVKM_ENGINE_NVENC1,
+        NVKM_ENGINE_NVENC2,
+        NVKM_ENGINE_NVENC_LAST = NVKM_ENGINE_NVENC2,
 
         NVKM_ENGINE_NVDEC,
         NVKM_ENGINE_PM,
@@ -102,6 +106,7 @@ struct nvkm_device {
                 NV_C0    = 0xc0,
                 NV_E0    = 0xe0,
                 GM100    = 0x110,
+                GP100    = 0x130,
         } card_type;
         u32 chipset;
         u8  chiprev;
@@ -136,7 +141,7 @@ struct nvkm_device {
         struct nvkm_volt *volt;
 
         struct nvkm_engine *bsp;
-        struct nvkm_engine *ce[3];
+        struct nvkm_engine *ce[6];
         struct nvkm_engine *cipher;
         struct nvkm_disp *disp;
         struct nvkm_dma *dma;
@@ -149,7 +154,7 @@ struct nvkm_device {
         struct nvkm_engine *mspdec;
         struct nvkm_engine *msppp;
         struct nvkm_engine *msvld;
-        struct nvkm_engine *nvenc[2];
+        struct nvkm_engine *nvenc[3];
         struct nvkm_engine *nvdec;
         struct nvkm_pm *pm;
         struct nvkm_engine *sec;
@@ -170,7 +175,6 @@ struct nvkm_device_func {
         void (*fini)(struct nvkm_device *, bool suspend);
         resource_size_t (*resource_addr)(struct nvkm_device *, unsigned bar);
         resource_size_t (*resource_size)(struct nvkm_device *, unsigned bar);
-        bool cpu_coherent;
 };
 
 struct nvkm_device_quirk {
@@ -206,7 +210,7 @@ struct nvkm_device_chip {
         int (*volt    )(struct nvkm_device *, int idx, struct nvkm_volt **);
 
         int (*bsp     )(struct nvkm_device *, int idx, struct nvkm_engine **);
-        int (*ce[3]   )(struct nvkm_device *, int idx, struct nvkm_engine **);
+        int (*ce[6]   )(struct nvkm_device *, int idx, struct nvkm_engine **);
         int (*cipher  )(struct nvkm_device *, int idx, struct nvkm_engine **);
         int (*disp    )(struct nvkm_device *, int idx, struct nvkm_disp **);
         int (*dma     )(struct nvkm_device *, int idx, struct nvkm_dma **);
@@ -219,7 +223,7 @@ struct nvkm_device_chip {
         int (*mspdec  )(struct nvkm_device *, int idx, struct nvkm_engine **);
         int (*msppp   )(struct nvkm_device *, int idx, struct nvkm_engine **);
         int (*msvld   )(struct nvkm_device *, int idx, struct nvkm_engine **);
-        int (*nvenc[2])(struct nvkm_device *, int idx, struct nvkm_engine **);
+        int (*nvenc[3])(struct nvkm_device *, int idx, struct nvkm_engine **);
         int (*nvdec   )(struct nvkm_device *, int idx, struct nvkm_engine **);
         int (*pm      )(struct nvkm_device *, int idx, struct nvkm_pm **);
         int (*sec     )(struct nvkm_device *, int idx, struct nvkm_engine **);
diff --git a/drivers/gpu/drm/nouveau/include/nvkm/core/tegra.h b/drivers/gpu/drm/nouveau/include/nvkm/core/tegra.h
index b5370cb56e3c..e5c9b6268dcc 100644
--- a/drivers/gpu/drm/nouveau/include/nvkm/core/tegra.h
+++ b/drivers/gpu/drm/nouveau/include/nvkm/core/tegra.h
@@ -28,6 +28,7 @@ struct nvkm_device_tegra {
         } iommu;
 
         int gpu_speedo;
+        int gpu_speedo_id;
 };
 
 struct nvkm_device_tegra_func {
diff --git a/drivers/gpu/drm/nouveau/include/nvkm/engine/ce.h b/drivers/gpu/drm/nouveau/include/nvkm/engine/ce.h
index 594d719ba41e..d3d26a1e215d 100644
--- a/drivers/gpu/drm/nouveau/include/nvkm/engine/ce.h
+++ b/drivers/gpu/drm/nouveau/include/nvkm/engine/ce.h
@@ -7,4 +7,6 @@ int gf100_ce_new(struct nvkm_device *, int, struct nvkm_engine **);
 int gk104_ce_new(struct nvkm_device *, int, struct nvkm_engine **);
 int gm107_ce_new(struct nvkm_device *, int, struct nvkm_engine **);
 int gm200_ce_new(struct nvkm_device *, int, struct nvkm_engine **);
+int gp100_ce_new(struct nvkm_device *, int, struct nvkm_engine **);
+int gp104_ce_new(struct nvkm_device *, int, struct nvkm_engine **);
 #endif
diff --git a/drivers/gpu/drm/nouveau/include/nvkm/engine/disp.h b/drivers/gpu/drm/nouveau/include/nvkm/engine/disp.h
index d4fdce27b297..e82049667ce4 100644
--- a/drivers/gpu/drm/nouveau/include/nvkm/engine/disp.h
+++ b/drivers/gpu/drm/nouveau/include/nvkm/engine/disp.h
@@ -32,4 +32,6 @@ int gk104_disp_new(struct nvkm_device *, int, struct nvkm_disp **);
 int gk110_disp_new(struct nvkm_device *, int, struct nvkm_disp **);
 int gm107_disp_new(struct nvkm_device *, int, struct nvkm_disp **);
 int gm200_disp_new(struct nvkm_device *, int, struct nvkm_disp **);
+int gp100_disp_new(struct nvkm_device *, int, struct nvkm_disp **);
+int gp104_disp_new(struct nvkm_device *, int, struct nvkm_disp **);
 #endif
diff --git a/drivers/gpu/drm/nouveau/include/nvkm/engine/fifo.h b/drivers/gpu/drm/nouveau/include/nvkm/engine/fifo.h
index 15ddfcf5e8db..ed92fec5292c 100644
--- a/drivers/gpu/drm/nouveau/include/nvkm/engine/fifo.h
+++ b/drivers/gpu/drm/nouveau/include/nvkm/engine/fifo.h
@@ -66,4 +66,5 @@ int gk20a_fifo_new(struct nvkm_device *, int, struct nvkm_fifo **);
 int gm107_fifo_new(struct nvkm_device *, int, struct nvkm_fifo **);
 int gm200_fifo_new(struct nvkm_device *, int, struct nvkm_fifo **);
 int gm20b_fifo_new(struct nvkm_device *, int, struct nvkm_fifo **);
+int gp100_fifo_new(struct nvkm_device *, int, struct nvkm_fifo **);
 #endif
diff --git a/drivers/gpu/drm/nouveau/include/nvkm/engine/gr.h b/drivers/gpu/drm/nouveau/include/nvkm/engine/gr.h
index 6515f5810a26..89cf99307828 100644
--- a/drivers/gpu/drm/nouveau/include/nvkm/engine/gr.h
+++ b/drivers/gpu/drm/nouveau/include/nvkm/engine/gr.h
@@ -42,4 +42,5 @@ int gk20a_gr_new(struct nvkm_device *, int, struct nvkm_gr **);
 int gm107_gr_new(struct nvkm_device *, int, struct nvkm_gr **);
 int gm200_gr_new(struct nvkm_device *, int, struct nvkm_gr **);
 int gm20b_gr_new(struct nvkm_device *, int, struct nvkm_gr **);
+int gp100_gr_new(struct nvkm_device *, int, struct nvkm_gr **);
 #endif
diff --git a/drivers/gpu/drm/nouveau/include/nvkm/subdev/bios.h b/drivers/gpu/drm/nouveau/include/nvkm/subdev/bios.h
index e39a1fea930b..a72f3290528a 100644
--- a/drivers/gpu/drm/nouveau/include/nvkm/subdev/bios.h
+++ b/drivers/gpu/drm/nouveau/include/nvkm/subdev/bios.h
@@ -7,6 +7,9 @@ struct nvkm_bios {
         u32 size;
         u8 *data;
 
+        u32 image0_size;
+        u32 imaged_addr;
+
         u32 bmp_offset;
         u32 bit_offset;
 
@@ -22,10 +25,9 @@ struct nvkm_bios {
 u8  nvbios_checksum(const u8 *data, int size);
 u16 nvbios_findstr(const u8 *data, int size, const char *str, int len);
 int nvbios_memcmp(struct nvkm_bios *, u32 addr, const char *, u32 len);
-
-#define nvbios_rd08(b,o) (b)->data[(o)]
-#define nvbios_rd16(b,o) get_unaligned_le16(&(b)->data[(o)])
-#define nvbios_rd32(b,o) get_unaligned_le32(&(b)->data[(o)])
+u8  nvbios_rd08(struct nvkm_bios *, u32 addr);
+u16 nvbios_rd16(struct nvkm_bios *, u32 addr);
+u32 nvbios_rd32(struct nvkm_bios *, u32 addr);
 
 int nvkm_bios_new(struct nvkm_device *, int, struct nvkm_bios **);
 #endif
diff --git a/drivers/gpu/drm/nouveau/include/nvkm/subdev/fb.h b/drivers/gpu/drm/nouveau/include/nvkm/subdev/fb.h
index 0a734fd06acf..3a410275fa71 100644
--- a/drivers/gpu/drm/nouveau/include/nvkm/subdev/fb.h
+++ b/drivers/gpu/drm/nouveau/include/nvkm/subdev/fb.h
@@ -56,6 +56,8 @@ struct nvkm_fb {
                 int regions;
         } tile;
 
+        u8 page;
+
         struct nvkm_memory *mmu_rd;
         struct nvkm_memory *mmu_wr;
 };
@@ -91,6 +93,8 @@ int gk104_fb_new(struct nvkm_device *, int, struct nvkm_fb **);
 int gk20a_fb_new(struct nvkm_device *, int, struct nvkm_fb **);
 int gm107_fb_new(struct nvkm_device *, int, struct nvkm_fb **);
 int gm200_fb_new(struct nvkm_device *, int, struct nvkm_fb **);
+int gp100_fb_new(struct nvkm_device *, int, struct nvkm_fb **);
+int gp104_fb_new(struct nvkm_device *, int, struct nvkm_fb **);
 
 #include <subdev/bios.h>
 #include <subdev/bios/ramcfg.h>
diff --git a/drivers/gpu/drm/nouveau/include/nvkm/subdev/ltc.h b/drivers/gpu/drm/nouveau/include/nvkm/subdev/ltc.h
index c6b90b6543b3..cd755baf9cab 100644
--- a/drivers/gpu/drm/nouveau/include/nvkm/subdev/ltc.h
+++ b/drivers/gpu/drm/nouveau/include/nvkm/subdev/ltc.h
@@ -38,4 +38,5 @@ int gk104_ltc_new(struct nvkm_device *, int, struct nvkm_ltc **);
 int gk20a_ltc_new(struct nvkm_device *, int, struct nvkm_ltc **);
 int gm107_ltc_new(struct nvkm_device *, int, struct nvkm_ltc **);
 int gm200_ltc_new(struct nvkm_device *, int, struct nvkm_ltc **);
+int gp100_ltc_new(struct nvkm_device *, int, struct nvkm_ltc **);
 #endif
diff --git a/drivers/gpu/drm/nouveau/include/nvkm/subdev/mc.h b/drivers/gpu/drm/nouveau/include/nvkm/subdev/mc.h
index 2e80682b2da1..27d25b18d85c 100644
--- a/drivers/gpu/drm/nouveau/include/nvkm/subdev/mc.h
+++ b/drivers/gpu/drm/nouveau/include/nvkm/subdev/mc.h
@@ -7,11 +7,14 @@ struct nvkm_mc {
         struct nvkm_subdev subdev;
 };
 
-void nvkm_mc_intr(struct nvkm_mc *, bool *handled);
-void nvkm_mc_intr_unarm(struct nvkm_mc *);
-void nvkm_mc_intr_rearm(struct nvkm_mc *);
-void nvkm_mc_reset(struct nvkm_mc *, enum nvkm_devidx);
-void nvkm_mc_unk260(struct nvkm_mc *, u32 data);
+void nvkm_mc_enable(struct nvkm_device *, enum nvkm_devidx);
+void nvkm_mc_disable(struct nvkm_device *, enum nvkm_devidx);
+void nvkm_mc_reset(struct nvkm_device *, enum nvkm_devidx);
+void nvkm_mc_intr(struct nvkm_device *, bool *handled);
+void nvkm_mc_intr_unarm(struct nvkm_device *);
+void nvkm_mc_intr_rearm(struct nvkm_device *);
+void nvkm_mc_intr_mask(struct nvkm_device *, enum nvkm_devidx, bool enable);
+void nvkm_mc_unk260(struct nvkm_device *, u32 data);
 
 int nv04_mc_new(struct nvkm_device *, int, struct nvkm_mc **);
 int nv11_mc_new(struct nvkm_device *, int, struct nvkm_mc **);
@@ -24,4 +27,5 @@ int gt215_mc_new(struct nvkm_device *, int, struct nvkm_mc **);
 int gf100_mc_new(struct nvkm_device *, int, struct nvkm_mc **);
 int gk104_mc_new(struct nvkm_device *, int, struct nvkm_mc **);
 int gk20a_mc_new(struct nvkm_device *, int, struct nvkm_mc **);
+int gp100_mc_new(struct nvkm_device *, int, struct nvkm_mc **);
 #endif
diff --git a/drivers/gpu/drm/nouveau/include/nvkm/subdev/pci.h b/drivers/gpu/drm/nouveau/include/nvkm/subdev/pci.h
index ddb913889d7e..e6523e2cea9f 100644
--- a/drivers/gpu/drm/nouveau/include/nvkm/subdev/pci.h
+++ b/drivers/gpu/drm/nouveau/include/nvkm/subdev/pci.h
@@ -47,6 +47,7 @@ int g94_pci_new(struct nvkm_device *, int, struct nvkm_pci **);
 int gf100_pci_new(struct nvkm_device *, int, struct nvkm_pci **);
 int gf106_pci_new(struct nvkm_device *, int, struct nvkm_pci **);
 int gk104_pci_new(struct nvkm_device *, int, struct nvkm_pci **);
+int gp100_pci_new(struct nvkm_device *, int, struct nvkm_pci **);
 
 /* pcie functions */
 int nvkm_pcie_set_link(struct nvkm_pci *, enum nvkm_pcie_speed, u8 width);
diff --git a/drivers/gpu/drm/nouveau/include/nvkm/subdev/secboot.h b/drivers/gpu/drm/nouveau/include/nvkm/subdev/secboot.h
index c6edd95a5b69..b04c38c07761 100644
--- a/drivers/gpu/drm/nouveau/include/nvkm/subdev/secboot.h
+++ b/drivers/gpu/drm/nouveau/include/nvkm/subdev/secboot.h
@@ -43,9 +43,8 @@ struct nvkm_secboot {
         const struct nvkm_secboot_func *func;
         struct nvkm_subdev subdev;
 
+        enum nvkm_devidx devidx;
         u32 base;
-        u32 irq_mask;
-        u32 enable_mask;
 };
 #define nvkm_secboot(p) container_of((p), struct nvkm_secboot, subdev)
 
diff --git a/drivers/gpu/drm/nouveau/include/nvkm/subdev/top.h b/drivers/gpu/drm/nouveau/include/nvkm/subdev/top.h
index 8fb575a92c48..71ebbfd4484f 100644
--- a/drivers/gpu/drm/nouveau/include/nvkm/subdev/top.h
+++ b/drivers/gpu/drm/nouveau/include/nvkm/subdev/top.h
@@ -8,10 +8,11 @@ struct nvkm_top {
         struct list_head device;
 };
 
-u32 nvkm_top_reset(struct nvkm_top *, enum nvkm_devidx);
-u32 nvkm_top_intr(struct nvkm_top *, u32 intr, u64 *subdevs);
-enum nvkm_devidx nvkm_top_fault(struct nvkm_top *, int fault);
-enum nvkm_devidx nvkm_top_engine(struct nvkm_top *, int, int *runl, int *engn);
+u32 nvkm_top_reset(struct nvkm_device *, enum nvkm_devidx);
+u32 nvkm_top_intr(struct nvkm_device *, u32 intr, u64 *subdevs);
+u32 nvkm_top_intr_mask(struct nvkm_device *, enum nvkm_devidx);
+enum nvkm_devidx nvkm_top_fault(struct nvkm_device *, int fault);
+enum nvkm_devidx nvkm_top_engine(struct nvkm_device *, int, int *runl, int *engn);
 
 int gk104_top_new(struct nvkm_device *, int, struct nvkm_top **);
 #endif
diff --git a/drivers/gpu/drm/nouveau/include/nvkm/subdev/volt.h b/drivers/gpu/drm/nouveau/include/nvkm/subdev/volt.h
index feff55cff05b..b765f4ffcde6 100644
--- a/drivers/gpu/drm/nouveau/include/nvkm/subdev/volt.h
+++ b/drivers/gpu/drm/nouveau/include/nvkm/subdev/volt.h
@@ -12,6 +12,9 @@ struct nvkm_volt {
                 u32 uv;
                 u8 vid;
         } vid[256];
+
+        u32 max_uv;
+        u32 min_uv;
 };
 
 int nvkm_volt_get(struct nvkm_volt *);
diff --git a/drivers/gpu/drm/nouveau/nouveau_abi16.c b/drivers/gpu/drm/nouveau/nouveau_abi16.c
index eb7de487a2b3..7bd4683216d0 100644
--- a/drivers/gpu/drm/nouveau/nouveau_abi16.c
+++ b/drivers/gpu/drm/nouveau/nouveau_abi16.c
@@ -100,6 +100,7 @@ nouveau_abi16_swclass(struct nouveau_drm *drm)
         case NV_DEVICE_INFO_V0_FERMI:
         case NV_DEVICE_INFO_V0_KEPLER:
         case NV_DEVICE_INFO_V0_MAXWELL:
+        case NV_DEVICE_INFO_V0_PASCAL:
                 return NVIF_CLASS_SW_GF100;
         }
 
diff --git a/drivers/gpu/drm/nouveau/nouveau_acpi.c b/drivers/gpu/drm/nouveau/nouveau_acpi.c
index db76b94e6e26..f2ad17aa33f0 100644
--- a/drivers/gpu/drm/nouveau/nouveau_acpi.c
+++ b/drivers/gpu/drm/nouveau/nouveau_acpi.c
@@ -45,6 +45,8 @@
 static struct nouveau_dsm_priv {
         bool dsm_detected;
         bool optimus_detected;
+        bool optimus_flags_detected;
+        bool optimus_skip_dsm;
         acpi_handle dhandle;
         acpi_handle rom_handle;
 } nouveau_dsm_priv;
@@ -57,9 +59,6 @@ bool nouveau_is_v1_dsm(void) {
         return nouveau_dsm_priv.dsm_detected;
 }
 
-#define NOUVEAU_DSM_HAS_MUX 0x1
-#define NOUVEAU_DSM_HAS_OPT 0x2
-
 #ifdef CONFIG_VGA_SWITCHEROO
 static const char nouveau_dsm_muid[] = {
         0xA0, 0xA0, 0x95, 0x9D, 0x60, 0x00, 0x48, 0x4D,
@@ -110,7 +109,7 @@ static int nouveau_optimus_dsm(acpi_handle handle, int func, int arg, uint32_t *
  * requirements on the fourth parameter, so a private implementation
  * instead of using acpi_check_dsm().
  */
-static int nouveau_check_optimus_dsm(acpi_handle handle)
+static int nouveau_dsm_get_optimus_functions(acpi_handle handle)
 {
         int result;
 
@@ -125,7 +124,9 @@ static int nouveau_check_optimus_dsm(acpi_handle handle)
          * ACPI Spec v4 9.14.1: if bit 0 is zero, no function is supported.
          * If the n-th bit is enabled, function n is supported
          */
-        return result & 1 && result & (1 << NOUVEAU_DSM_OPTIMUS_CAPS);
+        if (result & 1 && result & (1 << NOUVEAU_DSM_OPTIMUS_CAPS))
+                return result;
+        return 0;
 }
 
 static int nouveau_dsm(acpi_handle handle, int func, int arg)
@@ -212,26 +213,55 @@ static const struct vga_switcheroo_handler nouveau_dsm_handler = {
         .get_client_id = nouveau_dsm_get_client_id,
 };
 
-static int nouveau_dsm_pci_probe(struct pci_dev *pdev)
+/*
+ * Firmware supporting Windows 8 or later do not use _DSM to put the device into
+ * D3cold, they instead rely on disabling power resources on the parent.
+ */
+static bool nouveau_pr3_present(struct pci_dev *pdev)
+{
+        struct pci_dev *parent_pdev = pci_upstream_bridge(pdev);
+        struct acpi_device *parent_adev;
+
+        if (!parent_pdev)
+                return false;
+
+        parent_adev = ACPI_COMPANION(&parent_pdev->dev);
+        if (!parent_adev)
+                return false;
+
+        return acpi_has_method(parent_adev->handle, "_PR3");
+}
+
+static void nouveau_dsm_pci_probe(struct pci_dev *pdev, acpi_handle *dhandle_out,
+                                  bool *has_mux, bool *has_opt,
+                                  bool *has_opt_flags, bool *has_pr3)
 {
         acpi_handle dhandle;
-        int retval = 0;
+        bool supports_mux;
+        int optimus_funcs;
 
         dhandle = ACPI_HANDLE(&pdev->dev);
         if (!dhandle)
-                return false;
+                return;
 
         if (!acpi_has_method(dhandle, "_DSM"))
-                return false;
+                return;
+
+        supports_mux = acpi_check_dsm(dhandle, nouveau_dsm_muid, 0x00000102,
+                                      1 << NOUVEAU_DSM_POWER);
+        optimus_funcs = nouveau_dsm_get_optimus_functions(dhandle);
 
-        if (acpi_check_dsm(dhandle, nouveau_dsm_muid, 0x00000102,
-                           1 << NOUVEAU_DSM_POWER))
-                retval |= NOUVEAU_DSM_HAS_MUX;
+        /* Does not look like a Nvidia device. */
+        if (!supports_mux && !optimus_funcs)
+                return;
 
-        if (nouveau_check_optimus_dsm(dhandle))
-                retval |= NOUVEAU_DSM_HAS_OPT;
+        *dhandle_out = dhandle;
+        *has_mux = supports_mux;
+        *has_opt = !!optimus_funcs;
+        *has_opt_flags = optimus_funcs & (1 << NOUVEAU_DSM_OPTIMUS_FLAGS);
+        *has_pr3 = false;
 
-        if (retval & NOUVEAU_DSM_HAS_OPT) {
+        if (optimus_funcs) {
                 uint32_t result;
                 nouveau_optimus_dsm(dhandle, NOUVEAU_DSM_OPTIMUS_CAPS, 0,
                                     &result);
@@ -239,11 +269,9 @@ static int nouveau_dsm_pci_probe(struct pci_dev *pdev)
                          (result & OPTIMUS_ENABLED) ? "enabled" : "disabled",
                          (result & OPTIMUS_DYNAMIC_PWR_CAP) ? "dynamic power, " : "",
                          (result & OPTIMUS_HDA_CODEC_MASK) ? "hda bios codec supported" : "");
-        }
-        if (retval)
-                nouveau_dsm_priv.dhandle = dhandle;
 
-        return retval;
+                *has_pr3 = nouveau_pr3_present(pdev);
+        }
 }
 
 static bool nouveau_dsm_detect(void)
@@ -251,11 +279,13 @@ static bool nouveau_dsm_detect(void)
         char acpi_method_name[255] = { 0 };
         struct acpi_buffer buffer = {sizeof(acpi_method_name), acpi_method_name};
         struct pci_dev *pdev = NULL;
-        int has_dsm = 0;
-        int has_optimus = 0;
+        acpi_handle dhandle = NULL;
+        bool has_mux = false;
+        bool has_optimus = false;
+        bool has_optimus_flags = false;
+        bool has_power_resources = false;
         int vga_count = 0;
         bool guid_valid;
-        int retval;
         bool ret = false;
 
         /* lookup the MXM GUID */
@@ -268,32 +298,32 @@ static bool nouveau_dsm_detect(void)
         while ((pdev = pci_get_class(PCI_CLASS_DISPLAY_VGA << 8, pdev)) != NULL) {
                 vga_count++;
 
-                retval = nouveau_dsm_pci_probe(pdev);
-                if (retval & NOUVEAU_DSM_HAS_MUX)
-                        has_dsm |= 1;
-                if (retval & NOUVEAU_DSM_HAS_OPT)
-                        has_optimus = 1;
+                nouveau_dsm_pci_probe(pdev, &dhandle, &has_mux, &has_optimus,
+                                      &has_optimus_flags, &has_power_resources);
         }
 
         while ((pdev = pci_get_class(PCI_CLASS_DISPLAY_3D << 8, pdev)) != NULL) {
                 vga_count++;
 
-                retval = nouveau_dsm_pci_probe(pdev);
-                if (retval & NOUVEAU_DSM_HAS_MUX)
-                        has_dsm |= 1;
-                if (retval & NOUVEAU_DSM_HAS_OPT)
-                        has_optimus = 1;
+                nouveau_dsm_pci_probe(pdev, &dhandle, &has_mux, &has_optimus,
+                                      &has_optimus_flags, &has_power_resources);
         }
 
         /* find the optimus DSM or the old v1 DSM */
-        if (has_optimus == 1) {
+        if (has_optimus) {
+                nouveau_dsm_priv.dhandle = dhandle;
                 acpi_get_name(nouveau_dsm_priv.dhandle, ACPI_FULL_PATHNAME,
                         &buffer);
                 printk(KERN_INFO "VGA switcheroo: detected Optimus DSM method %s handle\n",
                         acpi_method_name);
+                if (has_power_resources)
+                        pr_info("nouveau: detected PR support, will not use DSM\n");
                 nouveau_dsm_priv.optimus_detected = true;
+                nouveau_dsm_priv.optimus_flags_detected = has_optimus_flags;
+                nouveau_dsm_priv.optimus_skip_dsm = has_power_resources;
                 ret = true;
-        } else if (vga_count == 2 && has_dsm && guid_valid) {
+        } else if (vga_count == 2 && has_mux && guid_valid) {
+                nouveau_dsm_priv.dhandle = dhandle;
                 acpi_get_name(nouveau_dsm_priv.dhandle, ACPI_FULL_PATHNAME,
                         &buffer);
                 printk(KERN_INFO "VGA switcheroo: detected DSM switching method %s handle\n",
@@ -321,11 +351,12 @@ void nouveau_register_dsm_handler(void)
 void nouveau_switcheroo_optimus_dsm(void)
 {
         u32 result = 0;
-        if (!nouveau_dsm_priv.optimus_detected)
+        if (!nouveau_dsm_priv.optimus_detected || nouveau_dsm_priv.optimus_skip_dsm)
                 return;
 
-        nouveau_optimus_dsm(nouveau_dsm_priv.dhandle, NOUVEAU_DSM_OPTIMUS_FLAGS,
-                            0x3, &result);
+        if (nouveau_dsm_priv.optimus_flags_detected)
+                nouveau_optimus_dsm(nouveau_dsm_priv.dhandle, NOUVEAU_DSM_OPTIMUS_FLAGS,
+                                    0x3, &result);
 
         nouveau_optimus_dsm(nouveau_dsm_priv.dhandle, NOUVEAU_DSM_OPTIMUS_CAPS,
                 NOUVEAU_DSM_OPTIMUS_SET_POWERDOWN, &result);
diff --git a/drivers/gpu/drm/nouveau/nouveau_bo.c b/drivers/gpu/drm/nouveau/nouveau_bo.c
index 5e3f3e826476..528bdeffb339 100644
--- a/drivers/gpu/drm/nouveau/nouveau_bo.c
+++ b/drivers/gpu/drm/nouveau/nouveau_bo.c
@@ -209,8 +209,7 @@ nouveau_bo_new(struct drm_device *dev, int size, int align,
         nvbo->tile_flags = tile_flags;
         nvbo->bo.bdev = &drm->ttm.bdev;
 
-        if (!nvxx_device(&drm->device)->func->cpu_coherent)
-                nvbo->force_coherent = flags & TTM_PL_FLAG_UNCACHED;
+        nvbo->force_coherent = flags & TTM_PL_FLAG_UNCACHED;
 
         nvbo->page_shift = 12;
         if (drm->client.vm) {
@@ -424,13 +423,7 @@ nouveau_bo_map(struct nouveau_bo *nvbo)
         if (ret)
                 return ret;
 
-        /*
-         * TTM buffers allocated using the DMA API already have a mapping, let's
-         * use it instead.
-         */
-        if (!nvbo->force_coherent)
-                ret = ttm_bo_kmap(&nvbo->bo, 0, nvbo->bo.mem.num_pages,
-                                  &nvbo->kmap);
+        ret = ttm_bo_kmap(&nvbo->bo, 0, nvbo->bo.mem.num_pages, &nvbo->kmap);
 
         ttm_bo_unreserve(&nvbo->bo);
         return ret;
@@ -442,12 +435,7 @@ nouveau_bo_unmap(struct nouveau_bo *nvbo)
         if (!nvbo)
                 return;
 
-        /*
-         * TTM buffers allocated using the DMA API already had a coherent
-         * mapping which we used, no need to unmap.
-         */
-        if (!nvbo->force_coherent)
-                ttm_bo_kunmap(&nvbo->kmap);
+        ttm_bo_kunmap(&nvbo->kmap);
 }
 
 void
@@ -506,35 +494,13 @@ nouveau_bo_validate(struct nouveau_bo *nvbo, bool interruptible,
         return 0;
 }
 
-static inline void *
-_nouveau_bo_mem_index(struct nouveau_bo *nvbo, unsigned index, void *mem, u8 sz)
-{
-        struct ttm_dma_tt *dma_tt;
-        u8 *m = mem;
-
-        index *= sz;
-
-        if (m) {
-                /* kmap'd address, return the corresponding offset */
-                m += index;
-        } else {
-                /* DMA-API mapping, lookup the right address */
-                dma_tt = (struct ttm_dma_tt *)nvbo->bo.ttm;
-                m = dma_tt->cpu_address[index / PAGE_SIZE];
-                m += index % PAGE_SIZE;
-        }
-
-        return m;
-}
-#define nouveau_bo_mem_index(o, i, m) _nouveau_bo_mem_index(o, i, m, sizeof(*m))
-
 void
 nouveau_bo_wr16(struct nouveau_bo *nvbo, unsigned index, u16 val)
 {
         bool is_iomem;
         u16 *mem = ttm_kmap_obj_virtual(&nvbo->kmap, &is_iomem);
 
-        mem = nouveau_bo_mem_index(nvbo, index, mem);
+        mem += index;
 
         if (is_iomem)
                 iowrite16_native(val, (void __force __iomem *)mem);
@@ -548,7 +514,7 @@ nouveau_bo_rd32(struct nouveau_bo *nvbo, unsigned index)
         bool is_iomem;
         u32 *mem = ttm_kmap_obj_virtual(&nvbo->kmap, &is_iomem);
 
-        mem = nouveau_bo_mem_index(nvbo, index, mem);
+        mem += index;
 
         if (is_iomem)
                 return ioread32_native((void __force __iomem *)mem);
@@ -562,7 +528,7 @@ nouveau_bo_wr32(struct nouveau_bo *nvbo, unsigned index, u32 val)
         bool is_iomem;
         u32 *mem = ttm_kmap_obj_virtual(&nvbo->kmap, &is_iomem);
 
-        mem = nouveau_bo_mem_index(nvbo, index, mem);
+        mem += index;
 
         if (is_iomem)
                 iowrite32_native(val, (void __force __iomem *)mem);
@@ -1082,7 +1048,6 @@ nouveau_bo_move_m2mf(struct ttm_buffer_object *bo, int evict, bool intr,
                                 ret = ttm_bo_move_accel_cleanup(bo,
                                                                 &fence->base,
                                                                 evict,
-                                                                no_wait_gpu,
                                                                 new_mem);
                                 nouveau_fence_unref(&fence);
                         }
@@ -1104,6 +1069,10 @@ nouveau_bo_move_init(struct nouveau_drm *drm)
                             struct ttm_mem_reg *, struct ttm_mem_reg *);
                 int (*init)(struct nouveau_channel *, u32 handle);
         } _methods[] = {
+                {  "COPY", 4, 0xc1b5, nve0_bo_move_copy, nve0_bo_move_init },
+                {  "GRCE", 0, 0xc1b5, nve0_bo_move_copy, nvc0_bo_move_init },
+                {  "COPY", 4, 0xc0b5, nve0_bo_move_copy, nve0_bo_move_init },
+                {  "GRCE", 0, 0xc0b5, nve0_bo_move_copy, nvc0_bo_move_init },
                 {  "COPY", 4, 0xb0b5, nve0_bo_move_copy, nve0_bo_move_init },
                 {  "GRCE", 0, 0xb0b5, nve0_bo_move_copy, nvc0_bo_move_init },
                 {  "COPY", 4, 0xa0b5, nve0_bo_move_copy, nve0_bo_move_init },
@@ -1289,6 +1258,10 @@ nouveau_bo_move(struct ttm_buffer_object *bo, bool evict, bool intr,
         struct nouveau_drm_tile *new_tile = NULL;
         int ret = 0;
 
+        ret = ttm_bo_wait(bo, intr, no_wait_gpu);
+        if (ret)
+                return ret;
+
         if (nvbo->pin_refcnt)
                 NV_WARN(drm, "Moving pinned object %p!\n", nvbo);
 
@@ -1324,7 +1297,7 @@ nouveau_bo_move(struct ttm_buffer_object *bo, bool evict, bool intr,
         /* Fallback to software copy. */
         ret = ttm_bo_wait(bo, intr, no_wait_gpu);
         if (ret == 0)
-                ret = ttm_bo_move_memcpy(bo, evict, no_wait_gpu, new_mem);
+                ret = ttm_bo_move_memcpy(bo, evict, intr, no_wait_gpu, new_mem);
 
 out:
         if (drm->device.info.family < NV_DEVICE_INFO_V0_TESLA) {
@@ -1488,14 +1461,6 @@ nouveau_ttm_tt_populate(struct ttm_tt *ttm)
         dev = drm->dev;
         pdev = device->dev;
 
-        /*
-         * Objects matching this condition have been marked as force_coherent,
-         * so use the DMA API for them.
-         */
-        if (!nvxx_device(&drm->device)->func->cpu_coherent &&
-            ttm->caching_state == tt_uncached)
-                return ttm_dma_populate(ttm_dma, dev->dev);
-
 #if IS_ENABLED(CONFIG_AGP)
         if (drm->agp.bridge) {
                 return ttm_agp_tt_populate(ttm);
@@ -1553,16 +1518,6 @@ nouveau_ttm_tt_unpopulate(struct ttm_tt *ttm)
         dev = drm->dev;
         pdev = device->dev;
 
-        /*
-         * Objects matching this condition have been marked as force_coherent,
-         * so use the DMA API for them.
-         */
-        if (!nvxx_device(&drm->device)->func->cpu_coherent &&
-            ttm->caching_state == tt_uncached) {
-                ttm_dma_unpopulate(ttm_dma, dev->dev);
-                return;
-        }
-
 #if IS_ENABLED(CONFIG_AGP)
         if (drm->agp.bridge) {
                 ttm_agp_tt_unpopulate(ttm);
diff --git a/drivers/gpu/drm/nouveau/nouveau_chan.c b/drivers/gpu/drm/nouveau/nouveau_chan.c
index b1d2527c5625..f9b3c811187e 100644
--- a/drivers/gpu/drm/nouveau/nouveau_chan.c
+++ b/drivers/gpu/drm/nouveau/nouveau_chan.c
@@ -191,7 +191,8 @@ static int
 nouveau_channel_ind(struct nouveau_drm *drm, struct nvif_device *device,
                     u32 engine, struct nouveau_channel **pchan)
 {
-        static const u16 oclasses[] = { MAXWELL_CHANNEL_GPFIFO_A,
+        static const u16 oclasses[] = { PASCAL_CHANNEL_GPFIFO_A,
+                                        MAXWELL_CHANNEL_GPFIFO_A,
                                         KEPLER_CHANNEL_GPFIFO_B,
                                         KEPLER_CHANNEL_GPFIFO_A,
                                         FERMI_CHANNEL_GPFIFO,
diff --git a/drivers/gpu/drm/nouveau/nouveau_display.c b/drivers/gpu/drm/nouveau/nouveau_display.c
index 7c77f960c8b8..afbf557b23d4 100644
--- a/drivers/gpu/drm/nouveau/nouveau_display.c
+++ b/drivers/gpu/drm/nouveau/nouveau_display.c
@@ -47,7 +47,7 @@ nouveau_display_vblank_handler(struct nvif_notify *notify)
 {
         struct nouveau_crtc *nv_crtc =
                 container_of(notify, typeof(*nv_crtc), vblank);
-        drm_handle_vblank(nv_crtc->base.dev, nv_crtc->index);
+        drm_crtc_handle_vblank(&nv_crtc->base);
         return NVIF_NOTIFY_KEEP;
 }
 
@@ -495,6 +495,8 @@ nouveau_display_create(struct drm_device *dev)
 
         if (nouveau_modeset != 2 && drm->vbios.dcb.entries) {
                 static const u16 oclass[] = {
+                        GP104_DISP,
+                        GP100_DISP,
                         GM200_DISP,
                         GM107_DISP,
                         GK110_DISP,
@@ -554,6 +556,7 @@ nouveau_display_destroy(struct drm_device *dev)
         nouveau_display_vblank_fini(dev);
 
         drm_kms_helper_poll_fini(dev);
+        drm_crtc_force_disable_all(dev);
         drm_mode_config_cleanup(dev);
 
         if (disp->dtor)
@@ -760,12 +763,11 @@ nouveau_crtc_page_flip(struct drm_crtc *crtc, struct drm_framebuffer *fb,
 
         /* Initialize a page flip struct */
         *s = (struct nouveau_page_flip_state)
-                { { }, event, nouveau_crtc(crtc)->index,
-                  fb->bits_per_pixel, fb->pitches[0], crtc->x, crtc->y,
+                { { }, event, crtc, fb->bits_per_pixel, fb->pitches[0],
                   new_bo->bo.offset };
 
         /* Keep vblanks on during flip, for the target crtc of this flip */
-        drm_vblank_get(dev, nouveau_crtc(crtc)->index);
+        drm_crtc_vblank_get(crtc);
 
         /* Emit a page flip */
         if (drm->device.info.family >= NV_DEVICE_INFO_V0_TESLA) {
@@ -810,7 +812,7 @@ nouveau_crtc_page_flip(struct drm_crtc *crtc, struct drm_framebuffer *fb,
         return 0;
 
 fail_unreserve:
-        drm_vblank_put(dev, nouveau_crtc(crtc)->index);
+        drm_crtc_vblank_put(crtc);
         ttm_bo_unreserve(&old_bo->bo);
 fail_unpin:
         mutex_unlock(&cli->mutex);
@@ -842,17 +844,17 @@ nouveau_finish_page_flip(struct nouveau_channel *chan,
         s = list_first_entry(&fctx->flip, struct nouveau_page_flip_state, head);
         if (s->event) {
                 if (drm->device.info.family < NV_DEVICE_INFO_V0_TESLA) {
-                        drm_arm_vblank_event(dev, s->crtc, s->event);
+                        drm_crtc_arm_vblank_event(s->crtc, s->event);
                 } else {
-                        drm_send_vblank_event(dev, s->crtc, s->event);
+                        drm_crtc_send_vblank_event(s->crtc, s->event);
 
                         /* Give up ownership of vblank for page-flipped crtc */
-                        drm_vblank_put(dev, s->crtc);
+                        drm_crtc_vblank_put(s->crtc);
                 }
         }
         else {
                 /* Give up ownership of vblank for page-flipped crtc */
-                drm_vblank_put(dev, s->crtc);
+                drm_crtc_vblank_put(s->crtc);
         }
 
         list_del(&s->head);
@@ -873,9 +875,10 @@ nouveau_flip_complete(struct nvif_notify *notify)
 
         if (!nouveau_finish_page_flip(chan, &state)) {
                 if (drm->device.info.family < NV_DEVICE_INFO_V0_TESLA) {
-                        nv_set_crtc_base(drm->dev, state.crtc, state.offset +
-                                         state.y * state.pitch +
-                                         state.x * state.bpp / 8);
+                        nv_set_crtc_base(drm->dev, drm_crtc_index(state.crtc),
+                                         state.offset + state.crtc->y *
+                                         state.pitch + state.crtc->x *
+                                         state.bpp / 8);
                 }
         }
 
diff --git a/drivers/gpu/drm/nouveau/nouveau_display.h b/drivers/gpu/drm/nouveau/nouveau_display.h
index 24273bacd885..0420ee861ea4 100644
--- a/drivers/gpu/drm/nouveau/nouveau_display.h
+++ b/drivers/gpu/drm/nouveau/nouveau_display.h
@@ -28,7 +28,8 @@ int nouveau_framebuffer_init(struct drm_device *, struct nouveau_framebuffer *,
 struct nouveau_page_flip_state {
         struct list_head head;
         struct drm_pending_vblank_event *event;
-        int crtc, bpp, pitch, x, y;
+        struct drm_crtc *crtc;
+        int bpp, pitch;
         u64 offset;
 };
 
diff --git a/drivers/gpu/drm/nouveau/nouveau_drm.c b/drivers/gpu/drm/nouveau/nouveau_drm.c
index 11f8dd9c0edb..66c1280c0f1f 100644
--- a/drivers/gpu/drm/nouveau/nouveau_drm.c
+++ b/drivers/gpu/drm/nouveau/nouveau_drm.c
@@ -22,13 +22,11 @@
  * Authors: Ben Skeggs
  */
 
-#include <linux/apple-gmux.h>
 #include <linux/console.h>
 #include <linux/delay.h>
 #include <linux/module.h>
 #include <linux/pci.h>
 #include <linux/pm_runtime.h>
-#include <linux/vgaarb.h>
 #include <linux/vga_switcheroo.h>
 
 #include "drmP.h"
@@ -200,6 +198,7 @@ nouveau_accel_init(struct nouveau_drm *drm)
                 case KEPLER_CHANNEL_GPFIFO_A:
                 case KEPLER_CHANNEL_GPFIFO_B:
                 case MAXWELL_CHANNEL_GPFIFO_A:
+                case PASCAL_CHANNEL_GPFIFO_A:
                         ret = nvc0_fence_create(drm);
                         break;
                 default:
@@ -315,16 +314,19 @@ static int nouveau_drm_probe(struct pci_dev *pdev,
         bool boot = false;
         int ret;
 
-        /*
-         * apple-gmux is needed on dual GPU MacBook Pro
-         * to probe the panel if we're the inactive GPU.
-         */
-        if (IS_ENABLED(CONFIG_VGA_ARB) && IS_ENABLED(CONFIG_VGA_SWITCHEROO) &&
-            apple_gmux_present() && pdev != vga_default_device() &&
-            !vga_switcheroo_handler_flags())
+        if (vga_switcheroo_client_probe_defer(pdev))
                 return -EPROBE_DEFER;
 
-        /* remove conflicting drivers (vesafb, efifb etc) */
+        /* We need to check that the chipset is supported before booting
+         * fbdev off the hardware, as there's no way to put it back.
+         */
+        ret = nvkm_device_pci_new(pdev, NULL, "error", true, false, 0, &device);
+        if (ret)
+                return ret;
+
+        nvkm_device_del(&device);
+
+        /* Remove conflicting drivers (vesafb, efifb etc). */
         aper = alloc_apertures(3);
         if (!aper)
                 return -ENOMEM;
@@ -438,6 +440,11 @@ nouveau_drm_load(struct drm_device *dev, unsigned long flags)
         nouveau_vga_init(drm);
 
         if (drm->device.info.family >= NV_DEVICE_INFO_V0_TESLA) {
+                if (!nvxx_device(&drm->device)->mmu) {
+                        ret = -ENOSYS;
+                        goto fail_device;
+                }
+
                 ret = nvkm_vm_new(nvxx_device(&drm->device), 0, (1ULL << 40),
                                   0x1000, NULL, &drm->client.vm);
                 if (ret)
@@ -498,7 +505,11 @@ nouveau_drm_unload(struct drm_device *dev)
 {
         struct nouveau_drm *drm = nouveau_drm(dev);
 
-        pm_runtime_get_sync(dev->dev);
+        if (nouveau_runtime_pm != 0) {
+                pm_runtime_get_sync(dev->dev);
+                pm_runtime_forbid(dev->dev);
+        }
+
         nouveau_fbcon_fini(dev);
         nouveau_accel_fini(drm);
         nouveau_hwmon_fini(dev);
@@ -970,7 +981,7 @@ driver_stub = {
         .gem_prime_vmap = nouveau_gem_prime_vmap,
         .gem_prime_vunmap = nouveau_gem_prime_vunmap,
 
-        .gem_free_object = nouveau_gem_object_del,
+        .gem_free_object_unlocked = nouveau_gem_object_del,
         .gem_open_object = nouveau_gem_object_open,
         .gem_close_object = nouveau_gem_object_close,
 
@@ -1078,7 +1089,6 @@ nouveau_drm_init(void)
         driver_pci = driver_stub;
         driver_pci.set_busid = drm_pci_set_busid;
         driver_platform = driver_stub;
-        driver_platform.set_busid = drm_platform_set_busid;
 
         nouveau_display_options();
 
diff --git a/drivers/gpu/drm/nouveau/nouveau_fence.h b/drivers/gpu/drm/nouveau/nouveau_fence.h
index 2e3a62d38fe9..64c4ce7115ad 100644
--- a/drivers/gpu/drm/nouveau/nouveau_fence.h
+++ b/drivers/gpu/drm/nouveau/nouveau_fence.h
@@ -57,7 +57,8 @@ struct nouveau_fence_priv {
         int  (*context_new)(struct nouveau_channel *);
         void (*context_del)(struct nouveau_channel *);
 
-        u32 contexts, context_base;
+        u32 contexts;
+        u64 context_base;
         bool uevent;
 };
 
diff --git a/drivers/gpu/drm/nouveau/nouveau_hwmon.c b/drivers/gpu/drm/nouveau/nouveau_hwmon.c
index 1ff4166af26e..71f764bf4cc6 100644
--- a/drivers/gpu/drm/nouveau/nouveau_hwmon.c
+++ b/drivers/gpu/drm/nouveau/nouveau_hwmon.c
@@ -535,6 +535,40 @@ static SENSOR_DEVICE_ATTR(in0_input, S_IRUGO,
                           nouveau_hwmon_get_in0_input, NULL, 0);
 
 static ssize_t
+nouveau_hwmon_get_in0_min(struct device *d,
+                            struct device_attribute *a, char *buf)
+{
+        struct drm_device *dev = dev_get_drvdata(d);
+        struct nouveau_drm *drm = nouveau_drm(dev);
+        struct nvkm_volt *volt = nvxx_volt(&drm->device);
+
+        if (!volt || !volt->min_uv)
+                return -ENODEV;
+
+        return sprintf(buf, "%i\n", volt->min_uv / 1000);
+}
+
+static SENSOR_DEVICE_ATTR(in0_min, S_IRUGO,
+                          nouveau_hwmon_get_in0_min, NULL, 0);
+
+static ssize_t
+nouveau_hwmon_get_in0_max(struct device *d,
+                            struct device_attribute *a, char *buf)
+{
+        struct drm_device *dev = dev_get_drvdata(d);
+        struct nouveau_drm *drm = nouveau_drm(dev);
+        struct nvkm_volt *volt = nvxx_volt(&drm->device);
+
+        if (!volt || !volt->max_uv)
+                return -ENODEV;
+
+        return sprintf(buf, "%i\n", volt->max_uv / 1000);
+}
+
+static SENSOR_DEVICE_ATTR(in0_max, S_IRUGO,
+                          nouveau_hwmon_get_in0_max, NULL, 0);
+
+static ssize_t
 nouveau_hwmon_get_in0_label(struct device *d,
                             struct device_attribute *a, char *buf)
 {
@@ -594,6 +628,8 @@ static struct attribute *hwmon_pwm_fan_attributes[] = {
 
 static struct attribute *hwmon_in0_attributes[] = {
         &sensor_dev_attr_in0_input.dev_attr.attr,
+        &sensor_dev_attr_in0_min.dev_attr.attr,
+        &sensor_dev_attr_in0_max.dev_attr.attr,
         &sensor_dev_attr_in0_label.dev_attr.attr,
         NULL
 };
diff --git a/drivers/gpu/drm/nouveau/nouveau_ttm.c b/drivers/gpu/drm/nouveau/nouveau_ttm.c
index bcee91497eb9..1825dbc33192 100644
--- a/drivers/gpu/drm/nouveau/nouveau_ttm.c
+++ b/drivers/gpu/drm/nouveau/nouveau_ttm.c
@@ -164,6 +164,7 @@ nouveau_gart_manager_new(struct ttm_mem_type_manager *man,
         case NV_DEVICE_INFO_V0_FERMI:
         case NV_DEVICE_INFO_V0_KEPLER:
         case NV_DEVICE_INFO_V0_MAXWELL:
+        case NV_DEVICE_INFO_V0_PASCAL:
                 node->memtype = (nvbo->tile_flags & 0xff00) >> 8;
                 break;
         default:
diff --git a/drivers/gpu/drm/nouveau/nouveau_usif.c b/drivers/gpu/drm/nouveau/nouveau_usif.c
index 675e9e077a95..08f9c6fa0f7f 100644
--- a/drivers/gpu/drm/nouveau/nouveau_usif.c
+++ b/drivers/gpu/drm/nouveau/nouveau_usif.c
@@ -212,7 +212,6 @@ usif_notify_get(struct drm_file *f, void *data, u32 size, void *argv, u32 argc)
         ntfy->p->base.event = &ntfy->p->e.base;
         ntfy->p->base.file_priv = f;
         ntfy->p->base.pid = current->pid;
-        ntfy->p->base.destroy =(void(*)(struct drm_pending_event *))kfree;
         ntfy->p->e.base.type = DRM_NOUVEAU_EVENT_NVIF;
         ntfy->p->e.base.length = sizeof(ntfy->p->e.base) + ntfy->reply;
 
diff --git a/drivers/gpu/drm/nouveau/nv04_fbcon.c b/drivers/gpu/drm/nouveau/nv04_fbcon.c
index 7d9248b8c664..da8fd5ff9d0f 100644
--- a/drivers/gpu/drm/nouveau/nv04_fbcon.c
+++ b/drivers/gpu/drm/nouveau/nv04_fbcon.c
@@ -107,11 +107,11 @@ nv04_fbcon_imageblit(struct fb_info *info, const struct fb_image *image)
                          ((image->dx + image->width) & 0xffff));
         OUT_RING(chan, bg);
         OUT_RING(chan, fg);
-        OUT_RING(chan, (image->height << 16) | image->width);
+        OUT_RING(chan, (image->height << 16) | ALIGN(image->width, 8));
         OUT_RING(chan, (image->height << 16) | image->width);
         OUT_RING(chan, (image->dy << 16) | (image->dx & 0xffff));
 
-        dsize = ALIGN(image->width * image->height, 32) >> 5;
+        dsize = ALIGN(ALIGN(image->width, 8) * image->height, 32) >> 5;
         while (dsize) {
                 int iter_len = dsize > 128 ? 128 : dsize;
 
diff --git a/drivers/gpu/drm/nouveau/nv50_display.c b/drivers/gpu/drm/nouveau/nv50_display.c
index 3ffc2b0057bf..7d0edcbcfca7 100644
--- a/drivers/gpu/drm/nouveau/nv50_display.c
+++ b/drivers/gpu/drm/nouveau/nv50_display.c
@@ -297,6 +297,8 @@ nv50_core_create(struct nvif_device *device, struct nvif_object *disp,
                 .pushbuf = 0xb0007d00,
         };
         static const s32 oclass[] = {
+                GP104_DISP_CORE_CHANNEL_DMA,
+                GP100_DISP_CORE_CHANNEL_DMA,
                 GM200_DISP_CORE_CHANNEL_DMA,
                 GM107_DISP_CORE_CHANNEL_DMA,
                 GK110_DISP_CORE_CHANNEL_DMA,
@@ -1346,21 +1348,22 @@ nv50_crtc_cursor_move(struct drm_crtc *crtc, int x, int y)
         return 0;
 }
 
-static void
+static int
 nv50_crtc_gamma_set(struct drm_crtc *crtc, u16 *r, u16 *g, u16 *b,
-                    uint32_t start, uint32_t size)
+                    uint32_t size)
 {
         struct nouveau_crtc *nv_crtc = nouveau_crtc(crtc);
-        u32 end = min_t(u32, start + size, 256);
         u32 i;
 
-        for (i = start; i < end; i++) {
+        for (i = 0; i < size; i++) {
                 nv_crtc->lut.r[i] = r[i];
                 nv_crtc->lut.g[i] = g[i];
                 nv_crtc->lut.b[i] = b[i];
         }
 
         nv50_crtc_lut_load(crtc);
+
+        return 0;
 }
 
 static void
diff --git a/drivers/gpu/drm/nouveau/nv50_fbcon.c b/drivers/gpu/drm/nouveau/nv50_fbcon.c
index 1aeb698e9707..af3d3c49411a 100644
--- a/drivers/gpu/drm/nouveau/nv50_fbcon.c
+++ b/drivers/gpu/drm/nouveau/nv50_fbcon.c
@@ -125,7 +125,7 @@ nv50_fbcon_imageblit(struct fb_info *info, const struct fb_image *image)
         OUT_RING(chan, 0);
         OUT_RING(chan, image->dy);
 
-        dwords = ALIGN(image->width * image->height, 32) >> 5;
+        dwords = ALIGN(ALIGN(image->width, 8) * image->height, 32) >> 5;
         while (dwords) {
                 int push = dwords > 2047 ? 2047 : dwords;
 
diff --git a/drivers/gpu/drm/nouveau/nvc0_fbcon.c b/drivers/gpu/drm/nouveau/nvc0_fbcon.c
index 839f4c8c1805..054b6a056d99 100644
--- a/drivers/gpu/drm/nouveau/nvc0_fbcon.c
+++ b/drivers/gpu/drm/nouveau/nvc0_fbcon.c
@@ -125,7 +125,7 @@ nvc0_fbcon_imageblit(struct fb_info *info, const struct fb_image *image)
         OUT_RING  (chan, 0);
         OUT_RING  (chan, image->dy);
 
-        dwords = ALIGN(image->width * image->height, 32) >> 5;
+        dwords = ALIGN(ALIGN(image->width, 8) * image->height, 32) >> 5;
         while (dwords) {
                 int push = dwords > 2047 ? 2047 : dwords;
 
diff --git a/drivers/gpu/drm/nouveau/nvkm/core/subdev.c b/drivers/gpu/drm/nouveau/nvkm/core/subdev.c
index b18557858f19..19044aba265e 100644
--- a/drivers/gpu/drm/nouveau/nvkm/core/subdev.c
+++ b/drivers/gpu/drm/nouveau/nvkm/core/subdev.c
@@ -57,6 +57,9 @@ nvkm_subdev_name[NVKM_SUBDEV_NR] = {
         [NVKM_ENGINE_CE0     ] = "ce0",
         [NVKM_ENGINE_CE1     ] = "ce1",
         [NVKM_ENGINE_CE2     ] = "ce2",
+        [NVKM_ENGINE_CE3     ] = "ce3",
+        [NVKM_ENGINE_CE4     ] = "ce4",
+        [NVKM_ENGINE_CE5     ] = "ce5",
         [NVKM_ENGINE_CIPHER  ] = "cipher",
         [NVKM_ENGINE_DISP    ] = "disp",
         [NVKM_ENGINE_DMAOBJ  ] = "dma",
@@ -71,6 +74,7 @@ nvkm_subdev_name[NVKM_SUBDEV_NR] = {
         [NVKM_ENGINE_MSVLD   ] = "msvld",
         [NVKM_ENGINE_NVENC0  ] = "nvenc0",
         [NVKM_ENGINE_NVENC1  ] = "nvenc1",
+        [NVKM_ENGINE_NVENC2  ] = "nvenc2",
         [NVKM_ENGINE_NVDEC   ] = "nvdec",
         [NVKM_ENGINE_PM      ] = "pm",
         [NVKM_ENGINE_SEC     ] = "sec",
@@ -105,7 +109,7 @@ nvkm_subdev_fini(struct nvkm_subdev *subdev, bool suspend)
                 }
         }
 
-        nvkm_mc_reset(device->mc, subdev->index);
+        nvkm_mc_reset(device, subdev->index);
 
         time = ktime_to_us(ktime_get()) - time;
         nvkm_trace(subdev, "%s completed in %lldus\n", action, time);
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/ce/Kbuild b/drivers/gpu/drm/nouveau/nvkm/engine/ce/Kbuild
index 9c19d59b47df..a4458a8eb30a 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/ce/Kbuild
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/ce/Kbuild
@@ -3,3 +3,5 @@ nvkm-y += nvkm/engine/ce/gf100.o
 nvkm-y += nvkm/engine/ce/gk104.o
 nvkm-y += nvkm/engine/ce/gm107.o
 nvkm-y += nvkm/engine/ce/gm200.o
+nvkm-y += nvkm/engine/ce/gp100.o
+nvkm-y += nvkm/engine/ce/gp104.o
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/ce/gp100.c b/drivers/gpu/drm/nouveau/nvkm/engine/ce/gp100.c
new file mode 100644
index 000000000000..c7710456bc30
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/ce/gp100.c
@@ -0,0 +1,102 @@
+/*
+ * Copyright 2015 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs
+ */
+#include "priv.h"
+#include <core/enum.h>
+
+#include <nvif/class.h>
+
+static const struct nvkm_enum
+gp100_ce_launcherr_report[] = {
+        { 0x0, "NO_ERR" },
+        { 0x1, "2D_LAYER_EXCEEDS_DEPTH" },
+        { 0x2, "INVALID_ALIGNMENT" },
+        { 0x3, "MEM2MEM_RECT_OUT_OF_BOUNDS" },
+        { 0x4, "SRC_LINE_EXCEEDS_PITCH" },
+        { 0x5, "SRC_LINE_EXCEEDS_NEG_PITCH" },
+        { 0x6, "DST_LINE_EXCEEDS_PITCH" },
+        { 0x7, "DST_LINE_EXCEEDS_NEG_PITCH" },
+        { 0x8, "BAD_SRC_PIXEL_COMP_REF" },
+        { 0x9, "INVALID_VALUE" },
+        { 0xa, "UNUSED_FIELD" },
+        { 0xb, "INVALID_OPERATION" },
+        { 0xc, "NO_RESOURCES" },
+        { 0xd, "INVALID_CONFIG" },
+        {}
+};
+
+static void
+gp100_ce_intr_launcherr(struct nvkm_engine *ce, const u32 base)
+{
+        struct nvkm_subdev *subdev = &ce->subdev;
+        struct nvkm_device *device = subdev->device;
+        u32 stat = nvkm_rd32(device, 0x104418 + base);
+        const struct nvkm_enum *en =
+                nvkm_enum_find(gp100_ce_launcherr_report, stat & 0x0000000f);
+        nvkm_warn(subdev, "LAUNCHERR %08x [%s]\n", stat, en ? en->name : "");
+}
+
+void
+gp100_ce_intr(struct nvkm_engine *ce)
+{
+        const u32 base = (ce->subdev.index - NVKM_ENGINE_CE0) * 0x80;
+        struct nvkm_subdev *subdev = &ce->subdev;
+        struct nvkm_device *device = subdev->device;
+        u32 mask = nvkm_rd32(device, 0x10440c + base);
+        u32 intr = nvkm_rd32(device, 0x104410 + base) & mask;
+        if (intr & 0x00000001) { //XXX: guess
+                nvkm_warn(subdev, "BLOCKPIPE\n");
+                nvkm_wr32(device, 0x104410 + base, 0x00000001);
+                intr &= ~0x00000001;
+        }
+        if (intr & 0x00000002) { //XXX: guess
+                nvkm_warn(subdev, "NONBLOCKPIPE\n");
+                nvkm_wr32(device, 0x104410 + base, 0x00000002);
+                intr &= ~0x00000002;
+        }
+        if (intr & 0x00000004) {
+                gp100_ce_intr_launcherr(ce, base);
+                nvkm_wr32(device, 0x104410 + base, 0x00000004);
+                intr &= ~0x00000004;
+        }
+        if (intr) {
+                nvkm_warn(subdev, "intr %08x\n", intr);
+                nvkm_wr32(device, 0x104410 + base, intr);
+        }
+}
+
+static const struct nvkm_engine_func
+gp100_ce = {
+        .intr = gp100_ce_intr,
+        .sclass = {
+                { -1, -1, PASCAL_DMA_COPY_A },
+                {}
+        }
+};
+
+int
+gp100_ce_new(struct nvkm_device *device, int index,
+             struct nvkm_engine **pengine)
+{
+        return nvkm_engine_new_(&gp100_ce, device, index, true, pengine);
+}
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/ce/gp104.c b/drivers/gpu/drm/nouveau/nvkm/engine/ce/gp104.c
new file mode 100644
index 000000000000..20e019788a53
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/ce/gp104.c
@@ -0,0 +1,44 @@
+/*
+ * Copyright 2015 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs
+ */
+#include "priv.h"
+#include <core/enum.h>
+
+#include <nvif/class.h>
+
+static const struct nvkm_engine_func
+gp104_ce = {
+        .intr = gp100_ce_intr,
+        .sclass = {
+                { -1, -1, PASCAL_DMA_COPY_B },
+                { -1, -1, PASCAL_DMA_COPY_A },
+                {}
+        }
+};
+
+int
+gp104_ce_new(struct nvkm_device *device, int index,
+             struct nvkm_engine **pengine)
+{
+        return nvkm_engine_new_(&gp104_ce, device, index, true, pengine);
+}
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/ce/priv.h b/drivers/gpu/drm/nouveau/nvkm/engine/ce/priv.h
index e2fa8b161943..2dce405976ad 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/ce/priv.h
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/ce/priv.h
@@ -4,4 +4,5 @@
 
 void gt215_ce_intr(struct nvkm_falcon *, struct nvkm_fifo_chan *);
 void gk104_ce_intr(struct nvkm_engine *);
+void gp100_ce_intr(struct nvkm_engine *);
 #endif
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/device/base.c b/drivers/gpu/drm/nouveau/nvkm/engine/device/base.c
index 4572debcb0c9..7218a067a6c5 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/device/base.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/device/base.c
@@ -2148,6 +2148,67 @@ nv12b_chipset = {
         .sw = gf100_sw_new,
 };
 
+static const struct nvkm_device_chip
+nv130_chipset = {
+        .name = "GP100",
+        .bar = gf100_bar_new,
+        .bios = nvkm_bios_new,
+        .bus = gf100_bus_new,
+        .devinit = gm200_devinit_new,
+        .fb = gp100_fb_new,
+        .fuse = gm107_fuse_new,
+        .gpio = gk104_gpio_new,
+        .i2c = gm200_i2c_new,
+        .ibus = gm200_ibus_new,
+        .imem = nv50_instmem_new,
+        .ltc = gp100_ltc_new,
+        .mc = gp100_mc_new,
+        .mmu = gf100_mmu_new,
+        .secboot = gm200_secboot_new,
+        .pci = gp100_pci_new,
+        .timer = gk20a_timer_new,
+        .top = gk104_top_new,
+        .ce[0] = gp100_ce_new,
+        .ce[1] = gp100_ce_new,
+        .ce[2] = gp100_ce_new,
+        .ce[3] = gp100_ce_new,
+        .ce[4] = gp100_ce_new,
+        .ce[5] = gp100_ce_new,
+        .dma = gf119_dma_new,
+        .disp = gp100_disp_new,
+        .fifo = gp100_fifo_new,
+        .gr = gp100_gr_new,
+        .sw = gf100_sw_new,
+};
+
+static const struct nvkm_device_chip
+nv134_chipset = {
+        .name = "GP104",
+        .bar = gf100_bar_new,
+        .bios = nvkm_bios_new,
+        .bus = gf100_bus_new,
+        .devinit = gm200_devinit_new,
+        .fb = gp104_fb_new,
+        .fuse = gm107_fuse_new,
+        .gpio = gk104_gpio_new,
+        .i2c = gm200_i2c_new,
+        .ibus = gm200_ibus_new,
+        .imem = nv50_instmem_new,
+        .ltc = gp100_ltc_new,
+        .mc = gp100_mc_new,
+        .mmu = gf100_mmu_new,
+        .pci = gp100_pci_new,
+        .timer = gk20a_timer_new,
+        .top = gk104_top_new,
+        .ce[0] = gp104_ce_new,
+        .ce[1] = gp104_ce_new,
+        .ce[2] = gp104_ce_new,
+        .ce[3] = gp104_ce_new,
+        .disp = gp104_disp_new,
+        .dma = gf119_dma_new,
+        .fifo = gp100_fifo_new,
+};
+
 static int
 nvkm_device_event_ctor(struct nvkm_object *object, void *data, u32 size,
                        struct nvkm_notify *notify)
@@ -2221,6 +2282,9 @@ nvkm_device_engine(struct nvkm_device *device, int index)
         _(CE0    , device->ce[0]   ,  device->ce[0]);
         _(CE1    , device->ce[1]   ,  device->ce[1]);
         _(CE2    , device->ce[2]   ,  device->ce[2]);
+        _(CE3    , device->ce[3]   ,  device->ce[3]);
+        _(CE4    , device->ce[4]   ,  device->ce[4]);
+        _(CE5    , device->ce[5]   ,  device->ce[5]);
         _(CIPHER , device->cipher  ,  device->cipher);
         _(DISP   , device->disp    , &device->disp->engine);
         _(DMAOBJ , device->dma     , &device->dma->engine);
@@ -2235,6 +2299,7 @@ nvkm_device_engine(struct nvkm_device *device, int index)
         _(MSVLD  , device->msvld   ,  device->msvld);
         _(NVENC0 , device->nvenc[0],  device->nvenc[0]);
         _(NVENC1 , device->nvenc[1],  device->nvenc[1]);
+        _(NVENC2 , device->nvenc[2],  device->nvenc[2]);
         _(NVDEC  , device->nvdec   ,  device->nvdec);
         _(PM     , device->pm      , &device->pm->engine);
         _(SEC    , device->sec     ,  device->sec);
@@ -2492,6 +2557,7 @@ nvkm_device_ctor(const struct nvkm_device_func *func,
                         case 0x100: device->card_type = NV_E0; break;
                         case 0x110:
                         case 0x120: device->card_type = GM100; break;
+                        case 0x130: device->card_type = GP100; break;
                         default:
                                 break;
                         }
@@ -2576,6 +2642,8 @@ nvkm_device_ctor(const struct nvkm_device_func *func,
                 case 0x124: device->chip = &nv124_chipset; break;
                 case 0x126: device->chip = &nv126_chipset; break;
                 case 0x12b: device->chip = &nv12b_chipset; break;
+                case 0x130: device->chip = &nv130_chipset; break;
+                case 0x134: device->chip = &nv134_chipset; break;
                 default:
                         nvdev_error(device, "unknown chipset (%08x)\n", boot0);
                         goto done;
@@ -2659,6 +2727,9 @@ nvkm_device_ctor(const struct nvkm_device_func *func,
                 _(NVKM_ENGINE_CE0     ,    ce[0]);
                 _(NVKM_ENGINE_CE1     ,    ce[1]);
                 _(NVKM_ENGINE_CE2     ,    ce[2]);
+                _(NVKM_ENGINE_CE3     ,    ce[3]);
+                _(NVKM_ENGINE_CE4     ,    ce[4]);
+                _(NVKM_ENGINE_CE5     ,    ce[5]);
                 _(NVKM_ENGINE_CIPHER  ,   cipher);
                 _(NVKM_ENGINE_DISP    ,     disp);
                 _(NVKM_ENGINE_DMAOBJ  ,      dma);
@@ -2673,6 +2744,7 @@ nvkm_device_ctor(const struct nvkm_device_func *func,
                 _(NVKM_ENGINE_MSVLD   ,    msvld);
                 _(NVKM_ENGINE_NVENC0  , nvenc[0]);
                 _(NVKM_ENGINE_NVENC1  , nvenc[1]);
+                _(NVKM_ENGINE_NVENC2  , nvenc[2]);
                 _(NVKM_ENGINE_NVDEC   ,    nvdec);
                 _(NVKM_ENGINE_PM      ,       pm);
                 _(NVKM_ENGINE_SEC     ,      sec);
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/device/pci.c b/drivers/gpu/drm/nouveau/nvkm/engine/device/pci.c
index 62ad0300cfa5..b1b693219db3 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/device/pci.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/device/pci.c
@@ -1614,7 +1614,6 @@ nvkm_device_pci_func = {
         .fini = nvkm_device_pci_fini,
         .resource_addr = nvkm_device_pci_resource_addr,
         .resource_size = nvkm_device_pci_resource_size,
-        .cpu_coherent = !IS_ENABLED(CONFIG_ARM),
 };
 
 int
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/device/tegra.c b/drivers/gpu/drm/nouveau/nvkm/engine/device/tegra.c
index ec12efb4689a..939682f18788 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/device/tegra.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/device/tegra.c
@@ -191,13 +191,11 @@ static irqreturn_t
 nvkm_device_tegra_intr(int irq, void *arg)
 {
         struct nvkm_device_tegra *tdev = arg;
-        struct nvkm_mc *mc = tdev->device.mc;
+        struct nvkm_device *device = &tdev->device;
         bool handled = false;
-        if (likely(mc)) {
-                nvkm_mc_intr_unarm(mc);
-                nvkm_mc_intr(mc, &handled);
-                nvkm_mc_intr_rearm(mc);
-        }
+        nvkm_mc_intr_unarm(device);
+        nvkm_mc_intr(device, &handled);
+        nvkm_mc_intr_rearm(device);
         return handled ? IRQ_HANDLED : IRQ_NONE;
 }
 
@@ -247,7 +245,6 @@ nvkm_device_tegra_func = {
         .fini = nvkm_device_tegra_fini,
         .resource_addr = nvkm_device_tegra_resource_addr,
         .resource_size = nvkm_device_tegra_resource_size,
-        .cpu_coherent = false,
 };
 
 int
@@ -313,6 +310,7 @@ nvkm_device_tegra_new(const struct nvkm_device_tegra_func *func,
                 goto remove;
 
         tdev->gpu_speedo = tegra_sku_info.gpu_speedo_value;
+        tdev->gpu_speedo_id = tegra_sku_info.gpu_speedo_id;
         ret = nvkm_device_ctor(&nvkm_device_tegra_func, NULL, &pdev->dev,
                                NVKM_DEVICE_TEGRA, pdev->id, NULL,
                                cfg, dbg, detect, mmio, subdev_mask,
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/device/user.c b/drivers/gpu/drm/nouveau/nvkm/engine/device/user.c
index 137066426ed7..79a8f71cf788 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/device/user.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/device/user.c
@@ -102,6 +102,7 @@ nvkm_udevice_info(struct nvkm_udevice *udev, void *data, u32 size)
         case NV_C0: args->v0.family = NV_DEVICE_INFO_V0_FERMI; break;
         case NV_E0: args->v0.family = NV_DEVICE_INFO_V0_KEPLER; break;
         case GM100: args->v0.family = NV_DEVICE_INFO_V0_MAXWELL; break;
+        case GP100: args->v0.family = NV_DEVICE_INFO_V0_PASCAL; break;
         default:
                 args->v0.family = 0;
                 break;
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/Kbuild b/drivers/gpu/drm/nouveau/nvkm/engine/disp/Kbuild
index e2a64ed14b22..77a52b54a31e 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/Kbuild
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/Kbuild
@@ -10,6 +10,8 @@ nvkm-y += nvkm/engine/disp/gk104.o
 nvkm-y += nvkm/engine/disp/gk110.o
 nvkm-y += nvkm/engine/disp/gm107.o
 nvkm-y += nvkm/engine/disp/gm200.o
+nvkm-y += nvkm/engine/disp/gp100.o
+nvkm-y += nvkm/engine/disp/gp104.o
 
 nvkm-y += nvkm/engine/disp/outp.o
 nvkm-y += nvkm/engine/disp/outpdp.o
@@ -45,12 +47,15 @@ nvkm-y += nvkm/engine/disp/rootgk104.o
 nvkm-y += nvkm/engine/disp/rootgk110.o
 nvkm-y += nvkm/engine/disp/rootgm107.o
 nvkm-y += nvkm/engine/disp/rootgm200.o
+nvkm-y += nvkm/engine/disp/rootgp100.o
+nvkm-y += nvkm/engine/disp/rootgp104.o
 
 nvkm-y += nvkm/engine/disp/channv50.o
 nvkm-y += nvkm/engine/disp/changf119.o
 
 nvkm-y += nvkm/engine/disp/dmacnv50.o
 nvkm-y += nvkm/engine/disp/dmacgf119.o
+nvkm-y += nvkm/engine/disp/dmacgp104.o
 
 nvkm-y += nvkm/engine/disp/basenv50.o
 nvkm-y += nvkm/engine/disp/baseg84.o
@@ -59,6 +64,7 @@ nvkm-y += nvkm/engine/disp/basegt215.o
 nvkm-y += nvkm/engine/disp/basegf119.o
 nvkm-y += nvkm/engine/disp/basegk104.o
 nvkm-y += nvkm/engine/disp/basegk110.o
+nvkm-y += nvkm/engine/disp/basegp104.o
 
 nvkm-y += nvkm/engine/disp/corenv50.o
 nvkm-y += nvkm/engine/disp/coreg84.o
@@ -70,6 +76,8 @@ nvkm-y += nvkm/engine/disp/coregk104.o
 nvkm-y += nvkm/engine/disp/coregk110.o
 nvkm-y += nvkm/engine/disp/coregm107.o
 nvkm-y += nvkm/engine/disp/coregm200.o
+nvkm-y += nvkm/engine/disp/coregp100.o
+nvkm-y += nvkm/engine/disp/coregp104.o
 
 nvkm-y += nvkm/engine/disp/ovlynv50.o
 nvkm-y += nvkm/engine/disp/ovlyg84.o
@@ -77,6 +85,7 @@ nvkm-y += nvkm/engine/disp/ovlygt200.o
 nvkm-y += nvkm/engine/disp/ovlygt215.o
 nvkm-y += nvkm/engine/disp/ovlygf119.o
 nvkm-y += nvkm/engine/disp/ovlygk104.o
+nvkm-y += nvkm/engine/disp/ovlygp104.o
 
 nvkm-y += nvkm/engine/disp/piocnv50.o
 nvkm-y += nvkm/engine/disp/piocgf119.o
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/basegp104.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/basegp104.c
new file mode 100644
index 000000000000..51688e37c54e
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/basegp104.c
@@ -0,0 +1,38 @@
+/*
+ * Copyright 2016 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs <bskeggs@redhat.com>
+ */
+#include "dmacnv50.h"
+#include "rootnv50.h"
+
+#include <nvif/class.h>
+
+const struct nv50_disp_dmac_oclass
+gp104_disp_base_oclass = {
+        .base.oclass = GK110_DISP_BASE_CHANNEL_DMA,
+        .base.minver = 0,
+        .base.maxver = 0,
+        .ctor = nv50_disp_base_new,
+        .func = &gp104_disp_dmac_func,
+        .mthd = &gf119_disp_base_chan_mthd,
+        .chid = 1,
+};
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/channv50.h b/drivers/gpu/drm/nouveau/nvkm/engine/disp/channv50.h
index aee374884c96..f5f683d9fd20 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/channv50.h
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/channv50.h
@@ -85,6 +85,7 @@ extern const struct nv50_disp_mthd_list gf119_disp_core_mthd_pior;
 extern const struct nv50_disp_chan_mthd gf119_disp_base_chan_mthd;
 
 extern const struct nv50_disp_chan_mthd gk104_disp_core_chan_mthd;
+extern const struct nv50_disp_chan_mthd gk104_disp_ovly_chan_mthd;
 
 struct nv50_disp_pioc_oclass {
         int (*ctor)(const struct nv50_disp_chan_func *,
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/coregf119.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/coregf119.c
index 6b1dc703dac7..21fbf89b6319 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/coregf119.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/coregf119.c
@@ -171,7 +171,7 @@ gf119_disp_core_chan_mthd = {
         }
 };
 
-static void
+void
 gf119_disp_core_fini(struct nv50_disp_dmac *chan)
 {
         struct nv50_disp *disp = chan->base.root->disp;
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/coregp100.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/coregp100.c
new file mode 100644
index 000000000000..d5dff6619d4d
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/coregp100.c
@@ -0,0 +1,38 @@
+/*
+ * Copyright 2015 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs <bskeggs@redhat.com>
+ */
+#include "dmacnv50.h"
+#include "rootnv50.h"
+
+#include <nvif/class.h>
+
+const struct nv50_disp_dmac_oclass
+gp100_disp_core_oclass = {
+        .base.oclass = GP100_DISP_CORE_CHANNEL_DMA,
+        .base.minver = 0,
+        .base.maxver = 0,
+        .ctor = nv50_disp_core_new,
+        .func = &gf119_disp_core_func,
+        .mthd = &gk104_disp_core_chan_mthd,
+        .chid = 0,
+};
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/coregp104.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/coregp104.c
new file mode 100644
index 000000000000..6922f4007b61
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/coregp104.c
@@ -0,0 +1,78 @@
+/*
+ * Copyright 2016 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs <bskeggs@redhat.com>
+ */
+#include "dmacnv50.h"
+#include "rootnv50.h"
+
+#include <subdev/timer.h>
+
+#include <nvif/class.h>
+
+static int
+gp104_disp_core_init(struct nv50_disp_dmac *chan)
+{
+        struct nv50_disp *disp = chan->base.root->disp;
+        struct nvkm_subdev *subdev = &disp->base.engine.subdev;
+        struct nvkm_device *device = subdev->device;
+
+        /* enable error reporting */
+        nvkm_mask(device, 0x6100a0, 0x00000001, 0x00000001);
+
+        /* initialise channel for dma command submission */
+        nvkm_wr32(device, 0x611494, chan->push);
+        nvkm_wr32(device, 0x611498, 0x00010000);
+        nvkm_wr32(device, 0x61149c, 0x00000001);
+        nvkm_mask(device, 0x610490, 0x00000010, 0x00000010);
+        nvkm_wr32(device, 0x640000, 0x00000000);
+        nvkm_wr32(device, 0x610490, 0x01000013);
+
+        /* wait for it to go inactive */
+        if (nvkm_msec(device, 2000,
+                if (!(nvkm_rd32(device, 0x610490) & 0x80000000))
+                        break;
+        ) < 0) {
+                nvkm_error(subdev, "core init: %08x\n",
+                           nvkm_rd32(device, 0x610490));
+                return -EBUSY;
+        }
+
+        return 0;
+}
+
+const struct nv50_disp_dmac_func
+gp104_disp_core_func = {
+        .init = gp104_disp_core_init,
+        .fini = gf119_disp_core_fini,
+        .bind = gf119_disp_dmac_bind,
+};
+
+const struct nv50_disp_dmac_oclass
+gp104_disp_core_oclass = {
+        .base.oclass = GP104_DISP_CORE_CHANNEL_DMA,
+        .base.minver = 0,
+        .base.maxver = 0,
+        .ctor = nv50_disp_core_new,
+        .func = &gp104_disp_core_func,
+        .mthd = &gk104_disp_core_chan_mthd,
+        .chid = 0,
+};
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/dmacgf119.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/dmacgf119.c
index 876b14549a58..a57f7cef307a 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/dmacgf119.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/dmacgf119.c
@@ -36,7 +36,7 @@ gf119_disp_dmac_bind(struct nv50_disp_dmac *chan,
                                  chan->base.chid << 27 | 0x00000001);
 }
 
-static void
+void
 gf119_disp_dmac_fini(struct nv50_disp_dmac *chan)
 {
         struct nv50_disp *disp = chan->base.root->disp;
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/dmacgp104.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/dmacgp104.c
new file mode 100644
index 000000000000..ad24c2c57696
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/dmacgp104.c
@@ -0,0 +1,66 @@
+/*
+ * Copyright 2016 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs <bskeggs@redhat.com>
+ */
+#include "dmacnv50.h"
+#include "rootnv50.h"
+
+#include <subdev/timer.h>
+
+static int
+gp104_disp_dmac_init(struct nv50_disp_dmac *chan)
+{
+        struct nv50_disp *disp = chan->base.root->disp;
+        struct nvkm_subdev *subdev = &disp->base.engine.subdev;
+        struct nvkm_device *device = subdev->device;
+        int chid = chan->base.chid;
+
+        /* enable error reporting */
+        nvkm_mask(device, 0x6100a0, 0x00000001 << chid, 0x00000001 << chid);
+
+        /* initialise channel for dma command submission */
+        nvkm_wr32(device, 0x611494 + (chid * 0x0010), chan->push);
+        nvkm_wr32(device, 0x611498 + (chid * 0x0010), 0x00010000);
+        nvkm_wr32(device, 0x61149c + (chid * 0x0010), 0x00000001);
+        nvkm_mask(device, 0x610490 + (chid * 0x0010), 0x00000010, 0x00000010);
+        nvkm_wr32(device, 0x640000 + (chid * 0x1000), 0x00000000);
+        nvkm_wr32(device, 0x610490 + (chid * 0x0010), 0x00000013);
+
+        /* wait for it to go inactive */
+        if (nvkm_msec(device, 2000,
+                if (!(nvkm_rd32(device, 0x610490 + (chid * 0x10)) & 0x80000000))
+                        break;
+        ) < 0) {
+                nvkm_error(subdev, "ch %d init: %08x\n", chid,
+                           nvkm_rd32(device, 0x610490 + (chid * 0x10)));
+                return -EBUSY;
+        }
+
+        return 0;
+}
+
+const struct nv50_disp_dmac_func
+gp104_disp_dmac_func = {
+        .init = gp104_disp_dmac_init,
+        .fini = gf119_disp_dmac_fini,
+        .bind = gf119_disp_dmac_bind,
+};
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/dmacnv50.h b/drivers/gpu/drm/nouveau/nvkm/engine/disp/dmacnv50.h
index fc84eb8b5c45..43ac05857853 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/dmacnv50.h
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/dmacnv50.h
@@ -25,8 +25,12 @@ int nv50_disp_dmac_bind(struct nv50_disp_dmac *, struct nvkm_object *, u32);
 extern const struct nv50_disp_dmac_func nv50_disp_core_func;
 
 extern const struct nv50_disp_dmac_func gf119_disp_dmac_func;
+void gf119_disp_dmac_fini(struct nv50_disp_dmac *);
 int gf119_disp_dmac_bind(struct nv50_disp_dmac *, struct nvkm_object *, u32);
 extern const struct nv50_disp_dmac_func gf119_disp_core_func;
+void gf119_disp_core_fini(struct nv50_disp_dmac *);
+
+extern const struct nv50_disp_dmac_func gp104_disp_dmac_func;
 
 struct nv50_disp_dmac_oclass {
         int (*ctor)(const struct nv50_disp_dmac_func *,
@@ -88,4 +92,10 @@ extern const struct nv50_disp_dmac_oclass gk110_disp_base_oclass;
 extern const struct nv50_disp_dmac_oclass gm107_disp_core_oclass;
 
 extern const struct nv50_disp_dmac_oclass gm200_disp_core_oclass;
+
+extern const struct nv50_disp_dmac_oclass gp100_disp_core_oclass;
+
+extern const struct nv50_disp_dmac_oclass gp104_disp_core_oclass;
+extern const struct nv50_disp_dmac_oclass gp104_disp_base_oclass;
+extern const struct nv50_disp_dmac_oclass gp104_disp_ovly_oclass;
 #endif
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/gf119.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/gf119.c
index 5dd34382f55a..29e84b241cca 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/gf119.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/gf119.c
@@ -76,12 +76,10 @@ exec_lookup(struct nv50_disp *disp, int head, int or, u32 ctrl,
         mask |= 0x0001 << or;
         mask |= 0x0100 << head;
 
-
         list_for_each_entry(outp, &disp->base.outp, head) {
                 if ((outp->info.hasht & 0xff) == type &&
                     (outp->info.hashm & mask) == mask) {
-                        *data = nvbios_outp_match(bios, outp->info.hasht,
-                                                        outp->info.hashm,
+                        *data = nvbios_outp_match(bios, outp->info.hasht, mask,
                                                   ver, hdr, cnt, len, info);
                         if (!*data)
                                 return NULL;
@@ -415,7 +413,7 @@ gf119_disp_intr_supervisor(struct work_struct *work)
         nvkm_wr32(device, 0x6101d0, 0x80000000);
 }
 
-static void
+void
 gf119_disp_intr_error(struct nv50_disp *disp, int chid)
 {
         struct nvkm_subdev *subdev = &disp->base.engine.subdev;
@@ -463,7 +461,7 @@ gf119_disp_intr(struct nv50_disp *disp)
                 u32 stat = nvkm_rd32(device, 0x61009c);
                 int chid = ffs(stat) - 1;
                 if (chid >= 0)
-                        gf119_disp_intr_error(disp, chid);
+                        disp->func->intr_error(disp, chid);
                 intr &= ~0x00000002;
         }
 
@@ -507,6 +505,7 @@ gf119_disp_new_(const struct nv50_disp_func *func, struct nvkm_device *device,
 static const struct nv50_disp_func
 gf119_disp = {
         .intr = gf119_disp_intr,
+        .intr_error = gf119_disp_intr_error,
         .uevent = &gf119_disp_chan_uevent,
         .super = gf119_disp_intr_supervisor,
         .root = &gf119_disp_root_oclass,
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/gk104.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/gk104.c
index a86384b8e388..37f145cf30d7 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/gk104.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/gk104.c
@@ -27,6 +27,7 @@
 static const struct nv50_disp_func
 gk104_disp = {
         .intr = gf119_disp_intr,
+        .intr_error = gf119_disp_intr_error,
         .uevent = &gf119_disp_chan_uevent,
         .super = gf119_disp_intr_supervisor,
         .root = &gk104_disp_root_oclass,
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/gk110.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/gk110.c
index 0d574c7e594a..e14ac946608c 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/gk110.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/gk110.c
@@ -27,6 +27,7 @@
 static const struct nv50_disp_func
 gk110_disp = {
         .intr = gf119_disp_intr,
+        .intr_error = gf119_disp_intr_error,
         .uevent = &gf119_disp_chan_uevent,
         .super = gf119_disp_intr_supervisor,
         .root = &gk110_disp_root_oclass,
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/gm107.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/gm107.c
index f4b9cf8574be..2f2437cc5891 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/gm107.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/gm107.c
@@ -27,6 +27,7 @@
 static const struct nv50_disp_func
 gm107_disp = {
         .intr = gf119_disp_intr,
+        .intr_error = gf119_disp_intr_error,
         .uevent = &gf119_disp_chan_uevent,
         .super = gf119_disp_intr_supervisor,
         .root = &gm107_disp_root_oclass,
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/gm200.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/gm200.c
index 67eec8620719..9f368d4ee61e 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/gm200.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/gm200.c
@@ -27,6 +27,7 @@
 static const struct nv50_disp_func
 gm200_disp = {
         .intr = gf119_disp_intr,
+        .intr_error = gf119_disp_intr_error,
         .uevent = &gf119_disp_chan_uevent,
         .super = gf119_disp_intr_supervisor,
         .root = &gm200_disp_root_oclass,
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/gp100.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/gp100.c
new file mode 100644
index 000000000000..4f81bf31435e
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/gp100.c
@@ -0,0 +1,55 @@
+/*
+ * Copyright 2015 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs <bskeggs@redhat.com>
+ */
+#include "nv50.h"
+#include "rootnv50.h"
+
+static const struct nv50_disp_func
+gp100_disp = {
+        .intr = gf119_disp_intr,
+        .intr_error = gf119_disp_intr_error,
+        .uevent = &gf119_disp_chan_uevent,
+        .super = gf119_disp_intr_supervisor,
+        .root = &gp100_disp_root_oclass,
+        .head.vblank_init = gf119_disp_vblank_init,
+        .head.vblank_fini = gf119_disp_vblank_fini,
+        .head.scanoutpos = gf119_disp_root_scanoutpos,
+        .outp.internal.crt = nv50_dac_output_new,
+        .outp.internal.tmds = nv50_sor_output_new,
+        .outp.internal.lvds = nv50_sor_output_new,
+        .outp.internal.dp = gm200_sor_dp_new,
+        .dac.nr = 3,
+        .dac.power = nv50_dac_power,
+        .dac.sense = nv50_dac_sense,
+        .sor.nr = 4,
+        .sor.power = nv50_sor_power,
+        .sor.hda_eld = gf119_hda_eld,
+        .sor.hdmi = gk104_hdmi_ctrl,
+        .sor.magic = gm200_sor_magic,
+};
+
+int
+gp100_disp_new(struct nvkm_device *device, int index, struct nvkm_disp **pdisp)
+{
+        return gf119_disp_new_(&gp100_disp, device, index, pdisp);
+}
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/gp104.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/gp104.c
new file mode 100644
index 000000000000..3bf3380336e4
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/gp104.c
@@ -0,0 +1,81 @@
+/*
+ * Copyright 2016 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs <bskeggs@redhat.com>
+ */
+#include "nv50.h"
+#include "rootnv50.h"
+
+static void
+gp104_disp_intr_error(struct nv50_disp *disp, int chid)
+{
+        struct nvkm_subdev *subdev = &disp->base.engine.subdev;
+        struct nvkm_device *device = subdev->device;
+        u32 mthd = nvkm_rd32(device, 0x6111f0 + (chid * 12));
+        u32 data = nvkm_rd32(device, 0x6111f4 + (chid * 12));
+        u32 unkn = nvkm_rd32(device, 0x6111f8 + (chid * 12));
+
+        nvkm_error(subdev, "chid %d mthd %04x data %08x %08x %08x\n",
+                   chid, (mthd & 0x0000ffc), data, mthd, unkn);
+
+        if (chid < ARRAY_SIZE(disp->chan)) {
+                switch (mthd & 0xffc) {
+                case 0x0080:
+                        nv50_disp_chan_mthd(disp->chan[chid], NV_DBG_ERROR);
+                        break;
+                default:
+                        break;
+                }
+        }
+
+        nvkm_wr32(device, 0x61009c, (1 << chid));
+        nvkm_wr32(device, 0x6111f0 + (chid * 12), 0x90000000);
+}
+
+static const struct nv50_disp_func
+gp104_disp = {
+        .intr = gf119_disp_intr,
+        .intr_error = gp104_disp_intr_error,
+        .uevent = &gf119_disp_chan_uevent,
+        .super = gf119_disp_intr_supervisor,
+        .root = &gp104_disp_root_oclass,
+        .head.vblank_init = gf119_disp_vblank_init,
+        .head.vblank_fini = gf119_disp_vblank_fini,
+        .head.scanoutpos = gf119_disp_root_scanoutpos,
+        .outp.internal.crt = nv50_dac_output_new,
+        .outp.internal.tmds = nv50_sor_output_new,
+        .outp.internal.lvds = nv50_sor_output_new,
+        .outp.internal.dp = gm200_sor_dp_new,
+        .dac.nr = 3,
+        .dac.power = nv50_dac_power,
+        .dac.sense = nv50_dac_sense,
+        .sor.nr = 4,
+        .sor.power = nv50_sor_power,
+        .sor.hda_eld = gf119_hda_eld,
+        .sor.hdmi = gk104_hdmi_ctrl,
+        .sor.magic = gm200_sor_magic,
+};
+
+int
+gp104_disp_new(struct nvkm_device *device, int index, struct nvkm_disp **pdisp)
+{
+        return gf119_disp_new_(&gp104_disp, device, index, pdisp);
+}
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/nv50.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/nv50.c
index fcb1b0c46d64..fbb8c7dc18fd 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/nv50.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/nv50.c
@@ -32,6 +32,7 @@
 #include <subdev/bios/init.h>
 #include <subdev/bios/pll.h>
 #include <subdev/devinit.h>
+#include <subdev/timer.h>
 
 static const struct nvkm_disp_oclass *
 nv50_disp_root_(struct nvkm_disp *base)
@@ -269,8 +270,7 @@ exec_lookup(struct nv50_disp *disp, int head, int or, u32 ctrl,
         list_for_each_entry(outp, &disp->base.outp, head) {
                 if ((outp->info.hasht & 0xff) == type &&
                     (outp->info.hashm & mask) == mask) {
-                        *data = nvbios_outp_match(bios, outp->info.hasht,
-                                                        outp->info.hashm,
+                        *data = nvbios_outp_match(bios, outp->info.hasht, mask,
                                                   ver, hdr, cnt, len, info);
                         if (!*data)
                                 return NULL;
@@ -426,6 +426,134 @@ exec_clkcmp(struct nv50_disp *disp, int head, int id, u32 pclk, u32 *conf)
         return outp;
 }
 
+static bool
+nv50_disp_dptmds_war(struct nvkm_device *device)
+{
+        switch (device->chipset) {
+        case 0x94:
+        case 0x96:
+        case 0x98:
+        case 0xaa:
+        case 0xac:
+                return true;
+        default:
+                break;
+        }
+        return false;
+}
+
+static bool
+nv50_disp_dptmds_war_needed(struct nv50_disp *disp, struct dcb_output *outp)
+{
+        struct nvkm_device *device = disp->base.engine.subdev.device;
+        const u32 soff = __ffs(outp->or) * 0x800;
+        if (nv50_disp_dptmds_war(device) && outp->type == DCB_OUTPUT_TMDS) {
+                switch (nvkm_rd32(device, 0x614300 + soff) & 0x00030000) {
+                case 0x00000000:
+                case 0x00030000:
+                        return true;
+                default:
+                        break;
+                }
+        }
+        return false;
+
+}
+
+static void
+nv50_disp_dptmds_war_2(struct nv50_disp *disp, struct dcb_output *outp)
+{
+        struct nvkm_device *device = disp->base.engine.subdev.device;
+        const u32 soff = __ffs(outp->or) * 0x800;
+
+        if (!nv50_disp_dptmds_war_needed(disp, outp))
+                return;
+
+        nvkm_mask(device, 0x00e840, 0x80000000, 0x80000000);
+        nvkm_mask(device, 0x614300 + soff, 0x03000000, 0x03000000);
+        nvkm_mask(device, 0x61c10c + soff, 0x00000001, 0x00000001);
+
+        nvkm_mask(device, 0x61c00c + soff, 0x0f000000, 0x00000000);
+        nvkm_mask(device, 0x61c008 + soff, 0xff000000, 0x14000000);
+        nvkm_usec(device, 400, NVKM_DELAY);
+        nvkm_mask(device, 0x61c008 + soff, 0xff000000, 0x00000000);
+        nvkm_mask(device, 0x61c00c + soff, 0x0f000000, 0x01000000);
+
+        if (nvkm_rd32(device, 0x61c004 + soff) & 0x00000001) {
+                u32 seqctl = nvkm_rd32(device, 0x61c030 + soff);
+                u32  pu_pc = seqctl & 0x0000000f;
+                nvkm_wr32(device, 0x61c040 + soff + pu_pc * 4, 0x1f008000);
+        }
+}
+
+static void
+nv50_disp_dptmds_war_3(struct nv50_disp *disp, struct dcb_output *outp)
+{
+        struct nvkm_device *device = disp->base.engine.subdev.device;
+        const u32 soff = __ffs(outp->or) * 0x800;
+        u32 sorpwr;
+
+        if (!nv50_disp_dptmds_war_needed(disp, outp))
+                return;
+
+        sorpwr = nvkm_rd32(device, 0x61c004 + soff);
+        if (sorpwr & 0x00000001) {
+                u32 seqctl = nvkm_rd32(device, 0x61c030 + soff);
+                u32  pd_pc = (seqctl & 0x00000f00) >> 8;
+                u32  pu_pc =  seqctl & 0x0000000f;
+
+                nvkm_wr32(device, 0x61c040 + soff + pd_pc * 4, 0x1f008000);
+
+                nvkm_msec(device, 2000,
+                        if (!(nvkm_rd32(device, 0x61c030 + soff) & 0x10000000))
+                                break;
+                );
+                nvkm_mask(device, 0x61c004 + soff, 0x80000001, 0x80000000);
+                nvkm_msec(device, 2000,
+                        if (!(nvkm_rd32(device, 0x61c030 + soff) & 0x10000000))
+                                break;
+                );
+
+                nvkm_wr32(device, 0x61c040 + soff + pd_pc * 4, 0x00002000);
+                nvkm_wr32(device, 0x61c040 + soff + pu_pc * 4, 0x1f000000);
+        }
+
+        nvkm_mask(device, 0x61c10c + soff, 0x00000001, 0x00000000);
+        nvkm_mask(device, 0x614300 + soff, 0x03000000, 0x00000000);
+
+        if (sorpwr & 0x00000001) {
+                nvkm_mask(device, 0x61c004 + soff, 0x80000001, 0x80000001);
+        }
+}
+
+static void
+nv50_disp_update_sppll1(struct nv50_disp *disp)
+{
+        struct nvkm_device *device = disp->base.engine.subdev.device;
+        bool used = false;
+        int sor;
+
+        if (!nv50_disp_dptmds_war(device))
+                return;
+
+        for (sor = 0; sor < disp->func->sor.nr; sor++) {
+                u32 clksor = nvkm_rd32(device, 0x614300 + (sor * 0x800));
+                switch (clksor & 0x03000000) {
+                case 0x02000000:
+                case 0x03000000:
+                        used = true;
+                        break;
+                default:
+                        break;
+                }
+        }
+
+        if (used)
+                return;
+
+        nvkm_mask(device, 0x00e840, 0x80000000, 0x00000000);
+}
+
 static void
 nv50_disp_intr_unk10_0(struct nv50_disp *disp, int head)
 {
@@ -679,6 +807,8 @@ nv50_disp_intr_unk20_2(struct nv50_disp *disp, int head)
 
         nvkm_mask(device, hreg, 0x0000000f, hval);
         nvkm_mask(device, oreg, mask, oval);
+
+        nv50_disp_dptmds_war_2(disp, &outp->info);
 }
 
 /* If programming a TMDS output on a SOR that can also be configured for
@@ -720,6 +850,7 @@ nv50_disp_intr_unk40_0(struct nv50_disp *disp, int head)
 
         if (outp->info.location == 0 && outp->info.type == DCB_OUTPUT_TMDS)
                 nv50_disp_intr_unk40_0_tmds(disp, &outp->info);
+        nv50_disp_dptmds_war_3(disp, &outp->info);
 }
 
 void
@@ -767,6 +898,7 @@ nv50_disp_intr_supervisor(struct work_struct *work)
                                 continue;
                         nv50_disp_intr_unk40_0(disp, head);
                 }
+                nv50_disp_update_sppll1(disp);
         }
 
         nvkm_wr32(device, 0x610030, 0x80000000);
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/nv50.h b/drivers/gpu/drm/nouveau/nvkm/engine/disp/nv50.h
index aecebd8717e5..1e1de6bfe85a 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/nv50.h
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/nv50.h
@@ -68,6 +68,7 @@ struct nv50_disp_func_outp {
 
 struct nv50_disp_func {
         void (*intr)(struct nv50_disp *);
+        void (*intr_error)(struct nv50_disp *, int chid);
 
         const struct nvkm_event_func *uevent;
         void (*super)(struct work_struct *);
@@ -114,4 +115,5 @@ void gf119_disp_vblank_init(struct nv50_disp *, int);
 void gf119_disp_vblank_fini(struct nv50_disp *, int);
 void gf119_disp_intr(struct nv50_disp *);
 void gf119_disp_intr_supervisor(struct work_struct *);
+void gf119_disp_intr_error(struct nv50_disp *, int);
 #endif
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/ovlygk104.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/ovlygk104.c
index 2e2dc0641ef2..2f0220b39f34 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/ovlygk104.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/ovlygk104.c
@@ -80,7 +80,7 @@ gk104_disp_ovly_mthd_base = {
         }
 };
 
-static const struct nv50_disp_chan_mthd
+const struct nv50_disp_chan_mthd
 gk104_disp_ovly_chan_mthd = {
         .name = "Overlay",
         .addr = 0x001000,
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/ovlygp104.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/ovlygp104.c
new file mode 100644
index 000000000000..97e2dd2d908e
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/ovlygp104.c
@@ -0,0 +1,38 @@
+/*
+ * Copyright 2012 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs
+ */
+#include "dmacnv50.h"
+#include "rootnv50.h"
+
+#include <nvif/class.h>
+
+const struct nv50_disp_dmac_oclass
+gp104_disp_ovly_oclass = {
+        .base.oclass = GK104_DISP_OVERLAY_CONTROL_DMA,
+        .base.minver = 0,
+        .base.maxver = 0,
+        .ctor = nv50_disp_ovly_new,
+        .func = &gp104_disp_dmac_func,
+        .mthd = &gk104_disp_ovly_chan_mthd,
+        .chid = 5,
+};
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/rootgp100.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/rootgp100.c
new file mode 100644
index 000000000000..ac8fdd728ec6
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/rootgp100.c
@@ -0,0 +1,58 @@
+/*
+ * Copyright 2015 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs <bskeggs@redhat.com>
+ */
+#include "rootnv50.h"
+#include "dmacnv50.h"
+
+#include <nvif/class.h>
+
+static const struct nv50_disp_root_func
+gp100_disp_root = {
+        .init = gf119_disp_root_init,
+        .fini = gf119_disp_root_fini,
+        .dmac = {
+                &gp100_disp_core_oclass,
+                &gk110_disp_base_oclass,
+                &gk104_disp_ovly_oclass,
+        },
+        .pioc = {
+                &gk104_disp_oimm_oclass,
+                &gk104_disp_curs_oclass,
+        },
+};
+
+static int
+gp100_disp_root_new(struct nvkm_disp *disp, const struct nvkm_oclass *oclass,
+                    void *data, u32 size, struct nvkm_object **pobject)
+{
+        return nv50_disp_root_new_(&gp100_disp_root, disp, oclass,
+                                   data, size, pobject);
+}
+
+const struct nvkm_disp_oclass
+gp100_disp_root_oclass = {
+        .base.oclass = GP100_DISP,
+        .base.minver = -1,
+        .base.maxver = -1,
+        .ctor = gp100_disp_root_new,
+};
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/rootgp104.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/rootgp104.c
new file mode 100644
index 000000000000..8443e04dc626
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/rootgp104.c
@@ -0,0 +1,58 @@
+/*
+ * Copyright 2016 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs <bskeggs@redhat.com>
+ */
+#include "rootnv50.h"
+#include "dmacnv50.h"
+
+#include <nvif/class.h>
+
+static const struct nv50_disp_root_func
+gp104_disp_root = {
+        .init = gf119_disp_root_init,
+        .fini = gf119_disp_root_fini,
+        .dmac = {
+                &gp104_disp_core_oclass,
+                &gp104_disp_base_oclass,
+                &gp104_disp_ovly_oclass,
+        },
+        .pioc = {
+                &gk104_disp_oimm_oclass,
+                &gk104_disp_curs_oclass,
+        },
+};
+
+static int
+gp104_disp_root_new(struct nvkm_disp *disp, const struct nvkm_oclass *oclass,
+                    void *data, u32 size, struct nvkm_object **pobject)
+{
+        return nv50_disp_root_new_(&gp104_disp_root, disp, oclass,
+                                   data, size, pobject);
+}
+
+const struct nvkm_disp_oclass
+gp104_disp_root_oclass = {
+        .base.oclass = GP104_DISP,
+        .base.minver = -1,
+        .base.maxver = -1,
+        .ctor = gp104_disp_root_new,
+};
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/rootnv50.h b/drivers/gpu/drm/nouveau/nvkm/engine/disp/rootnv50.h
index cb449ed8d92c..ad00f1724b72 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/rootnv50.h
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/rootnv50.h
@@ -40,4 +40,6 @@ extern const struct nvkm_disp_oclass gk104_disp_root_oclass;
 extern const struct nvkm_disp_oclass gk110_disp_root_oclass;
 extern const struct nvkm_disp_oclass gm107_disp_root_oclass;
 extern const struct nvkm_disp_oclass gm200_disp_root_oclass;
+extern const struct nvkm_disp_oclass gp100_disp_root_oclass;
+extern const struct nvkm_disp_oclass gp104_disp_root_oclass;
 #endif
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/Kbuild b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/Kbuild
index 65e5d291ecda..98651a43bc12 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/Kbuild
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/Kbuild
@@ -13,6 +13,7 @@ nvkm-y += nvkm/engine/fifo/gk20a.o
 nvkm-y += nvkm/engine/fifo/gm107.o
 nvkm-y += nvkm/engine/fifo/gm200.o
 nvkm-y += nvkm/engine/fifo/gm20b.o
+nvkm-y += nvkm/engine/fifo/gp100.o
 
 nvkm-y += nvkm/engine/fifo/chan.o
 nvkm-y += nvkm/engine/fifo/channv50.o
@@ -31,3 +32,4 @@ nvkm-y += nvkm/engine/fifo/gpfifogf100.o
 nvkm-y += nvkm/engine/fifo/gpfifogk104.o
 nvkm-y += nvkm/engine/fifo/gpfifogk110.o
 nvkm-y += nvkm/engine/fifo/gpfifogm200.o
+nvkm-y += nvkm/engine/fifo/gpfifogp100.o
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/changk104.h b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/changk104.h
index e06f4d46f802..230f64e5f731 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/changk104.h
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/changk104.h
@@ -27,4 +27,5 @@ int gk104_fifo_gpfifo_new(struct nvkm_fifo *, const struct nvkm_oclass *,
 extern const struct nvkm_fifo_chan_oclass gk104_fifo_gpfifo_oclass;
 extern const struct nvkm_fifo_chan_oclass gk110_fifo_gpfifo_oclass;
 extern const struct nvkm_fifo_chan_oclass gm200_fifo_gpfifo_oclass;
+extern const struct nvkm_fifo_chan_oclass gp100_fifo_gpfifo_oclass;
 #endif
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.c
index 743f3a189f28..103c0afaaa6d 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gk104.c
@@ -329,7 +329,7 @@ gk104_fifo_intr_fault(struct gk104_fifo *fifo, int unit)
         }
 
         if (eu == NULL) {
-                enum nvkm_devidx engidx = nvkm_top_fault(device->top, unit);
+                enum nvkm_devidx engidx = nvkm_top_fault(device, unit);
                 if (engidx < NVKM_SUBDEV_NR) {
                         const char *src = nvkm_subdev_name[engidx];
                         char *dst = en;
@@ -589,7 +589,6 @@ gk104_fifo_oneinit(struct nvkm_fifo *base)
         struct gk104_fifo *fifo = gk104_fifo(base);
         struct nvkm_subdev *subdev = &fifo->base.engine.subdev;
         struct nvkm_device *device = subdev->device;
-        struct nvkm_top *top = device->top;
         int engn, runl, pbid, ret, i, j;
         enum nvkm_devidx engidx;
         u32 *map;
@@ -608,7 +607,7 @@ gk104_fifo_oneinit(struct nvkm_fifo *base)
 
         /* Determine runlist configuration from topology device info. */
         i = 0;
-        while ((int)(engidx = nvkm_top_engine(top, i++, &runl, &engn)) >= 0) {
+        while ((int)(engidx = nvkm_top_engine(device, i++, &runl, &engn)) >= 0) {
                 /* Determine which PBDMA handles requests for this engine. */
                 for (j = 0, pbid = -1; j < fifo->pbdma_nr; j++) {
                         if (map[j] & (1 << runl)) {
@@ -617,8 +616,8 @@ gk104_fifo_oneinit(struct nvkm_fifo *base)
                         }
                 }
 
-                nvkm_debug(subdev, "engine %2d: runlist %2d pbdma %2d\n",
-                           engn, runl, pbid);
+                nvkm_debug(subdev, "engine %2d: runlist %2d pbdma %2d (%s)\n",
+                           engn, runl, pbid, nvkm_subdev_name[engidx]);
 
                 fifo->engine[engn].engine = nvkm_device_engine(device, engidx);
                 fifo->engine[engn].runl = runl;
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gp100.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gp100.c
new file mode 100644
index 000000000000..eff83f7fb705
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gp100.c
@@ -0,0 +1,67 @@
+/*
+ * Copyright 2016 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs
+ */
+#include "gk104.h"
+#include "changk104.h"
+
+static const struct nvkm_enum
+gp100_fifo_fault_engine[] = {
+        { 0x01, "DISPLAY" },
+        { 0x03, "IFB", NULL, NVKM_ENGINE_IFB },
+        { 0x04, "BAR1", NULL, NVKM_SUBDEV_BAR },
+        { 0x05, "BAR2", NULL, NVKM_SUBDEV_INSTMEM },
+        { 0x06, "HOST0" },
+        { 0x07, "HOST1" },
+        { 0x08, "HOST2" },
+        { 0x09, "HOST3" },
+        { 0x0a, "HOST4" },
+        { 0x0b, "HOST5" },
+        { 0x0c, "HOST6" },
+        { 0x0d, "HOST7" },
+        { 0x0e, "HOST8" },
+        { 0x0f, "HOST9" },
+        { 0x10, "HOST10" },
+        { 0x13, "PERF" },
+        { 0x17, "PMU" },
+        { 0x18, "PTP" },
+        { 0x1f, "PHYSICAL" },
+        {}
+};
+
+static const struct gk104_fifo_func
+gp100_fifo = {
+        .fault.engine = gp100_fifo_fault_engine,
+        .fault.reason = gk104_fifo_fault_reason,
+        .fault.hubclient = gk104_fifo_fault_hubclient,
+        .fault.gpcclient = gk104_fifo_fault_gpcclient,
+        .chan = {
+                &gp100_fifo_gpfifo_oclass,
+                NULL
+        },
+};
+
+int
+gp100_fifo_new(struct nvkm_device *device, int index, struct nvkm_fifo **pfifo)
+{
+        return gk104_fifo_new_(&gp100_fifo, device, index, 4096, pfifo);
+}
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogp100.c b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogp100.c
new file mode 100644
index 000000000000..1530a9217aea
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/fifo/gpfifogp100.c
@@ -0,0 +1,34 @@
+/*
+ * Copyright 2016 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs
+ */
+#include "changk104.h"
+
+#include <nvif/class.h>
+
+const struct nvkm_fifo_chan_oclass
+gp100_fifo_gpfifo_oclass = {
+        .base.oclass = PASCAL_CHANNEL_GPFIFO_A,
+        .base.minver = 0,
+        .base.maxver = 0,
+        .ctor = gk104_fifo_gpfifo_new,
+};
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/Kbuild b/drivers/gpu/drm/nouveau/nvkm/engine/gr/Kbuild
index 290ed0db8047..f1c494182248 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/Kbuild
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/Kbuild
@@ -31,6 +31,7 @@ nvkm-y += nvkm/engine/gr/gk20a.o
 nvkm-y += nvkm/engine/gr/gm107.o
 nvkm-y += nvkm/engine/gr/gm200.o
 nvkm-y += nvkm/engine/gr/gm20b.o
+nvkm-y += nvkm/engine/gr/gp100.o
 
 nvkm-y += nvkm/engine/gr/ctxnv40.o
 nvkm-y += nvkm/engine/gr/ctxnv50.o
@@ -48,3 +49,4 @@ nvkm-y += nvkm/engine/gr/ctxgk20a.o
 nvkm-y += nvkm/engine/gr/ctxgm107.o
 nvkm-y += nvkm/engine/gr/ctxgm200.o
 nvkm-y += nvkm/engine/gr/ctxgm20b.o
+nvkm-y += nvkm/engine/gr/ctxgp100.o
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgf100.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgf100.c
index b02d8f50ea6a..bc77eea351a5 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgf100.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgf100.c
@@ -1240,7 +1240,7 @@ gf100_grctx_generate_main(struct gf100_gr *gr, struct gf100_grctx *info)
         const struct gf100_grctx_func *grctx = gr->func->grctx;
         u32 idle_timeout;
 
-        nvkm_mc_unk260(device->mc, 0);
+        nvkm_mc_unk260(device, 0);
 
         gf100_gr_mmio(gr, grctx->hub);
         gf100_gr_mmio(gr, grctx->gpc);
@@ -1264,7 +1264,7 @@ gf100_grctx_generate_main(struct gf100_gr *gr, struct gf100_grctx *info)
         gf100_gr_icmd(gr, grctx->icmd);
         nvkm_wr32(device, 0x404154, idle_timeout);
         gf100_gr_mthd(gr, grctx->mthd);
-        nvkm_mc_unk260(device->mc, 1);
+        nvkm_mc_unk260(device, 1);
 }
 
 int
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgf100.h b/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgf100.h
index ac895edce164..52048b5a5274 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgf100.h
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgf100.h
@@ -101,6 +101,8 @@ void gm200_grctx_generate_405b60(struct gf100_gr *);
 
 extern const struct gf100_grctx_func gm20b_grctx;
 
+extern const struct gf100_grctx_func gp100_grctx;
+
 /* context init value lists */
 
 extern const struct gf100_gr_pack gf100_grctx_pack_icmd[];
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgf117.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgf117.c
index f521de11a299..c925ade5880e 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgf117.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgf117.c
@@ -226,7 +226,7 @@ gf117_grctx_generate_main(struct gf100_gr *gr, struct gf100_grctx *info)
         u32 idle_timeout;
         int i;
 
-        nvkm_mc_unk260(device->mc, 0);
+        nvkm_mc_unk260(device, 0);
 
         gf100_gr_mmio(gr, grctx->hub);
         gf100_gr_mmio(gr, grctx->gpc);
@@ -253,7 +253,7 @@ gf117_grctx_generate_main(struct gf100_gr *gr, struct gf100_grctx *info)
         gf100_gr_icmd(gr, grctx->icmd);
         nvkm_wr32(device, 0x404154, idle_timeout);
         gf100_gr_mthd(gr, grctx->mthd);
-        nvkm_mc_unk260(device->mc, 1);
+        nvkm_mc_unk260(device, 1);
 }
 
 const struct gf100_grctx_func
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgk104.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgk104.c
index 9ba337778ef5..c46b3fdf7203 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgk104.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgk104.c
@@ -950,7 +950,7 @@ gk104_grctx_generate_main(struct gf100_gr *gr, struct gf100_grctx *info)
         u32 idle_timeout;
         int i;
 
-        nvkm_mc_unk260(device->mc, 0);
+        nvkm_mc_unk260(device, 0);
 
         gf100_gr_mmio(gr, grctx->hub);
         gf100_gr_mmio(gr, grctx->gpc);
@@ -979,7 +979,7 @@ gk104_grctx_generate_main(struct gf100_gr *gr, struct gf100_grctx *info)
         gf100_gr_icmd(gr, grctx->icmd);
         nvkm_wr32(device, 0x404154, idle_timeout);
         gf100_gr_mthd(gr, grctx->mthd);
-        nvkm_mc_unk260(device->mc, 1);
+        nvkm_mc_unk260(device, 1);
 
         nvkm_mask(device, 0x418800, 0x00200000, 0x00200000);
         nvkm_mask(device, 0x41be10, 0x00800000, 0x00800000);
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgp100.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgp100.c
new file mode 100644
index 000000000000..3d1ae7ddf7dd
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/ctxgp100.c
@@ -0,0 +1,179 @@
+/*
+ * Copyright 2016 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs <bskeggs@redhat.com>
+ */
+#include "ctxgf100.h"
+
+#include <subdev/fb.h>
+
+/*******************************************************************************
+ * PGRAPH context implementation
+ ******************************************************************************/
+
+static void
+gp100_grctx_generate_pagepool(struct gf100_grctx *info)
+{
+        const struct gf100_grctx_func *grctx = info->gr->func->grctx;
+        const u32 access = NV_MEM_ACCESS_RW | NV_MEM_ACCESS_SYS;
+        const int s = 8;
+        const int b = mmio_vram(info, grctx->pagepool_size, (1 << s), access);
+        mmio_refn(info, 0x40800c, 0x00000000, s, b);
+        mmio_wr32(info, 0x408010, 0x80000000);
+        mmio_refn(info, 0x419004, 0x00000000, s, b);
+        mmio_wr32(info, 0x419008, 0x00000000);
+}
+
+static void
+gp100_grctx_generate_attrib(struct gf100_grctx *info)
+{
+        struct gf100_gr *gr = info->gr;
+        const struct gf100_grctx_func *grctx = gr->func->grctx;
+        const u32  alpha = grctx->alpha_nr;
+        const u32 attrib = grctx->attrib_nr;
+        const u32 pertpc = 0x20 * (grctx->attrib_nr_max + grctx->alpha_nr_max);
+        const u32   size = roundup(gr->tpc_total * pertpc, 0x80);
+        const u32 access = NV_MEM_ACCESS_RW;
+        const int s = 12;
+        const int b = mmio_vram(info, size, (1 << s), access);
+        const int max_batches = 0xffff;
+        u32 ao = 0;
+        u32 bo = ao + grctx->alpha_nr_max * gr->tpc_total;
+        int gpc, ppc, n = 0;
+
+        mmio_refn(info, 0x418810, 0x80000000, s, b);
+        mmio_refn(info, 0x419848, 0x10000000, s, b);
+        mmio_refn(info, 0x419c2c, 0x10000000, s, b);
+        mmio_refn(info, 0x419b00, 0x00000000, s, b);
+        mmio_wr32(info, 0x419b04, 0x80000000 | size >> 7);
+        mmio_wr32(info, 0x405830, attrib);
+        mmio_wr32(info, 0x40585c, alpha);
+        mmio_wr32(info, 0x4064c4, ((alpha / 4) << 16) | max_batches);
+
+        for (gpc = 0; gpc < gr->gpc_nr; gpc++) {
+                for (ppc = 0; ppc < gr->ppc_nr[gpc]; ppc++, n++) {
+                        const u32 as =  alpha * gr->ppc_tpc_nr[gpc][ppc];
+                        const u32 bs = attrib * gr->ppc_tpc_nr[gpc][ppc];
+                        const u32 u = 0x418ea0 + (n * 0x04);
+                        const u32 o = PPC_UNIT(gpc, ppc, 0);
+                        if (!(gr->ppc_mask[gpc] & (1 << ppc)))
+                                continue;
+                        mmio_wr32(info, o + 0xc0, bs);
+                        mmio_wr32(info, o + 0xf4, bo);
+                        mmio_wr32(info, o + 0xf0, bs);
+                        bo += grctx->attrib_nr_max * gr->ppc_tpc_nr[gpc][ppc];
+                        mmio_wr32(info, o + 0xe4, as);
+                        mmio_wr32(info, o + 0xf8, ao);
+                        ao += grctx->alpha_nr_max * gr->ppc_tpc_nr[gpc][ppc];
+                        mmio_wr32(info, u, bs);
+                }
+        }
+
+        mmio_wr32(info, 0x418eec, 0x00000000);
+        mmio_wr32(info, 0x41befc, 0x00000000);
+}
+
+static void
+gp100_grctx_generate_405b60(struct gf100_gr *gr)
+{
+        struct nvkm_device *device = gr->base.engine.subdev.device;
+        const u32 dist_nr = DIV_ROUND_UP(gr->tpc_total, 4);
+        u32 dist[TPC_MAX / 4] = {};
+        u32 gpcs[GPC_MAX * 2] = {};
+        u8  tpcnr[GPC_MAX];
+        int tpc, gpc, i;
+
+        memcpy(tpcnr, gr->tpc_nr, sizeof(gr->tpc_nr));
+
+        /* won't result in the same distribution as the binary driver where
+         * some of the gpcs have more tpcs than others, but this shall do
+         * for the moment.  the code for earlier gpus has this issue too.
+         */
+        for (gpc = -1, i = 0; i < gr->tpc_total; i++) {
+                do {
+                        gpc = (gpc + 1) % gr->gpc_nr;
+                } while(!tpcnr[gpc]);
+                tpc = gr->tpc_nr[gpc] - tpcnr[gpc]--;
+
+                dist[i / 4] |= ((gpc << 4) | tpc) << ((i % 4) * 8);
+                gpcs[gpc + (gr->gpc_nr * (tpc / 4))] |= i << (tpc * 8);
+        }
+
+        for (i = 0; i < dist_nr; i++)
+                nvkm_wr32(device, 0x405b60 + (i * 4), dist[i]);
+        for (i = 0; i < gr->gpc_nr * 2; i++)
+                nvkm_wr32(device, 0x405ba0 + (i * 4), gpcs[i]);
+}
+
+static void
+gp100_grctx_generate_main(struct gf100_gr *gr, struct gf100_grctx *info)
+{
+        struct nvkm_device *device = gr->base.engine.subdev.device;
+        const struct gf100_grctx_func *grctx = gr->func->grctx;
+        u32 idle_timeout, tmp;
+        int i;
+
+        gf100_gr_mmio(gr, gr->fuc_sw_ctx);
+
+        idle_timeout = nvkm_mask(device, 0x404154, 0xffffffff, 0x00000000);
+
+        grctx->pagepool(info);
+        grctx->bundle(info);
+        grctx->attrib(info);
+        grctx->unkn(gr);
+
+        gm200_grctx_generate_tpcid(gr);
+        gf100_grctx_generate_r406028(gr);
+        gk104_grctx_generate_r418bb8(gr);
+
+        for (i = 0; i < 8; i++)
+                nvkm_wr32(device, 0x4064d0 + (i * 0x04), 0x00000000);
+        nvkm_wr32(device, 0x406500, 0x00000000);
+
+        nvkm_wr32(device, 0x405b00, (gr->tpc_total << 8) | gr->gpc_nr);
+
+        for (tmp = 0, i = 0; i < gr->gpc_nr; i++)
+                tmp |= ((1 << gr->tpc_nr[i]) - 1) << (i * 5);
+        nvkm_wr32(device, 0x4041c4, tmp);
+
+        gp100_grctx_generate_405b60(gr);
+
+        gf100_gr_icmd(gr, gr->fuc_bundle);
+        nvkm_wr32(device, 0x404154, idle_timeout);
+        gf100_gr_mthd(gr, gr->fuc_method);
+}
+
+const struct gf100_grctx_func
+gp100_grctx = {
+        .main  = gp100_grctx_generate_main,
+        .unkn  = gk104_grctx_generate_unkn,
+        .bundle = gm107_grctx_generate_bundle,
+        .bundle_size = 0x3000,
+        .bundle_min_gpm_fifo_depth = 0x180,
+        .bundle_token_limit = 0x1080,
+        .pagepool = gp100_grctx_generate_pagepool,
+        .pagepool_size = 0x20000,
+        .attrib = gp100_grctx_generate_attrib,
+        .attrib_nr_max = 0x660,
+        .attrib_nr = 0x440,
+        .alpha_nr_max = 0xc00,
+        .alpha_nr = 0x800,
+};
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.c
index ae9ab5b1ab97..157919c788e6 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.c
@@ -1457,24 +1457,30 @@ gf100_gr_init_ctxctl(struct gf100_gr *gr)
         struct nvkm_device *device = subdev->device;
         struct nvkm_secboot *sb = device->secboot;
         int i;
+        int ret = 0;
 
         if (gr->firmware) {
                 /* load fuc microcode */
-                nvkm_mc_unk260(device->mc, 0);
+                nvkm_mc_unk260(device, 0);
 
                 /* securely-managed falcons must be reset using secure boot */
                 if (nvkm_secboot_is_managed(sb, NVKM_SECBOOT_FALCON_FECS))
-                        nvkm_secboot_reset(sb, NVKM_SECBOOT_FALCON_FECS);
+                        ret = nvkm_secboot_reset(sb, NVKM_SECBOOT_FALCON_FECS);
                 else
                         gf100_gr_init_fw(gr, 0x409000, &gr->fuc409c,
                                          &gr->fuc409d);
+                if (ret)
+                        return ret;
+
                 if (nvkm_secboot_is_managed(sb, NVKM_SECBOOT_FALCON_GPCCS))
-                        nvkm_secboot_reset(sb, NVKM_SECBOOT_FALCON_GPCCS);
+                        ret = nvkm_secboot_reset(sb, NVKM_SECBOOT_FALCON_GPCCS);
                 else
                         gf100_gr_init_fw(gr, 0x41a000, &gr->fuc41ac,
                                          &gr->fuc41ad);
+                if (ret)
+                        return ret;
 
-                nvkm_mc_unk260(device->mc, 1);
+                nvkm_mc_unk260(device, 1);
 
                 /* start both of them running */
                 nvkm_wr32(device, 0x409840, 0xffffffff);
@@ -1576,7 +1582,7 @@ gf100_gr_init_ctxctl(struct gf100_gr *gr)
         }
 
         /* load HUB microcode */
-        nvkm_mc_unk260(device->mc, 0);
+        nvkm_mc_unk260(device, 0);
         nvkm_wr32(device, 0x4091c0, 0x01000000);
         for (i = 0; i < gr->func->fecs.ucode->data.size / 4; i++)
                 nvkm_wr32(device, 0x4091c4, gr->func->fecs.ucode->data.data[i]);
@@ -1599,7 +1605,7 @@ gf100_gr_init_ctxctl(struct gf100_gr *gr)
                         nvkm_wr32(device, 0x41a188, i >> 6);
                 nvkm_wr32(device, 0x41a184, gr->func->gpccs.ucode->code.data[i]);
         }
-        nvkm_mc_unk260(device->mc, 1);
+        nvkm_mc_unk260(device, 1);
 
         /* load register lists */
         gf100_gr_init_csdata(gr, grctx->hub, 0x409000, 0x000, 0x000000);
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.h b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.h
index 2b98abdb9270..268b8d60ff73 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.h
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gf100.h
@@ -292,4 +292,6 @@ extern const struct gf100_gr_init gm107_gr_init_l1c_0[];
 extern const struct gf100_gr_init gm107_gr_init_wwdx_0[];
 extern const struct gf100_gr_init gm107_gr_init_cbm_0[];
 void gm107_gr_init_bios(struct gf100_gr *);
+
+void gm200_gr_init_gpc_mmu(struct gf100_gr *);
 #endif
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gk20a.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gk20a.c
index 4ca8ed15191c..de8b806b88fd 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gk20a.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gk20a.c
@@ -361,6 +361,5 @@ gk20a_gr_new(struct nvkm_device *device, int index, struct nvkm_gr **pgr)
         if (ret)
                 return ret;
 
-
         return 0;
 }
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gm200.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gm200.c
index 4dfa4513bb6c..6435f1257572 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gm200.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gm200.c
@@ -38,7 +38,7 @@ gm200_gr_rops(struct gf100_gr *gr)
         return nvkm_rd32(gr->base.engine.subdev.device, 0x12006c);
 }
 
-static void
+void
 gm200_gr_init_gpc_mmu(struct gf100_gr *gr)
 {
         struct nvkm_device *device = gr->base.engine.subdev.device;
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/gp100.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gp100.c
new file mode 100644
index 000000000000..26ad79def0ff
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/gp100.c
@@ -0,0 +1,171 @@
+/*
+ * Copyright 2016 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs <bskeggs@redhat.com>
+ */
+#include "gf100.h"
+#include "ctxgf100.h"
+
+#include <nvif/class.h>
+
+/*******************************************************************************
+ * PGRAPH engine/subdev functions
+ ******************************************************************************/
+
+static void
+gp100_gr_init_rop_active_fbps(struct gf100_gr *gr)
+{
+        struct nvkm_device *device = gr->base.engine.subdev.device;
+        /*XXX: otherwise identical to gm200 aside from mask.. do everywhere? */
+        const u32 fbp_count = nvkm_rd32(device, 0x12006c) & 0x0000000f;
+        nvkm_mask(device, 0x408850, 0x0000000f, fbp_count); /* zrop */
+        nvkm_mask(device, 0x408958, 0x0000000f, fbp_count); /* crop */
+}
+
+static int
+gp100_gr_init(struct gf100_gr *gr)
+{
+        struct nvkm_device *device = gr->base.engine.subdev.device;
+        const u32 magicgpc918 = DIV_ROUND_UP(0x00800000, gr->tpc_total);
+        u32 data[TPC_MAX / 8] = {};
+        u8  tpcnr[GPC_MAX];
+        int gpc, tpc, rop;
+        int i;
+
+        gr->func->init_gpc_mmu(gr);
+
+        gf100_gr_mmio(gr, gr->fuc_sw_nonctx);
+
+        nvkm_wr32(device, GPC_UNIT(0, 0x3018), 0x00000001);
+
+        memset(data, 0x00, sizeof(data));
+        memcpy(tpcnr, gr->tpc_nr, sizeof(gr->tpc_nr));
+        for (i = 0, gpc = -1; i < gr->tpc_total; i++) {
+                do {
+                        gpc = (gpc + 1) % gr->gpc_nr;
+                } while (!tpcnr[gpc]);
+                tpc = gr->tpc_nr[gpc] - tpcnr[gpc]--;
+
+                data[i / 8] |= tpc << ((i % 8) * 4);
+        }
+
+        nvkm_wr32(device, GPC_BCAST(0x0980), data[0]);
+        nvkm_wr32(device, GPC_BCAST(0x0984), data[1]);
+        nvkm_wr32(device, GPC_BCAST(0x0988), data[2]);
+        nvkm_wr32(device, GPC_BCAST(0x098c), data[3]);
+
+        for (gpc = 0; gpc < gr->gpc_nr; gpc++) {
+                nvkm_wr32(device, GPC_UNIT(gpc, 0x0914),
+                          gr->screen_tile_row_offset << 8 | gr->tpc_nr[gpc]);
+                nvkm_wr32(device, GPC_UNIT(gpc, 0x0910), 0x00040000 |
+                                                         gr->tpc_total);
+                nvkm_wr32(device, GPC_UNIT(gpc, 0x0918), magicgpc918);
+        }
+
+        nvkm_wr32(device, GPC_BCAST(0x3fd4), magicgpc918);
+        nvkm_wr32(device, GPC_BCAST(0x08ac), nvkm_rd32(device, 0x100800));
+        nvkm_wr32(device, GPC_BCAST(0x033c), nvkm_rd32(device, 0x100804));
+
+        gr->func->init_rop_active_fbps(gr);
+
+        nvkm_wr32(device, 0x400500, 0x00010001);
+        nvkm_wr32(device, 0x400100, 0xffffffff);
+        nvkm_wr32(device, 0x40013c, 0xffffffff);
+        nvkm_wr32(device, 0x400124, 0x00000002);
+        nvkm_wr32(device, 0x409c24, 0x000f0002);
+        nvkm_wr32(device, 0x405848, 0xc0000000);
+        nvkm_mask(device, 0x40584c, 0x00000000, 0x00000001);
+        nvkm_wr32(device, 0x404000, 0xc0000000);
+        nvkm_wr32(device, 0x404600, 0xc0000000);
+        nvkm_wr32(device, 0x408030, 0xc0000000);
+        nvkm_wr32(device, 0x404490, 0xc0000000);
+        nvkm_wr32(device, 0x406018, 0xc0000000);
+        nvkm_wr32(device, 0x407020, 0x40000000);
+        nvkm_wr32(device, 0x405840, 0xc0000000);
+        nvkm_wr32(device, 0x405844, 0x00ffffff);
+        nvkm_mask(device, 0x419cc0, 0x00000008, 0x00000008);
+
+        nvkm_mask(device, 0x419c9c, 0x00010000, 0x00010000);
+        nvkm_mask(device, 0x419c9c, 0x00020000, 0x00020000);
+
+        gr->func->init_ppc_exceptions(gr);
+
+        for (gpc = 0; gpc < gr->gpc_nr; gpc++) {
+                nvkm_wr32(device, GPC_UNIT(gpc, 0x0420), 0xc0000000);
+                nvkm_wr32(device, GPC_UNIT(gpc, 0x0900), 0xc0000000);
+                nvkm_wr32(device, GPC_UNIT(gpc, 0x1028), 0xc0000000);
+                nvkm_wr32(device, GPC_UNIT(gpc, 0x0824), 0xc0000000);
+                for (tpc = 0; tpc < gr->tpc_nr[gpc]; tpc++) {
+                        nvkm_wr32(device, TPC_UNIT(gpc, tpc, 0x508), 0xffffffff);
+                        nvkm_wr32(device, TPC_UNIT(gpc, tpc, 0x50c), 0xffffffff);
+                        nvkm_wr32(device, TPC_UNIT(gpc, tpc, 0x224), 0xc0000000);
+                        nvkm_wr32(device, TPC_UNIT(gpc, tpc, 0x48c), 0xc0000000);
+                        nvkm_wr32(device, TPC_UNIT(gpc, tpc, 0x084), 0xc0000000);
+                        nvkm_wr32(device, TPC_UNIT(gpc, tpc, 0x430), 0xc0000000);
+                        nvkm_wr32(device, TPC_UNIT(gpc, tpc, 0x644), 0x00dffffe);
+                        nvkm_wr32(device, TPC_UNIT(gpc, tpc, 0x64c), 0x00000105);
+                }
+                nvkm_wr32(device, GPC_UNIT(gpc, 0x2c90), 0xffffffff);
+                nvkm_wr32(device, GPC_UNIT(gpc, 0x2c94), 0xffffffff);
+        }
+
+        for (rop = 0; rop < gr->rop_nr; rop++) {
+                nvkm_wr32(device, ROP_UNIT(rop, 0x144), 0x40000000);
+                nvkm_wr32(device, ROP_UNIT(rop, 0x070), 0x40000000);
+                nvkm_wr32(device, ROP_UNIT(rop, 0x204), 0xffffffff);
+                nvkm_wr32(device, ROP_UNIT(rop, 0x208), 0xffffffff);
+        }
+
+        nvkm_wr32(device, 0x400108, 0xffffffff);
+        nvkm_wr32(device, 0x400138, 0xffffffff);
+        nvkm_wr32(device, 0x400118, 0xffffffff);
+        nvkm_wr32(device, 0x400130, 0xffffffff);
+        nvkm_wr32(device, 0x40011c, 0xffffffff);
+        nvkm_wr32(device, 0x400134, 0xffffffff);
+
+        gf100_gr_zbc_init(gr);
+
+        return gf100_gr_init_ctxctl(gr);
+}
+
+static const struct gf100_gr_func
+gp100_gr = {
+        .init = gp100_gr_init,
+        .init_gpc_mmu = gm200_gr_init_gpc_mmu,
+        .init_rop_active_fbps = gp100_gr_init_rop_active_fbps,
+        .init_ppc_exceptions = gk104_gr_init_ppc_exceptions,
+        .rops = gm200_gr_rops,
+        .ppc_nr = 2,
+        .grctx = &gp100_grctx,
+        .sclass = {
+                { -1, -1, FERMI_TWOD_A },
+                { -1, -1, KEPLER_INLINE_TO_MEMORY_B },
+                { -1, -1, PASCAL_A, &gf100_fermi },
+                { -1, -1, PASCAL_COMPUTE_A },
+                {}
+        }
+};
+
+int
+gp100_gr_new(struct nvkm_device *device, int index, struct nvkm_gr **pgr)
+{
+        return gm200_gr_new_(&gp100_gr, device, index, pgr);
+}
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/nv30.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/nv30.c
index 69de8c6259fe..f1e15a4d4f64 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/nv30.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/nv30.c
@@ -76,8 +76,8 @@ nv30_gr_chan_new(struct nvkm_gr *base, struct nvkm_fifo_chan *fifoch,
                 nvkm_wo32(chan->inst, i, 0x00040004);
         for (i = 0x1f18; i <= 0x3088 ; i += 16) {
                 nvkm_wo32(chan->inst, i + 0, 0x10700ff9);
-                nvkm_wo32(chan->inst, i + 1, 0x0436086c);
-                nvkm_wo32(chan->inst, i + 2, 0x000c001b);
+                nvkm_wo32(chan->inst, i + 4, 0x0436086c);
+                nvkm_wo32(chan->inst, i + 8, 0x000c001b);
         }
         for (i = 0x30b8; i < 0x30c8; i += 4)
                 nvkm_wo32(chan->inst, i, 0x0000ffff);
diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/gr/nv34.c b/drivers/gpu/drm/nouveau/nvkm/engine/gr/nv34.c
index 2207dac23981..300f5ed5de0b 100644
--- a/drivers/gpu/drm/nouveau/nvkm/engine/gr/nv34.c
+++ b/drivers/gpu/drm/nouveau/nvkm/engine/gr/nv34.c
@@ -75,8 +75,8 @@ nv34_gr_chan_new(struct nvkm_gr *base, struct nvkm_fifo_chan *fifoch,
                 nvkm_wo32(chan->inst, i, 0x00040004);
         for (i = 0x15ac; i <= 0x271c ; i += 16) {
                 nvkm_wo32(chan->inst, i + 0, 0x10700ff9);
-                nvkm_wo32(chan->inst, i + 1, 0x0436086c);
-                nvkm_wo32(chan->inst, i + 2, 0x000c001b);
+                nvkm_wo32(chan->inst, i + 4, 0x0436086c);
+                nvkm_wo32(chan->inst, i + 8, 0x000c001b);
         }
         for (i = 0x274c; i < 0x275c; i += 4)
                 nvkm_wo32(chan->inst, i, 0x0000ffff);
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/bios/base.c b/drivers/gpu/drm/nouveau/nvkm/subdev/bios/base.c
index e15b9627b07e..f3c30b2a788e 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/bios/base.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/bios/base.c
@@ -26,6 +26,49 @@
 #include <subdev/bios.h>
 #include <subdev/bios/bmp.h>
 #include <subdev/bios/bit.h>
+#include <subdev/bios/image.h>
+
+static bool
+nvbios_addr(struct nvkm_bios *bios, u32 *addr, u8 size)
+{
+        u32 p = *addr;
+
+        if (*addr > bios->image0_size && bios->imaged_addr) {
+                *addr -= bios->image0_size;
+                *addr += bios->imaged_addr;
+        }
+
+        if (unlikely(*addr + size >= bios->size)) {
+                nvkm_error(&bios->subdev, "OOB %d %08x %08x\n", size, p, *addr);
+                return false;
+        }
+
+        return true;
+}
+
+u8
+nvbios_rd08(struct nvkm_bios *bios, u32 addr)
+{
+        if (likely(nvbios_addr(bios, &addr, 1)))
+                return bios->data[addr];
+        return 0x00;
+}
+
+u16
+nvbios_rd16(struct nvkm_bios *bios, u32 addr)
+{
+        if (likely(nvbios_addr(bios, &addr, 2)))
+                return get_unaligned_le16(&bios->data[addr]);
+        return 0x0000;
+}
+
+u32
+nvbios_rd32(struct nvkm_bios *bios, u32 addr)
+{
+        if (likely(nvbios_addr(bios, &addr, 4)))
+                return get_unaligned_le32(&bios->data[addr]);
+        return 0x00000000;
+}
 
 u8
 nvbios_checksum(const u8 *data, int size)
@@ -100,8 +143,9 @@ int
 nvkm_bios_new(struct nvkm_device *device, int index, struct nvkm_bios **pbios)
 {
         struct nvkm_bios *bios;
+        struct nvbios_image image;
         struct bit_entry bit_i;
-        int ret;
+        int ret, idx = 0;
 
         if (!(bios = *pbios = kzalloc(sizeof(*bios), GFP_KERNEL)))
                 return -ENOMEM;
@@ -111,6 +155,19 @@ nvkm_bios_new(struct nvkm_device *device, int index, struct nvkm_bios **pbios)
         if (ret)
                 return ret;
 
+        /* Some tables have weird pointers that need adjustment before
+         * they're dereferenced.  I'm not entirely sure why...
+         */
+        if (nvbios_image(bios, idx++, &image)) {
+                bios->image0_size = image.size;
+                while (nvbios_image(bios, idx++, &image)) {
+                        if (image.type == 0xe0) {
+                                bios->imaged_addr = image.base;
+                                break;
+                        }
+                }
+        }
+
         /* detect type of vbios we're dealing with */
         bios->bmp_offset = nvbios_findstr(bios->data, bios->size,
                                           "\xff\x7f""NV\0", 5);
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/bios/dp.c b/drivers/gpu/drm/nouveau/nvkm/subdev/bios/dp.c
index 05332476354a..d89e78c4e689 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/bios/dp.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/bios/dp.c
@@ -40,6 +40,7 @@ nvbios_dp_table(struct nvkm_bios *bios, u8 *ver, u8 *hdr, u8 *cnt, u8 *len)
                                 case 0x30:
                                 case 0x40:
                                 case 0x41:
+                                case 0x42:
                                         *hdr = nvbios_rd08(bios, data + 0x01);
                                         *len = nvbios_rd08(bios, data + 0x02);
                                         *cnt = nvbios_rd08(bios, data + 0x03);
@@ -70,6 +71,7 @@ nvbios_dpout_entry(struct nvkm_bios *bios, u8 idx,
                         break;
                 case 0x40:
                 case 0x41:
+                case 0x42:
                         *hdr = nvbios_rd08(bios, data + 0x04);
                         *cnt = 0;
                         *len = 0;
@@ -109,6 +111,7 @@ nvbios_dpout_parse(struct nvkm_bios *bios, u8 idx,
                         break;
                 case 0x40:
                 case 0x41:
+                case 0x42:
                         info->flags     = nvbios_rd08(bios, data + 0x04);
                         info->script[0] = nvbios_rd16(bios, data + 0x05);
                         info->script[1] = nvbios_rd16(bios, data + 0x07);
@@ -180,6 +183,11 @@ nvbios_dpcfg_parse(struct nvkm_bios *bios, u16 outp, u8 idx,
                         info->pe    = nvbios_rd08(bios, data + 0x02);
                         info->tx_pu = nvbios_rd08(bios, data + 0x03);
                         break;
+                case 0x42:
+                        info->dc    = nvbios_rd08(bios, data + 0x00);
+                        info->pe    = nvbios_rd08(bios, data + 0x01);
+                        info->tx_pu = nvbios_rd08(bios, data + 0x02);
+                        break;
                 default:
                         data = 0x0000;
                         break;
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/bios/image.c b/drivers/gpu/drm/nouveau/nvkm/subdev/bios/image.c
index 74b14cf09308..1dbff7aeafec 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/bios/image.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/bios/image.c
@@ -68,11 +68,16 @@ nvbios_imagen(struct nvkm_bios *bios, struct nvbios_image *image)
 bool
 nvbios_image(struct nvkm_bios *bios, int idx, struct nvbios_image *image)
 {
+        u32 imaged_addr = bios->imaged_addr;
         memset(image, 0x00, sizeof(*image));
+        bios->imaged_addr = 0;
         do {
                 image->base += image->size;
-                if (image->last || !nvbios_imagen(bios, image))
+                if (image->last || !nvbios_imagen(bios, image)) {
+                        bios->imaged_addr = imaged_addr;
                         return false;
+                }
         } while(idx--);
+        bios->imaged_addr = imaged_addr;
         return true;
 }
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/bios/pll.c b/drivers/gpu/drm/nouveau/nvkm/subdev/bios/pll.c
index 91a7dc56e406..2ca23a9157ab 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/bios/pll.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/bios/pll.c
@@ -77,15 +77,17 @@ g84_pll_mapping[] = {
         {}
 };
 
-static u16
+static u32
 pll_limits_table(struct nvkm_bios *bios, u8 *ver, u8 *hdr, u8 *cnt, u8 *len)
 {
         struct bit_entry bit_C;
-        u16 data = 0x0000;
+        u32 data = 0x0000;
 
         if (!bit_entry(bios, 'C', &bit_C)) {
                 if (bit_C.version == 1 && bit_C.length >= 10)
                         data = nvbios_rd16(bios, bit_C.offset + 8);
+                if (bit_C.version == 2 && bit_C.length >= 4)
+                        data = nvbios_rd32(bios, bit_C.offset + 0);
                 if (data) {
                         *ver = nvbios_rd08(bios, data + 0);
                         *hdr = nvbios_rd08(bios, data + 1);
@@ -137,12 +139,12 @@ pll_map(struct nvkm_bios *bios)
         }
 }
 
-static u16
+static u32
 pll_map_reg(struct nvkm_bios *bios, u32 reg, u32 *type, u8 *ver, u8 *len)
 {
         struct pll_mapping *map;
         u8  hdr, cnt;
-        u16 data;
+        u32 data;
 
         data = pll_limits_table(bios, ver, &hdr, &cnt, len);
         if (data && *ver >= 0x30) {
@@ -160,7 +162,7 @@ pll_map_reg(struct nvkm_bios *bios, u32 reg, u32 *type, u8 *ver, u8 *len)
         map = pll_map(bios);
         while (map && map->reg) {
                 if (map->reg == reg && *ver >= 0x20) {
-                        u16 addr = (data += hdr);
+                        u32 addr = (data += hdr);
                         *type = map->type;
                         while (cnt--) {
                                 if (nvbios_rd32(bios, data) == map->reg)
@@ -179,12 +181,12 @@ pll_map_reg(struct nvkm_bios *bios, u32 reg, u32 *type, u8 *ver, u8 *len)
         return 0x0000;
 }
 
-static u16
+static u32
 pll_map_type(struct nvkm_bios *bios, u8 type, u32 *reg, u8 *ver, u8 *len)
 {
         struct pll_mapping *map;
         u8  hdr, cnt;
-        u16 data;
+        u32 data;
 
         data = pll_limits_table(bios, ver, &hdr, &cnt, len);
         if (data && *ver >= 0x30) {
@@ -202,7 +204,7 @@ pll_map_type(struct nvkm_bios *bios, u8 type, u32 *reg, u8 *ver, u8 *len)
         map = pll_map(bios);
         while (map && map->reg) {
                 if (map->type == type && *ver >= 0x20) {
-                        u16 addr = (data += hdr);
+                        u32 addr = (data += hdr);
                         *reg = map->reg;
                         while (cnt--) {
                                 if (nvbios_rd32(bios, data) == map->reg)
@@ -228,7 +230,7 @@ nvbios_pll_parse(struct nvkm_bios *bios, u32 type, struct nvbios_pll *info)
         struct nvkm_device *device = subdev->device;
         u8  ver, len;
         u32 reg = type;
-        u16 data;
+        u32 data;
 
         if (type > PLL_MAX) {
                 reg  = type;
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/bios/pmu.c b/drivers/gpu/drm/nouveau/nvkm/subdev/bios/pmu.c
index c268e5afe852..b4a308f3cf7b 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/bios/pmu.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/bios/pmu.c
@@ -26,21 +26,6 @@
 #include <subdev/bios/image.h>
 #include <subdev/bios/pmu.h>
 
-static u32
-weirdo_pointer(struct nvkm_bios *bios, u32 data)
-{
-        struct nvbios_image image;
-        int idx = 0;
-        if (nvbios_image(bios, idx++, &image)) {
-                data -= image.size;
-                while (nvbios_image(bios, idx++, &image)) {
-                        if (image.type == 0xe0)
-                                return image.base + data;
-                }
-        }
-        return 0;
-}
-
 u32
 nvbios_pmuTe(struct nvkm_bios *bios, u8 *ver, u8 *hdr, u8 *cnt, u8 *len)
 {
@@ -50,7 +35,7 @@ nvbios_pmuTe(struct nvkm_bios *bios, u8 *ver, u8 *hdr, u8 *cnt, u8 *len)
         if (!bit_entry(bios, 'p', &bit_p)) {
                 if (bit_p.version == 2 && bit_p.length >= 4)
                         data = nvbios_rd32(bios, bit_p.offset + 0x00);
-                if ((data = weirdo_pointer(bios, data))) {
+                if (data) {
                         *ver = nvbios_rd08(bios, data + 0x00); /* maybe? */
                         *hdr = nvbios_rd08(bios, data + 0x01);
                         *len = nvbios_rd08(bios, data + 0x02);
@@ -97,8 +82,7 @@ nvbios_pmuRm(struct nvkm_bios *bios, u8 type, struct nvbios_pmuR *info)
         u32 data;
         memset(info, 0x00, sizeof(*info));
         while ((data = nvbios_pmuEp(bios, idx++, &ver, &hdr, &pmuE))) {
-                if ( pmuE.type == type &&
-                    (data = weirdo_pointer(bios, pmuE.data))) {
+                if (pmuE.type == type && (data = pmuE.data)) {
                         info->init_addr_pmu = nvbios_rd32(bios, data + 0x08);
                         info->args_addr_pmu = nvbios_rd32(bios, data + 0x0c);
                         info->boot_addr     = data + 0x30;
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/bios/rammap.c b/drivers/gpu/drm/nouveau/nvkm/subdev/bios/rammap.c
index d0ae7454764e..b57c370c725d 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/bios/rammap.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/bios/rammap.c
@@ -30,11 +30,11 @@ nvbios_rammapTe(struct nvkm_bios *bios, u8 *ver, u8 *hdr,
                 u8 *cnt, u8 *len, u8 *snr, u8 *ssz)
 {
         struct bit_entry bit_P;
-        u16 rammap = 0x0000;
+        u32 rammap = 0x0000;
 
         if (!bit_entry(bios, 'P', &bit_P)) {
                 if (bit_P.version == 2)
-                        rammap = nvbios_rd16(bios, bit_P.offset + 4);
+                        rammap = nvbios_rd32(bios, bit_P.offset + 4);
 
                 if (rammap) {
                         *ver = nvbios_rd08(bios, rammap + 0);
@@ -61,7 +61,7 @@ nvbios_rammapEe(struct nvkm_bios *bios, int idx,
                 u8 *ver, u8 *hdr, u8 *cnt, u8 *len)
 {
         u8  snr, ssz;
-        u16 rammap = nvbios_rammapTe(bios, ver, hdr, cnt, len, &snr, &ssz);
+        u32 rammap = nvbios_rammapTe(bios, ver, hdr, cnt, len, &snr, &ssz);
         if (rammap && idx < *cnt) {
                 rammap = rammap + *hdr + (idx * (*len + (snr * ssz)));
                 *hdr = *len;
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/clk/gf100.c b/drivers/gpu/drm/nouveau/nvkm/subdev/clk/gf100.c
index 78c449b417b7..89d5543118cf 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/clk/gf100.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/clk/gf100.c
@@ -99,7 +99,7 @@ read_div(struct gf100_clk *clk, int doff, u32 dsrc, u32 dctl)
 {
         struct nvkm_device *device = clk->base.subdev.device;
         u32 ssrc = nvkm_rd32(device, dsrc + (doff * 4));
-        u32 sctl = nvkm_rd32(device, dctl + (doff * 4));
+        u32 sclk, sctl, sdiv = 2;
 
         switch (ssrc & 0x00000003) {
         case 0:
@@ -109,13 +109,21 @@ read_div(struct gf100_clk *clk, int doff, u32 dsrc, u32 dctl)
         case 2:
                 return 100000;
         case 3:
-                if (sctl & 0x80000000) {
-                        u32 sclk = read_vco(clk, dsrc + (doff * 4));
-                        u32 sdiv = (sctl & 0x0000003f) + 2;
-                        return (sclk * 2) / sdiv;
+                sclk = read_vco(clk, dsrc + (doff * 4));
+
+                /* Memclk has doff of 0 despite its alt. location */
+                if (doff <= 2) {
+                        sctl = nvkm_rd32(device, dctl + (doff * 4));
+
+                        if (sctl & 0x80000000) {
+                                if (ssrc & 0x100)
+                                        sctl >>= 8;
+
+                                sdiv = (sctl & 0x3f) + 2;
+                        }
                 }
 
-                return read_vco(clk, dsrc + (doff * 4));
+                return (sclk * 2) / sdiv;
         default:
                 return 0;
         }
@@ -366,11 +374,17 @@ gf100_clk_prog_2(struct gf100_clk *clk, int idx)
                 if (info->coef) {
                         nvkm_wr32(device, addr + 0x04, info->coef);
                         nvkm_mask(device, addr + 0x00, 0x00000001, 0x00000001);
+
+                        /* Test PLL lock */
+                        nvkm_mask(device, addr + 0x00, 0x00000010, 0x00000000);
                         nvkm_msec(device, 2000,
                                 if (nvkm_rd32(device, addr + 0x00) & 0x00020000)
                                         break;
                         );
-                        nvkm_mask(device, addr + 0x00, 0x00020004, 0x00000004);
+                        nvkm_mask(device, addr + 0x00, 0x00000010, 0x00000010);
+
+                        /* Enable sync mode */
+                        nvkm_mask(device, addr + 0x00, 0x00000004, 0x00000004);
                 }
         }
 }
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/clk/gk104.c b/drivers/gpu/drm/nouveau/nvkm/subdev/clk/gk104.c
index 975c401bccab..06bc0d2d6ae1 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/clk/gk104.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/clk/gk104.c
@@ -393,11 +393,17 @@ gk104_clk_prog_2(struct gk104_clk *clk, int idx)
         if (info->coef) {
                 nvkm_wr32(device, addr + 0x04, info->coef);
                 nvkm_mask(device, addr + 0x00, 0x00000001, 0x00000001);
+
+                /* Test PLL lock */
+                nvkm_mask(device, addr + 0x00, 0x00000010, 0x00000000);
                 nvkm_msec(device, 2000,
                         if (nvkm_rd32(device, addr + 0x00) & 0x00020000)
                                 break;
                 );
-                nvkm_mask(device, addr + 0x00, 0x00020004, 0x00000004);
+                nvkm_mask(device, addr + 0x00, 0x00000010, 0x00000010);
+
+                /* Enable sync mode */
+                nvkm_mask(device, addr + 0x00, 0x00000004, 0x00000004);
         }
 }
 
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/clk/gk20a.c b/drivers/gpu/drm/nouveau/nvkm/subdev/clk/gk20a.c
index 5f0ee24e31b8..218893e3e5f9 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/clk/gk20a.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/clk/gk20a.c
@@ -28,69 +28,6 @@
 #include <core/tegra.h>
 #include <subdev/timer.h>
 
-#define KHZ (1000)
-#define MHZ (KHZ * 1000)
-
-#define MASK(w) ((1 << w) - 1)
-
-#define GPCPLL_CFG              (SYS_GPCPLL_CFG_BASE + 0)
-#define GPCPLL_CFG_ENABLE       BIT(0)
-#define GPCPLL_CFG_IDDQ         BIT(1)
-#define GPCPLL_CFG_LOCK_DET_OFF BIT(4)
-#define GPCPLL_CFG_LOCK         BIT(17)
-
-#define GPCPLL_COEFF            (SYS_GPCPLL_CFG_BASE + 4)
-#define GPCPLL_COEFF_M_SHIFT    0
-#define GPCPLL_COEFF_M_WIDTH    8
-#define GPCPLL_COEFF_N_SHIFT    8
-#define GPCPLL_COEFF_N_WIDTH    8
-#define GPCPLL_COEFF_P_SHIFT    16
-#define GPCPLL_COEFF_P_WIDTH    6
-
-#define GPCPLL_CFG2                     (SYS_GPCPLL_CFG_BASE + 0xc)
-#define GPCPLL_CFG2_SETUP2_SHIFT        16
-#define GPCPLL_CFG2_PLL_STEPA_SHIFT     24
-
-#define GPCPLL_CFG3                     (SYS_GPCPLL_CFG_BASE + 0x18)
-#define GPCPLL_CFG3_PLL_STEPB_SHIFT     16
-
-#define GPC_BCASE_GPCPLL_CFG_BASE               0x00132800
-#define GPCPLL_NDIV_SLOWDOWN                    (SYS_GPCPLL_CFG_BASE + 0x1c)
-#define GPCPLL_NDIV_SLOWDOWN_NDIV_LO_SHIFT      0
-#define GPCPLL_NDIV_SLOWDOWN_NDIV_MID_SHIFT     8
-#define GPCPLL_NDIV_SLOWDOWN_STEP_SIZE_LO2MID_SHIFT     16
-#define GPCPLL_NDIV_SLOWDOWN_SLOWDOWN_USING_PLL_SHIFT   22
-#define GPCPLL_NDIV_SLOWDOWN_EN_DYNRAMP_SHIFT   31
-
-#define SEL_VCO                         (SYS_GPCPLL_CFG_BASE + 0x100)
-#define SEL_VCO_GPC2CLK_OUT_SHIFT       0
-
-#define GPC2CLK_OUT                     (SYS_GPCPLL_CFG_BASE + 0x250)
-#define GPC2CLK_OUT_SDIV14_INDIV4_WIDTH 1
-#define GPC2CLK_OUT_SDIV14_INDIV4_SHIFT 31
-#define GPC2CLK_OUT_SDIV14_INDIV4_MODE  1
-#define GPC2CLK_OUT_VCODIV_WIDTH        6
-#define GPC2CLK_OUT_VCODIV_SHIFT        8
-#define GPC2CLK_OUT_VCODIV1             0
-#define GPC2CLK_OUT_VCODIV_MASK         (MASK(GPC2CLK_OUT_VCODIV_WIDTH) << \
-                                        GPC2CLK_OUT_VCODIV_SHIFT)
-#define GPC2CLK_OUT_BYPDIV_WIDTH        6
-#define GPC2CLK_OUT_BYPDIV_SHIFT        0
-#define GPC2CLK_OUT_BYPDIV31            0x3c
-#define GPC2CLK_OUT_INIT_MASK   ((MASK(GPC2CLK_OUT_SDIV14_INDIV4_WIDTH) << \
-                GPC2CLK_OUT_SDIV14_INDIV4_SHIFT)\
-                | (MASK(GPC2CLK_OUT_VCODIV_WIDTH) << GPC2CLK_OUT_VCODIV_SHIFT)\
-                | (MASK(GPC2CLK_OUT_BYPDIV_WIDTH) << GPC2CLK_OUT_BYPDIV_SHIFT))
-#define GPC2CLK_OUT_INIT_VAL    ((GPC2CLK_OUT_SDIV14_INDIV4_MODE << \
-                GPC2CLK_OUT_SDIV14_INDIV4_SHIFT) \
-                | (GPC2CLK_OUT_VCODIV1 << GPC2CLK_OUT_VCODIV_SHIFT) \
-                | (GPC2CLK_OUT_BYPDIV31 << GPC2CLK_OUT_BYPDIV_SHIFT))
-
-#define GPC_BCAST_NDIV_SLOWDOWN_DEBUG   (GPC_BCASE_GPCPLL_CFG_BASE + 0xa0)
-#define GPC_BCAST_NDIV_SLOWDOWN_DEBUG_PLL_DYNRAMP_DONE_SYNCED_SHIFT     24
-#define GPC_BCAST_NDIV_SLOWDOWN_DEBUG_PLL_DYNRAMP_DONE_SYNCED_MASK \
-            (0x1 << GPC_BCAST_NDIV_SLOWDOWN_DEBUG_PLL_DYNRAMP_DONE_SYNCED_SHIFT)
-
 static const u8 _pl_to_div[] = {
 /* PL:   0, 1, 2, 3, 4, 5, 6,  7,  8,  9, 10, 11, 12, 13, 14 */
 /* p: */ 1, 2, 3, 4, 5, 6, 8, 10, 12, 16, 12, 16, 20, 24, 32,
@@ -124,7 +61,7 @@ static const struct gk20a_clk_pllg_params gk20a_pllg_params = {
         .min_pl = 1, .max_pl = 32,
 };
 
-static void
+void
 gk20a_pllg_read_mnp(struct gk20a_clk *clk, struct gk20a_pll *pll)
 {
         struct nvkm_device *device = clk->base.subdev.device;
@@ -136,20 +73,33 @@ gk20a_pllg_read_mnp(struct gk20a_clk *clk, struct gk20a_pll *pll)
         pll->pl = (val >> GPCPLL_COEFF_P_SHIFT) & MASK(GPCPLL_COEFF_P_WIDTH);
 }
 
-static u32
-gk20a_pllg_calc_rate(struct gk20a_clk *clk)
+void
+gk20a_pllg_write_mnp(struct gk20a_clk *clk, const struct gk20a_pll *pll)
+{
+        struct nvkm_device *device = clk->base.subdev.device;
+        u32 val;
+
+        val = (pll->m & MASK(GPCPLL_COEFF_M_WIDTH)) << GPCPLL_COEFF_M_SHIFT;
+        val |= (pll->n & MASK(GPCPLL_COEFF_N_WIDTH)) << GPCPLL_COEFF_N_SHIFT;
+        val |= (pll->pl & MASK(GPCPLL_COEFF_P_WIDTH)) << GPCPLL_COEFF_P_SHIFT;
+        nvkm_wr32(device, GPCPLL_COEFF, val);
+}
+
+u32
+gk20a_pllg_calc_rate(struct gk20a_clk *clk, struct gk20a_pll *pll)
 {
         u32 rate;
         u32 divider;
 
-        rate = clk->parent_rate * clk->pll.n;
-        divider = clk->pll.m * clk->pl_to_div(clk->pll.pl);
+        rate = clk->parent_rate * pll->n;
+        divider = pll->m * clk->pl_to_div(pll->pl);
 
         return rate / divider / 2;
 }
 
-static int
-gk20a_pllg_calc_mnp(struct gk20a_clk *clk, unsigned long rate)
+int
+gk20a_pllg_calc_mnp(struct gk20a_clk *clk, unsigned long rate,
+                    struct gk20a_pll *pll)
 {
         struct nvkm_subdev *subdev = &clk->base.subdev;
         u32 target_clk_f, ref_clk_f, target_freq;
@@ -163,16 +113,13 @@ gk20a_pllg_calc_mnp(struct gk20a_clk *clk, unsigned long rate)
         target_clk_f = rate * 2 / KHZ;
         ref_clk_f = clk->parent_rate / KHZ;
 
-        max_vco_f = clk->params->max_vco;
+        target_vco_f = target_clk_f + target_clk_f / 50;
+        max_vco_f = max(clk->params->max_vco, target_vco_f);
         min_vco_f = clk->params->min_vco;
         best_m = clk->params->max_m;
         best_n = clk->params->min_n;
         best_pl = clk->params->min_pl;
 
-        target_vco_f = target_clk_f + target_clk_f / 50;
-        if (max_vco_f < target_vco_f)
-                max_vco_f = target_vco_f;
-
         /* min_pl <= high_pl <= max_pl */
         high_pl = (max_vco_f + target_vco_f - 1) / target_vco_f;
         high_pl = min(high_pl, clk->params->max_pl);
@@ -195,9 +142,7 @@ gk20a_pllg_calc_mnp(struct gk20a_clk *clk, unsigned long rate)
                 target_vco_f = target_clk_f * clk->pl_to_div(pl);
 
                 for (m = clk->params->min_m; m <= clk->params->max_m; m++) {
-                        u32 u_f, vco_f;
-
-                        u_f = ref_clk_f / m;
+                        u32 u_f = ref_clk_f / m;
 
                         if (u_f < clk->params->min_u)
                                 break;
@@ -211,6 +156,8 @@ gk20a_pllg_calc_mnp(struct gk20a_clk *clk, unsigned long rate)
                                 break;
 
                         for (; n <= n2; n++) {
+                                u32 vco_f;
+
                                 if (n < clk->params->min_n)
                                         continue;
                                 if (n > clk->params->max_n)
@@ -247,16 +194,16 @@ found_match:
                            "no best match for target @ %dMHz on gpc_pll",
                            target_clk_f / KHZ);
 
-        clk->pll.m = best_m;
-        clk->pll.n = best_n;
-        clk->pll.pl = best_pl;
+        pll->m = best_m;
+        pll->n = best_n;
+        pll->pl = best_pl;
 
-        target_freq = gk20a_pllg_calc_rate(clk);
+        target_freq = gk20a_pllg_calc_rate(clk, pll);
 
         nvkm_debug(subdev,
-                   "actual target freq %d MHz, M %d, N %d, PL %d(div%d)\n",
-                   target_freq / MHZ, clk->pll.m, clk->pll.n, clk->pll.pl,
-                   clk->pl_to_div(clk->pll.pl));
+                   "actual target freq %d KHz, M %d, N %d, PL %d(div%d)\n",
+                   target_freq / KHZ, pll->m, pll->n, pll->pl,
+                   clk->pl_to_div(pll->pl));
         return 0;
 }
 
@@ -265,45 +212,36 @@ gk20a_pllg_slide(struct gk20a_clk *clk, u32 n)
 {
         struct nvkm_subdev *subdev = &clk->base.subdev;
         struct nvkm_device *device = subdev->device;
-        u32 val;
-        int ramp_timeout;
+        struct gk20a_pll pll;
+        int ret = 0;
 
         /* get old coefficients */
-        val = nvkm_rd32(device, GPCPLL_COEFF);
+        gk20a_pllg_read_mnp(clk, &pll);
         /* do nothing if NDIV is the same */
-        if (n == ((val >> GPCPLL_COEFF_N_SHIFT) & MASK(GPCPLL_COEFF_N_WIDTH)))
+        if (n == pll.n)
                 return 0;
 
-        /* setup */
-        nvkm_mask(device, GPCPLL_CFG2, 0xff << GPCPLL_CFG2_PLL_STEPA_SHIFT,
-                0x2b << GPCPLL_CFG2_PLL_STEPA_SHIFT);
-        nvkm_mask(device, GPCPLL_CFG3, 0xff << GPCPLL_CFG3_PLL_STEPB_SHIFT,
-                0xb << GPCPLL_CFG3_PLL_STEPB_SHIFT);
-
         /* pll slowdown mode */
         nvkm_mask(device, GPCPLL_NDIV_SLOWDOWN,
                 BIT(GPCPLL_NDIV_SLOWDOWN_SLOWDOWN_USING_PLL_SHIFT),
                 BIT(GPCPLL_NDIV_SLOWDOWN_SLOWDOWN_USING_PLL_SHIFT));
 
         /* new ndiv ready for ramp */
-        val = nvkm_rd32(device, GPCPLL_COEFF);
-        val &= ~(MASK(GPCPLL_COEFF_N_WIDTH) << GPCPLL_COEFF_N_SHIFT);
-        val |= (n & MASK(GPCPLL_COEFF_N_WIDTH)) << GPCPLL_COEFF_N_SHIFT;
+        pll.n = n;
         udelay(1);
-        nvkm_wr32(device, GPCPLL_COEFF, val);
+        gk20a_pllg_write_mnp(clk, &pll);
 
         /* dynamic ramp to new ndiv */
-        val = nvkm_rd32(device, GPCPLL_NDIV_SLOWDOWN);
-        val |= 0x1 << GPCPLL_NDIV_SLOWDOWN_EN_DYNRAMP_SHIFT;
         udelay(1);
-        nvkm_wr32(device, GPCPLL_NDIV_SLOWDOWN, val);
+        nvkm_mask(device, GPCPLL_NDIV_SLOWDOWN,
+                  BIT(GPCPLL_NDIV_SLOWDOWN_EN_DYNRAMP_SHIFT),
+                  BIT(GPCPLL_NDIV_SLOWDOWN_EN_DYNRAMP_SHIFT));
 
-        for (ramp_timeout = 500; ramp_timeout > 0; ramp_timeout--) {
-                udelay(1);
-                val = nvkm_rd32(device, GPC_BCAST_NDIV_SLOWDOWN_DEBUG);
-                if (val & GPC_BCAST_NDIV_SLOWDOWN_DEBUG_PLL_DYNRAMP_DONE_SYNCED_MASK)
-                        break;
-        }
+        /* wait for ramping to complete */
+        if (nvkm_wait_usec(device, 500, GPC_BCAST_NDIV_SLOWDOWN_DEBUG,
+                GPC_BCAST_NDIV_SLOWDOWN_DEBUG_PLL_DYNRAMP_DONE_SYNCED_MASK,
+                GPC_BCAST_NDIV_SLOWDOWN_DEBUG_PLL_DYNRAMP_DONE_SYNCED_MASK) < 0)
+                ret = -ETIMEDOUT;
 
         /* exit slowdown mode */
         nvkm_mask(device, GPCPLL_NDIV_SLOWDOWN,
@@ -311,21 +249,35 @@ gk20a_pllg_slide(struct gk20a_clk *clk, u32 n)
                 BIT(GPCPLL_NDIV_SLOWDOWN_EN_DYNRAMP_SHIFT), 0);
         nvkm_rd32(device, GPCPLL_NDIV_SLOWDOWN);
 
-        if (ramp_timeout <= 0) {
-                nvkm_error(subdev, "gpcpll dynamic ramp timeout\n");
-                return -ETIMEDOUT;
-        }
-
-        return 0;
+        return ret;
 }
 
-static void
+static int
 gk20a_pllg_enable(struct gk20a_clk *clk)
 {
         struct nvkm_device *device = clk->base.subdev.device;
+        u32 val;
 
         nvkm_mask(device, GPCPLL_CFG, GPCPLL_CFG_ENABLE, GPCPLL_CFG_ENABLE);
         nvkm_rd32(device, GPCPLL_CFG);
+
+        /* enable lock detection */
+        val = nvkm_rd32(device, GPCPLL_CFG);
+        if (val & GPCPLL_CFG_LOCK_DET_OFF) {
+                val &= ~GPCPLL_CFG_LOCK_DET_OFF;
+                nvkm_wr32(device, GPCPLL_CFG, val);
+        }
+
+        /* wait for lock */
+        if (nvkm_wait_usec(device, 300, GPCPLL_CFG, GPCPLL_CFG_LOCK,
+                           GPCPLL_CFG_LOCK) < 0)
+                return -ETIMEDOUT;
+
+        /* switch to VCO mode */
+        nvkm_mask(device, SEL_VCO, BIT(SEL_VCO_GPC2CLK_OUT_SHIFT),
+                BIT(SEL_VCO_GPC2CLK_OUT_SHIFT));
+
+        return 0;
 }
 
 static void
@@ -333,117 +285,81 @@ gk20a_pllg_disable(struct gk20a_clk *clk)
 {
         struct nvkm_device *device = clk->base.subdev.device;
 
+        /* put PLL in bypass before disabling it */
+        nvkm_mask(device, SEL_VCO, BIT(SEL_VCO_GPC2CLK_OUT_SHIFT), 0);
+
         nvkm_mask(device, GPCPLL_CFG, GPCPLL_CFG_ENABLE, 0);
         nvkm_rd32(device, GPCPLL_CFG);
 }
 
 static int
-_gk20a_pllg_program_mnp(struct gk20a_clk *clk, bool allow_slide)
+gk20a_pllg_program_mnp(struct gk20a_clk *clk, const struct gk20a_pll *pll)
 {
         struct nvkm_subdev *subdev = &clk->base.subdev;
         struct nvkm_device *device = subdev->device;
-        u32 val, cfg;
-        struct gk20a_pll old_pll;
-        u32 n_lo;
-
-        /* get old coefficients */
-        gk20a_pllg_read_mnp(clk, &old_pll);
-
-        /* do NDIV slide if there is no change in M and PL */
-        cfg = nvkm_rd32(device, GPCPLL_CFG);
-        if (allow_slide && clk->pll.m == old_pll.m &&
-            clk->pll.pl == old_pll.pl && (cfg & GPCPLL_CFG_ENABLE)) {
-                return gk20a_pllg_slide(clk, clk->pll.n);
-        }
-
-        /* slide down to NDIV_LO */
-        if (allow_slide && (cfg & GPCPLL_CFG_ENABLE)) {
-                int ret;
-
-                n_lo = DIV_ROUND_UP(old_pll.m * clk->params->min_vco,
-                                    clk->parent_rate / KHZ);
-                ret = gk20a_pllg_slide(clk, n_lo);
+        struct gk20a_pll cur_pll;
+        int ret;
 
-                if (ret)
-                        return ret;
-        }
+        gk20a_pllg_read_mnp(clk, &cur_pll);
 
-        /* split FO-to-bypass jump in halfs by setting out divider 1:2 */
+        /* split VCO-to-bypass jump in half by setting out divider 1:2 */
         nvkm_mask(device, GPC2CLK_OUT, GPC2CLK_OUT_VCODIV_MASK,
-                0x2 << GPC2CLK_OUT_VCODIV_SHIFT);
-
-        /* put PLL in bypass before programming it */
-        val = nvkm_rd32(device, SEL_VCO);
-        val &= ~(BIT(SEL_VCO_GPC2CLK_OUT_SHIFT));
+                  GPC2CLK_OUT_VCODIV2 << GPC2CLK_OUT_VCODIV_SHIFT);
+        /* Intentional 2nd write to assure linear divider operation */
+        nvkm_mask(device, GPC2CLK_OUT, GPC2CLK_OUT_VCODIV_MASK,
+                  GPC2CLK_OUT_VCODIV2 << GPC2CLK_OUT_VCODIV_SHIFT);
+        nvkm_rd32(device, GPC2CLK_OUT);
         udelay(2);
-        nvkm_wr32(device, SEL_VCO, val);
-
-        /* get out from IDDQ */
-        val = nvkm_rd32(device, GPCPLL_CFG);
-        if (val & GPCPLL_CFG_IDDQ) {
-                val &= ~GPCPLL_CFG_IDDQ;
-                nvkm_wr32(device, GPCPLL_CFG, val);
-                nvkm_rd32(device, GPCPLL_CFG);
-                udelay(2);
-        }
 
         gk20a_pllg_disable(clk);
 
-        nvkm_debug(subdev, "%s: m=%d n=%d pl=%d\n", __func__,
-                   clk->pll.m, clk->pll.n, clk->pll.pl);
-
-        n_lo = DIV_ROUND_UP(clk->pll.m * clk->params->min_vco,
-                            clk->parent_rate / KHZ);
-        val = clk->pll.m << GPCPLL_COEFF_M_SHIFT;
-        val |= (allow_slide ? n_lo : clk->pll.n) << GPCPLL_COEFF_N_SHIFT;
-        val |= clk->pll.pl << GPCPLL_COEFF_P_SHIFT;
-        nvkm_wr32(device, GPCPLL_COEFF, val);
+        gk20a_pllg_write_mnp(clk, pll);
 
-        gk20a_pllg_enable(clk);
-
-        val = nvkm_rd32(device, GPCPLL_CFG);
-        if (val & GPCPLL_CFG_LOCK_DET_OFF) {
-                val &= ~GPCPLL_CFG_LOCK_DET_OFF;
-                nvkm_wr32(device, GPCPLL_CFG, val);
-        }
-
-        if (nvkm_usec(device, 300,
-                if (nvkm_rd32(device, GPCPLL_CFG) & GPCPLL_CFG_LOCK)
-                        break;
-        ) < 0)
-                return -ETIMEDOUT;
-
-        /* switch to VCO mode */
-        nvkm_mask(device, SEL_VCO, BIT(SEL_VCO_GPC2CLK_OUT_SHIFT),
-                  BIT(SEL_VCO_GPC2CLK_OUT_SHIFT));
+        ret = gk20a_pllg_enable(clk);
+        if (ret)
+                return ret;
 
         /* restore out divider 1:1 */
-        val = nvkm_rd32(device, GPC2CLK_OUT);
-        if ((val & GPC2CLK_OUT_VCODIV_MASK) !=
-            (GPC2CLK_OUT_VCODIV1 << GPC2CLK_OUT_VCODIV_SHIFT)) {
-                val &= ~GPC2CLK_OUT_VCODIV_MASK;
-                val |= GPC2CLK_OUT_VCODIV1 << GPC2CLK_OUT_VCODIV_SHIFT;
-                udelay(2);
-                nvkm_wr32(device, GPC2CLK_OUT, val);
-                /* Intentional 2nd write to assure linear divider operation */
-                nvkm_wr32(device, GPC2CLK_OUT, val);
-                nvkm_rd32(device, GPC2CLK_OUT);
-        }
+        udelay(2);
+        nvkm_mask(device, GPC2CLK_OUT, GPC2CLK_OUT_VCODIV_MASK,
+                  GPC2CLK_OUT_VCODIV1 << GPC2CLK_OUT_VCODIV_SHIFT);
+        /* Intentional 2nd write to assure linear divider operation */
+        nvkm_mask(device, GPC2CLK_OUT, GPC2CLK_OUT_VCODIV_MASK,
+                  GPC2CLK_OUT_VCODIV1 << GPC2CLK_OUT_VCODIV_SHIFT);
+        nvkm_rd32(device, GPC2CLK_OUT);
 
-        /* slide up to new NDIV */
-        return allow_slide ? gk20a_pllg_slide(clk, clk->pll.n) : 0;
+        return 0;
 }
 
 static int
-gk20a_pllg_program_mnp(struct gk20a_clk *clk)
+gk20a_pllg_program_mnp_slide(struct gk20a_clk *clk, const struct gk20a_pll *pll)
 {
-        int err;
+        struct gk20a_pll cur_pll;
+        int ret;
 
-        err = _gk20a_pllg_program_mnp(clk, true);
-        if (err)
-                err = _gk20a_pllg_program_mnp(clk, false);
+        if (gk20a_pllg_is_enabled(clk)) {
+                gk20a_pllg_read_mnp(clk, &cur_pll);
+
+                /* just do NDIV slide if there is no change to M and PL */
+                if (pll->m == cur_pll.m && pll->pl == cur_pll.pl)
+                        return gk20a_pllg_slide(clk, pll->n);
+
+                /* slide down to current NDIV_LO */
+                cur_pll.n = gk20a_pllg_n_lo(clk, &cur_pll);
+                ret = gk20a_pllg_slide(clk, cur_pll.n);
+                if (ret)
+                        return ret;
+        }
+
+        /* program MNP with the new clock parameters and new NDIV_LO */
+        cur_pll = *pll;
+        cur_pll.n = gk20a_pllg_n_lo(clk, &cur_pll);
+        ret = gk20a_pllg_program_mnp(clk, &cur_pll);
+        if (ret)
+                return ret;
 
-        return err;
+        /* slide up to new NDIV */
+        return gk20a_pllg_slide(clk, pll->n);
 }
 
 static struct nvkm_pstate
@@ -546,13 +462,14 @@ gk20a_clk_read(struct nvkm_clk *base, enum nv_clk_src src)
         struct gk20a_clk *clk = gk20a_clk(base);
         struct nvkm_subdev *subdev = &clk->base.subdev;
         struct nvkm_device *device = subdev->device;
+        struct gk20a_pll pll;
 
         switch (src) {
         case nv_clk_src_crystal:
                 return device->crystal;
         case nv_clk_src_gpc:
-                gk20a_pllg_read_mnp(clk, &clk->pll);
-                return gk20a_pllg_calc_rate(clk) / GK20A_CLK_GPC_MDIV;
+                gk20a_pllg_read_mnp(clk, &pll);
+                return gk20a_pllg_calc_rate(clk, &pll) / GK20A_CLK_GPC_MDIV;
         default:
                 nvkm_error(subdev, "invalid clock source %d\n", src);
                 return -EINVAL;
@@ -565,15 +482,20 @@ gk20a_clk_calc(struct nvkm_clk *base, struct nvkm_cstate *cstate)
         struct gk20a_clk *clk = gk20a_clk(base);
 
         return gk20a_pllg_calc_mnp(clk, cstate->domain[nv_clk_src_gpc] *
-                                         GK20A_CLK_GPC_MDIV);
+                                         GK20A_CLK_GPC_MDIV, &clk->pll);
 }
 
 int
 gk20a_clk_prog(struct nvkm_clk *base)
 {
         struct gk20a_clk *clk = gk20a_clk(base);
+        int ret;
+
+        ret = gk20a_pllg_program_mnp_slide(clk, &clk->pll);
+        if (ret)
+                ret = gk20a_pllg_program_mnp(clk, &clk->pll);
 
-        return gk20a_pllg_program_mnp(clk);
+        return ret;
 }
 
 void
@@ -581,29 +503,62 @@ gk20a_clk_tidy(struct nvkm_clk *base)
 {
 }
 
+int
+gk20a_clk_setup_slide(struct gk20a_clk *clk)
+{
+        struct nvkm_subdev *subdev = &clk->base.subdev;
+        struct nvkm_device *device = subdev->device;
+        u32 step_a, step_b;
+
+        switch (clk->parent_rate) {
+        case 12000000:
+        case 12800000:
+        case 13000000:
+                step_a = 0x2b;
+                step_b = 0x0b;
+                break;
+        case 19200000:
+                step_a = 0x12;
+                step_b = 0x08;
+                break;
+        case 38400000:
+                step_a = 0x04;
+                step_b = 0x05;
+                break;
+        default:
+                nvkm_error(subdev, "invalid parent clock rate %u KHz",
+                           clk->parent_rate / KHZ);
+                return -EINVAL;
+        }
+
+        nvkm_mask(device, GPCPLL_CFG2, 0xff << GPCPLL_CFG2_PLL_STEPA_SHIFT,
+                step_a << GPCPLL_CFG2_PLL_STEPA_SHIFT);
+        nvkm_mask(device, GPCPLL_CFG3, 0xff << GPCPLL_CFG3_PLL_STEPB_SHIFT,
+                step_b << GPCPLL_CFG3_PLL_STEPB_SHIFT);
+
+        return 0;
+}
+
 void
 gk20a_clk_fini(struct nvkm_clk *base)
 {
         struct nvkm_device *device = base->subdev.device;
         struct gk20a_clk *clk = gk20a_clk(base);
-        u32 val;
 
         /* slide to VCO min */
-        val = nvkm_rd32(device, GPCPLL_CFG);
-        if (val & GPCPLL_CFG_ENABLE) {
+        if (gk20a_pllg_is_enabled(clk)) {
                 struct gk20a_pll pll;
                 u32 n_lo;
 
                 gk20a_pllg_read_mnp(clk, &pll);
-                n_lo = DIV_ROUND_UP(pll.m * clk->params->min_vco,
-                                    clk->parent_rate / KHZ);
+                n_lo = gk20a_pllg_n_lo(clk, &pll);
                 gk20a_pllg_slide(clk, n_lo);
         }
 
-        /* put PLL in bypass before disabling it */
-        nvkm_mask(device, SEL_VCO, BIT(SEL_VCO_GPC2CLK_OUT_SHIFT), 0);
-
         gk20a_pllg_disable(clk);
+
+        /* set IDDQ */
+        nvkm_mask(device, GPCPLL_CFG, GPCPLL_CFG_IDDQ, 1);
 }
 
 static int
@@ -614,9 +569,18 @@ gk20a_clk_init(struct nvkm_clk *base)
         struct nvkm_device *device = subdev->device;
         int ret;
 
+        /* get out from IDDQ */
+        nvkm_mask(device, GPCPLL_CFG, GPCPLL_CFG_IDDQ, 0);
+        nvkm_rd32(device, GPCPLL_CFG);
+        udelay(5);
+
         nvkm_mask(device, GPC2CLK_OUT, GPC2CLK_OUT_INIT_MASK,
                   GPC2CLK_OUT_INIT_VAL);
 
+        ret = gk20a_clk_setup_slide(clk);
+        if (ret)
+                return ret;
+
         /* Start with lowest frequency */
         base->func->calc(base, &base->func->pstates[0].base);
         ret = base->func->prog(&clk->base);
@@ -646,7 +610,7 @@ gk20a_clk = {
 };
 
 int
-_gk20a_clk_ctor(struct nvkm_device *device, int index,
+gk20a_clk_ctor(struct nvkm_device *device, int index,
                 const struct nvkm_clk_func *func,
                 const struct gk20a_clk_pllg_params *params,
                 struct gk20a_clk *clk)
@@ -685,7 +649,7 @@ gk20a_clk_new(struct nvkm_device *device, int index, struct nvkm_clk **pclk)
                 return -ENOMEM;
         *pclk = &clk->base;
 
-        ret = _gk20a_clk_ctor(device, index, &gk20a_clk, &gk20a_pllg_params,
+        ret = gk20a_clk_ctor(device, index, &gk20a_clk, &gk20a_pllg_params,
                               clk);
 
         clk->pl_to_div = pl_to_div;
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/clk/gk20a.h b/drivers/gpu/drm/nouveau/nvkm/subdev/clk/gk20a.h
index 13c46740197d..0d1450972162 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/clk/gk20a.h
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/clk/gk20a.h
@@ -24,9 +24,79 @@
 #ifndef __NVKM_CLK_GK20A_H__
 #define __NVKM_CLK_GK20A_H__
 
+#define KHZ (1000)
+#define MHZ (KHZ * 1000)
+
+#define MASK(w) ((1 << (w)) - 1)
+
 #define GK20A_CLK_GPC_MDIV 1000
 
 #define SYS_GPCPLL_CFG_BASE     0x00137000
+#define GPCPLL_CFG              (SYS_GPCPLL_CFG_BASE + 0)
+#define GPCPLL_CFG_ENABLE       BIT(0)
+#define GPCPLL_CFG_IDDQ         BIT(1)
+#define GPCPLL_CFG_LOCK_DET_OFF BIT(4)
+#define GPCPLL_CFG_LOCK         BIT(17)
+
+#define GPCPLL_CFG2             (SYS_GPCPLL_CFG_BASE + 0xc)
+#define GPCPLL_CFG2_SETUP2_SHIFT        16
+#define GPCPLL_CFG2_PLL_STEPA_SHIFT     24
+
+#define GPCPLL_CFG3                     (SYS_GPCPLL_CFG_BASE + 0x18)
+#define GPCPLL_CFG3_VCO_CTRL_SHIFT              0
+#define GPCPLL_CFG3_VCO_CTRL_WIDTH              9
+#define GPCPLL_CFG3_VCO_CTRL_MASK               \
+        (MASK(GPCPLL_CFG3_VCO_CTRL_WIDTH) << GPCPLL_CFG3_VCO_CTRL_SHIFT)
+#define GPCPLL_CFG3_PLL_STEPB_SHIFT             16
+#define GPCPLL_CFG3_PLL_STEPB_WIDTH             8
+
+#define GPCPLL_COEFF            (SYS_GPCPLL_CFG_BASE + 4)
+#define GPCPLL_COEFF_M_SHIFT    0
+#define GPCPLL_COEFF_M_WIDTH    8
+#define GPCPLL_COEFF_N_SHIFT    8
+#define GPCPLL_COEFF_N_WIDTH    8
+#define GPCPLL_COEFF_N_MASK     \
+        (MASK(GPCPLL_COEFF_N_WIDTH) << GPCPLL_COEFF_N_SHIFT)
+#define GPCPLL_COEFF_P_SHIFT    16
+#define GPCPLL_COEFF_P_WIDTH    6
+
+#define GPCPLL_NDIV_SLOWDOWN                    (SYS_GPCPLL_CFG_BASE + 0x1c)
+#define GPCPLL_NDIV_SLOWDOWN_NDIV_LO_SHIFT      0
+#define GPCPLL_NDIV_SLOWDOWN_NDIV_MID_SHIFT     8
+#define GPCPLL_NDIV_SLOWDOWN_STEP_SIZE_LO2MID_SHIFT     16
+#define GPCPLL_NDIV_SLOWDOWN_SLOWDOWN_USING_PLL_SHIFT   22
+#define GPCPLL_NDIV_SLOWDOWN_EN_DYNRAMP_SHIFT   31
+
+#define GPC_BCAST_GPCPLL_CFG_BASE               0x00132800
+#define GPC_BCAST_NDIV_SLOWDOWN_DEBUG   (GPC_BCAST_GPCPLL_CFG_BASE + 0xa0)
+#define GPC_BCAST_NDIV_SLOWDOWN_DEBUG_PLL_DYNRAMP_DONE_SYNCED_SHIFT     24
+#define GPC_BCAST_NDIV_SLOWDOWN_DEBUG_PLL_DYNRAMP_DONE_SYNCED_MASK \
+        (0x1 << GPC_BCAST_NDIV_SLOWDOWN_DEBUG_PLL_DYNRAMP_DONE_SYNCED_SHIFT)
+
+#define SEL_VCO                         (SYS_GPCPLL_CFG_BASE + 0x100)
+#define SEL_VCO_GPC2CLK_OUT_SHIFT       0
+
+#define GPC2CLK_OUT                     (SYS_GPCPLL_CFG_BASE + 0x250)
+#define GPC2CLK_OUT_SDIV14_INDIV4_WIDTH 1
+#define GPC2CLK_OUT_SDIV14_INDIV4_SHIFT 31
+#define GPC2CLK_OUT_SDIV14_INDIV4_MODE  1
+#define GPC2CLK_OUT_VCODIV_WIDTH        6
+#define GPC2CLK_OUT_VCODIV_SHIFT        8
+#define GPC2CLK_OUT_VCODIV1             0
+#define GPC2CLK_OUT_VCODIV2             2
+#define GPC2CLK_OUT_VCODIV_MASK         (MASK(GPC2CLK_OUT_VCODIV_WIDTH) << \
+                                        GPC2CLK_OUT_VCODIV_SHIFT)
+#define GPC2CLK_OUT_BYPDIV_WIDTH        6
+#define GPC2CLK_OUT_BYPDIV_SHIFT        0
+#define GPC2CLK_OUT_BYPDIV31            0x3c
+#define GPC2CLK_OUT_INIT_MASK   ((MASK(GPC2CLK_OUT_SDIV14_INDIV4_WIDTH) << \
+                GPC2CLK_OUT_SDIV14_INDIV4_SHIFT)\
+                | (MASK(GPC2CLK_OUT_VCODIV_WIDTH) << GPC2CLK_OUT_VCODIV_SHIFT)\
+                | (MASK(GPC2CLK_OUT_BYPDIV_WIDTH) << GPC2CLK_OUT_BYPDIV_SHIFT))
+#define GPC2CLK_OUT_INIT_VAL    ((GPC2CLK_OUT_SDIV14_INDIV4_MODE << \
+                GPC2CLK_OUT_SDIV14_INDIV4_SHIFT) \
+                | (GPC2CLK_OUT_VCODIV1 << GPC2CLK_OUT_VCODIV_SHIFT) \
+                | (GPC2CLK_OUT_BYPDIV31 << GPC2CLK_OUT_BYPDIV_SHIFT))
 
 /* All frequencies in Khz */
 struct gk20a_clk_pllg_params {
@@ -54,7 +124,29 @@ struct gk20a_clk {
 };
 #define gk20a_clk(p) container_of((p), struct gk20a_clk, base)
 
-int _gk20a_clk_ctor(struct nvkm_device *, int, const struct nvkm_clk_func *,
+u32 gk20a_pllg_calc_rate(struct gk20a_clk *, struct gk20a_pll *);
+int gk20a_pllg_calc_mnp(struct gk20a_clk *, unsigned long, struct gk20a_pll *);
+void gk20a_pllg_read_mnp(struct gk20a_clk *, struct gk20a_pll *);
+void gk20a_pllg_write_mnp(struct gk20a_clk *, const struct gk20a_pll *);
+
+static inline bool
+gk20a_pllg_is_enabled(struct gk20a_clk *clk)
+{
+        struct nvkm_device *device = clk->base.subdev.device;
+        u32 val;
+
+        val = nvkm_rd32(device, GPCPLL_CFG);
+        return val & GPCPLL_CFG_ENABLE;
+}
+
+static inline u32
+gk20a_pllg_n_lo(struct gk20a_clk *clk, struct gk20a_pll *pll)
+{
+        return DIV_ROUND_UP(pll->m * clk->params->min_vco,
+                            clk->parent_rate / KHZ);
+}
+
+int gk20a_clk_ctor(struct nvkm_device *, int, const struct nvkm_clk_func *,
                     const struct gk20a_clk_pllg_params *, struct gk20a_clk *);
 void gk20a_clk_fini(struct nvkm_clk *);
 int gk20a_clk_read(struct nvkm_clk *, enum nv_clk_src);
@@ -62,4 +154,6 @@ int gk20a_clk_calc(struct nvkm_clk *, struct nvkm_cstate *);
 int gk20a_clk_prog(struct nvkm_clk *);
 void gk20a_clk_tidy(struct nvkm_clk *);
 
+int gk20a_clk_setup_slide(struct gk20a_clk *);
+
 #endif
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/clk/gm20b.c b/drivers/gpu/drm/nouveau/nvkm/subdev/clk/gm20b.c
index 71b2bbb61973..b284e949f732 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/clk/gm20b.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/clk/gm20b.c
@@ -21,20 +21,123 @@
  */
 
 #include <subdev/clk.h>
+#include <subdev/volt.h>
+#include <subdev/timer.h>
 #include <core/device.h>
+#include <core/tegra.h>
 
 #include "priv.h"
 #include "gk20a.h"
 
-#define KHZ (1000)
-#define MHZ (KHZ * 1000)
-
-#define MASK(w) ((1 << w) - 1)
+#define GPCPLL_CFG_SYNC_MODE    BIT(2)
 
 #define BYPASSCTRL_SYS  (SYS_GPCPLL_CFG_BASE + 0x340)
 #define BYPASSCTRL_SYS_GPCPLL_SHIFT     0
 #define BYPASSCTRL_SYS_GPCPLL_WIDTH     1
 
+#define GPCPLL_CFG2_SDM_DIN_SHIFT       0
+#define GPCPLL_CFG2_SDM_DIN_WIDTH       8
+#define GPCPLL_CFG2_SDM_DIN_MASK        \
+        (MASK(GPCPLL_CFG2_SDM_DIN_WIDTH) << GPCPLL_CFG2_SDM_DIN_SHIFT)
+#define GPCPLL_CFG2_SDM_DIN_NEW_SHIFT   8
+#define GPCPLL_CFG2_SDM_DIN_NEW_WIDTH   15
+#define GPCPLL_CFG2_SDM_DIN_NEW_MASK    \
+        (MASK(GPCPLL_CFG2_SDM_DIN_NEW_WIDTH) << GPCPLL_CFG2_SDM_DIN_NEW_SHIFT)
+#define GPCPLL_CFG2_SETUP2_SHIFT        16
+#define GPCPLL_CFG2_PLL_STEPA_SHIFT     24
+
+#define GPCPLL_DVFS0    (SYS_GPCPLL_CFG_BASE + 0x10)
+#define GPCPLL_DVFS0_DFS_COEFF_SHIFT    0
+#define GPCPLL_DVFS0_DFS_COEFF_WIDTH    7
+#define GPCPLL_DVFS0_DFS_COEFF_MASK     \
+        (MASK(GPCPLL_DVFS0_DFS_COEFF_WIDTH) << GPCPLL_DVFS0_DFS_COEFF_SHIFT)
+#define GPCPLL_DVFS0_DFS_DET_MAX_SHIFT  8
+#define GPCPLL_DVFS0_DFS_DET_MAX_WIDTH  7
+#define GPCPLL_DVFS0_DFS_DET_MAX_MASK   \
+        (MASK(GPCPLL_DVFS0_DFS_DET_MAX_WIDTH) << GPCPLL_DVFS0_DFS_DET_MAX_SHIFT)
+
+#define GPCPLL_DVFS1            (SYS_GPCPLL_CFG_BASE + 0x14)
+#define GPCPLL_DVFS1_DFS_EXT_DET_SHIFT          0
+#define GPCPLL_DVFS1_DFS_EXT_DET_WIDTH          7
+#define GPCPLL_DVFS1_DFS_EXT_STRB_SHIFT         7
+#define GPCPLL_DVFS1_DFS_EXT_STRB_WIDTH         1
+#define GPCPLL_DVFS1_DFS_EXT_CAL_SHIFT          8
+#define GPCPLL_DVFS1_DFS_EXT_CAL_WIDTH          7
+#define GPCPLL_DVFS1_DFS_EXT_SEL_SHIFT          15
+#define GPCPLL_DVFS1_DFS_EXT_SEL_WIDTH          1
+#define GPCPLL_DVFS1_DFS_CTRL_SHIFT             16
+#define GPCPLL_DVFS1_DFS_CTRL_WIDTH             12
+#define GPCPLL_DVFS1_EN_SDM_SHIFT               28
+#define GPCPLL_DVFS1_EN_SDM_WIDTH               1
+#define GPCPLL_DVFS1_EN_SDM_BIT                 BIT(28)
+#define GPCPLL_DVFS1_EN_DFS_SHIFT               29
+#define GPCPLL_DVFS1_EN_DFS_WIDTH               1
+#define GPCPLL_DVFS1_EN_DFS_BIT                 BIT(29)
+#define GPCPLL_DVFS1_EN_DFS_CAL_SHIFT           30
+#define GPCPLL_DVFS1_EN_DFS_CAL_WIDTH           1
+#define GPCPLL_DVFS1_EN_DFS_CAL_BIT             BIT(30)
+#define GPCPLL_DVFS1_DFS_CAL_DONE_SHIFT         31
+#define GPCPLL_DVFS1_DFS_CAL_DONE_WIDTH         1
+#define GPCPLL_DVFS1_DFS_CAL_DONE_BIT           BIT(31)
+
+#define GPC_BCAST_GPCPLL_DVFS2  (GPC_BCAST_GPCPLL_CFG_BASE + 0x20)
+#define GPC_BCAST_GPCPLL_DVFS2_DFS_EXT_STROBE_BIT       BIT(16)
+
+#define GPCPLL_CFG3_PLL_DFS_TESTOUT_SHIFT       24
+#define GPCPLL_CFG3_PLL_DFS_TESTOUT_WIDTH       7
+
+#define DFS_DET_RANGE   6       /* -2^6 ... 2^6-1 */
+#define SDM_DIN_RANGE   12      /* -2^12 ... 2^12-1 */
+
+struct gm20b_clk_dvfs_params {
+        s32 coeff_slope;
+        s32 coeff_offs;
+        u32 vco_ctrl;
+};
+
+static const struct gm20b_clk_dvfs_params gm20b_dvfs_params = {
+        .coeff_slope = -165230,
+        .coeff_offs = 214007,
+        .vco_ctrl = 0x7 << 3,
+};
+
+/*
+ * base.n is now the *integer* part of the N factor.
+ * sdm_din contains n's decimal part.
+ */
+struct gm20b_pll {
+        struct gk20a_pll base;
+        u32 sdm_din;
+};
+
+struct gm20b_clk_dvfs {
+        u32 dfs_coeff;
+        s32 dfs_det_max;
+        s32 dfs_ext_cal;
+};
+
+struct gm20b_clk {
+        /* currently applied parameters */
+        struct gk20a_clk base;
+        struct gm20b_clk_dvfs dvfs;
+        u32 uv;
+
+        /* new parameters to apply */
+        struct gk20a_pll new_pll;
+        struct gm20b_clk_dvfs new_dvfs;
+        u32 new_uv;
+
+        const struct gm20b_clk_dvfs_params *dvfs_params;
+
+        /* fused parameters */
+        s32 uvdet_slope;
+        s32 uvdet_offs;
+
+        /* safe frequency we can use at minimum voltage */
+        u32 safe_fmax_vmin;
+};
+#define gm20b_clk(p) container_of((gk20a_clk(p)), struct gm20b_clk, base)
+
 static u32 pl_to_div(u32 pl)
 {
         return pl;
@@ -53,6 +156,484 @@ static const struct gk20a_clk_pllg_params gm20b_pllg_params = {
         .min_pl = 1, .max_pl = 31,
 };
 
+static void
+gm20b_pllg_read_mnp(struct gm20b_clk *clk, struct gm20b_pll *pll)
+{
+        struct nvkm_subdev *subdev = &clk->base.base.subdev;
+        struct nvkm_device *device = subdev->device;
+        u32 val;
+
+        gk20a_pllg_read_mnp(&clk->base, &pll->base);
+        val = nvkm_rd32(device, GPCPLL_CFG2);
+        pll->sdm_din = (val >> GPCPLL_CFG2_SDM_DIN_SHIFT) &
+                       MASK(GPCPLL_CFG2_SDM_DIN_WIDTH);
+}
+
+static void
+gm20b_pllg_write_mnp(struct gm20b_clk *clk, const struct gm20b_pll *pll)
+{
+        struct nvkm_device *device = clk->base.base.subdev.device;
+
+        nvkm_mask(device, GPCPLL_CFG2, GPCPLL_CFG2_SDM_DIN_MASK,
+                  pll->sdm_din << GPCPLL_CFG2_SDM_DIN_SHIFT);
+        gk20a_pllg_write_mnp(&clk->base, &pll->base);
+}
+
+/*
+ * Determine DFS_COEFF for the requested voltage. Always select external
+ * calibration override equal to the voltage, and set maximum detection
+ * limit "0" (to make sure that PLL output remains under F/V curve when
+ * voltage increases).
+ */
+static void
+gm20b_dvfs_calc_det_coeff(struct gm20b_clk *clk, s32 uv,
+                          struct gm20b_clk_dvfs *dvfs)
+{
+        struct nvkm_subdev *subdev = &clk->base.base.subdev;
+        const struct gm20b_clk_dvfs_params *p = clk->dvfs_params;
+        u32 coeff;
+        /* Work with mv as uv would likely trigger an overflow */
+        s32 mv = DIV_ROUND_CLOSEST(uv, 1000);
+
+        /* coeff = slope * voltage + offset */
+        coeff = DIV_ROUND_CLOSEST(mv * p->coeff_slope, 1000) + p->coeff_offs;
+        coeff = DIV_ROUND_CLOSEST(coeff, 1000);
+        dvfs->dfs_coeff = min_t(u32, coeff, MASK(GPCPLL_DVFS0_DFS_COEFF_WIDTH));
+
+        dvfs->dfs_ext_cal = DIV_ROUND_CLOSEST(uv - clk->uvdet_offs,
+                                             clk->uvdet_slope);
+        /* should never happen */
+        if (abs(dvfs->dfs_ext_cal) >= BIT(DFS_DET_RANGE))
+                nvkm_error(subdev, "dfs_ext_cal overflow!\n");
+
+        dvfs->dfs_det_max = 0;
+
+        nvkm_debug(subdev, "%s uv: %d coeff: %x, ext_cal: %d, det_max: %d\n",
+                   __func__, uv, dvfs->dfs_coeff, dvfs->dfs_ext_cal,
+                   dvfs->dfs_det_max);
+}
+
+/*
+ * Solve equation for integer and fractional part of the effective NDIV:
+ *
+ * n_eff = n_int + 1/2 + (SDM_DIN / 2^(SDM_DIN_RANGE + 1)) +
+ *         (DVFS_COEFF * DVFS_DET_DELTA) / 2^DFS_DET_RANGE
+ *
+ * The SDM_DIN LSB is finally shifted out, since it is not accessible by sw.
+ */
+static void
+gm20b_dvfs_calc_ndiv(struct gm20b_clk *clk, u32 n_eff, u32 *n_int, u32 *sdm_din)
+{
+        struct nvkm_subdev *subdev = &clk->base.base.subdev;
+        const struct gk20a_clk_pllg_params *p = clk->base.params;
+        u32 n;
+        s32 det_delta;
+        u32 rem, rem_range;
+
+        /* calculate current ext_cal and subtract previous one */
+        det_delta = DIV_ROUND_CLOSEST(((s32)clk->uv) - clk->uvdet_offs,
+                                      clk->uvdet_slope);
+        det_delta -= clk->dvfs.dfs_ext_cal;
+        det_delta = min(det_delta, clk->dvfs.dfs_det_max);
+        det_delta *= clk->dvfs.dfs_coeff;
+
+        /* integer part of n */
+        n = (n_eff << DFS_DET_RANGE) - det_delta;
+        /* should never happen! */
+        if (n <= 0) {
+                nvkm_error(subdev, "ndiv <= 0 - setting to 1...\n");
+                n = 1 << DFS_DET_RANGE;
+        }
+        if (n >> DFS_DET_RANGE > p->max_n) {
+                nvkm_error(subdev, "ndiv > max_n - setting to max_n...\n");
+                n = p->max_n << DFS_DET_RANGE;
+        }
+        *n_int = n >> DFS_DET_RANGE;
+
+        /* fractional part of n */
+        rem = ((u32)n) & MASK(DFS_DET_RANGE);
+        rem_range = SDM_DIN_RANGE + 1 - DFS_DET_RANGE;
+        /* subtract 2^SDM_DIN_RANGE to account for the 1/2 of the equation */
+        rem = (rem << rem_range) - BIT(SDM_DIN_RANGE);
+        /* lose 8 LSB and clip - sdm_din only keeps the most significant byte */
+        *sdm_din = (rem >> BITS_PER_BYTE) & MASK(GPCPLL_CFG2_SDM_DIN_WIDTH);
+
+        nvkm_debug(subdev, "%s n_eff: %d, n_int: %d, sdm_din: %d\n", __func__,
+                   n_eff, *n_int, *sdm_din);
+}
+
+static int
+gm20b_pllg_slide(struct gm20b_clk *clk, u32 n)
+{
+        struct nvkm_subdev *subdev = &clk->base.base.subdev;
+        struct nvkm_device *device = subdev->device;
+        struct gm20b_pll pll;
+        u32 n_int, sdm_din;
+        int ret = 0;
+
+        /* calculate the new n_int/sdm_din for this n/uv */
+        gm20b_dvfs_calc_ndiv(clk, n, &n_int, &sdm_din);
+
+        /* get old coefficients */
+        gm20b_pllg_read_mnp(clk, &pll);
+        /* do nothing if NDIV is the same */
+        if (n_int == pll.base.n && sdm_din == pll.sdm_din)
+                return 0;
+
+        /* pll slowdown mode */
+        nvkm_mask(device, GPCPLL_NDIV_SLOWDOWN,
+                BIT(GPCPLL_NDIV_SLOWDOWN_SLOWDOWN_USING_PLL_SHIFT),
+                BIT(GPCPLL_NDIV_SLOWDOWN_SLOWDOWN_USING_PLL_SHIFT));
+
+        /* new ndiv ready for ramp */
+        /* in DVFS mode SDM is updated via "new" field */
+        nvkm_mask(device, GPCPLL_CFG2, GPCPLL_CFG2_SDM_DIN_NEW_MASK,
+                  sdm_din << GPCPLL_CFG2_SDM_DIN_NEW_SHIFT);
+        pll.base.n = n_int;
+        udelay(1);
+        gk20a_pllg_write_mnp(&clk->base, &pll.base);
+
+        /* dynamic ramp to new ndiv */
+        udelay(1);
+        nvkm_mask(device, GPCPLL_NDIV_SLOWDOWN,
+                  BIT(GPCPLL_NDIV_SLOWDOWN_EN_DYNRAMP_SHIFT),
+                  BIT(GPCPLL_NDIV_SLOWDOWN_EN_DYNRAMP_SHIFT));
+
+        /* wait for ramping to complete */
+        if (nvkm_wait_usec(device, 500, GPC_BCAST_NDIV_SLOWDOWN_DEBUG,
+                GPC_BCAST_NDIV_SLOWDOWN_DEBUG_PLL_DYNRAMP_DONE_SYNCED_MASK,
+                GPC_BCAST_NDIV_SLOWDOWN_DEBUG_PLL_DYNRAMP_DONE_SYNCED_MASK) < 0)
+                ret = -ETIMEDOUT;
+
+        /* in DVFS mode complete SDM update */
+        nvkm_mask(device, GPCPLL_CFG2, GPCPLL_CFG2_SDM_DIN_MASK,
+                  sdm_din << GPCPLL_CFG2_SDM_DIN_SHIFT);
+
+        /* exit slowdown mode */
+        nvkm_mask(device, GPCPLL_NDIV_SLOWDOWN,
+                BIT(GPCPLL_NDIV_SLOWDOWN_SLOWDOWN_USING_PLL_SHIFT) |
+                BIT(GPCPLL_NDIV_SLOWDOWN_EN_DYNRAMP_SHIFT), 0);
+        nvkm_rd32(device, GPCPLL_NDIV_SLOWDOWN);
+
+        return ret;
+}
+
+static int
+gm20b_pllg_enable(struct gm20b_clk *clk)
+{
+        struct nvkm_device *device = clk->base.base.subdev.device;
+
+        nvkm_mask(device, GPCPLL_CFG, GPCPLL_CFG_ENABLE, GPCPLL_CFG_ENABLE);
+        nvkm_rd32(device, GPCPLL_CFG);
+
+        /* In DVFS mode lock cannot be used - so just delay */
+        udelay(40);
+
+        /* set SYNC_MODE for glitchless switch out of bypass */
+        nvkm_mask(device, GPCPLL_CFG, GPCPLL_CFG_SYNC_MODE,
+                       GPCPLL_CFG_SYNC_MODE);
+        nvkm_rd32(device, GPCPLL_CFG);
+
+        /* switch to VCO mode */
+        nvkm_mask(device, SEL_VCO, BIT(SEL_VCO_GPC2CLK_OUT_SHIFT),
+                  BIT(SEL_VCO_GPC2CLK_OUT_SHIFT));
+
+        return 0;
+}
+
+static void
+gm20b_pllg_disable(struct gm20b_clk *clk)
+{
+        struct nvkm_device *device = clk->base.base.subdev.device;
+
+        /* put PLL in bypass before disabling it */
+        nvkm_mask(device, SEL_VCO, BIT(SEL_VCO_GPC2CLK_OUT_SHIFT), 0);
+
+        /* clear SYNC_MODE before disabling PLL */
+        nvkm_mask(device, GPCPLL_CFG, GPCPLL_CFG_SYNC_MODE, 0);
+
+        nvkm_mask(device, GPCPLL_CFG, GPCPLL_CFG_ENABLE, 0);
+        nvkm_rd32(device, GPCPLL_CFG);
+}
+
+static int
+gm20b_pllg_program_mnp(struct gm20b_clk *clk, const struct gk20a_pll *pll)
+{
+        struct nvkm_subdev *subdev = &clk->base.base.subdev;
+        struct nvkm_device *device = subdev->device;
+        struct gm20b_pll cur_pll;
+        u32 n_int, sdm_din;
+        /* if we only change pdiv, we can do a glitchless transition */
+        bool pdiv_only;
+        int ret;
+
+        gm20b_dvfs_calc_ndiv(clk, pll->n, &n_int, &sdm_din);
+        gm20b_pllg_read_mnp(clk, &cur_pll);
+        pdiv_only = cur_pll.base.n == n_int && cur_pll.sdm_din == sdm_din &&
+                    cur_pll.base.m == pll->m;
+
+        /* need full sequence if clock not enabled yet */
+        if (!gk20a_pllg_is_enabled(&clk->base))
+                pdiv_only = false;
+
+        /* split VCO-to-bypass jump in half by setting out divider 1:2 */
+        nvkm_mask(device, GPC2CLK_OUT, GPC2CLK_OUT_VCODIV_MASK,
+                  GPC2CLK_OUT_VCODIV2 << GPC2CLK_OUT_VCODIV_SHIFT);
+        /* Intentional 2nd write to assure linear divider operation */
+        nvkm_mask(device, GPC2CLK_OUT, GPC2CLK_OUT_VCODIV_MASK,
+                  GPC2CLK_OUT_VCODIV2 << GPC2CLK_OUT_VCODIV_SHIFT);
+        nvkm_rd32(device, GPC2CLK_OUT);
+        udelay(2);
+
+        if (pdiv_only) {
+                u32 old = cur_pll.base.pl;
+                u32 new = pll->pl;
+
+                /*
+                 * we can do a glitchless transition only if the old and new PL
+                 * parameters share at least one bit set to 1. If this is not
+                 * the case, calculate and program an interim PL that will allow
+                 * us to respect that rule.
+                 */
+                if ((old & new) == 0) {
+                        cur_pll.base.pl = min(old | BIT(ffs(new) - 1),
+                                              new | BIT(ffs(old) - 1));
+                        gk20a_pllg_write_mnp(&clk->base, &cur_pll.base);
+                }
+
+                cur_pll.base.pl = new;
+                gk20a_pllg_write_mnp(&clk->base, &cur_pll.base);
+        } else {
+                /* disable before programming if more than pdiv changes */
+                gm20b_pllg_disable(clk);
+
+                cur_pll.base = *pll;
+                cur_pll.base.n = n_int;
+                cur_pll.sdm_din = sdm_din;
+                gm20b_pllg_write_mnp(clk, &cur_pll);
+
+                ret = gm20b_pllg_enable(clk);
+                if (ret)
+                        return ret;
+        }
+
+        /* restore out divider 1:1 */
+        udelay(2);
+        nvkm_mask(device, GPC2CLK_OUT, GPC2CLK_OUT_VCODIV_MASK,
+                  GPC2CLK_OUT_VCODIV1 << GPC2CLK_OUT_VCODIV_SHIFT);
+        /* Intentional 2nd write to assure linear divider operation */
+        nvkm_mask(device, GPC2CLK_OUT, GPC2CLK_OUT_VCODIV_MASK,
+                  GPC2CLK_OUT_VCODIV1 << GPC2CLK_OUT_VCODIV_SHIFT);
+        nvkm_rd32(device, GPC2CLK_OUT);
+
+        return 0;
+}
+
+static int
+gm20b_pllg_program_mnp_slide(struct gm20b_clk *clk, const struct gk20a_pll *pll)
+{
+        struct gk20a_pll cur_pll;
+        int ret;
+
+        if (gk20a_pllg_is_enabled(&clk->base)) {
+                gk20a_pllg_read_mnp(&clk->base, &cur_pll);
+
+                /* just do NDIV slide if there is no change to M and PL */
+                if (pll->m == cur_pll.m && pll->pl == cur_pll.pl)
+                        return gm20b_pllg_slide(clk, pll->n);
+
+                /* slide down to current NDIV_LO */
+                cur_pll.n = gk20a_pllg_n_lo(&clk->base, &cur_pll);
+                ret = gm20b_pllg_slide(clk, cur_pll.n);
+                if (ret)
+                        return ret;
+        }
+
+        /* program MNP with the new clock parameters and new NDIV_LO */
+        cur_pll = *pll;
+        cur_pll.n = gk20a_pllg_n_lo(&clk->base, &cur_pll);
+        ret = gm20b_pllg_program_mnp(clk, &cur_pll);
+        if (ret)
+                return ret;
+
+        /* slide up to new NDIV */
+        return gm20b_pllg_slide(clk, pll->n);
+}
+
+static int
+gm20b_clk_calc(struct nvkm_clk *base, struct nvkm_cstate *cstate)
+{
+        struct gm20b_clk *clk = gm20b_clk(base);
+        struct nvkm_subdev *subdev = &base->subdev;
+        struct nvkm_volt *volt = base->subdev.device->volt;
+        int ret;
+
+        ret = gk20a_pllg_calc_mnp(&clk->base, cstate->domain[nv_clk_src_gpc] *
+                                             GK20A_CLK_GPC_MDIV, &clk->new_pll);
+        if (ret)
+                return ret;
+
+        clk->new_uv = volt->vid[cstate->voltage].uv;
+        gm20b_dvfs_calc_det_coeff(clk, clk->new_uv, &clk->new_dvfs);
+
+        nvkm_debug(subdev, "%s uv: %d uv\n", __func__, clk->new_uv);
+
+        return 0;
+}
+
+/*
+ * Compute PLL parameters that are always safe for the current voltage
+ */
+static void
+gm20b_dvfs_calc_safe_pll(struct gm20b_clk *clk, struct gk20a_pll *pll)
+{
+        u32 rate = gk20a_pllg_calc_rate(&clk->base, pll) / KHZ;
+        u32 parent_rate = clk->base.parent_rate / KHZ;
+        u32 nmin, nsafe;
+
+        /* remove a safe margin of 10% */
+        if (rate > clk->safe_fmax_vmin)
+                rate = rate * (100 - 10) / 100;
+
+        /* gpc2clk */
+        rate *= 2;
+
+        nmin = DIV_ROUND_UP(pll->m * clk->base.params->min_vco, parent_rate);
+        nsafe = pll->m * rate / (clk->base.parent_rate);
+
+        if (nsafe < nmin) {
+                pll->pl = DIV_ROUND_UP(nmin * parent_rate, pll->m * rate);
+                nsafe = nmin;
+        }
+
+        pll->n = nsafe;
+}
+
+static void
+gm20b_dvfs_program_coeff(struct gm20b_clk *clk, u32 coeff)
+{
+        struct nvkm_device *device = clk->base.base.subdev.device;
+
+        /* strobe to read external DFS coefficient */
+        nvkm_mask(device, GPC_BCAST_GPCPLL_DVFS2,
+                  GPC_BCAST_GPCPLL_DVFS2_DFS_EXT_STROBE_BIT,
+                  GPC_BCAST_GPCPLL_DVFS2_DFS_EXT_STROBE_BIT);
+
+        nvkm_mask(device, GPCPLL_DVFS0, GPCPLL_DVFS0_DFS_COEFF_MASK,
+                  coeff << GPCPLL_DVFS0_DFS_COEFF_SHIFT);
+
+        udelay(1);
+        nvkm_mask(device, GPC_BCAST_GPCPLL_DVFS2,
+                  GPC_BCAST_GPCPLL_DVFS2_DFS_EXT_STROBE_BIT, 0);
+}
+
+static void
+gm20b_dvfs_program_ext_cal(struct gm20b_clk *clk, u32 dfs_det_cal)
+{
+        struct nvkm_device *device = clk->base.base.subdev.device;
+        u32 val;
+
+        nvkm_mask(device, GPC_BCAST_GPCPLL_DVFS2, MASK(DFS_DET_RANGE + 1),
+                  dfs_det_cal);
+        udelay(1);
+
+        val = nvkm_rd32(device, GPCPLL_DVFS1);
+        if (!(val & BIT(25))) {
+                /* Use external value to overwrite calibration value */
+                val |= BIT(25) | BIT(16);
+                nvkm_wr32(device, GPCPLL_DVFS1, val);
+        }
+}
+
+static void
+gm20b_dvfs_program_dfs_detection(struct gm20b_clk *clk,
+                                 struct gm20b_clk_dvfs *dvfs)
+{
+        struct nvkm_device *device = clk->base.base.subdev.device;
+
+        /* strobe to read external DFS coefficient */
+        nvkm_mask(device, GPC_BCAST_GPCPLL_DVFS2,
+                  GPC_BCAST_GPCPLL_DVFS2_DFS_EXT_STROBE_BIT,
+                  GPC_BCAST_GPCPLL_DVFS2_DFS_EXT_STROBE_BIT);
+
+        nvkm_mask(device, GPCPLL_DVFS0,
+                  GPCPLL_DVFS0_DFS_COEFF_MASK | GPCPLL_DVFS0_DFS_DET_MAX_MASK,
+                  dvfs->dfs_coeff << GPCPLL_DVFS0_DFS_COEFF_SHIFT |
+                  dvfs->dfs_det_max << GPCPLL_DVFS0_DFS_DET_MAX_SHIFT);
+
+        udelay(1);
+        nvkm_mask(device, GPC_BCAST_GPCPLL_DVFS2,
+                  GPC_BCAST_GPCPLL_DVFS2_DFS_EXT_STROBE_BIT, 0);
+
+        gm20b_dvfs_program_ext_cal(clk, dvfs->dfs_ext_cal);
+}
+
+static int
+gm20b_clk_prog(struct nvkm_clk *base)
+{
+        struct gm20b_clk *clk = gm20b_clk(base);
+        u32 cur_freq;
+        int ret;
+
+        /* No change in DVFS settings? */
+        if (clk->uv == clk->new_uv)
+                goto prog;
+
+        /*
+         * Interim step for changing DVFS detection settings: low enough
+         * frequency to be safe at at DVFS coeff = 0.
+         *
+         * 1. If voltage is increasing:
+         * - safe frequency target matches the lowest - old - frequency
+         * - DVFS settings are still old
+         * - Voltage already increased to new level by volt, but maximum
+         *   detection limit assures PLL output remains under F/V curve
+         *
+         * 2. If voltage is decreasing:
+         * - safe frequency target matches the lowest - new - frequency
+         * - DVFS settings are still old
+         * - Voltage is also old, it will be lowered by volt afterwards
+         *
+         * Interim step can be skipped if old frequency is below safe minimum,
+         * i.e., it is low enough to be safe at any voltage in operating range
+         * with zero DVFS coefficient.
+         */
+        cur_freq = nvkm_clk_read(&clk->base.base, nv_clk_src_gpc);
+        if (cur_freq > clk->safe_fmax_vmin) {
+                struct gk20a_pll pll_safe;
+
+                if (clk->uv < clk->new_uv)
+                        /* voltage will raise: safe frequency is current one */
+                        pll_safe = clk->base.pll;
+                else
+                        /* voltage will drop: safe frequency is new one */
+                        pll_safe = clk->new_pll;
+
+                gm20b_dvfs_calc_safe_pll(clk, &pll_safe);
+                ret = gm20b_pllg_program_mnp_slide(clk, &pll_safe);
+                if (ret)
+                        return ret;
+        }
+
+        /*
+         * DVFS detection settings transition:
+         * - Set DVFS coefficient zero
+         * - Set calibration level to new voltage
+         * - Set DVFS coefficient to match new voltage
+         */
+        gm20b_dvfs_program_coeff(clk, 0);
+        gm20b_dvfs_program_ext_cal(clk, clk->new_dvfs.dfs_ext_cal);
+        gm20b_dvfs_program_coeff(clk, clk->new_dvfs.dfs_coeff);
+        gm20b_dvfs_program_dfs_detection(clk, &clk->new_dvfs);
+
+prog:
+        clk->uv = clk->new_uv;
+        clk->dvfs = clk->new_dvfs;
+        clk->base.pll = clk->new_pll;
+
+        return gm20b_pllg_program_mnp_slide(clk, &clk->base.pll);
+}
+
 static struct nvkm_pstate
 gm20b_pstates[] = {
         {
@@ -133,9 +714,99 @@ gm20b_pstates[] = {
                         .voltage = 12,
                 },
         },
-
 };
 
+static void
+gm20b_clk_fini(struct nvkm_clk *base)
+{
+        struct nvkm_device *device = base->subdev.device;
+        struct gm20b_clk *clk = gm20b_clk(base);
+
+        /* slide to VCO min */
+        if (gk20a_pllg_is_enabled(&clk->base)) {
+                struct gk20a_pll pll;
+                u32 n_lo;
+
+                gk20a_pllg_read_mnp(&clk->base, &pll);
+                n_lo = gk20a_pllg_n_lo(&clk->base, &pll);
+                gm20b_pllg_slide(clk, n_lo);
+        }
+
+        gm20b_pllg_disable(clk);
+
+        /* set IDDQ */
+        nvkm_mask(device, GPCPLL_CFG, GPCPLL_CFG_IDDQ, 1);
+}
+
+static int
+gm20b_clk_init_dvfs(struct gm20b_clk *clk)
+{
+        struct nvkm_subdev *subdev = &clk->base.base.subdev;
+        struct nvkm_device *device = subdev->device;
+        bool fused = clk->uvdet_offs && clk->uvdet_slope;
+        static const s32 ADC_SLOPE_UV = 10000; /* default ADC detection slope */
+        u32 data;
+        int ret;
+
+        /* Enable NA DVFS */
+        nvkm_mask(device, GPCPLL_DVFS1, GPCPLL_DVFS1_EN_DFS_BIT,
+                  GPCPLL_DVFS1_EN_DFS_BIT);
+
+        /* Set VCO_CTRL */
+        if (clk->dvfs_params->vco_ctrl)
+                nvkm_mask(device, GPCPLL_CFG3, GPCPLL_CFG3_VCO_CTRL_MASK,
+                      clk->dvfs_params->vco_ctrl << GPCPLL_CFG3_VCO_CTRL_SHIFT);
+
+        if (fused) {
+                /* Start internal calibration, but ignore results */
+                nvkm_mask(device, GPCPLL_DVFS1, GPCPLL_DVFS1_EN_DFS_CAL_BIT,
+                          GPCPLL_DVFS1_EN_DFS_CAL_BIT);
+
+                /* got uvdev parameters from fuse, skip calibration */
+                goto calibrated;
+        }
+
+        /*
+         * If calibration parameters are not fused, start internal calibration,
+         * wait for completion, and use results along with default slope to
+         * calculate ADC offset during boot.
+         */
+        nvkm_mask(device, GPCPLL_DVFS1, GPCPLL_DVFS1_EN_DFS_CAL_BIT,
+                          GPCPLL_DVFS1_EN_DFS_CAL_BIT);
+
+        /* Wait for internal calibration done (spec < 2us). */
+        ret = nvkm_wait_usec(device, 10, GPCPLL_DVFS1,
+                             GPCPLL_DVFS1_DFS_CAL_DONE_BIT,
+                             GPCPLL_DVFS1_DFS_CAL_DONE_BIT);
+        if (ret < 0) {
+                nvkm_error(subdev, "GPCPLL calibration timeout\n");
+                return -ETIMEDOUT;
+        }
+
+        data = nvkm_rd32(device, GPCPLL_CFG3) >>
+                         GPCPLL_CFG3_PLL_DFS_TESTOUT_SHIFT;
+        data &= MASK(GPCPLL_CFG3_PLL_DFS_TESTOUT_WIDTH);
+
+        clk->uvdet_slope = ADC_SLOPE_UV;
+        clk->uvdet_offs = ((s32)clk->uv) - data * ADC_SLOPE_UV;
+
+        nvkm_debug(subdev, "calibrated DVFS parameters: offs %d, slope %d\n",
+                   clk->uvdet_offs, clk->uvdet_slope);
+
+calibrated:
+        /* Compute and apply initial DVFS parameters */
+        gm20b_dvfs_calc_det_coeff(clk, clk->uv, &clk->dvfs);
+        gm20b_dvfs_program_coeff(clk, 0);
+        gm20b_dvfs_program_ext_cal(clk, clk->dvfs.dfs_ext_cal);
+        gm20b_dvfs_program_coeff(clk, clk->dvfs.dfs_coeff);
+        gm20b_dvfs_program_dfs_detection(clk, &clk->new_dvfs);
+
+        return 0;
+}
+
+/* Forward declaration to detect speedo >=1 in gm20b_clk_init() */
+static const struct nvkm_clk_func gm20b_clk;
+
 static int
 gm20b_clk_init(struct nvkm_clk *base)
 {
@@ -143,15 +814,56 @@ gm20b_clk_init(struct nvkm_clk *base)
         struct nvkm_subdev *subdev = &clk->base.subdev;
         struct nvkm_device *device = subdev->device;
         int ret;
+        u32 data;
+
+        /* get out from IDDQ */
+        nvkm_mask(device, GPCPLL_CFG, GPCPLL_CFG_IDDQ, 0);
+        nvkm_rd32(device, GPCPLL_CFG);
+        udelay(5);
+
+        nvkm_mask(device, GPC2CLK_OUT, GPC2CLK_OUT_INIT_MASK,
+                  GPC2CLK_OUT_INIT_VAL);
 
         /* Set the global bypass control to VCO */
         nvkm_mask(device, BYPASSCTRL_SYS,
                MASK(BYPASSCTRL_SYS_GPCPLL_WIDTH) << BYPASSCTRL_SYS_GPCPLL_SHIFT,
                0);
 
+        ret = gk20a_clk_setup_slide(clk);
+        if (ret)
+                return ret;
+
+        /* If not fused, set RAM SVOP PDP data 0x2, and enable fuse override */
+        data = nvkm_rd32(device, 0x021944);
+        if (!(data & 0x3)) {
+                data |= 0x2;
+                nvkm_wr32(device, 0x021944, data);
+
+                data = nvkm_rd32(device, 0x021948);
+                data |=  0x1;
+                nvkm_wr32(device, 0x021948, data);
+        }
+
+        /* Disable idle slow down  */
+        nvkm_mask(device, 0x20160, 0x003f0000, 0x0);
+
+        /* speedo >= 1? */
+        if (clk->base.func == &gm20b_clk) {
+                struct gm20b_clk *_clk = gm20b_clk(base);
+                struct nvkm_volt *volt = device->volt;
+
+                /* Get current voltage */
+                _clk->uv = nvkm_volt_get(volt);
+
+                /* Initialize DVFS */
+                ret = gm20b_clk_init_dvfs(_clk);
+                if (ret)
+                        return ret;
+        }
+
         /* Start with lowest frequency */
         base->func->calc(base, &base->func->pstates[0].base);
-        ret = base->func->prog(&clk->base);
+        ret = base->func->prog(base);
         if (ret) {
                 nvkm_error(subdev, "cannot initialize clock\n");
                 return ret;
@@ -169,6 +881,7 @@ gm20b_clk_speedo0 = {
         .prog = gk20a_clk_prog,
         .tidy = gk20a_clk_tidy,
         .pstates = gm20b_pstates,
+        /* Speedo 0 only supports 12 voltages */
         .nr_pstates = ARRAY_SIZE(gm20b_pstates) - 1,
         .domains = {
                 { nv_clk_src_crystal, 0xff },
@@ -177,8 +890,26 @@ gm20b_clk_speedo0 = {
         },
 };
 
-int
-gm20b_clk_new(struct nvkm_device *device, int index, struct nvkm_clk **pclk)
+static const struct nvkm_clk_func
+gm20b_clk = {
+        .init = gm20b_clk_init,
+        .fini = gm20b_clk_fini,
+        .read = gk20a_clk_read,
+        .calc = gm20b_clk_calc,
+        .prog = gm20b_clk_prog,
+        .tidy = gk20a_clk_tidy,
+        .pstates = gm20b_pstates,
+        .nr_pstates = ARRAY_SIZE(gm20b_pstates),
+        .domains = {
+                { nv_clk_src_crystal, 0xff },
+                { nv_clk_src_gpc, 0xff, 0, "core", GK20A_CLK_GPC_MDIV },
+                { nv_clk_src_max },
+        },
+};
+
+static int
+gm20b_clk_new_speedo0(struct nvkm_device *device, int index,
+                      struct nvkm_clk **pclk)
 {
         struct gk20a_clk *clk;
         int ret;
@@ -188,11 +919,156 @@ gm20b_clk_new(struct nvkm_device *device, int index, struct nvkm_clk **pclk)
                 return -ENOMEM;
         *pclk = &clk->base;
 
-        ret = _gk20a_clk_ctor(device, index, &gm20b_clk_speedo0,
-                              &gm20b_pllg_params, clk);
+        ret = gk20a_clk_ctor(device, index, &gm20b_clk_speedo0,
+                             &gm20b_pllg_params, clk);
 
         clk->pl_to_div = pl_to_div;
         clk->div_to_pl = div_to_pl;
 
         return ret;
 }
+
+/* FUSE register */
+#define FUSE_RESERVED_CALIB0    0x204
+#define FUSE_RESERVED_CALIB0_INTERCEPT_FRAC_SHIFT       0
+#define FUSE_RESERVED_CALIB0_INTERCEPT_FRAC_WIDTH       4
+#define FUSE_RESERVED_CALIB0_INTERCEPT_INT_SHIFT        4
+#define FUSE_RESERVED_CALIB0_INTERCEPT_INT_WIDTH        10
+#define FUSE_RESERVED_CALIB0_SLOPE_FRAC_SHIFT           14
+#define FUSE_RESERVED_CALIB0_SLOPE_FRAC_WIDTH           10
+#define FUSE_RESERVED_CALIB0_SLOPE_INT_SHIFT            24
+#define FUSE_RESERVED_CALIB0_SLOPE_INT_WIDTH            6
+#define FUSE_RESERVED_CALIB0_FUSE_REV_SHIFT             30
+#define FUSE_RESERVED_CALIB0_FUSE_REV_WIDTH             2
+
+static int
+gm20b_clk_init_fused_params(struct gm20b_clk *clk)
+{
+        struct nvkm_subdev *subdev = &clk->base.base.subdev;
+        u32 val = 0;
+        u32 rev = 0;
+
+#if IS_ENABLED(CONFIG_ARCH_TEGRA)
+        tegra_fuse_readl(FUSE_RESERVED_CALIB0, &val);
+        rev = (val >> FUSE_RESERVED_CALIB0_FUSE_REV_SHIFT) &
+              MASK(FUSE_RESERVED_CALIB0_FUSE_REV_WIDTH);
+#endif
+
+        /* No fused parameters, we will calibrate later */
+        if (rev == 0)
+                return -EINVAL;
+
+        /* Integer part in mV + fractional part in uV */
+        clk->uvdet_slope = ((val >> FUSE_RESERVED_CALIB0_SLOPE_INT_SHIFT) &
+                        MASK(FUSE_RESERVED_CALIB0_SLOPE_INT_WIDTH)) * 1000 +
+                        ((val >> FUSE_RESERVED_CALIB0_SLOPE_FRAC_SHIFT) &
+                        MASK(FUSE_RESERVED_CALIB0_SLOPE_FRAC_WIDTH));
+
+        /* Integer part in mV + fractional part in 100uV */
+        clk->uvdet_offs = ((val >> FUSE_RESERVED_CALIB0_INTERCEPT_INT_SHIFT) &
+                        MASK(FUSE_RESERVED_CALIB0_INTERCEPT_INT_WIDTH)) * 1000 +
+                        ((val >> FUSE_RESERVED_CALIB0_INTERCEPT_FRAC_SHIFT) &
+                         MASK(FUSE_RESERVED_CALIB0_INTERCEPT_FRAC_WIDTH)) * 100;
+
+        nvkm_debug(subdev, "fused calibration data: slope %d, offs %d\n",
+                   clk->uvdet_slope, clk->uvdet_offs);
+        return 0;
+}
+
+static int
+gm20b_clk_init_safe_fmax(struct gm20b_clk *clk)
+{
+        struct nvkm_subdev *subdev = &clk->base.base.subdev;
+        struct nvkm_volt *volt = subdev->device->volt;
+        struct nvkm_pstate *pstates = clk->base.base.func->pstates;
+        int nr_pstates = clk->base.base.func->nr_pstates;
+        int vmin, id = 0;
+        u32 fmax = 0;
+        int i;
+
+        /* find lowest voltage we can use */
+        vmin = volt->vid[0].uv;
+        for (i = 1; i < volt->vid_nr; i++) {
+                if (volt->vid[i].uv <= vmin) {
+                        vmin = volt->vid[i].uv;
+                        id = volt->vid[i].vid;
+                }
+        }
+
+        /* find max frequency at this voltage */
+        for (i = 0; i < nr_pstates; i++)
+                if (pstates[i].base.voltage == id)
+                        fmax = max(fmax,
+                                   pstates[i].base.domain[nv_clk_src_gpc]);
+
+        if (!fmax) {
+                nvkm_error(subdev, "failed to evaluate safe fmax\n");
+                return -EINVAL;
+        }
+
+        /* we are safe at 90% of the max frequency */
+        clk->safe_fmax_vmin = fmax * (100 - 10) / 100;
+        nvkm_debug(subdev, "safe fmax @ vmin = %u Khz\n", clk->safe_fmax_vmin);
+
+        return 0;
+}
+
+int
+gm20b_clk_new(struct nvkm_device *device, int index, struct nvkm_clk **pclk)
+{
+        struct nvkm_device_tegra *tdev = device->func->tegra(device);
+        struct gm20b_clk *clk;
+        struct nvkm_subdev *subdev;
+        struct gk20a_clk_pllg_params *clk_params;
+        int ret;
+
+        /* Speedo 0 GPUs cannot use noise-aware PLL */
+        if (tdev->gpu_speedo_id == 0)
+                return gm20b_clk_new_speedo0(device, index, pclk);
+
+        /* Speedo >= 1, use NAPLL */
+        clk = kzalloc(sizeof(*clk) + sizeof(*clk_params), GFP_KERNEL);
+        if (!clk)
+                return -ENOMEM;
+        *pclk = &clk->base.base;
+        subdev = &clk->base.base.subdev;
+
+        /* duplicate the clock parameters since we will patch them below */
+        clk_params = (void *) (clk + 1);
+        *clk_params = gm20b_pllg_params;
+        ret = gk20a_clk_ctor(device, index, &gm20b_clk, clk_params,
+                             &clk->base);
+        if (ret)
+                return ret;
+
+        /*
+         * NAPLL can only work with max_u, clamp the m range so
+         * gk20a_pllg_calc_mnp always uses it
+         */
+        clk_params->max_m = clk_params->min_m = DIV_ROUND_UP(clk_params->max_u,
+                                                (clk->base.parent_rate / KHZ));
+        if (clk_params->max_m == 0) {
+                nvkm_warn(subdev, "cannot use NAPLL, using legacy clock...\n");
+                kfree(clk);
+                return gm20b_clk_new_speedo0(device, index, pclk);
+        }
+
+        clk->base.pl_to_div = pl_to_div;
+        clk->base.div_to_pl = div_to_pl;
+
+        clk->dvfs_params = &gm20b_dvfs_params;
+
+        ret = gm20b_clk_init_fused_params(clk);
+        /*
+         * we will calibrate during init - should never happen on
+         * prod parts
+         */
+        if (ret)
+                nvkm_warn(subdev, "no fused calibration parameters\n");
+
+        ret = gm20b_clk_init_safe_fmax(clk);
+        if (ret)
+                return ret;
+
+        return 0;
+}
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/Kbuild b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/Kbuild
index 842d5de96d73..edcc157e6ac8 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/Kbuild
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/Kbuild
@@ -24,6 +24,8 @@ nvkm-y += nvkm/subdev/fb/gk104.o
 nvkm-y += nvkm/subdev/fb/gk20a.o
 nvkm-y += nvkm/subdev/fb/gm107.o
 nvkm-y += nvkm/subdev/fb/gm200.o
+nvkm-y += nvkm/subdev/fb/gp100.o
+nvkm-y += nvkm/subdev/fb/gp104.o
 
 nvkm-y += nvkm/subdev/fb/ram.o
 nvkm-y += nvkm/subdev/fb/ramnv04.o
@@ -41,6 +43,7 @@ nvkm-y += nvkm/subdev/fb/rammcp77.o
 nvkm-y += nvkm/subdev/fb/ramgf100.o
 nvkm-y += nvkm/subdev/fb/ramgk104.o
 nvkm-y += nvkm/subdev/fb/ramgm107.o
+nvkm-y += nvkm/subdev/fb/ramgp100.o
 nvkm-y += nvkm/subdev/fb/sddr2.o
 nvkm-y += nvkm/subdev/fb/sddr3.o
 nvkm-y += nvkm/subdev/fb/gddr3.o
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/base.c b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/base.c
index ce90242b8cce..a7049c041594 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/base.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/base.c
@@ -25,6 +25,7 @@
 #include "ram.h"
 
 #include <core/memory.h>
+#include <core/option.h>
 #include <subdev/bios.h>
 #include <subdev/bios/M0203.h>
 #include <engine/gr.h>
@@ -134,6 +135,10 @@ nvkm_fb_init(struct nvkm_subdev *subdev)
 
         if (fb->func->init)
                 fb->func->init(fb);
+        if (fb->func->init_page)
+                fb->func->init_page(fb);
+        if (fb->func->init_unkn)
+                fb->func->init_unkn(fb);
         return 0;
 }
 
@@ -171,6 +176,7 @@ nvkm_fb_ctor(const struct nvkm_fb_func *func, struct nvkm_device *device,
         nvkm_subdev_ctor(&nvkm_fb, device, index, &fb->subdev);
         fb->func = func;
         fb->tile.regions = fb->func->tile.regions;
+        fb->page = nvkm_longopt(device->cfgopt, "NvFbBigPage", 0);
 }
 
 int
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gf100.c b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gf100.c
index e649ead5ccfc..76433cc66fff 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gf100.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gf100.c
@@ -72,6 +72,22 @@ gf100_fb_oneinit(struct nvkm_fb *fb)
 }
 
 void
+gf100_fb_init_page(struct nvkm_fb *fb)
+{
+        struct nvkm_device *device = fb->subdev.device;
+        switch (fb->page) {
+        case 16:
+                nvkm_mask(device, 0x100c80, 0x00000001, 0x00000001);
+                break;
+        case 17:
+        default:
+                nvkm_mask(device, 0x100c80, 0x00000001, 0x00000000);
+                fb->page = 17;
+                break;
+        }
+}
+
+void
 gf100_fb_init(struct nvkm_fb *base)
 {
         struct gf100_fb *fb = gf100_fb(base);
@@ -79,8 +95,6 @@ gf100_fb_init(struct nvkm_fb *base)
 
         if (fb->r100c10_page)
                 nvkm_wr32(device, 0x100c10, fb->r100c10 >> 8);
-
-        nvkm_mask(device, 0x100c80, 0x00000001, 0x00000000); /* 128KiB lpg */
 }
 
 void *
@@ -125,6 +139,7 @@ gf100_fb = {
         .dtor = gf100_fb_dtor,
         .oneinit = gf100_fb_oneinit,
         .init = gf100_fb_init,
+        .init_page = gf100_fb_init_page,
         .intr = gf100_fb_intr,
         .ram_new = gf100_ram_new,
         .memtype_valid = gf100_fb_memtype_valid,
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gf100.h b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gf100.h
index 2160e5a39c9a..449f431644b3 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gf100.h
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gf100.h
@@ -14,4 +14,6 @@ int gf100_fb_new_(const struct nvkm_fb_func *, struct nvkm_device *,
 void *gf100_fb_dtor(struct nvkm_fb *);
 void gf100_fb_init(struct nvkm_fb *);
 void gf100_fb_intr(struct nvkm_fb *);
+
+void gp100_fb_init(struct nvkm_fb *);
 #endif
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gk104.c b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gk104.c
index b41f0f70038c..4245e2e6e604 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gk104.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gk104.c
@@ -29,6 +29,7 @@ gk104_fb = {
         .dtor = gf100_fb_dtor,
         .oneinit = gf100_fb_oneinit,
         .init = gf100_fb_init,
+        .init_page = gf100_fb_init_page,
         .intr = gf100_fb_intr,
         .ram_new = gk104_ram_new,
         .memtype_valid = gf100_fb_memtype_valid,
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gk20a.c b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gk20a.c
index 7306f7dfc3b9..f815fe2bbf08 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gk20a.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gk20a.c
@@ -27,7 +27,6 @@ static void
 gk20a_fb_init(struct nvkm_fb *fb)
 {
         struct nvkm_device *device = fb->subdev.device;
-        nvkm_mask(device, 0x100c80, 0x00000001, 0x00000000); /* 128KiB lpg */
         nvkm_wr32(device, 0x100cc8, nvkm_memory_addr(fb->mmu_wr) >> 8);
         nvkm_wr32(device, 0x100ccc, nvkm_memory_addr(fb->mmu_rd) >> 8);
 }
@@ -36,6 +35,7 @@ static const struct nvkm_fb_func
 gk20a_fb = {
         .oneinit = gf100_fb_oneinit,
         .init = gk20a_fb_init,
+        .init_page = gf100_fb_init_page,
         .memtype_valid = gf100_fb_memtype_valid,
 };
 
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gm107.c b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gm107.c
index 4869fdb753c9..db699025f546 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gm107.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gm107.c
@@ -29,6 +29,7 @@ gm107_fb = {
         .dtor = gf100_fb_dtor,
         .oneinit = gf100_fb_oneinit,
         .init = gf100_fb_init,
+        .init_page = gf100_fb_init_page,
         .intr = gf100_fb_intr,
         .ram_new = gm107_ram_new,
         .memtype_valid = gf100_fb_memtype_valid,
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gm200.c b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gm200.c
index 44f5716f64d8..62f653240be3 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gm200.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gm200.c
@@ -26,6 +26,24 @@
 
 #include <core/memory.h>
 
+void
+gm200_fb_init_page(struct nvkm_fb *fb)
+{
+        struct nvkm_device *device = fb->subdev.device;
+        switch (fb->page) {
+        case 16:
+                nvkm_mask(device, 0x100c80, 0x00000801, 0x00000001);
+                break;
+        case 17:
+                nvkm_mask(device, 0x100c80, 0x00000801, 0x00000000);
+                break;
+        default:
+                nvkm_mask(device, 0x100c80, 0x00000800, 0x00000800);
+                fb->page = 0;
+                break;
+        }
+}
+
 static void
 gm200_fb_init(struct nvkm_fb *base)
 {
@@ -48,6 +66,7 @@ gm200_fb = {
         .dtor = gf100_fb_dtor,
         .oneinit = gf100_fb_oneinit,
         .init = gm200_fb_init,
+        .init_page = gm200_fb_init_page,
         .intr = gf100_fb_intr,
         .ram_new = gm107_ram_new,
         .memtype_valid = gf100_fb_memtype_valid,
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gp100.c b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gp100.c
new file mode 100644
index 000000000000..98474aec1921
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gp100.c
@@ -0,0 +1,69 @@
+/*
+ * Copyright 2016 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs <bskeggs@redhat.com>
+ */
+#include "gf100.h"
+#include "ram.h"
+
+#include <core/memory.h>
+
+static void
+gp100_fb_init_unkn(struct nvkm_fb *base)
+{
+        struct nvkm_device *device = gf100_fb(base)->base.subdev.device;
+        nvkm_wr32(device, 0x1fac80, nvkm_rd32(device, 0x100c80));
+        nvkm_wr32(device, 0x1facc4, nvkm_rd32(device, 0x100cc4));
+        nvkm_wr32(device, 0x1facc8, nvkm_rd32(device, 0x100cc8));
+        nvkm_wr32(device, 0x1faccc, nvkm_rd32(device, 0x100ccc));
+}
+
+void
+gp100_fb_init(struct nvkm_fb *base)
+{
+        struct gf100_fb *fb = gf100_fb(base);
+        struct nvkm_device *device = fb->base.subdev.device;
+
+        if (fb->r100c10_page)
+                nvkm_wr32(device, 0x100c10, fb->r100c10 >> 8);
+
+        nvkm_wr32(device, 0x100cc8, nvkm_memory_addr(fb->base.mmu_wr) >> 8);
+        nvkm_wr32(device, 0x100ccc, nvkm_memory_addr(fb->base.mmu_rd) >> 8);
+        nvkm_mask(device, 0x100cc4, 0x00060000,
+                  max(nvkm_memory_size(fb->base.mmu_rd) >> 16, (u64)2) << 17);
+}
+
+static const struct nvkm_fb_func
+gp100_fb = {
+        .dtor = gf100_fb_dtor,
+        .oneinit = gf100_fb_oneinit,
+        .init = gp100_fb_init,
+        .init_page = gm200_fb_init_page,
+        .init_unkn = gp100_fb_init_unkn,
+        .ram_new = gp100_ram_new,
+        .memtype_valid = gf100_fb_memtype_valid,
+};
+
+int
+gp100_fb_new(struct nvkm_device *device, int index, struct nvkm_fb **pfb)
+{
+        return gf100_fb_new_(&gp100_fb, device, index, pfb);
+}
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gp104.c b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gp104.c
new file mode 100644
index 000000000000..92cb71861bec
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/gp104.c
@@ -0,0 +1,43 @@
+/*
+ * Copyright 2016 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs <bskeggs@redhat.com>
+ */
+#include "gf100.h"
+#include "ram.h"
+
+#include <core/memory.h>
+
+static const struct nvkm_fb_func
+gp104_fb = {
+        .dtor = gf100_fb_dtor,
+        .oneinit = gf100_fb_oneinit,
+        .init = gp100_fb_init,
+        .init_page = gm200_fb_init_page,
+        .ram_new = gp100_ram_new,
+        .memtype_valid = gf100_fb_memtype_valid,
+};
+
+int
+gp104_fb_new(struct nvkm_device *device, int index, struct nvkm_fb **pfb)
+{
+        return gf100_fb_new_(&gp104_fb, device, index, pfb);
+}
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/priv.h b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/priv.h
index d97d640e60a0..e905d44fa1d5 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/priv.h
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/priv.h
@@ -8,6 +8,8 @@ struct nvkm_fb_func {
         void *(*dtor)(struct nvkm_fb *);
         int (*oneinit)(struct nvkm_fb *);
         void (*init)(struct nvkm_fb *);
+        void (*init_page)(struct nvkm_fb *);
+        void (*init_unkn)(struct nvkm_fb *);
         void (*intr)(struct nvkm_fb *);
 
         struct {
@@ -60,5 +62,8 @@ void nv46_fb_tile_init(struct nvkm_fb *, int i, u32 addr, u32 size,
                        u32 pitch, u32 flags, struct nvkm_fb_tile *);
 
 int gf100_fb_oneinit(struct nvkm_fb *);
+void gf100_fb_init_page(struct nvkm_fb *);
 bool gf100_fb_memtype_valid(struct nvkm_fb *, u32);
+
+void gm200_fb_init_page(struct nvkm_fb *);
 #endif
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ram.h b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ram.h
index f816cbf2ced3..b9ec0ae6723a 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ram.h
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ram.h
@@ -47,4 +47,5 @@ int mcp77_ram_new(struct nvkm_fb *, struct nvkm_ram **);
 int gf100_ram_new(struct nvkm_fb *, struct nvkm_ram **);
 int gk104_ram_new(struct nvkm_fb *, struct nvkm_ram **);
 int gm107_ram_new(struct nvkm_fb *, struct nvkm_ram **);
+int gp100_ram_new(struct nvkm_fb *, struct nvkm_ram **);
 #endif
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramgp100.c b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramgp100.c
new file mode 100644
index 000000000000..f3be408b5e5e
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/fb/ramgp100.c
@@ -0,0 +1,146 @@
+/*
+ * Copyright 2013 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs
+ */
+#include "ram.h"
+
+#include <subdev/bios.h>
+#include <subdev/bios/init.h>
+#include <subdev/bios/rammap.h>
+
+static int
+gp100_ram_init(struct nvkm_ram *ram)
+{
+        struct nvkm_subdev *subdev = &ram->fb->subdev;
+        struct nvkm_device *device = subdev->device;
+        struct nvkm_bios *bios = device->bios;
+        u8  ver, hdr, cnt, len, snr, ssz;
+        u32 data;
+        int i;
+
+        /* run a bunch of tables from rammap table.  there's actually
+         * individual pointers for each rammap entry too, but, nvidia
+         * seem to just run the last two entries' scripts early on in
+         * their init, and never again.. we'll just run 'em all once
+         * for now.
+         *
+         * i strongly suspect that each script is for a separate mode
+         * (likely selected by 0x9a065c's lower bits?), and the
+         * binary driver skips the one that's already been setup by
+         * the init tables.
+         */
+        data = nvbios_rammapTe(bios, &ver, &hdr, &cnt, &len, &snr, &ssz);
+        if (!data || hdr < 0x15)
+                return -EINVAL;
+
+        cnt  = nvbios_rd08(bios, data + 0x14); /* guess at count */
+        data = nvbios_rd32(bios, data + 0x10); /* guess u32... */
+        if (cnt) {
+                u32 save = nvkm_rd32(device, 0x9a065c) & 0x000000f0;
+                for (i = 0; i < cnt; i++, data += 4) {
+                        if (i != save >> 4) {
+                                nvkm_mask(device, 0x9a065c, 0x000000f0, i << 4);
+                                nvbios_exec(&(struct nvbios_init) {
+                                                .subdev = subdev,
+                                                .bios = bios,
+                                                .offset = nvbios_rd32(bios, data),
+                                                .execute = 1,
+                                            });
+                        }
+                }
+                nvkm_mask(device, 0x9a065c, 0x000000f0, save);
+        }
+
+        nvkm_mask(device, 0x9a0584, 0x11000000, 0x00000000);
+        nvkm_wr32(device, 0x10ecc0, 0xffffffff);
+        nvkm_mask(device, 0x9a0160, 0x00000010, 0x00000010);
+        return 0;
+}
+
+static const struct nvkm_ram_func
+gp100_ram_func = {
+        .init = gp100_ram_init,
+        .get = gf100_ram_get,
+        .put = gf100_ram_put,
+};
+
+int
+gp100_ram_new(struct nvkm_fb *fb, struct nvkm_ram **pram)
+{
+        struct nvkm_ram *ram;
+        struct nvkm_subdev *subdev = &fb->subdev;
+        struct nvkm_device *device = subdev->device;
+        enum nvkm_ram_type type = nvkm_fb_bios_memtype(device->bios);
+        const u32 rsvd_head = ( 256 * 1024); /* vga memory */
+        const u32 rsvd_tail = (1024 * 1024); /* vbios etc */
+        u32 fbpa_num = nvkm_rd32(device, 0x022438), fbpa;
+        u32 fbio_opt = nvkm_rd32(device, 0x021c14);
+        u64 part, size = 0, comm = ~0ULL;
+        bool mixed = false;
+        int ret;
+
+        nvkm_debug(subdev, "022438: %08x\n", fbpa_num);
+        nvkm_debug(subdev, "021c14: %08x\n", fbio_opt);
+        for (fbpa = 0; fbpa < fbpa_num; fbpa++) {
+                if (!(fbio_opt & (1 << fbpa))) {
+                        part = nvkm_rd32(device, 0x90020c + (fbpa * 0x4000));
+                        nvkm_debug(subdev, "fbpa %02x: %lld MiB\n", fbpa, part);
+                        part = part << 20;
+                        if (part != comm) {
+                                if (comm != ~0ULL)
+                                        mixed = true;
+                                comm = min(comm, part);
+                        }
+                        size = size + part;
+                }
+        }
+
+        ret = nvkm_ram_new_(&gp100_ram_func, fb, type, size, 0, &ram);
+        *pram = ram;
+        if (ret)
+                return ret;
+
+        nvkm_mm_fini(&ram->vram);
+
+        if (mixed) {
+                ret = nvkm_mm_init(&ram->vram, rsvd_head >> NVKM_RAM_MM_SHIFT,
+                                   ((comm * fbpa_num) - rsvd_head) >>
+                                   NVKM_RAM_MM_SHIFT, 1);
+                if (ret)
+                        return ret;
+
+                ret = nvkm_mm_init(&ram->vram, (0x1000000000ULL + comm) >>
+                                   NVKM_RAM_MM_SHIFT,
+                                   (size - (comm * fbpa_num) - rsvd_tail) >>
+                                   NVKM_RAM_MM_SHIFT, 1);
+                if (ret)
+                        return ret;
+        } else {
+                ret = nvkm_mm_init(&ram->vram, rsvd_head >> NVKM_RAM_MM_SHIFT,
+                                   (size - rsvd_head - rsvd_tail) >>
+                                   NVKM_RAM_MM_SHIFT, 1);
+                if (ret)
+                        return ret;
+        }
+
+        return 0;
+}
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/ltc/Kbuild b/drivers/gpu/drm/nouveau/nvkm/subdev/ltc/Kbuild
index 932b366598aa..12d6f4f102cb 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/ltc/Kbuild
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/ltc/Kbuild
@@ -3,3 +3,4 @@ nvkm-y += nvkm/subdev/ltc/gf100.o
 nvkm-y += nvkm/subdev/ltc/gk104.o
 nvkm-y += nvkm/subdev/ltc/gm107.o
 nvkm-y += nvkm/subdev/ltc/gm200.o
+nvkm-y += nvkm/subdev/ltc/gp100.o
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/ltc/gf100.c b/drivers/gpu/drm/nouveau/nvkm/subdev/ltc/gf100.c
index c9eb677967a8..4a0fa0a9b802 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/ltc/gf100.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/ltc/gf100.c
@@ -23,7 +23,6 @@
  */
 #include "priv.h"
 
-#include <core/enum.h>
 #include <subdev/fb.h>
 #include <subdev/timer.h>
 
@@ -71,7 +70,7 @@ gf100_ltc_zbc_clear_depth(struct nvkm_ltc *ltc, int i, const u32 depth)
         nvkm_wr32(device, 0x17ea58, depth);
 }
 
-static const struct nvkm_bitfield
+const struct nvkm_bitfield
 gf100_ltc_lts_intr_name[] = {
         { 0x00000001, "IDLE_ERROR_IQ" },
         { 0x00000002, "IDLE_ERROR_CBC" },
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/ltc/gm107.c b/drivers/gpu/drm/nouveau/nvkm/subdev/ltc/gm107.c
index 389fb13a1998..ec0a3844b2d1 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/ltc/gm107.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/ltc/gm107.c
@@ -68,18 +68,22 @@ gm107_ltc_zbc_clear_depth(struct nvkm_ltc *ltc, int i, const u32 depth)
         nvkm_wr32(device, 0x17e34c, depth);
 }
 
-static void
+void
 gm107_ltc_intr_lts(struct nvkm_ltc *ltc, int c, int s)
 {
         struct nvkm_subdev *subdev = &ltc->subdev;
         struct nvkm_device *device = subdev->device;
         u32 base = 0x140400 + (c * 0x2000) + (s * 0x200);
-        u32 stat = nvkm_rd32(device, base + 0x00c);
+        u32 intr = nvkm_rd32(device, base + 0x00c);
+        u16 stat = intr & 0x0000ffff;
+        char msg[128];
 
         if (stat) {
-                nvkm_error(subdev, "LTC%d_LTS%d: %08x\n", c, s, stat);
-                nvkm_wr32(device, base + 0x00c, stat);
+                nvkm_snprintbf(msg, sizeof(msg), gf100_ltc_lts_intr_name, stat);
+                nvkm_error(subdev, "LTC%d_LTS%d: %08x [%s]\n", c, s, intr, msg);
         }
+
+        nvkm_wr32(device, base + 0x00c, intr);
 }
 
 void
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/ltc/gp100.c b/drivers/gpu/drm/nouveau/nvkm/subdev/ltc/gp100.c
new file mode 100644
index 000000000000..0bdfb2f40266
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/ltc/gp100.c
@@ -0,0 +1,75 @@
+/*
+ * Copyright 2016 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs
+ */
+#include "priv.h"
+
+static void
+gp100_ltc_intr(struct nvkm_ltc *ltc)
+{
+        struct nvkm_device *device = ltc->subdev.device;
+        u32 mask;
+
+        mask = nvkm_rd32(device, 0x0001c0);
+        while (mask) {
+                u32 s, c = __ffs(mask);
+                for (s = 0; s < ltc->lts_nr; s++)
+                        gm107_ltc_intr_lts(ltc, c, s);
+                mask &= ~(1 << c);
+        }
+}
+
+static int
+gp100_ltc_oneinit(struct nvkm_ltc *ltc)
+{
+        struct nvkm_device *device = ltc->subdev.device;
+        ltc->ltc_nr = nvkm_rd32(device, 0x12006c);
+        ltc->lts_nr = nvkm_rd32(device, 0x17e280) >> 28;
+        /*XXX: tagram allocation - TBD */
+        return nvkm_mm_init(&ltc->tags, 0, 0, 1);
+}
+
+static void
+gp100_ltc_init(struct nvkm_ltc *ltc)
+{
+        /*XXX: PMU LS call to setup tagram address */
+}
+
+static const struct nvkm_ltc_func
+gp100_ltc = {
+        .oneinit = gp100_ltc_oneinit,
+        .init = gp100_ltc_init,
+        .intr = gp100_ltc_intr,
+        .cbc_clear = gm107_ltc_cbc_clear,
+        .cbc_wait = gm107_ltc_cbc_wait,
+        .zbc = 16,
+        .zbc_clear_color = gm107_ltc_zbc_clear_color,
+        .zbc_clear_depth = gm107_ltc_zbc_clear_depth,
+        .invalidate = gf100_ltc_invalidate,
+        .flush = gf100_ltc_flush,
+};
+
+int
+gp100_ltc_new(struct nvkm_device *device, int index, struct nvkm_ltc **pltc)
+{
+        return nvkm_ltc_new_(&gp100_ltc, device, index, pltc);
+}
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/ltc/priv.h b/drivers/gpu/drm/nouveau/nvkm/subdev/ltc/priv.h
index 6d81c695ed0d..8b95f96e3ffa 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/ltc/priv.h
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/ltc/priv.h
@@ -2,6 +2,7 @@
 #define __NVKM_LTC_PRIV_H__
 #define nvkm_ltc(p) container_of((p), struct nvkm_ltc, subdev)
 #include <subdev/ltc.h>
+#include <core/enum.h>
 
 int nvkm_ltc_new_(const struct nvkm_ltc_func *, struct nvkm_device *,
                   int index, struct nvkm_ltc **);
@@ -31,8 +32,10 @@ void gf100_ltc_zbc_clear_color(struct nvkm_ltc *, int, const u32[4]);
 void gf100_ltc_zbc_clear_depth(struct nvkm_ltc *, int, const u32);
 void gf100_ltc_invalidate(struct nvkm_ltc *);
 void gf100_ltc_flush(struct nvkm_ltc *);
+extern const struct nvkm_bitfield gf100_ltc_lts_intr_name[];
 
 void gm107_ltc_intr(struct nvkm_ltc *);
+void gm107_ltc_intr_lts(struct nvkm_ltc *, int ltc, int lts);
 void gm107_ltc_cbc_clear(struct nvkm_ltc *, u32, u32);
 void gm107_ltc_cbc_wait(struct nvkm_ltc *);
 void gm107_ltc_zbc_clear_color(struct nvkm_ltc *, int, const u32[4]);
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mc/Kbuild b/drivers/gpu/drm/nouveau/nvkm/subdev/mc/Kbuild
index 49695ac7be2e..12943f92c206 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/mc/Kbuild
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mc/Kbuild
@@ -10,3 +10,4 @@ nvkm-y += nvkm/subdev/mc/gt215.o
 nvkm-y += nvkm/subdev/mc/gf100.o
 nvkm-y += nvkm/subdev/mc/gk104.o
 nvkm-y += nvkm/subdev/mc/gk20a.o
+nvkm-y += nvkm/subdev/mc/gp100.o
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mc/base.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mc/base.c
index 350a8caa84c8..6b25e25f9eba 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/mc/base.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mc/base.c
@@ -27,43 +27,67 @@
 #include <subdev/top.h>
 
 void
-nvkm_mc_unk260(struct nvkm_mc *mc, u32 data)
+nvkm_mc_unk260(struct nvkm_device *device, u32 data)
 {
-        if (mc->func->unk260)
+        struct nvkm_mc *mc = device->mc;
+        if (likely(mc) && mc->func->unk260)
                 mc->func->unk260(mc, data);
 }
 
 void
-nvkm_mc_intr_unarm(struct nvkm_mc *mc)
+nvkm_mc_intr_mask(struct nvkm_device *device, enum nvkm_devidx devidx, bool en)
 {
-        return mc->func->intr_unarm(mc);
+        struct nvkm_mc *mc = device->mc;
+        const struct nvkm_mc_map *map;
+        if (likely(mc) && mc->func->intr_mask) {
+                u32 mask = nvkm_top_intr_mask(device, devidx);
+                for (map = mc->func->intr; !mask && map->stat; map++) {
+                        if (map->unit == devidx)
+                                mask = map->stat;
+                }
+                mc->func->intr_mask(mc, mask, en ? mask : 0);
+        }
+}
+
+void
+nvkm_mc_intr_unarm(struct nvkm_device *device)
+{
+        struct nvkm_mc *mc = device->mc;
+        if (likely(mc))
+                mc->func->intr_unarm(mc);
 }
 
 void
-nvkm_mc_intr_rearm(struct nvkm_mc *mc)
+nvkm_mc_intr_rearm(struct nvkm_device *device)
 {
-        return mc->func->intr_rearm(mc);
+        struct nvkm_mc *mc = device->mc;
+        if (likely(mc))
+                mc->func->intr_rearm(mc);
 }
 
 static u32
-nvkm_mc_intr_mask(struct nvkm_mc *mc)
+nvkm_mc_intr_stat(struct nvkm_mc *mc)
 {
-        u32 intr = mc->func->intr_mask(mc);
+        u32 intr = mc->func->intr_stat(mc);
         if (WARN_ON_ONCE(intr == 0xffffffff))
                 intr = 0; /* likely fallen off the bus */
         return intr;
 }
 
 void
-nvkm_mc_intr(struct nvkm_mc *mc, bool *handled)
+nvkm_mc_intr(struct nvkm_device *device, bool *handled)
 {
-        struct nvkm_device *device = mc->subdev.device;
+        struct nvkm_mc *mc = device->mc;
         struct nvkm_subdev *subdev;
-        const struct nvkm_mc_map *map = mc->func->intr;
-        u32 stat, intr = nvkm_mc_intr_mask(mc);
+        const struct nvkm_mc_map *map;
+        u32 stat, intr;
         u64 subdevs;
 
-        stat = nvkm_top_intr(device->top, intr, &subdevs);
+        if (unlikely(!mc))
+                return;
+
+        intr = nvkm_mc_intr_stat(mc);
+        stat = nvkm_top_intr(device, intr, &subdevs);
         while (subdevs) {
                 enum nvkm_devidx subidx = __ffs64(subdevs);
                 subdev = nvkm_device_subdev(device, subidx);
@@ -72,14 +96,13 @@ nvkm_mc_intr(struct nvkm_mc *mc, bool *handled)
                 subdevs &= ~BIT_ULL(subidx);
         }
 
-        while (map->stat) {
+        for (map = mc->func->intr; map->stat; map++) {
                 if (intr & map->stat) {
                         subdev = nvkm_device_subdev(device, map->unit);
                         if (subdev)
                                 nvkm_subdev_intr(subdev);
                         stat &= ~map->stat;
                 }
-                map++;
         }
 
         if (stat)
@@ -87,22 +110,32 @@ nvkm_mc_intr(struct nvkm_mc *mc, bool *handled)
         *handled = intr != 0;
 }
 
-static void
-nvkm_mc_reset_(struct nvkm_mc *mc, enum nvkm_devidx devidx)
+static u32
+nvkm_mc_reset_mask(struct nvkm_device *device, bool isauto,
+                   enum nvkm_devidx devidx)
 {
-        struct nvkm_device *device = mc->subdev.device;
+        struct nvkm_mc *mc = device->mc;
         const struct nvkm_mc_map *map;
-        u64 pmc_enable;
-
-        if (!(pmc_enable = nvkm_top_reset(device->top, devidx))) {
-                for (map = mc->func->reset; map && map->stat; map++) {
-                        if (map->unit == devidx) {
-                                pmc_enable = map->stat;
-                                break;
+        u64 pmc_enable = 0;
+        if (likely(mc)) {
+                if (!(pmc_enable = nvkm_top_reset(device, devidx))) {
+                        for (map = mc->func->reset; map && map->stat; map++) {
+                                if (!isauto || !map->noauto) {
+                                        if (map->unit == devidx) {
+                                                pmc_enable = map->stat;
+                                                break;
+                                        }
+                                }
                         }
                 }
         }
+        return pmc_enable;
+}
 
+void
+nvkm_mc_reset(struct nvkm_device *device, enum nvkm_devidx devidx)
+{
+        u64 pmc_enable = nvkm_mc_reset_mask(device, true, devidx);
         if (pmc_enable) {
                 nvkm_mask(device, 0x000200, pmc_enable, 0x00000000);
                 nvkm_mask(device, 0x000200, pmc_enable, pmc_enable);
@@ -111,17 +144,27 @@ nvkm_mc_reset_(struct nvkm_mc *mc, enum nvkm_devidx devidx)
 }
 
 void
-nvkm_mc_reset(struct nvkm_mc *mc, enum nvkm_devidx devidx)
+nvkm_mc_disable(struct nvkm_device *device, enum nvkm_devidx devidx)
 {
-        if (likely(mc))
-                nvkm_mc_reset_(mc, devidx);
+        u64 pmc_enable = nvkm_mc_reset_mask(device, false, devidx);
+        if (pmc_enable)
+                nvkm_mask(device, 0x000200, pmc_enable, 0x00000000);
+}
+
+void
+nvkm_mc_enable(struct nvkm_device *device, enum nvkm_devidx devidx)
+{
+        u64 pmc_enable = nvkm_mc_reset_mask(device, false, devidx);
+        if (pmc_enable) {
+                nvkm_mask(device, 0x000200, pmc_enable, pmc_enable);
+                nvkm_rd32(device, 0x000200);
+        }
 }
 
 static int
 nvkm_mc_fini(struct nvkm_subdev *subdev, bool suspend)
 {
-        struct nvkm_mc *mc = nvkm_mc(subdev);
-        nvkm_mc_intr_unarm(mc);
+        nvkm_mc_intr_unarm(subdev->device);
         return 0;
 }
 
@@ -131,7 +174,7 @@ nvkm_mc_init(struct nvkm_subdev *subdev)
         struct nvkm_mc *mc = nvkm_mc(subdev);
         if (mc->func->init)
                 mc->func->init(mc);
-        nvkm_mc_intr_rearm(mc);
+        nvkm_mc_intr_rearm(subdev->device);
         return 0;
 }
 
@@ -148,16 +191,21 @@ nvkm_mc = {
         .fini = nvkm_mc_fini,
 };
 
+void
+nvkm_mc_ctor(const struct nvkm_mc_func *func, struct nvkm_device *device,
+             int index, struct nvkm_mc *mc)
+{
+        nvkm_subdev_ctor(&nvkm_mc, device, index, &mc->subdev);
+        mc->func = func;
+}
+
 int
 nvkm_mc_new_(const struct nvkm_mc_func *func, struct nvkm_device *device,
              int index, struct nvkm_mc **pmc)
 {
         struct nvkm_mc *mc;
-
         if (!(mc = *pmc = kzalloc(sizeof(*mc), GFP_KERNEL)))
                 return -ENOMEM;
-
-        nvkm_subdev_ctor(&nvkm_mc, device, index, &mc->subdev);
-        mc->func = func;
+        nvkm_mc_ctor(func, device, index, *pmc);
         return 0;
 }
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mc/g84.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mc/g84.c
index 5c85b47f071d..c3d66ef5dc12 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/mc/g84.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mc/g84.c
@@ -57,7 +57,7 @@ g84_mc = {
         .intr = g84_mc_intr,
         .intr_unarm = nv04_mc_intr_unarm,
         .intr_rearm = nv04_mc_intr_rearm,
-        .intr_mask = nv04_mc_intr_mask,
+        .intr_stat = nv04_mc_intr_stat,
         .reset = g84_mc_reset,
 };
 
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mc/g98.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mc/g98.c
index 0280b43cc10c..93ad4982ce5f 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/mc/g98.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mc/g98.c
@@ -57,7 +57,7 @@ g98_mc = {
         .intr = g98_mc_intr,
         .intr_unarm = nv04_mc_intr_unarm,
         .intr_rearm = nv04_mc_intr_rearm,
-        .intr_mask = nv04_mc_intr_mask,
+        .intr_stat = nv04_mc_intr_stat,
         .reset = g98_mc_reset,
 };
 
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mc/gf100.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mc/gf100.c
index 8397e223bd43..d2c4d6033abb 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/mc/gf100.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mc/gf100.c
@@ -76,7 +76,7 @@ gf100_mc_intr_rearm(struct nvkm_mc *mc)
 }
 
 u32
-gf100_mc_intr_mask(struct nvkm_mc *mc)
+gf100_mc_intr_stat(struct nvkm_mc *mc)
 {
         struct nvkm_device *device = mc->subdev.device;
         u32 intr0 = nvkm_rd32(device, 0x000100);
@@ -85,6 +85,14 @@ gf100_mc_intr_mask(struct nvkm_mc *mc)
 }
 
 void
+gf100_mc_intr_mask(struct nvkm_mc *mc, u32 mask, u32 stat)
+{
+        struct nvkm_device *device = mc->subdev.device;
+        nvkm_mask(device, 0x000640, mask, stat);
+        nvkm_mask(device, 0x000644, mask, stat);
+}
+
+void
 gf100_mc_unk260(struct nvkm_mc *mc, u32 data)
 {
         nvkm_wr32(mc->subdev.device, 0x000260, data);
@@ -97,6 +105,7 @@ gf100_mc = {
         .intr_unarm = gf100_mc_intr_unarm,
         .intr_rearm = gf100_mc_intr_rearm,
         .intr_mask = gf100_mc_intr_mask,
+        .intr_stat = gf100_mc_intr_stat,
         .reset = gf100_mc_reset,
         .unk260 = gf100_mc_unk260,
 };
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mc/gk104.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mc/gk104.c
index 317464212c7d..7b8c6ecad1a5 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/mc/gk104.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mc/gk104.c
@@ -26,6 +26,7 @@
 const struct nvkm_mc_map
 gk104_mc_reset[] = {
         { 0x00000100, NVKM_ENGINE_FIFO },
+        { 0x00002000, NVKM_SUBDEV_PMU, true },
         {}
 };
 
@@ -53,6 +54,7 @@ gk104_mc = {
         .intr_unarm = gf100_mc_intr_unarm,
         .intr_rearm = gf100_mc_intr_rearm,
         .intr_mask = gf100_mc_intr_mask,
+        .intr_stat = gf100_mc_intr_stat,
         .reset = gk104_mc_reset,
         .unk260 = gf100_mc_unk260,
 };
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mc/gk20a.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mc/gk20a.c
index 60b044f517ed..ca1bf3279dbe 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/mc/gk20a.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mc/gk20a.c
@@ -30,6 +30,7 @@ gk20a_mc = {
         .intr_unarm = gf100_mc_intr_unarm,
         .intr_rearm = gf100_mc_intr_rearm,
         .intr_mask = gf100_mc_intr_mask,
+        .intr_stat = gf100_mc_intr_stat,
         .reset = gk104_mc_reset,
 };
 
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mc/gp100.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mc/gp100.c
new file mode 100644
index 000000000000..4d22f4abd6de
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mc/gp100.c
@@ -0,0 +1,103 @@
+/*
+ * Copyright 2012 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs
+ */
+#define gp100_mc(p) container_of((p), struct gp100_mc, base)
+#include "priv.h"
+
+struct gp100_mc {
+        struct nvkm_mc base;
+        spinlock_t lock;
+        bool intr;
+        u32 mask;
+};
+
+static void
+gp100_mc_intr_update(struct gp100_mc *mc)
+{
+        struct nvkm_device *device = mc->base.subdev.device;
+        u32 mask = mc->intr ? mc->mask : 0, i;
+        for (i = 0; i < 2; i++) {
+                nvkm_wr32(device, 0x000180 + (i * 0x04), ~mask);
+                nvkm_wr32(device, 0x000160 + (i * 0x04),  mask);
+        }
+}
+
+static void
+gp100_mc_intr_unarm(struct nvkm_mc *base)
+{
+        struct gp100_mc *mc = gp100_mc(base);
+        unsigned long flags;
+        spin_lock_irqsave(&mc->lock, flags);
+        mc->intr = false;
+        gp100_mc_intr_update(mc);
+        spin_unlock_irqrestore(&mc->lock, flags);
+}
+
+static void
+gp100_mc_intr_rearm(struct nvkm_mc *base)
+{
+        struct gp100_mc *mc = gp100_mc(base);
+        unsigned long flags;
+        spin_lock_irqsave(&mc->lock, flags);
+        mc->intr = true;
+        gp100_mc_intr_update(mc);
+        spin_unlock_irqrestore(&mc->lock, flags);
+}
+
+static void
+gp100_mc_intr_mask(struct nvkm_mc *base, u32 mask, u32 intr)
+{
+        struct gp100_mc *mc = gp100_mc(base);
+        unsigned long flags;
+        spin_lock_irqsave(&mc->lock, flags);
+        mc->mask = (mc->mask & ~mask) | intr;
+        gp100_mc_intr_update(mc);
+        spin_unlock_irqrestore(&mc->lock, flags);
+}
+
+static const struct nvkm_mc_func
+gp100_mc = {
+        .init = nv50_mc_init,
+        .intr = gk104_mc_intr,
+        .intr_unarm = gp100_mc_intr_unarm,
+        .intr_rearm = gp100_mc_intr_rearm,
+        .intr_mask = gp100_mc_intr_mask,
+        .intr_stat = gf100_mc_intr_stat,
+        .reset = gk104_mc_reset,
+};
+
+int
+gp100_mc_new(struct nvkm_device *device, int index, struct nvkm_mc **pmc)
+{
+        struct gp100_mc *mc;
+
+        if (!(mc = kzalloc(sizeof(*mc), GFP_KERNEL)))
+                return -ENOMEM;
+        nvkm_mc_ctor(&gp100_mc, device, index, &mc->base);
+        *pmc = &mc->base;
+
+        spin_lock_init(&mc->lock);
+        mc->intr = false;
+        mc->mask = 0x7fffffff;
+        return 0;
+}
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mc/gt215.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mc/gt215.c
index aad0ba95bf18..99d50a3d956f 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/mc/gt215.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mc/gt215.c
@@ -53,13 +53,20 @@ gt215_mc_intr[] = {
         {},
 };
 
+static void
+gt215_mc_intr_mask(struct nvkm_mc *mc, u32 mask, u32 stat)
+{
+        nvkm_mask(mc->subdev.device, 0x000640, mask, stat);
+}
+
 static const struct nvkm_mc_func
 gt215_mc = {
         .init = nv50_mc_init,
         .intr = gt215_mc_intr,
         .intr_unarm = nv04_mc_intr_unarm,
         .intr_rearm = nv04_mc_intr_rearm,
-        .intr_mask = nv04_mc_intr_mask,
+        .intr_mask = gt215_mc_intr_mask,
+        .intr_stat = nv04_mc_intr_stat,
         .reset = gt215_mc_reset,
 };
 
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mc/nv04.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mc/nv04.c
index a062624e906b..6509defd1460 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/mc/nv04.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mc/nv04.c
@@ -56,7 +56,7 @@ nv04_mc_intr_rearm(struct nvkm_mc *mc)
 }
 
 u32
-nv04_mc_intr_mask(struct nvkm_mc *mc)
+nv04_mc_intr_stat(struct nvkm_mc *mc)
 {
         return nvkm_rd32(mc->subdev.device, 0x000100);
 }
@@ -75,7 +75,7 @@ nv04_mc = {
         .intr = nv04_mc_intr,
         .intr_unarm = nv04_mc_intr_unarm,
         .intr_rearm = nv04_mc_intr_rearm,
-        .intr_mask = nv04_mc_intr_mask,
+        .intr_stat = nv04_mc_intr_stat,
         .reset = nv04_mc_reset,
 };
 
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mc/nv11.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mc/nv11.c
index 55f0b9166b52..9213107901e6 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/mc/nv11.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mc/nv11.c
@@ -39,7 +39,7 @@ nv11_mc = {
         .intr = nv11_mc_intr,
         .intr_unarm = nv04_mc_intr_unarm,
         .intr_rearm = nv04_mc_intr_rearm,
-        .intr_mask = nv04_mc_intr_mask,
+        .intr_stat = nv04_mc_intr_stat,
         .reset = nv04_mc_reset,
 };
 
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mc/nv17.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mc/nv17.c
index c40fa67f79a5..64bf5bbf8146 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/mc/nv17.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mc/nv17.c
@@ -48,7 +48,7 @@ nv17_mc = {
         .intr = nv17_mc_intr,
         .intr_unarm = nv04_mc_intr_unarm,
         .intr_rearm = nv04_mc_intr_rearm,
-        .intr_mask = nv04_mc_intr_mask,
+        .intr_stat = nv04_mc_intr_stat,
         .reset = nv17_mc_reset,
 };
 
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mc/nv44.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mc/nv44.c
index cc56271db564..65fa44a64b98 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/mc/nv44.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mc/nv44.c
@@ -43,7 +43,7 @@ nv44_mc = {
         .intr = nv17_mc_intr,
         .intr_unarm = nv04_mc_intr_unarm,
         .intr_rearm = nv04_mc_intr_rearm,
-        .intr_mask = nv04_mc_intr_mask,
+        .intr_stat = nv04_mc_intr_stat,
         .reset = nv17_mc_reset,
 };
 
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mc/nv50.c b/drivers/gpu/drm/nouveau/nvkm/subdev/mc/nv50.c
index 343b6078580d..fe93b4fd7100 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/mc/nv50.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mc/nv50.c
@@ -50,7 +50,7 @@ nv50_mc = {
         .intr = nv50_mc_intr,
         .intr_unarm = nv04_mc_intr_unarm,
         .intr_rearm = nv04_mc_intr_rearm,
-        .intr_mask = nv04_mc_intr_mask,
+        .intr_stat = nv04_mc_intr_stat,
         .reset = nv17_mc_reset,
 };
 
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/mc/priv.h b/drivers/gpu/drm/nouveau/nvkm/subdev/mc/priv.h
index a12038118512..4f0576a06d24 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/mc/priv.h
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/mc/priv.h
@@ -3,12 +3,15 @@
 #define nvkm_mc(p) container_of((p), struct nvkm_mc, subdev)
 #include <subdev/mc.h>
 
+void nvkm_mc_ctor(const struct nvkm_mc_func *, struct nvkm_device *,
+                  int index, struct nvkm_mc *);
 int nvkm_mc_new_(const struct nvkm_mc_func *, struct nvkm_device *,
                  int index, struct nvkm_mc **);
 
 struct nvkm_mc_map {
         u32 stat;
         u32 unit;
+        bool noauto;
 };
 
 struct nvkm_mc_func {
@@ -18,8 +21,10 @@ struct nvkm_mc_func {
         void (*intr_unarm)(struct nvkm_mc *);
         /* enable reporting of interrupts to host */
         void (*intr_rearm)(struct nvkm_mc *);
+        /* (un)mask delivery of specific interrupts */
+        void (*intr_mask)(struct nvkm_mc *, u32 mask, u32 stat);
         /* retrieve pending interrupt mask (NV_PMC_INTR) */
-        u32 (*intr_mask)(struct nvkm_mc *);
+        u32 (*intr_stat)(struct nvkm_mc *);
         const struct nvkm_mc_map *reset;
         void (*unk260)(struct nvkm_mc *, u32);
 };
@@ -27,7 +32,7 @@ struct nvkm_mc_func {
 void nv04_mc_init(struct nvkm_mc *);
 void nv04_mc_intr_unarm(struct nvkm_mc *);
 void nv04_mc_intr_rearm(struct nvkm_mc *);
-u32 nv04_mc_intr_mask(struct nvkm_mc *);
+u32 nv04_mc_intr_stat(struct nvkm_mc *);
 extern const struct nvkm_mc_map nv04_mc_reset[];
 
 extern const struct nvkm_mc_map nv17_mc_intr[];
@@ -39,7 +44,8 @@ void nv50_mc_init(struct nvkm_mc *);
 
 void gf100_mc_intr_unarm(struct nvkm_mc *);
 void gf100_mc_intr_rearm(struct nvkm_mc *);
-u32 gf100_mc_intr_mask(struct nvkm_mc *);
+void gf100_mc_intr_mask(struct nvkm_mc *, u32, u32);
+u32 gf100_mc_intr_stat(struct nvkm_mc *);
 void gf100_mc_unk260(struct nvkm_mc *, u32);
 
 extern const struct nvkm_mc_map gk104_mc_intr[];
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/pci/Kbuild b/drivers/gpu/drm/nouveau/nvkm/subdev/pci/Kbuild
index 3c2519fdeb81..2a31b7d66a6d 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/pci/Kbuild
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/pci/Kbuild
@@ -10,3 +10,4 @@ nvkm-y += nvkm/subdev/pci/g94.o
 nvkm-y += nvkm/subdev/pci/gf100.o
 nvkm-y += nvkm/subdev/pci/gf106.o
 nvkm-y += nvkm/subdev/pci/gk104.o
+nvkm-y += nvkm/subdev/pci/gp100.o
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/pci/base.c b/drivers/gpu/drm/nouveau/nvkm/subdev/pci/base.c
index 6b0328bd7eed..eb9b278198b2 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/pci/base.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/pci/base.c
@@ -69,15 +69,13 @@ static irqreturn_t
 nvkm_pci_intr(int irq, void *arg)
 {
         struct nvkm_pci *pci = arg;
-        struct nvkm_mc *mc = pci->subdev.device->mc;
+        struct nvkm_device *device = pci->subdev.device;
         bool handled = false;
-        if (likely(mc)) {
-                nvkm_mc_intr_unarm(mc);
-                if (pci->msi)
-                        pci->func->msi_rearm(pci);
-                nvkm_mc_intr(mc, &handled);
-                nvkm_mc_intr_rearm(mc);
-        }
+        nvkm_mc_intr_unarm(device);
+        if (pci->msi)
+                pci->func->msi_rearm(pci);
+        nvkm_mc_intr(device, &handled);
+        nvkm_mc_intr_rearm(device);
         return handled ? IRQ_HANDLED : IRQ_NONE;
 }
 
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/pci/gp100.c b/drivers/gpu/drm/nouveau/nvkm/subdev/pci/gp100.c
new file mode 100644
index 000000000000..82c5234a06ff
--- /dev/null
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/pci/gp100.c
@@ -0,0 +1,44 @@
+/*
+ * Copyright 2015 Red Hat Inc.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
+ * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ *
+ * Authors: Ben Skeggs <bskeggs@redhat.com>
+ */
+#include "priv.h"
+
+static void
+gp100_pci_msi_rearm(struct nvkm_pci *pci)
+{
+        nvkm_pci_wr32(pci, 0x0704, 0x00000000);
+}
+
+static const struct nvkm_pci_func
+gp100_pci_func = {
+        .rd32 = nv40_pci_rd32,
+        .wr08 = nv40_pci_wr08,
+        .wr32 = nv40_pci_wr32,
+        .msi_rearm = gp100_pci_msi_rearm,
+};
+
+int
+gp100_pci_new(struct nvkm_device *device, int index, struct nvkm_pci **ppci)
+{
+        return nvkm_pci_new_(&gp100_pci_func, device, index, ppci);
+}
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/secboot/base.c b/drivers/gpu/drm/nouveau/nvkm/subdev/secboot/base.c
index 213fdba6cfa0..314be2192b7d 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/secboot/base.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/secboot/base.c
@@ -19,8 +19,9 @@
  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
  * DEALINGS IN THE SOFTWARE.
  */
-
 #include "priv.h"
+
+#include <subdev/mc.h>
 #include <subdev/timer.h>
 
 static const char *
@@ -70,12 +71,11 @@ nvkm_secboot_falcon_enable(struct nvkm_secboot *sb)
         int ret;
 
         /* enable engine */
-        nvkm_mask(device, 0x200, sb->enable_mask, sb->enable_mask);
-        nvkm_rd32(device, 0x200);
+        nvkm_mc_enable(device, sb->devidx);
         ret = nvkm_wait_msec(device, 10, sb->base + 0x10c, 0x6, 0x0);
         if (ret < 0) {
-                nvkm_mask(device, 0x200, sb->enable_mask, 0x0);
                 nvkm_error(&sb->subdev, "Falcon mem scrubbing timeout\n");
+                nvkm_mc_disable(device, sb->devidx);
                 return ret;
         }
 
@@ -85,8 +85,7 @@ nvkm_secboot_falcon_enable(struct nvkm_secboot *sb)
 
         /* enable IRQs */
         nvkm_wr32(device, sb->base + 0x010, 0xff);
-        nvkm_mask(device, 0x640, sb->irq_mask, sb->irq_mask);
-        nvkm_mask(device, 0x644, sb->irq_mask, sb->irq_mask);
+        nvkm_mc_intr_mask(device, sb->devidx, true);
 
         return 0;
 }
@@ -97,14 +96,13 @@ nvkm_secboot_falcon_disable(struct nvkm_secboot *sb)
         struct nvkm_device *device = sb->subdev.device;
 
         /* disable IRQs and wait for any previous code to complete */
-        nvkm_mask(device, 0x644, sb->irq_mask, 0x0);
-        nvkm_mask(device, 0x640, sb->irq_mask, 0x0);
+        nvkm_mc_intr_mask(device, sb->devidx, false);
         nvkm_wr32(device, sb->base + 0x014, 0xff);
 
         falcon_wait_idle(device, sb->base);
 
         /* disable engine */
-        nvkm_mask(device, 0x200, sb->enable_mask, 0x0);
+        nvkm_mc_disable(device, sb->devidx);
 
         return 0;
 }
@@ -216,14 +214,7 @@ nvkm_secboot_oneinit(struct nvkm_subdev *subdev)
                 return ret;
         }
 
-        /*
-         * Build all blobs - the same blobs can be used to perform secure boot
-         * multiple times
-         */
-        if (sb->func->prepare_blobs)
-                ret = sb->func->prepare_blobs(sb);
-
-        return ret;
+        return 0;
 }
 
 static int
@@ -270,9 +261,8 @@ nvkm_secboot_ctor(const struct nvkm_secboot_func *func,
         /* setup the performing falcon's base address and masks */
         switch (func->boot_falcon) {
         case NVKM_SECBOOT_FALCON_PMU:
+                sb->devidx = NVKM_SUBDEV_PMU;
                 sb->base = 0x10a000;
-                sb->irq_mask = 0x1000000;
-                sb->enable_mask = 0x2000;
                 break;
         default:
                 nvkm_error(&sb->subdev, "invalid secure boot falcon\n");
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/secboot/gm200.c b/drivers/gpu/drm/nouveau/nvkm/subdev/secboot/gm200.c
index cc100dc940ea..f1e2dc914366 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/secboot/gm200.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/secboot/gm200.c
@@ -860,6 +860,8 @@ gm200_secboot_prepare_ls_blob(struct gm200_secboot *gsb)
 
         /* Write LS blob */
         ret = ls_ucode_mgr_write_wpr(gsb, &mgr, gsb->ls_blob);
+        if (ret)
+                nvkm_gpuobj_del(&gsb->ls_blob);
 
 cleanup:
         ls_ucode_mgr_cleanup(&mgr);
@@ -1023,29 +1025,34 @@ gm20x_secboot_prepare_blobs(struct gm200_secboot *gsb)
         int ret;
 
         /* Load and prepare the managed falcon's firmwares */
-        ret = gm200_secboot_prepare_ls_blob(gsb);
-        if (ret)
-                return ret;
+        if (!gsb->ls_blob) {
+                ret = gm200_secboot_prepare_ls_blob(gsb);
+                if (ret)
+                        return ret;
+        }
 
         /* Load the HS firmware that will load the LS firmwares */
-        ret = gm200_secboot_prepare_hs_blob(gsb, "acr/ucode_load",
-                                            &gsb->acr_load_blob,
-                                            &gsb->acr_load_bl_desc, true);
-        if (ret)
-                return ret;
+        if (!gsb->acr_load_blob) {
+                ret = gm200_secboot_prepare_hs_blob(gsb, "acr/ucode_load",
+                                                &gsb->acr_load_blob,
+                                                &gsb->acr_load_bl_desc, true);
+                if (ret)
+                        return ret;
+        }
 
         /* Load the HS firmware bootloader */
-        ret = gm200_secboot_prepare_hsbl_blob(gsb);
-        if (ret)
-                return ret;
+        if (!gsb->hsbl_blob) {
+                ret = gm200_secboot_prepare_hsbl_blob(gsb);
+                if (ret)
+                        return ret;
+        }
 
         return 0;
 }
 
 static int
-gm200_secboot_prepare_blobs(struct nvkm_secboot *sb)
+gm200_secboot_prepare_blobs(struct gm200_secboot *gsb)
 {
-        struct gm200_secboot *gsb = gm200_secboot(sb);
         int ret;
 
         ret = gm20x_secboot_prepare_blobs(gsb);
@@ -1053,15 +1060,37 @@ gm200_secboot_prepare_blobs(struct nvkm_secboot *sb)
                 return ret;
 
         /* dGPU only: load the HS firmware that unprotects the WPR region */
-        ret = gm200_secboot_prepare_hs_blob(gsb, "acr/ucode_unload",
-                                            &gsb->acr_unload_blob,
-                                            &gsb->acr_unload_bl_desc, false);
-        if (ret)
-                return ret;
+        if (!gsb->acr_unload_blob) {
+                ret = gm200_secboot_prepare_hs_blob(gsb, "acr/ucode_unload",
+                                               &gsb->acr_unload_blob,
+                                               &gsb->acr_unload_bl_desc, false);
+                if (ret)
+                        return ret;
+        }
 
         return 0;
 }
 
+static int
+gm200_secboot_blobs_ready(struct gm200_secboot *gsb)
+{
+        struct nvkm_subdev *subdev = &gsb->base.subdev;
+        int ret;
+
+        /* firmware already loaded, nothing to do... */
+        if (gsb->firmware_ok)
+                return 0;
+
+        ret = gsb->func->prepare_blobs(gsb);
+        if (ret) {
+                nvkm_error(subdev, "failed to load secure firmware\n");
+                return ret;
+        }
+
+        gsb->firmware_ok = true;
+
+        return 0;
+}
 
 
 /*
@@ -1234,6 +1263,11 @@ gm200_secboot_reset(struct nvkm_secboot *sb, enum nvkm_secboot_falcon falcon)
         struct gm200_secboot *gsb = gm200_secboot(sb);
         int ret;
 
+        /* Make sure all blobs are ready */
+        ret = gm200_secboot_blobs_ready(gsb);
+        if (ret)
+                return ret;
+
         /*
          * Dummy GM200 implementation: perform secure boot each time we are
          * called on FECS. Since only FECS and GPCCS are managed and started
@@ -1373,7 +1407,6 @@ gm200_secboot = {
         .dtor = gm200_secboot_dtor,
         .init = gm200_secboot_init,
         .fini = gm200_secboot_fini,
-        .prepare_blobs = gm200_secboot_prepare_blobs,
         .reset = gm200_secboot_reset,
         .start = gm200_secboot_start,
         .managed_falcons = BIT(NVKM_SECBOOT_FALCON_FECS) |
@@ -1415,6 +1448,7 @@ gm200_secboot_func = {
         .bl_desc_size = sizeof(struct gm200_flcn_bl_desc),
         .fixup_bl_desc = gm200_secboot_fixup_bl_desc,
         .fixup_hs_desc = gm200_secboot_fixup_hs_desc,
+        .prepare_blobs = gm200_secboot_prepare_blobs,
 };
 
 int
@@ -1487,3 +1521,19 @@ MODULE_FIRMWARE("nvidia/gm206/gr/sw_ctx.bin");
 MODULE_FIRMWARE("nvidia/gm206/gr/sw_nonctx.bin");
 MODULE_FIRMWARE("nvidia/gm206/gr/sw_bundle_init.bin");
 MODULE_FIRMWARE("nvidia/gm206/gr/sw_method_init.bin");
+
+MODULE_FIRMWARE("nvidia/gp100/acr/bl.bin");
+MODULE_FIRMWARE("nvidia/gp100/acr/ucode_load.bin");
+MODULE_FIRMWARE("nvidia/gp100/acr/ucode_unload.bin");
+MODULE_FIRMWARE("nvidia/gp100/gr/fecs_bl.bin");
+MODULE_FIRMWARE("nvidia/gp100/gr/fecs_inst.bin");
+MODULE_FIRMWARE("nvidia/gp100/gr/fecs_data.bin");
+MODULE_FIRMWARE("nvidia/gp100/gr/fecs_sig.bin");
+MODULE_FIRMWARE("nvidia/gp100/gr/gpccs_bl.bin");
+MODULE_FIRMWARE("nvidia/gp100/gr/gpccs_inst.bin");
+MODULE_FIRMWARE("nvidia/gp100/gr/gpccs_data.bin");
+MODULE_FIRMWARE("nvidia/gp100/gr/gpccs_sig.bin");
+MODULE_FIRMWARE("nvidia/gp100/gr/sw_ctx.bin");
+MODULE_FIRMWARE("nvidia/gp100/gr/sw_nonctx.bin");
+MODULE_FIRMWARE("nvidia/gp100/gr/sw_bundle_init.bin");
+MODULE_FIRMWARE("nvidia/gp100/gr/sw_method_init.bin");
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/secboot/gm20b.c b/drivers/gpu/drm/nouveau/nvkm/subdev/secboot/gm20b.c
index 684320484b70..d5395ebfe8d3 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/secboot/gm20b.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/secboot/gm20b.c
@@ -42,6 +42,32 @@ struct gm20b_flcn_bl_desc {
         u32 data_size;
 };
 
+static int
+gm20b_secboot_prepare_blobs(struct gm200_secboot *gsb)
+{
+        struct nvkm_subdev *subdev = &gsb->base.subdev;
+        int acr_size;
+        int ret;
+
+        ret = gm20x_secboot_prepare_blobs(gsb);
+        if (ret)
+                return ret;
+
+        acr_size = gsb->acr_load_blob->size;
+        /*
+         * On Tegra the WPR region is set by the bootloader. It is illegal for
+         * the HS blob to be larger than this region.
+         */
+        if (acr_size > gsb->wpr_size) {
+                nvkm_error(subdev, "WPR region too small for FW blob!\n");
+                nvkm_error(subdev, "required: %dB\n", acr_size);
+                nvkm_error(subdev, "WPR size: %dB\n", gsb->wpr_size);
+                return -ENOSPC;
+        }
+
+        return 0;
+}
+
 /**
  * gm20b_secboot_fixup_bl_desc - adapt BL descriptor to format used by GM20B FW
  *
@@ -88,6 +114,7 @@ gm20b_secboot_func = {
         .bl_desc_size = sizeof(struct gm20b_flcn_bl_desc),
         .fixup_bl_desc = gm20b_secboot_fixup_bl_desc,
         .fixup_hs_desc = gm20b_secboot_fixup_hs_desc,
+        .prepare_blobs = gm20b_secboot_prepare_blobs,
 };
 
 
@@ -147,32 +174,6 @@ gm20b_tegra_read_wpr(struct gm200_secboot *gsb)
 #endif
 
 static int
-gm20b_secboot_prepare_blobs(struct nvkm_secboot *sb)
-{
-        struct gm200_secboot *gsb = gm200_secboot(sb);
-        int acr_size;
-        int ret;
-
-        ret = gm20x_secboot_prepare_blobs(gsb);
-        if (ret)
-                return ret;
-
-        acr_size = gsb->acr_load_blob->size;
-        /*
-         * On Tegra the WPR region is set by the bootloader. It is illegal for
-         * the HS blob to be larger than this region.
-         */
-        if (acr_size > gsb->wpr_size) {
-                nvkm_error(&sb->subdev, "WPR region too small for FW blob!\n");
-                nvkm_error(&sb->subdev, "required: %dB\n", acr_size);
-                nvkm_error(&sb->subdev, "WPR size: %dB\n", gsb->wpr_size);
-                return -ENOSPC;
-        }
-
-        return 0;
-}
-
-static int
 gm20b_secboot_init(struct nvkm_secboot *sb)
 {
         struct gm200_secboot *gsb = gm200_secboot(sb);
@@ -189,7 +190,6 @@ static const struct nvkm_secboot_func
 gm20b_secboot = {
         .dtor = gm200_secboot_dtor,
         .init = gm20b_secboot_init,
-        .prepare_blobs = gm20b_secboot_prepare_blobs,
         .reset = gm200_secboot_reset,
         .start = gm200_secboot_start,
         .managed_falcons = BIT(NVKM_SECBOOT_FALCON_FECS),
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/secboot/priv.h b/drivers/gpu/drm/nouveau/nvkm/subdev/secboot/priv.h
index f2b09dee7c5d..a9a8a0e1017e 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/secboot/priv.h
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/secboot/priv.h
@@ -30,7 +30,6 @@ struct nvkm_secboot_func {
         int (*init)(struct nvkm_secboot *);
         int (*fini)(struct nvkm_secboot *, bool suspend);
         void *(*dtor)(struct nvkm_secboot *);
-        int (*prepare_blobs)(struct nvkm_secboot *);
         int (*reset)(struct nvkm_secboot *, enum nvkm_secboot_falcon);
         int (*start)(struct nvkm_secboot *, enum nvkm_secboot_falcon);
 
@@ -147,10 +146,8 @@ struct hsflcn_acr_desc {
  * @inst:               instance block for HS falcon
  * @pgd:                page directory for the HS falcon
  * @vm:                 address space used by the HS falcon
- * @bl_desc_size:       size of the BL descriptor used by this chip.
- * @fixup_bl_desc:      hook that generates the proper BL descriptor format from
- *                      the generic GM200 format into a data array of size
- *                      bl_desc_size
+ * @falcon_state:       current state of the managed falcons
+ * @firmware_ok:        whether the firmware blobs have been created
  */
 struct gm200_secboot {
         struct nvkm_secboot base;
@@ -196,9 +193,19 @@ struct gm200_secboot {
                 RUNNING,
         } falcon_state[NVKM_SECBOOT_FALCON_END];
 
+        bool firmware_ok;
 };
 #define gm200_secboot(sb) container_of(sb, struct gm200_secboot, base)
 
+/**
+ * Contains functions we wish to abstract between GM200-like implementations
+ * @bl_desc_size:       size of the BL descriptor used by this chip.
+ * @fixup_bl_desc:      hook that generates the proper BL descriptor format from
+ *                      the generic GM200 format into a data array of size
+ *                      bl_desc_size
+ * @fixup_hs_desc:      hook that twiddles the HS descriptor before it is used
+ * @prepare_blobs:      prepares the various blobs needed for secure booting
+ */
 struct gm200_secboot_func {
         /*
          * Size of the bootloader descriptor for this chip. A block of this
@@ -214,6 +221,7 @@ struct gm200_secboot_func {
          * we want the HS FW to set up.
          */
         void (*fixup_hs_desc)(struct gm200_secboot *, struct hsflcn_acr_desc *);
+        int (*prepare_blobs)(struct gm200_secboot *);
 };
 
 int gm200_secboot_init(struct nvkm_secboot *);
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/top/base.c b/drivers/gpu/drm/nouveau/nvkm/subdev/top/base.c
index a1b264664aad..fe063d5728e2 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/top/base.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/top/base.c
@@ -41,8 +41,9 @@ nvkm_top_device_new(struct nvkm_top *top)
 }
 
 u32
-nvkm_top_reset(struct nvkm_top *top, enum nvkm_devidx index)
+nvkm_top_reset(struct nvkm_device *device, enum nvkm_devidx index)
 {
+        struct nvkm_top *top = device->top;
         struct nvkm_top_device *info;
 
         if (top) {
@@ -56,8 +57,25 @@ nvkm_top_reset(struct nvkm_top *top, enum nvkm_devidx index)
 }
 
 u32
-nvkm_top_intr(struct nvkm_top *top, u32 intr, u64 *psubdevs)
+nvkm_top_intr_mask(struct nvkm_device *device, enum nvkm_devidx devidx)
 {
+        struct nvkm_top *top = device->top;
+        struct nvkm_top_device *info;
+
+        if (top) {
+                list_for_each_entry(info, &top->device, head) {
+                        if (info->index == devidx && info->intr >= 0)
+                                return BIT(info->intr);
+                }
+        }
+
+        return 0;
+}
+
+u32
+nvkm_top_intr(struct nvkm_device *device, u32 intr, u64 *psubdevs)
+{
+        struct nvkm_top *top = device->top;
         struct nvkm_top_device *info;
         u64 subdevs = 0;
         u32 handled = 0;
@@ -78,8 +96,9 @@ nvkm_top_intr(struct nvkm_top *top, u32 intr, u64 *psubdevs)
 }
 
 enum nvkm_devidx
-nvkm_top_fault(struct nvkm_top *top, int fault)
+nvkm_top_fault(struct nvkm_device *device, int fault)
 {
+        struct nvkm_top *top = device->top;
         struct nvkm_top_device *info;
 
         list_for_each_entry(info, &top->device, head) {
@@ -91,8 +110,9 @@ nvkm_top_fault(struct nvkm_top *top, int fault)
 }
 
 enum nvkm_devidx
-nvkm_top_engine(struct nvkm_top *top, int index, int *runl, int *engn)
+nvkm_top_engine(struct nvkm_device *device, int index, int *runl, int *engn)
 {
+        struct nvkm_top *top = device->top;
         struct nvkm_top_device *info;
         int n = 0;
 
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/top/gk104.c b/drivers/gpu/drm/nouveau/nvkm/subdev/top/gk104.c
index e06acc340e99..efac3402f9dd 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/top/gk104.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/top/gk104.c
@@ -29,7 +29,7 @@ gk104_top_oneinit(struct nvkm_top *top)
         struct nvkm_subdev *subdev = &top->subdev;
         struct nvkm_device *device = subdev->device;
         struct nvkm_top_device *info = NULL;
-        u32 data, type;
+        u32 data, type, inst;
         int i;
 
         for (i = 0; i < 64; i++) {
@@ -37,6 +37,7 @@ gk104_top_oneinit(struct nvkm_top *top)
                         if (!(info = nvkm_top_device_new(top)))
                                 return -ENOMEM;
                         type = ~0;
+                        inst = 0;
                 }
 
                 data = nvkm_rd32(device, 0x022700 + (i * 0x04));
@@ -45,6 +46,7 @@ gk104_top_oneinit(struct nvkm_top *top)
                 case 0x00000000: /* NOT_VALID */
                         continue;
                 case 0x00000001: /* DATA */
+                        inst        = (data & 0x3c000000) >> 26;
                         info->addr  = (data & 0x00fff000);
                         info->fault = (data & 0x000000f8) >> 3;
                         break;
@@ -67,27 +69,32 @@ gk104_top_oneinit(struct nvkm_top *top)
                         continue;
 
                 /* Translate engine type to NVKM engine identifier. */
+#define A_(A) if (inst == 0) info->index = NVKM_ENGINE_##A
+#define B_(A) if (inst + NVKM_ENGINE_##A##0 < NVKM_ENGINE_##A##_LAST + 1)      \
+                info->index = NVKM_ENGINE_##A##0 + inst
                 switch (type) {
-                case 0x00000000: info->index = NVKM_ENGINE_GR; break;
-                case 0x00000001: info->index = NVKM_ENGINE_CE0; break;
-                case 0x00000002: info->index = NVKM_ENGINE_CE1; break;
-                case 0x00000003: info->index = NVKM_ENGINE_CE2; break;
-                case 0x00000008: info->index = NVKM_ENGINE_MSPDEC; break;
-                case 0x00000009: info->index = NVKM_ENGINE_MSPPP; break;
-                case 0x0000000a: info->index = NVKM_ENGINE_MSVLD; break;
-                case 0x0000000b: info->index = NVKM_ENGINE_MSENC; break;
-                case 0x0000000c: info->index = NVKM_ENGINE_VIC; break;
-                case 0x0000000d: info->index = NVKM_ENGINE_SEC; break;
-                case 0x0000000e: info->index = NVKM_ENGINE_NVENC0; break;
-                case 0x0000000f: info->index = NVKM_ENGINE_NVENC1; break;
-                case 0x00000010: info->index = NVKM_ENGINE_NVDEC; break;
+                case 0x00000000: A_(GR    ); break;
+                case 0x00000001: A_(CE0   ); break;
+                case 0x00000002: A_(CE1   ); break;
+                case 0x00000003: A_(CE2   ); break;
+                case 0x00000008: A_(MSPDEC); break;
+                case 0x00000009: A_(MSPPP ); break;
+                case 0x0000000a: A_(MSVLD ); break;
+                case 0x0000000b: A_(MSENC ); break;
+                case 0x0000000c: A_(VIC   ); break;
+                case 0x0000000d: A_(SEC   ); break;
+                case 0x0000000e: B_(NVENC ); break;
+                case 0x0000000f: A_(NVENC1); break;
+                case 0x00000010: A_(NVDEC ); break;
+                case 0x00000013: B_(CE    ); break;
                         break;
                 default:
                         break;
                 }
 
-                nvkm_debug(subdev, "%02x (%8s): addr %06x fault %2d engine %2d "
-                                   "runlist %2d intr %2d reset %2d\n", type,
+                nvkm_debug(subdev, "%02x.%d (%8s): addr %06x fault %2d "
+                                   "engine %2d runlist %2d intr %2d "
+                                   "reset %2d\n", type, inst,
                            info->index == NVKM_SUBDEV_NR ? NULL :
                                           nvkm_subdev_name[info->index],
                            info->addr, info->fault, info->engine, info->runlist,
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/volt/base.c b/drivers/gpu/drm/nouveau/nvkm/subdev/volt/base.c
index 6b2d7531a7ff..1c3d23b0e84a 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/volt/base.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/volt/base.c
@@ -120,6 +120,8 @@ nvkm_volt_parse_bios(struct nvkm_bios *bios, struct nvkm_volt *volt)
 
         data = nvbios_volt_parse(bios, &ver, &hdr, &cnt, &len, &info);
         if (data && info.vidmask && info.base && info.step) {
+                volt->min_uv = info.min;
+                volt->max_uv = info.max;
                 for (i = 0; i < info.vidmask + 1; i++) {
                         if (info.base >= info.min &&
                                 info.base <= info.max) {
@@ -131,6 +133,8 @@ nvkm_volt_parse_bios(struct nvkm_bios *bios, struct nvkm_volt *volt)
                 }
                 volt->vid_mask = info.vidmask;
         } else if (data && info.vidmask) {
+                volt->min_uv = 0xffffffff;
+                volt->max_uv = 0;
                 for (i = 0; i < cnt; i++) {
                         data = nvbios_volt_entry_parse(bios, i, &ver, &hdr,
                                                        &ivid);
@@ -138,9 +142,14 @@ nvkm_volt_parse_bios(struct nvkm_bios *bios, struct nvkm_volt *volt)
                                 volt->vid[volt->vid_nr].uv = ivid.voltage;
                                 volt->vid[volt->vid_nr].vid = ivid.vid;
                                 volt->vid_nr++;
+                                volt->min_uv = min(volt->min_uv, ivid.voltage);
+                                volt->max_uv = max(volt->max_uv, ivid.voltage);
                         }
                 }
                 volt->vid_mask = info.vidmask;
+        } else if (data && info.type == NVBIOS_VOLT_PWM) {
+                volt->min_uv = info.base;
+                volt->max_uv = info.base + info.pwm_range;
         }
 }
 
@@ -181,8 +190,11 @@ nvkm_volt_ctor(const struct nvkm_volt_func *func, struct nvkm_device *device,
         volt->func = func;
 
         /* Assuming the non-bios device should build the voltage table later */
-        if (bios)
+        if (bios) {
                 nvkm_volt_parse_bios(bios, volt);
+                nvkm_debug(&volt->subdev, "min: %iuv max: %iuv\n",
+                           volt->min_uv, volt->max_uv);
+        }
 
         if (volt->vid_nr) {
                 for (i = 0; i < volt->vid_nr; i++) {
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/volt/gk20a.c b/drivers/gpu/drm/nouveau/nvkm/subdev/volt/gk20a.c
index d554455326da..ce5d83cdc7cf 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/volt/gk20a.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/volt/gk20a.c
@@ -77,18 +77,19 @@ gk20a_volt_get_cvb_t_voltage(int speedo, int temp, int s_scale, int t_scale,
         return mv;
 }
 
-int
+static int
 gk20a_volt_calc_voltage(const struct cvb_coef *coef, int speedo)
 {
+        static const int v_scale = 1000;
         int mv;
 
         mv = gk20a_volt_get_cvb_t_voltage(speedo, -10, 100, 10, coef);
-        mv = DIV_ROUND_UP(mv, 1000);
+        mv = DIV_ROUND_UP(mv, v_scale);
 
         return mv * 1000;
 }
 
-int
+static int
 gk20a_volt_vid_get(struct nvkm_volt *base)
 {
         struct gk20a_volt *volt = gk20a_volt(base);
@@ -103,7 +104,7 @@ gk20a_volt_vid_get(struct nvkm_volt *base)
         return -EINVAL;
 }
 
-int
+static int
 gk20a_volt_vid_set(struct nvkm_volt *base, u8 vid)
 {
         struct gk20a_volt *volt = gk20a_volt(base);
@@ -113,7 +114,7 @@ gk20a_volt_vid_set(struct nvkm_volt *base, u8 vid)
         return regulator_set_voltage(volt->vdd, volt->base.vid[vid].uv, 1200000);
 }
 
-int
+static int
 gk20a_volt_set_id(struct nvkm_volt *base, u8 id, int condition)
 {
         struct gk20a_volt *volt = gk20a_volt(base);
@@ -143,9 +144,9 @@ gk20a_volt = {
 };
 
 int
-_gk20a_volt_ctor(struct nvkm_device *device, int index,
-                 const struct cvb_coef *coefs, int nb_coefs,
-                 struct gk20a_volt *volt)
+gk20a_volt_ctor(struct nvkm_device *device, int index,
+                const struct cvb_coef *coefs, int nb_coefs,
+                int vmin, struct gk20a_volt *volt)
 {
         struct nvkm_device_tegra *tdev = device->func->tegra(device);
         int i, uv;
@@ -160,9 +161,9 @@ _gk20a_volt_ctor(struct nvkm_device *device, int index,
         volt->base.vid_nr = nb_coefs;
         for (i = 0; i < volt->base.vid_nr; i++) {
                 volt->base.vid[i].vid = i;
-                volt->base.vid[i].uv =
-                        gk20a_volt_calc_voltage(&coefs[i],
-                                                tdev->gpu_speedo);
+                volt->base.vid[i].uv = max(
+                        gk20a_volt_calc_voltage(&coefs[i], tdev->gpu_speedo),
+                        vmin);
                 nvkm_debug(&volt->base.subdev, "%2d: vid=%d, uv=%d\n", i,
                            volt->base.vid[i].vid, volt->base.vid[i].uv);
         }
@@ -180,6 +181,6 @@ gk20a_volt_new(struct nvkm_device *device, int index, struct nvkm_volt **pvolt)
                 return -ENOMEM;
         *pvolt = &volt->base;
 
-        return _gk20a_volt_ctor(device, index, gk20a_cvb_coef,
-                                ARRAY_SIZE(gk20a_cvb_coef), volt);
+        return gk20a_volt_ctor(device, index, gk20a_cvb_coef,
+                               ARRAY_SIZE(gk20a_cvb_coef), 0, volt);
 }
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/volt/gk20a.h b/drivers/gpu/drm/nouveau/nvkm/subdev/volt/gk20a.h
index 0fa3b502bcf8..6a6c97f9684e 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/volt/gk20a.h
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/volt/gk20a.h
@@ -37,13 +37,8 @@ struct gk20a_volt {
         struct regulator *vdd;
 };
 
-int _gk20a_volt_ctor(struct nvkm_device *device, int index,
-                     const struct cvb_coef *coefs, int nb_coefs,
-                     struct gk20a_volt *volt);
-
-int gk20a_volt_calc_voltage(const struct cvb_coef *coef, int speedo);
-int gk20a_volt_vid_get(struct nvkm_volt *volt);
-int gk20a_volt_vid_set(struct nvkm_volt *volt, u8 vid);
-int gk20a_volt_set_id(struct nvkm_volt *volt, u8 id, int condition);
+int gk20a_volt_ctor(struct nvkm_device *device, int index,
+                    const struct cvb_coef *coefs, int nb_coefs,
+                    int vmin, struct gk20a_volt *volt);
 
 #endif
diff --git a/drivers/gpu/drm/nouveau/nvkm/subdev/volt/gm20b.c b/drivers/gpu/drm/nouveau/nvkm/subdev/volt/gm20b.c
index 49b5ecb701e4..74db4d28930f 100644
--- a/drivers/gpu/drm/nouveau/nvkm/subdev/volt/gm20b.c
+++ b/drivers/gpu/drm/nouveau/nvkm/subdev/volt/gm20b.c
@@ -41,16 +41,52 @@ const struct cvb_coef gm20b_cvb_coef[] = {
         /* 921600 */ { 2647676, -106455, 1632 },
 };
 
+static const struct cvb_coef gm20b_na_cvb_coef[] = {
+        /* KHz,         c0,     c1,   c2,    c3,     c4,   c5 */
+        /*  76800 */ {  814294, 8144, -940, 808, -21583, 226 },
+        /* 153600 */ {  856185, 8144, -940, 808, -21583, 226 },
+        /* 230400 */ {  898077, 8144, -940, 808, -21583, 226 },
+        /* 307200 */ {  939968, 8144, -940, 808, -21583, 226 },
+        /* 384000 */ {  981860, 8144, -940, 808, -21583, 226 },
+        /* 460800 */ { 1023751, 8144, -940, 808, -21583, 226 },
+        /* 537600 */ { 1065642, 8144, -940, 808, -21583, 226 },
+        /* 614400 */ { 1107534, 8144, -940, 808, -21583, 226 },
+        /* 691200 */ { 1149425, 8144, -940, 808, -21583, 226 },
+        /* 768000 */ { 1191317, 8144, -940, 808, -21583, 226 },
+        /* 844800 */ { 1233208, 8144, -940, 808, -21583, 226 },
+        /* 921600 */ { 1275100, 8144, -940, 808, -21583, 226 },
+        /* 998400 */ { 1316991, 8144, -940, 808, -21583, 226 },
+};
+
+const u32 speedo_to_vmin[] = {
+        /*   0,      1,      2,      3,      4, */
+        950000, 840000, 818750, 840000, 810000,
+};
+
 int
 gm20b_volt_new(struct nvkm_device *device, int index, struct nvkm_volt **pvolt)
 {
+        struct nvkm_device_tegra *tdev = device->func->tegra(device);
         struct gk20a_volt *volt;
+        u32 vmin;
+
+        if (tdev->gpu_speedo_id >= ARRAY_SIZE(speedo_to_vmin)) {
+                nvdev_error(device, "unsupported speedo %d\n",
+                            tdev->gpu_speedo_id);
+                return -EINVAL;
+        }
 
         volt = kzalloc(sizeof(*volt), GFP_KERNEL);
         if (!volt)
                 return -ENOMEM;
         *pvolt = &volt->base;
 
-        return _gk20a_volt_ctor(device, index, gm20b_cvb_coef,
-                                ARRAY_SIZE(gm20b_cvb_coef), volt);
+        vmin = speedo_to_vmin[tdev->gpu_speedo_id];
+
+        if (tdev->gpu_speedo_id >= 1)
+                return gk20a_volt_ctor(device, index, gm20b_na_cvb_coef,
+                                     ARRAY_SIZE(gm20b_na_cvb_coef), vmin, volt);
+        else
+                return gk20a_volt_ctor(device, index, gm20b_cvb_coef,
+                                        ARRAY_SIZE(gm20b_cvb_coef), vmin, volt);
 }
diff --git a/drivers/gpu/drm/omapdrm/Kconfig b/drivers/gpu/drm/omapdrm/Kconfig
index 336ad4de9981..556f81f6b2c7 100644
--- a/drivers/gpu/drm/omapdrm/Kconfig
+++ b/drivers/gpu/drm/omapdrm/Kconfig
@@ -4,11 +4,6 @@ config DRM_OMAP
         depends on ARCH_OMAP2PLUS || ARCH_MULTIPLATFORM
         select OMAP2_DSS
         select DRM_KMS_HELPER
-        select DRM_KMS_FB_HELPER
-        select FB_SYS_FILLRECT
-        select FB_SYS_COPYAREA
-        select FB_SYS_IMAGEBLIT
-        select FB_SYS_FOPS
         default n
         help
           DRM display driver for OMAP2/3/4 based boards.
diff --git a/drivers/gpu/drm/omapdrm/displays/Kconfig b/drivers/gpu/drm/omapdrm/displays/Kconfig
index 2a618afe0f53..c226da145fb3 100644
--- a/drivers/gpu/drm/omapdrm/displays/Kconfig
+++ b/drivers/gpu/drm/omapdrm/displays/Kconfig
@@ -1,80 +1,80 @@
 menu "OMAPDRM External Display Device Drivers"
 
-config DISPLAY_ENCODER_OPA362
+config DRM_OMAP_ENCODER_OPA362
         tristate "OPA362 external analog amplifier"
         help
           Driver for OPA362 external analog TV amplifier controlled
           through a GPIO.
 
-config DISPLAY_ENCODER_TFP410
+config DRM_OMAP_ENCODER_TFP410
         tristate "TFP410 DPI to DVI Encoder"
         help
           Driver for TFP410 DPI to DVI encoder.
 
-config DISPLAY_ENCODER_TPD12S015
+config DRM_OMAP_ENCODER_TPD12S015
         tristate "TPD12S015 HDMI ESD protection and level shifter"
         help
           Driver for TPD12S015, which offers HDMI ESD protection and level
           shifting.
 
-config DISPLAY_CONNECTOR_DVI
+config DRM_OMAP_CONNECTOR_DVI
         tristate "DVI Connector"
         depends on I2C
         help
           Driver for a generic DVI connector.
 
-config DISPLAY_CONNECTOR_HDMI
+config DRM_OMAP_CONNECTOR_HDMI
         tristate "HDMI Connector"
         help
           Driver for a generic HDMI connector.
 
-config DISPLAY_CONNECTOR_ANALOG_TV
+config DRM_OMAP_CONNECTOR_ANALOG_TV
         tristate "Analog TV Connector"
         help
           Driver for a generic analog TV connector.
 
-config DISPLAY_PANEL_DPI
+config DRM_OMAP_PANEL_DPI
         tristate "Generic DPI panel"
         help
           Driver for generic DPI panels.
 
-config DISPLAY_PANEL_DSI_CM
+config DRM_OMAP_PANEL_DSI_CM
         tristate "Generic DSI Command Mode Panel"
         depends on BACKLIGHT_CLASS_DEVICE
         help
           Driver for generic DSI command mode panels.
 
-config DISPLAY_PANEL_SONY_ACX565AKM
+config DRM_OMAP_PANEL_SONY_ACX565AKM
         tristate "ACX565AKM Panel"
         depends on SPI && BACKLIGHT_CLASS_DEVICE
         help
           This is the LCD panel used on Nokia N900
 
-config DISPLAY_PANEL_LGPHILIPS_LB035Q02
+config DRM_OMAP_PANEL_LGPHILIPS_LB035Q02
         tristate "LG.Philips LB035Q02 LCD Panel"
         depends on SPI
         help
           LCD Panel used on the Gumstix Overo Palo35
 
-config DISPLAY_PANEL_SHARP_LS037V7DW01
+config DRM_OMAP_PANEL_SHARP_LS037V7DW01
         tristate "Sharp LS037V7DW01 LCD Panel"
         depends on BACKLIGHT_CLASS_DEVICE
         help
           LCD Panel used in TI's SDP3430 and EVM boards
 
-config DISPLAY_PANEL_TPO_TD028TTEC1
+config DRM_OMAP_PANEL_TPO_TD028TTEC1
         tristate "TPO TD028TTEC1 LCD Panel"
         depends on SPI
         help
           LCD panel used in Openmoko.
 
-config DISPLAY_PANEL_TPO_TD043MTEA1
+config DRM_OMAP_PANEL_TPO_TD043MTEA1
         tristate "TPO TD043MTEA1 LCD Panel"
         depends on SPI
         help
           LCD Panel used in OMAP3 Pandora
 
-config DISPLAY_PANEL_NEC_NL8048HL11
+config DRM_OMAP_PANEL_NEC_NL8048HL11
         tristate "NEC NL8048HL11 Panel"
         depends on SPI
         depends on BACKLIGHT_CLASS_DEVICE
diff --git a/drivers/gpu/drm/omapdrm/displays/Makefile b/drivers/gpu/drm/omapdrm/displays/Makefile
index 9aa176bfbf2e..46baafb1a83e 100644
--- a/drivers/gpu/drm/omapdrm/displays/Makefile
+++ b/drivers/gpu/drm/omapdrm/displays/Makefile
@@ -1,14 +1,14 @@
-obj-$(CONFIG_DISPLAY_ENCODER_OPA362) += encoder-opa362.o
-obj-$(CONFIG_DISPLAY_ENCODER_TFP410) += encoder-tfp410.o
-obj-$(CONFIG_DISPLAY_ENCODER_TPD12S015) += encoder-tpd12s015.o
-obj-$(CONFIG_DISPLAY_CONNECTOR_DVI) += connector-dvi.o
-obj-$(CONFIG_DISPLAY_CONNECTOR_HDMI) += connector-hdmi.o
-obj-$(CONFIG_DISPLAY_CONNECTOR_ANALOG_TV) += connector-analog-tv.o
-obj-$(CONFIG_DISPLAY_PANEL_DPI) += panel-dpi.o
-obj-$(CONFIG_DISPLAY_PANEL_DSI_CM) += panel-dsi-cm.o
-obj-$(CONFIG_DISPLAY_PANEL_SONY_ACX565AKM) += panel-sony-acx565akm.o
-obj-$(CONFIG_DISPLAY_PANEL_LGPHILIPS_LB035Q02) += panel-lgphilips-lb035q02.o
-obj-$(CONFIG_DISPLAY_PANEL_SHARP_LS037V7DW01) += panel-sharp-ls037v7dw01.o
-obj-$(CONFIG_DISPLAY_PANEL_TPO_TD028TTEC1) += panel-tpo-td028ttec1.o
-obj-$(CONFIG_DISPLAY_PANEL_TPO_TD043MTEA1) += panel-tpo-td043mtea1.o
-obj-$(CONFIG_DISPLAY_PANEL_NEC_NL8048HL11) += panel-nec-nl8048hl11.o
+obj-$(CONFIG_DRM_OMAP_ENCODER_OPA362) += encoder-opa362.o
+obj-$(CONFIG_DRM_OMAP_ENCODER_TFP410) += encoder-tfp410.o
+obj-$(CONFIG_DRM_OMAP_ENCODER_TPD12S015) += encoder-tpd12s015.o
+obj-$(CONFIG_DRM_OMAP_CONNECTOR_DVI) += connector-dvi.o
+obj-$(CONFIG_DRM_OMAP_CONNECTOR_HDMI) += connector-hdmi.o
+obj-$(CONFIG_DRM_OMAP_CONNECTOR_ANALOG_TV) += connector-analog-tv.o
+obj-$(CONFIG_DRM_OMAP_PANEL_DPI) += panel-dpi.o
+obj-$(CONFIG_DRM_OMAP_PANEL_DSI_CM) += panel-dsi-cm.o
+obj-$(CONFIG_DRM_OMAP_PANEL_SONY_ACX565AKM) += panel-sony-acx565akm.o
+obj-$(CONFIG_DRM_OMAP_PANEL_LGPHILIPS_LB035Q02) += panel-lgphilips-lb035q02.o
+obj-$(CONFIG_DRM_OMAP_PANEL_SHARP_LS037V7DW01) += panel-sharp-ls037v7dw01.o
+obj-$(CONFIG_DRM_OMAP_PANEL_TPO_TD028TTEC1) += panel-tpo-td028ttec1.o
+obj-$(CONFIG_DRM_OMAP_PANEL_TPO_TD043MTEA1) += panel-tpo-td043mtea1.o
+obj-$(CONFIG_DRM_OMAP_PANEL_NEC_NL8048HL11) += panel-nec-nl8048hl11.o
diff --git a/drivers/gpu/drm/omapdrm/displays/connector-analog-tv.c b/drivers/gpu/drm/omapdrm/displays/connector-analog-tv.c
index 8511c648a15c..3485d1ecd655 100644
--- a/drivers/gpu/drm/omapdrm/displays/connector-analog-tv.c
+++ b/drivers/gpu/drm/omapdrm/displays/connector-analog-tv.c
@@ -14,9 +14,10 @@
 #include <linux/platform_device.h>
 #include <linux/of.h>
 
-#include <video/omapdss.h>
 #include <video/omap-panel-data.h>
 
+#include "../dss/omapdss.h"
+
 struct panel_drv_data {
         struct omap_dss_device dssdev;
         struct omap_dss_device *in;
@@ -25,7 +26,6 @@ struct panel_drv_data {
 
         struct omap_video_timings timings;
 
-        enum omap_dss_venc_type connector_type;
         bool invert_polarity;
 };
 
@@ -45,10 +45,6 @@ static const struct omap_video_timings tvc_pal_timings = {
 
 static const struct of_device_id tvc_of_match[];
 
-struct tvc_of_data {
-        enum omap_dss_venc_type connector_type;
-};
-
 #define to_panel_data(x) container_of(x, struct panel_drv_data, dssdev)
 
 static int tvc_connect(struct omap_dss_device *dssdev)
@@ -99,7 +95,7 @@ static int tvc_enable(struct omap_dss_device *dssdev)
         in->ops.atv->set_timings(in, &ddata->timings);
 
         if (!ddata->dev->of_node) {
-                in->ops.atv->set_type(in, ddata->connector_type);
+                in->ops.atv->set_type(in, OMAP_DSS_VENC_TYPE_COMPOSITE);
 
                 in->ops.atv->invert_vid_out_polarity(in,
                         ddata->invert_polarity);
@@ -207,7 +203,6 @@ static int tvc_probe_pdata(struct platform_device *pdev)
 
         ddata->in = in;
 
-        ddata->connector_type = pdata->connector_type;
         ddata->invert_polarity = pdata->invert_polarity;
 
         dssdev = &ddata->dssdev;
diff --git a/drivers/gpu/drm/omapdrm/displays/connector-dvi.c b/drivers/gpu/drm/omapdrm/displays/connector-dvi.c
index 747f26a55e43..684b7aeda411 100644
--- a/drivers/gpu/drm/omapdrm/displays/connector-dvi.c
+++ b/drivers/gpu/drm/omapdrm/displays/connector-dvi.c
@@ -15,10 +15,10 @@
 #include <linux/slab.h>
 
 #include <drm/drm_edid.h>
-
-#include <video/omapdss.h>
 #include <video/omap-panel-data.h>
 
+#include "../dss/omapdss.h"
+
 static const struct omap_video_timings dvic_default_timings = {
         .x_res          = 640,
         .y_res          = 480,
@@ -255,6 +255,7 @@ static int dvic_probe_of(struct platform_device *pdev)
         adapter_node = of_parse_phandle(node, "ddc-i2c-bus", 0);
         if (adapter_node) {
                 adapter = of_get_i2c_adapter_by_node(adapter_node);
+                of_node_put(adapter_node);
                 if (adapter == NULL) {
                         dev_err(&pdev->dev, "failed to parse ddc-i2c-bus\n");
                         omap_dss_put_device(ddata->in);
diff --git a/drivers/gpu/drm/omapdrm/displays/connector-hdmi.c b/drivers/gpu/drm/omapdrm/displays/connector-hdmi.c
index 667ca4a24ece..7bdf83af9797 100644
--- a/drivers/gpu/drm/omapdrm/displays/connector-hdmi.c
+++ b/drivers/gpu/drm/omapdrm/displays/connector-hdmi.c
@@ -17,10 +17,10 @@
 #include <linux/of_gpio.h>
 
 #include <drm/drm_edid.h>
-
-#include <video/omapdss.h>
 #include <video/omap-panel-data.h>
 
+#include "../dss/omapdss.h"
+
 static const struct omap_video_timings hdmic_default_timings = {
         .x_res          = 640,
         .y_res          = 480,
diff --git a/drivers/gpu/drm/omapdrm/displays/encoder-opa362.c b/drivers/gpu/drm/omapdrm/displays/encoder-opa362.c
index 9594ff7a2b0c..fe4e7ec3bab0 100644
--- a/drivers/gpu/drm/omapdrm/displays/encoder-opa362.c
+++ b/drivers/gpu/drm/omapdrm/displays/encoder-opa362.c
@@ -18,9 +18,8 @@
 #include <linux/module.h>
 #include <linux/platform_device.h>
 #include <linux/slab.h>
-#include <linux/of_gpio.h>
 
-#include <video/omapdss.h>
+#include "../dss/omapdss.h"
 
 struct panel_drv_data {
         struct omap_dss_device dssdev;
diff --git a/drivers/gpu/drm/omapdrm/displays/encoder-tfp410.c b/drivers/gpu/drm/omapdrm/displays/encoder-tfp410.c
index 671806ca7d6a..d768217cefe0 100644
--- a/drivers/gpu/drm/omapdrm/displays/encoder-tfp410.c
+++ b/drivers/gpu/drm/omapdrm/displays/encoder-tfp410.c
@@ -15,8 +15,7 @@
 #include <linux/slab.h>
 #include <linux/of_gpio.h>
 
-#include <video/omapdss.h>
-#include <video/omap-panel-data.h>
+#include "../dss/omapdss.h"
 
 struct panel_drv_data {
         struct omap_dss_device dssdev;
diff --git a/drivers/gpu/drm/omapdrm/displays/encoder-tpd12s015.c b/drivers/gpu/drm/omapdrm/displays/encoder-tpd12s015.c
index 916a89978387..46855c8f5cbf 100644
--- a/drivers/gpu/drm/omapdrm/displays/encoder-tpd12s015.c
+++ b/drivers/gpu/drm/omapdrm/displays/encoder-tpd12s015.c
@@ -16,8 +16,7 @@
 #include <linux/platform_device.h>
 #include <linux/gpio/consumer.h>
 
-#include <video/omapdss.h>
-#include <video/omap-panel-data.h>
+#include "../dss/omapdss.h"
 
 struct panel_drv_data {
         struct omap_dss_device dssdev;
diff --git a/drivers/gpu/drm/omapdrm/displays/panel-dpi.c b/drivers/gpu/drm/omapdrm/displays/panel-dpi.c
index 7c2331be8d15..7f16f985ab22 100644
--- a/drivers/gpu/drm/omapdrm/displays/panel-dpi.c
+++ b/drivers/gpu/drm/omapdrm/displays/panel-dpi.c
@@ -15,11 +15,13 @@
 #include <linux/slab.h>
 #include <linux/of.h>
 #include <linux/of_gpio.h>
+#include <linux/regulator/consumer.h>
 
-#include <video/omapdss.h>
 #include <video/omap-panel-data.h>
 #include <video/of_display_timing.h>
 
+#include "../dss/omapdss.h"
+
 struct panel_drv_data {
         struct omap_dss_device dssdev;
         struct omap_dss_device *in;
@@ -32,6 +34,7 @@ struct panel_drv_data {
         int backlight_gpio;
 
         struct gpio_desc *enable_gpio;
+        struct regulator *vcc_supply;
 };
 
 #define to_panel_data(p) container_of(p, struct panel_drv_data, dssdev)
@@ -83,6 +86,12 @@ static int panel_dpi_enable(struct omap_dss_device *dssdev)
         if (r)
                 return r;
 
+        r = regulator_enable(ddata->vcc_supply);
+        if (r) {
+                in->ops.dpi->disable(in);
+                return r;
+        }
+
         gpiod_set_value_cansleep(ddata->enable_gpio, 1);
 
         if (gpio_is_valid(ddata->backlight_gpio))
@@ -105,6 +114,7 @@ static void panel_dpi_disable(struct omap_dss_device *dssdev)
                 gpio_set_value_cansleep(ddata->backlight_gpio, 0);
 
         gpiod_set_value_cansleep(ddata->enable_gpio, 0);
+        regulator_disable(ddata->vcc_supply);
 
         in->ops.dpi->disable(in);
 
@@ -213,6 +223,20 @@ static int panel_dpi_probe_of(struct platform_device *pdev)
 
         ddata->enable_gpio = gpio;
 
+        /*
+         * Many different panels are supported by this driver and there are
+         * probably very different needs for their reset pins in regards to
+         * timing and order relative to the enable gpio. So for now it's just
+         * ensured that the reset line isn't active.
+         */
+        gpio = devm_gpiod_get_optional(&pdev->dev, "reset", GPIOD_OUT_LOW);
+        if (IS_ERR(gpio))
+                return PTR_ERR(gpio);
+
+        ddata->vcc_supply = devm_regulator_get(&pdev->dev, "vcc");
+        if (IS_ERR(ddata->vcc_supply))
+                return PTR_ERR(ddata->vcc_supply);
+
         ddata->backlight_gpio = -ENOENT;
 
         r = of_get_display_timing(node, "panel-timing", &timing);
diff --git a/drivers/gpu/drm/omapdrm/displays/panel-dsi-cm.c b/drivers/gpu/drm/omapdrm/displays/panel-dsi-cm.c
index 2b118071b5a1..0eae8afaed90 100644
--- a/drivers/gpu/drm/omapdrm/displays/panel-dsi-cm.c
+++ b/drivers/gpu/drm/omapdrm/displays/panel-dsi-cm.c
@@ -25,10 +25,10 @@
 #include <linux/of_device.h>
 #include <linux/of_gpio.h>
 
-#include <video/omapdss.h>
-#include <video/omap-panel-data.h>
 #include <video/mipi_display.h>
 
+#include "../dss/omapdss.h"
+
 /* DSI Virtual channel. Hardcoded for now. */
 #define TCH 0
 
@@ -1284,8 +1284,7 @@ static int dsicm_probe(struct platform_device *pdev)
         return 0;
 
 err_sysfs_create:
-        if (bldev != NULL)
-                backlight_device_unregister(bldev);
+        backlight_device_unregister(bldev);
 err_bl:
         destroy_workqueue(ddata->workqueue);
 err_reg:
diff --git a/drivers/gpu/drm/omapdrm/displays/panel-lgphilips-lb035q02.c b/drivers/gpu/drm/omapdrm/displays/panel-lgphilips-lb035q02.c
index ac680e1de603..6dfb96cea293 100644
--- a/drivers/gpu/drm/omapdrm/displays/panel-lgphilips-lb035q02.c
+++ b/drivers/gpu/drm/omapdrm/displays/panel-lgphilips-lb035q02.c
@@ -17,8 +17,7 @@
 #include <linux/gpio.h>
 #include <linux/gpio/consumer.h>
 
-#include <video/omapdss.h>
-#include <video/omap-panel-data.h>
+#include "../dss/omapdss.h"
 
 static struct omap_video_timings lb035q02_timings = {
         .x_res = 320,
@@ -51,9 +50,6 @@ struct panel_drv_data {
 
         struct omap_video_timings videomode;
 
-        /* used for non-DT boot, to be removed */
-        int backlight_gpio;
-
         struct gpio_desc *enable_gpio;
 };
 
@@ -171,9 +167,6 @@ static int lb035q02_enable(struct omap_dss_device *dssdev)
         if (ddata->enable_gpio)
                 gpiod_set_value_cansleep(ddata->enable_gpio, 1);
 
-        if (gpio_is_valid(ddata->backlight_gpio))
-                gpio_set_value_cansleep(ddata->backlight_gpio, 1);
-
         dssdev->state = OMAP_DSS_DISPLAY_ACTIVE;
 
         return 0;
@@ -190,9 +183,6 @@ static void lb035q02_disable(struct omap_dss_device *dssdev)
         if (ddata->enable_gpio)
                 gpiod_set_value_cansleep(ddata->enable_gpio, 0);
 
-        if (gpio_is_valid(ddata->backlight_gpio))
-                gpio_set_value_cansleep(ddata->backlight_gpio, 0);
-
         in->ops.dpi->disable(in);
 
         dssdev->state = OMAP_DSS_DISPLAY_DISABLED;
@@ -256,8 +246,6 @@ static int lb035q02_probe_of(struct spi_device *spi)
 
         ddata->enable_gpio = gpio;
 
-        ddata->backlight_gpio = -ENOENT;
-
         in = omapdss_of_find_source_for_first_ep(node);
         if (IS_ERR(in)) {
                 dev_err(&spi->dev, "failed to find video source\n");
@@ -290,13 +278,6 @@ static int lb035q02_panel_spi_probe(struct spi_device *spi)
         if (r)
                 return r;
 
-        if (gpio_is_valid(ddata->backlight_gpio)) {
-                r = devm_gpio_request_one(&spi->dev, ddata->backlight_gpio,
-                                GPIOF_OUT_INIT_LOW, "panel backlight");
-                if (r)
-                        goto err_gpio;
-        }
-
         ddata->videomode = lb035q02_timings;
 
         dssdev = &ddata->dssdev;
@@ -316,7 +297,6 @@ static int lb035q02_panel_spi_probe(struct spi_device *spi)
         return 0;
 
 err_reg:
-err_gpio:
         omap_dss_put_device(ddata->in);
         return r;
 }
diff --git a/drivers/gpu/drm/omapdrm/displays/panel-nec-nl8048hl11.c b/drivers/gpu/drm/omapdrm/displays/panel-nec-nl8048hl11.c
index 38d2920a95e6..fc4c238c9583 100644
--- a/drivers/gpu/drm/omapdrm/displays/panel-nec-nl8048hl11.c
+++ b/drivers/gpu/drm/omapdrm/displays/panel-nec-nl8048hl11.c
@@ -18,7 +18,7 @@
 #include <linux/gpio/consumer.h>
 #include <linux/of_gpio.h>
 
-#include <video/omapdss.h>
+#include "../dss/omapdss.h"
 
 struct panel_drv_data {
         struct omap_dss_device  dssdev;
diff --git a/drivers/gpu/drm/omapdrm/displays/panel-sharp-ls037v7dw01.c b/drivers/gpu/drm/omapdrm/displays/panel-sharp-ls037v7dw01.c
index 4363fffc87e3..3d3efc561ea9 100644
--- a/drivers/gpu/drm/omapdrm/displays/panel-sharp-ls037v7dw01.c
+++ b/drivers/gpu/drm/omapdrm/displays/panel-sharp-ls037v7dw01.c
@@ -13,11 +13,11 @@
 #include <linux/gpio/consumer.h>
 #include <linux/module.h>
 #include <linux/of.h>
-#include <linux/of_gpio.h>
 #include <linux/platform_device.h>
 #include <linux/slab.h>
 #include <linux/regulator/consumer.h>
-#include <video/omapdss.h>
+
+#include "../dss/omapdss.h"
 
 struct panel_drv_data {
         struct omap_dss_device dssdev;
diff --git a/drivers/gpu/drm/omapdrm/displays/panel-sony-acx565akm.c b/drivers/gpu/drm/omapdrm/displays/panel-sony-acx565akm.c
index deb416736aad..157c512205d1 100644
--- a/drivers/gpu/drm/omapdrm/displays/panel-sony-acx565akm.c
+++ b/drivers/gpu/drm/omapdrm/displays/panel-sony-acx565akm.c
@@ -33,9 +33,10 @@
 #include <linux/of.h>
 #include <linux/of_gpio.h>
 
-#include <video/omapdss.h>
 #include <video/omap-panel-data.h>
 
+#include "../dss/omapdss.h"
+
 #define MIPID_CMD_READ_DISP_ID          0x04
 #define MIPID_CMD_READ_RED              0x06
 #define MIPID_CMD_READ_GREEN            0x07
diff --git a/drivers/gpu/drm/omapdrm/displays/panel-tpo-td028ttec1.c b/drivers/gpu/drm/omapdrm/displays/panel-tpo-td028ttec1.c
index bd8d85041926..e859b3f893f7 100644
--- a/drivers/gpu/drm/omapdrm/displays/panel-tpo-td028ttec1.c
+++ b/drivers/gpu/drm/omapdrm/displays/panel-tpo-td028ttec1.c
@@ -28,7 +28,8 @@
 #include <linux/delay.h>
 #include <linux/spi/spi.h>
 #include <linux/gpio.h>
-#include <video/omapdss.h>
+
+#include "../dss/omapdss.h"
 
 struct panel_drv_data {
         struct omap_dss_device dssdev;
diff --git a/drivers/gpu/drm/omapdrm/displays/panel-tpo-td043mtea1.c b/drivers/gpu/drm/omapdrm/displays/panel-tpo-td043mtea1.c
index d93175b03a12..66c6bbe6472b 100644
--- a/drivers/gpu/drm/omapdrm/displays/panel-tpo-td043mtea1.c
+++ b/drivers/gpu/drm/omapdrm/displays/panel-tpo-td043mtea1.c
@@ -19,7 +19,7 @@
 #include <linux/slab.h>
 #include <linux/of_gpio.h>
 
-#include <video/omapdss.h>
+#include "../dss/omapdss.h"
 
 #define TPO_R02_MODE(x)         ((x) & 7)
 #define TPO_R02_MODE_800x480    7
diff --git a/drivers/gpu/drm/omapdrm/dss/core.c b/drivers/gpu/drm/omapdrm/dss/core.c
index 7e4e5bebabbe..6a3ebfcd7223 100644
--- a/drivers/gpu/drm/omapdrm/dss/core.c
+++ b/drivers/gpu/drm/omapdrm/dss/core.c
@@ -35,8 +35,7 @@
 #include <linux/suspend.h>
 #include <linux/slab.h>
 
-#include <video/omapdss.h>
-
+#include "omapdss.h"
 #include "dss.h"
 #include "dss_features.h"
 
@@ -196,8 +195,6 @@ static int __init omap_dss_probe(struct platform_device *pdev)
                 core.default_display_name = def_disp_name;
         else if (pdata->default_display_name)
                 core.default_display_name = pdata->default_display_name;
-        else if (pdata->default_device)
-                core.default_display_name = pdata->default_device->name;
 
         return 0;
 
diff --git a/drivers/gpu/drm/omapdrm/dss/dispc.c b/drivers/gpu/drm/omapdrm/dss/dispc.c
index f83608b69e68..535240fba671 100644
--- a/drivers/gpu/drm/omapdrm/dss/dispc.c
+++ b/drivers/gpu/drm/omapdrm/dss/dispc.c
@@ -41,8 +41,7 @@
 #include <linux/of.h>
 #include <linux/component.h>
 
-#include <video/omapdss.h>
-
+#include "omapdss.h"
 #include "dss.h"
 #include "dss_features.h"
 #include "dispc.h"
@@ -113,9 +112,14 @@ struct dispc_features {
          * never both, we can just use this flag for now.
          */
         bool reverse_ilace_field_order:1;
+
+        bool has_gamma_table:1;
+
+        bool has_gamma_i734_bug:1;
 };
 
 #define DISPC_MAX_NR_FIFOS 5
+#define DISPC_MAX_CHANNEL_GAMMA 4
 
 static struct {
         struct platform_device *pdev;
@@ -135,6 +139,8 @@ static struct {
         bool            ctx_valid;
         u32             ctx[DISPC_SZ_REGS / sizeof(u32)];
 
+        u32 *gamma_table[DISPC_MAX_CHANNEL_GAMMA];
+
         const struct dispc_features *feat;
 
         bool is_enabled;
@@ -178,11 +184,19 @@ struct dispc_reg_field {
         u8 low;
 };
 
+struct dispc_gamma_desc {
+        u32 len;
+        u32 bits;
+        u16 reg;
+        bool has_index;
+};
+
 static const struct {
         const char *name;
         u32 vsync_irq;
         u32 framedone_irq;
         u32 sync_lost_irq;
+        struct dispc_gamma_desc gamma;
         struct dispc_reg_field reg_desc[DISPC_MGR_FLD_NUM];
 } mgr_desc[] = {
         [OMAP_DSS_CHANNEL_LCD] = {
@@ -190,6 +204,12 @@ static const struct {
                 .vsync_irq      = DISPC_IRQ_VSYNC,
                 .framedone_irq  = DISPC_IRQ_FRAMEDONE,
                 .sync_lost_irq  = DISPC_IRQ_SYNC_LOST,
+                .gamma          = {
+                        .len    = 256,
+                        .bits   = 8,
+                        .reg    = DISPC_GAMMA_TABLE0,
+                        .has_index = true,
+                },
                 .reg_desc       = {
                         [DISPC_MGR_FLD_ENABLE]          = { DISPC_CONTROL,  0,  0 },
                         [DISPC_MGR_FLD_STNTFT]          = { DISPC_CONTROL,  3,  3 },
@@ -207,6 +227,12 @@ static const struct {
                 .vsync_irq      = DISPC_IRQ_EVSYNC_ODD | DISPC_IRQ_EVSYNC_EVEN,
                 .framedone_irq  = DISPC_IRQ_FRAMEDONETV,
                 .sync_lost_irq  = DISPC_IRQ_SYNC_LOST_DIGIT,
+                .gamma          = {
+                        .len    = 1024,
+                        .bits   = 10,
+                        .reg    = DISPC_GAMMA_TABLE2,
+                        .has_index = false,
+                },
                 .reg_desc       = {
                         [DISPC_MGR_FLD_ENABLE]          = { DISPC_CONTROL,  1,  1 },
                         [DISPC_MGR_FLD_STNTFT]          = { },
@@ -224,6 +250,12 @@ static const struct {
                 .vsync_irq      = DISPC_IRQ_VSYNC2,
                 .framedone_irq  = DISPC_IRQ_FRAMEDONE2,
                 .sync_lost_irq  = DISPC_IRQ_SYNC_LOST2,
+                .gamma          = {
+                        .len    = 256,
+                        .bits   = 8,
+                        .reg    = DISPC_GAMMA_TABLE1,
+                        .has_index = true,
+                },
                 .reg_desc       = {
                         [DISPC_MGR_FLD_ENABLE]          = { DISPC_CONTROL2,  0,  0 },
                         [DISPC_MGR_FLD_STNTFT]          = { DISPC_CONTROL2,  3,  3 },
@@ -241,6 +273,12 @@ static const struct {
                 .vsync_irq      = DISPC_IRQ_VSYNC3,
                 .framedone_irq  = DISPC_IRQ_FRAMEDONE3,
                 .sync_lost_irq  = DISPC_IRQ_SYNC_LOST3,
+                .gamma          = {
+                        .len    = 256,
+                        .bits   = 8,
+                        .reg    = DISPC_GAMMA_TABLE3,
+                        .has_index = true,
+                },
                 .reg_desc       = {
                         [DISPC_MGR_FLD_ENABLE]          = { DISPC_CONTROL3,  0,  0 },
                         [DISPC_MGR_FLD_STNTFT]          = { DISPC_CONTROL3,  3,  3 },
@@ -1084,20 +1122,6 @@ static u32 dispc_ovl_get_burst_size(enum omap_plane plane)
         return unit * 8;
 }
 
-void dispc_enable_gamma_table(bool enable)
-{
-        /*
-         * This is partially implemented to support only disabling of
-         * the gamma table.
-         */
-        if (enable) {
-                DSSWARN("Gamma table enabling for TV not yet supported");
-                return;
-        }
-
-        REG_FLD_MOD(DISPC_CONFIG, enable, 9, 9);
-}
-
 static void dispc_mgr_enable_cpr(enum omap_channel channel, bool enable)
 {
         if (channel == OMAP_DSS_CHANNEL_DIGIT)
@@ -3299,30 +3323,21 @@ static void dispc_mgr_get_lcd_divisor(enum omap_channel channel, int *lck_div,
 
 static unsigned long dispc_fclk_rate(void)
 {
-        struct dss_pll *pll;
-        unsigned long r = 0;
+        unsigned long r;
+        enum dss_clk_source src;
+
+        src = dss_get_dispc_clk_source();
 
-        switch (dss_get_dispc_clk_source()) {
-        case OMAP_DSS_CLK_SRC_FCK:
+        if (src == DSS_CLK_SRC_FCK) {
                 r = dss_get_dispc_clk_rate();
-                break;
-        case OMAP_DSS_CLK_SRC_DSI_PLL_HSDIV_DISPC:
-                pll = dss_pll_find("dsi0");
-                if (!pll)
-                        pll = dss_pll_find("video0");
+        } else {
+                struct dss_pll *pll;
+                unsigned clkout_idx;
 
-                r = pll->cinfo.clkout[0];
-                break;
-        case OMAP_DSS_CLK_SRC_DSI2_PLL_HSDIV_DISPC:
-                pll = dss_pll_find("dsi1");
-                if (!pll)
-                        pll = dss_pll_find("video1");
+                pll = dss_pll_find_by_src(src);
+                clkout_idx = dss_pll_get_clkout_idx_for_src(src);
 
-                r = pll->cinfo.clkout[0];
-                break;
-        default:
-                BUG();
-                return 0;
+                r = pll->cinfo.clkout[clkout_idx];
         }
 
         return r;
@@ -3330,43 +3345,31 @@ static unsigned long dispc_fclk_rate(void)
 
 static unsigned long dispc_mgr_lclk_rate(enum omap_channel channel)
 {
-        struct dss_pll *pll;
         int lcd;
         unsigned long r;
-        u32 l;
-
-        if (dss_mgr_is_lcd(channel)) {
-                l = dispc_read_reg(DISPC_DIVISORo(channel));
+        enum dss_clk_source src;
 
-                lcd = FLD_GET(l, 23, 16);
+        /* for TV, LCLK rate is the FCLK rate */
+        if (!dss_mgr_is_lcd(channel))
+                return dispc_fclk_rate();
 
-                switch (dss_get_lcd_clk_source(channel)) {
-                case OMAP_DSS_CLK_SRC_FCK:
-                        r = dss_get_dispc_clk_rate();
-                        break;
-                case OMAP_DSS_CLK_SRC_DSI_PLL_HSDIV_DISPC:
-                        pll = dss_pll_find("dsi0");
-                        if (!pll)
-                                pll = dss_pll_find("video0");
+        src = dss_get_lcd_clk_source(channel);
 
-                        r = pll->cinfo.clkout[0];
-                        break;
-                case OMAP_DSS_CLK_SRC_DSI2_PLL_HSDIV_DISPC:
-                        pll = dss_pll_find("dsi1");
-                        if (!pll)
-                                pll = dss_pll_find("video1");
+        if (src == DSS_CLK_SRC_FCK) {
+                r = dss_get_dispc_clk_rate();
+        } else {
+                struct dss_pll *pll;
+                unsigned clkout_idx;
 
-                        r = pll->cinfo.clkout[0];
-                        break;
-                default:
-                        BUG();
-                        return 0;
-                }
+                pll = dss_pll_find_by_src(src);
+                clkout_idx = dss_pll_get_clkout_idx_for_src(src);
 
-                return r / lcd;
-        } else {
-                return dispc_fclk_rate();
+                r = pll->cinfo.clkout[clkout_idx];
         }
+
+        lcd = REG_GET(DISPC_DIVISORo(channel), 23, 16);
+
+        return r / lcd;
 }
 
 static unsigned long dispc_mgr_pclk_rate(enum omap_channel channel)
@@ -3426,15 +3429,14 @@ static unsigned long dispc_plane_lclk_rate(enum omap_plane plane)
 static void dispc_dump_clocks_channel(struct seq_file *s, enum omap_channel channel)
 {
         int lcd, pcd;
-        enum omap_dss_clk_source lcd_clk_src;
+        enum dss_clk_source lcd_clk_src;
 
         seq_printf(s, "- %s -\n", mgr_desc[channel].name);
 
         lcd_clk_src = dss_get_lcd_clk_source(channel);
 
-        seq_printf(s, "%s clk source = %s (%s)\n", mgr_desc[channel].name,
-                dss_get_generic_clk_source_name(lcd_clk_src),
-                dss_feat_get_clk_source_name(lcd_clk_src));
+        seq_printf(s, "%s clk source = %s\n", mgr_desc[channel].name,
+                dss_get_clk_source_name(lcd_clk_src));
 
         dispc_mgr_get_lcd_divisor(channel, &lcd, &pcd);
 
@@ -3448,16 +3450,15 @@ void dispc_dump_clocks(struct seq_file *s)
 {
         int lcd;
         u32 l;
-        enum omap_dss_clk_source dispc_clk_src = dss_get_dispc_clk_source();
+        enum dss_clk_source dispc_clk_src = dss_get_dispc_clk_source();
 
         if (dispc_runtime_get())
                 return;
 
         seq_printf(s, "- DISPC -\n");
 
-        seq_printf(s, "dispc fclk source = %s (%s)\n",
-                        dss_get_generic_clk_source_name(dispc_clk_src),
-                        dss_feat_get_clk_source_name(dispc_clk_src));
+        seq_printf(s, "dispc fclk source = %s\n",
+                        dss_get_clk_source_name(dispc_clk_src));
 
         seq_printf(s, "fck\t\t%-16lu\n", dispc_fclk_rate());
 
@@ -3814,6 +3815,139 @@ void dispc_disable_sidle(void)
         REG_FLD_MOD(DISPC_SYSCONFIG, 1, 4, 3);  /* SIDLEMODE: no idle */
 }
 
+u32 dispc_mgr_gamma_size(enum omap_channel channel)
+{
+        const struct dispc_gamma_desc *gdesc = &mgr_desc[channel].gamma;
+
+        if (!dispc.feat->has_gamma_table)
+                return 0;
+
+        return gdesc->len;
+}
+EXPORT_SYMBOL(dispc_mgr_gamma_size);
+
+static void dispc_mgr_write_gamma_table(enum omap_channel channel)
+{
+        const struct dispc_gamma_desc *gdesc = &mgr_desc[channel].gamma;
+        u32 *table = dispc.gamma_table[channel];
+        unsigned int i;
+
+        DSSDBG("%s: channel %d\n", __func__, channel);
+
+        for (i = 0; i < gdesc->len; ++i) {
+                u32 v = table[i];
+
+                if (gdesc->has_index)
+                        v |= i << 24;
+                else if (i == 0)
+                        v |= 1 << 31;
+
+                dispc_write_reg(gdesc->reg, v);
+        }
+}
+
+static void dispc_restore_gamma_tables(void)
+{
+        DSSDBG("%s()\n", __func__);
+
+        if (!dispc.feat->has_gamma_table)
+                return;
+
+        dispc_mgr_write_gamma_table(OMAP_DSS_CHANNEL_LCD);
+
+        dispc_mgr_write_gamma_table(OMAP_DSS_CHANNEL_DIGIT);
+
+        if (dss_has_feature(FEAT_MGR_LCD2))
+                dispc_mgr_write_gamma_table(OMAP_DSS_CHANNEL_LCD2);
+
+        if (dss_has_feature(FEAT_MGR_LCD3))
+                dispc_mgr_write_gamma_table(OMAP_DSS_CHANNEL_LCD3);
+}
+
+static const struct drm_color_lut dispc_mgr_gamma_default_lut[] = {
+        { .red = 0, .green = 0, .blue = 0, },
+        { .red = U16_MAX, .green = U16_MAX, .blue = U16_MAX, },
+};
+
+void dispc_mgr_set_gamma(enum omap_channel channel,
+                         const struct drm_color_lut *lut,
+                         unsigned int length)
+{
+        const struct dispc_gamma_desc *gdesc = &mgr_desc[channel].gamma;
+        u32 *table = dispc.gamma_table[channel];
+        uint i;
+
+        DSSDBG("%s: channel %d, lut len %u, hw len %u\n", __func__,
+               channel, length, gdesc->len);
+
+        if (!dispc.feat->has_gamma_table)
+                return;
+
+        if (lut == NULL || length < 2) {
+                lut = dispc_mgr_gamma_default_lut;
+                length = ARRAY_SIZE(dispc_mgr_gamma_default_lut);
+        }
+
+        for (i = 0; i < length - 1; ++i) {
+                uint first = i * (gdesc->len - 1) / (length - 1);
+                uint last = (i + 1) * (gdesc->len - 1) / (length - 1);
+                uint w = last - first;
+                u16 r, g, b;
+                uint j;
+
+                if (w == 0)
+                        continue;
+
+                for (j = 0; j <= w; j++) {
+                        r = (lut[i].red * (w - j) + lut[i+1].red * j) / w;
+                        g = (lut[i].green * (w - j) + lut[i+1].green * j) / w;
+                        b = (lut[i].blue * (w - j) + lut[i+1].blue * j) / w;
+
+                        r >>= 16 - gdesc->bits;
+                        g >>= 16 - gdesc->bits;
+                        b >>= 16 - gdesc->bits;
+
+                        table[first + j] = (r << (gdesc->bits * 2)) |
+                                (g << gdesc->bits) | b;
+                }
+        }
+
+        if (dispc.is_enabled)
+                dispc_mgr_write_gamma_table(channel);
+}
+EXPORT_SYMBOL(dispc_mgr_set_gamma);
+
+static int dispc_init_gamma_tables(void)
+{
+        int channel;
+
+        if (!dispc.feat->has_gamma_table)
+                return 0;
+
+        for (channel = 0; channel < ARRAY_SIZE(dispc.gamma_table); channel++) {
+                const struct dispc_gamma_desc *gdesc = &mgr_desc[channel].gamma;
+                u32 *gt;
+
+                if (channel == OMAP_DSS_CHANNEL_LCD2 &&
+                    !dss_has_feature(FEAT_MGR_LCD2))
+                        continue;
+
+                if (channel == OMAP_DSS_CHANNEL_LCD3 &&
+                    !dss_has_feature(FEAT_MGR_LCD3))
+                        continue;
+
+                gt = devm_kmalloc_array(&dispc.pdev->dev, gdesc->len,
+                                           sizeof(u32), GFP_KERNEL);
+                if (!gt)
+                        return -ENOMEM;
+
+                dispc.gamma_table[channel] = gt;
+
+                dispc_mgr_set_gamma(channel, NULL, 0);
+        }
+        return 0;
+}
+
 static void _omap_dispc_initial_config(void)
 {
         u32 l;
@@ -3829,8 +3963,15 @@ static void _omap_dispc_initial_config(void)
                 dispc.core_clk_rate = dispc_fclk_rate();
         }
 
-        /* FUNCGATED */
-        if (dss_has_feature(FEAT_FUNCGATED))
+        /* Use gamma table mode, instead of palette mode */
+        if (dispc.feat->has_gamma_table)
+                REG_FLD_MOD(DISPC_CONFIG, 1, 3, 3);
+
+        /* For older DSS versions (FEAT_FUNCGATED) this enables
+         * func-clock auto-gating. For newer versions
+         * (dispc.feat->has_gamma_table) this enables tv-out gamma tables.
+         */
+        if (dss_has_feature(FEAT_FUNCGATED) || dispc.feat->has_gamma_table)
                 REG_FLD_MOD(DISPC_CONFIG, 1, 9, 9);
 
         dispc_setup_color_conv_coef();
@@ -3934,6 +4075,8 @@ static const struct dispc_features omap44xx_dispc_feats = {
         .has_writeback          =       true,
         .supports_double_pixel  =       true,
         .reverse_ilace_field_order =    true,
+        .has_gamma_table        =       true,
+        .has_gamma_i734_bug     =       true,
 };
 
 static const struct dispc_features omap54xx_dispc_feats = {
@@ -3959,6 +4102,8 @@ static const struct dispc_features omap54xx_dispc_feats = {
         .has_writeback          =       true,
         .supports_double_pixel  =       true,
         .reverse_ilace_field_order =    true,
+        .has_gamma_table        =       true,
+        .has_gamma_i734_bug     =       true,
 };
 
 static int dispc_init_features(struct platform_device *pdev)
@@ -4050,6 +4195,168 @@ void dispc_free_irq(void *dev_id)
 }
 EXPORT_SYMBOL(dispc_free_irq);
 
+/*
+ * Workaround for errata i734 in DSS dispc
+ *  - LCD1 Gamma Correction Is Not Working When GFX Pipe Is Disabled
+ *
+ * For gamma tables to work on LCD1 the GFX plane has to be used at
+ * least once after DSS HW has come out of reset. The workaround
+ * sets up a minimal LCD setup with GFX plane and waits for one
+ * vertical sync irq before disabling the setup and continuing with
+ * the context restore. The physical outputs are gated during the
+ * operation. This workaround requires that gamma table's LOADMODE
+ * is set to 0x2 in DISPC_CONTROL1 register.
+ *
+ * For details see:
+ * OMAP543x Multimedia Device Silicon Revision 2.0 Silicon Errata
+ * Literature Number: SWPZ037E
+ * Or some other relevant errata document for the DSS IP version.
+ */
+
+static const struct dispc_errata_i734_data {
+        struct omap_video_timings timings;
+        struct omap_overlay_info ovli;
+        struct omap_overlay_manager_info mgri;
+        struct dss_lcd_mgr_config lcd_conf;
+} i734 = {
+        .timings = {
+                .x_res = 8, .y_res = 1,
+                .pixelclock = 16000000,
+                .hsw = 8, .hfp = 4, .hbp = 4,
+                .vsw = 1, .vfp = 1, .vbp = 1,
+                .vsync_level = OMAPDSS_SIG_ACTIVE_LOW,
+                .hsync_level = OMAPDSS_SIG_ACTIVE_LOW,
+                .interlace = false,
+                .data_pclk_edge = OMAPDSS_DRIVE_SIG_RISING_EDGE,
+                .de_level = OMAPDSS_SIG_ACTIVE_HIGH,
+                .sync_pclk_edge = OMAPDSS_DRIVE_SIG_RISING_EDGE,
+                .double_pixel = false,
+        },
+        .ovli = {
+                .screen_width = 1,
+                .width = 1, .height = 1,
+                .color_mode = OMAP_DSS_COLOR_RGB24U,
+                .rotation = OMAP_DSS_ROT_0,
+                .rotation_type = OMAP_DSS_ROT_DMA,
+                .mirror = 0,
+                .pos_x = 0, .pos_y = 0,
+                .out_width = 0, .out_height = 0,
+                .global_alpha = 0xff,
+                .pre_mult_alpha = 0,
+                .zorder = 0,
+        },
+        .mgri = {
+                .default_color = 0,
+                .trans_enabled = false,
+                .partial_alpha_enabled = false,
+                .cpr_enable = false,
+        },
+        .lcd_conf = {
+                .io_pad_mode = DSS_IO_PAD_MODE_BYPASS,
+                .stallmode = false,
+                .fifohandcheck = false,
+                .clock_info = {
+                        .lck_div = 1,
+                        .pck_div = 2,
+                },
+                .video_port_width = 24,
+                .lcden_sig_polarity = 0,
+        },
+};
+
+static struct i734_buf {
+        size_t size;
+        dma_addr_t paddr;
+        void *vaddr;
+} i734_buf;
+
+static int dispc_errata_i734_wa_init(void)
+{
+        if (!dispc.feat->has_gamma_i734_bug)
+                return 0;
+
+        i734_buf.size = i734.ovli.width * i734.ovli.height *
+                color_mode_to_bpp(i734.ovli.color_mode) / 8;
+
+        i734_buf.vaddr = dma_alloc_writecombine(&dispc.pdev->dev, i734_buf.size,
+                                                &i734_buf.paddr, GFP_KERNEL);
+        if (!i734_buf.vaddr) {
+                dev_err(&dispc.pdev->dev, "%s: dma_alloc_writecombine failed",
+                        __func__);
+                return -ENOMEM;
+        }
+
+        return 0;
+}
+
+static void dispc_errata_i734_wa_fini(void)
+{
+        if (!dispc.feat->has_gamma_i734_bug)
+                return;
+
+        dma_free_writecombine(&dispc.pdev->dev, i734_buf.size, i734_buf.vaddr,
+                              i734_buf.paddr);
+}
+
+static void dispc_errata_i734_wa(void)
+{
+        u32 framedone_irq = dispc_mgr_get_framedone_irq(OMAP_DSS_CHANNEL_LCD);
+        struct omap_overlay_info ovli;
+        struct dss_lcd_mgr_config lcd_conf;
+        u32 gatestate;
+        unsigned int count;
+
+        if (!dispc.feat->has_gamma_i734_bug)
+                return;
+
+        gatestate = REG_GET(DISPC_CONFIG, 8, 4);
+
+        ovli = i734.ovli;
+        ovli.paddr = i734_buf.paddr;
+        lcd_conf = i734.lcd_conf;
+
+        /* Gate all LCD1 outputs */
+        REG_FLD_MOD(DISPC_CONFIG, 0x1f, 8, 4);
+
+        /* Setup and enable GFX plane */
+        dispc_ovl_set_channel_out(OMAP_DSS_GFX, OMAP_DSS_CHANNEL_LCD);
+        dispc_ovl_setup(OMAP_DSS_GFX, &ovli, false, &i734.timings, false);
+        dispc_ovl_enable(OMAP_DSS_GFX, true);
+
+        /* Set up and enable display manager for LCD1 */
+        dispc_mgr_setup(OMAP_DSS_CHANNEL_LCD, &i734.mgri);
+        dispc_calc_clock_rates(dss_get_dispc_clk_rate(),
+                               &lcd_conf.clock_info);
+        dispc_mgr_set_lcd_config(OMAP_DSS_CHANNEL_LCD, &lcd_conf);
+        dispc_mgr_set_timings(OMAP_DSS_CHANNEL_LCD, &i734.timings);
+
+        dispc_clear_irqstatus(framedone_irq);
+
+        /* Enable and shut the channel to produce just one frame */
+        dispc_mgr_enable(OMAP_DSS_CHANNEL_LCD, true);
+        dispc_mgr_enable(OMAP_DSS_CHANNEL_LCD, false);
+
+        /* Busy wait for framedone. We can't fiddle with irq handlers
+         * in PM resume. Typically the loop runs less than 5 times and
+         * waits less than a micro second.
+         */
+        count = 0;
+        while (!(dispc_read_irqstatus() & framedone_irq)) {
+                if (count++ > 10000) {
+                        dev_err(&dispc.pdev->dev, "%s: framedone timeout\n",
+                                __func__);
+                        break;
+                }
+        }
+        dispc_ovl_enable(OMAP_DSS_GFX, false);
+
+        /* Clear all irq bits before continuing */
+        dispc_clear_irqstatus(0xffffffff);
+
+        /* Restore the original state to LCD1 output gates */
+        REG_FLD_MOD(DISPC_CONFIG, gatestate, 8, 4);
+}
+
 /* DISPC HW IP initialisation */
 static int dispc_bind(struct device *dev, struct device *master, void *data)
 {
@@ -4067,6 +4374,10 @@ static int dispc_bind(struct device *dev, struct device *master, void *data)
         if (r)
                 return r;
 
+        r = dispc_errata_i734_wa_init();
+        if (r)
+                return r;
+
         dispc_mem = platform_get_resource(dispc.pdev, IORESOURCE_MEM, 0);
         if (!dispc_mem) {
                 DSSERR("can't get IORESOURCE_MEM DISPC\n");
@@ -4100,6 +4411,10 @@ static int dispc_bind(struct device *dev, struct device *master, void *data)
                 }
         }
 
+        r = dispc_init_gamma_tables();
+        if (r)
+                return r;
+
         pm_runtime_enable(&pdev->dev);
 
         r = dispc_runtime_get();
@@ -4127,6 +4442,8 @@ static void dispc_unbind(struct device *dev, struct device *master,
                                void *data)
 {
         pm_runtime_disable(dev);
+
+        dispc_errata_i734_wa_fini();
 }
 
 static const struct component_ops dispc_component_ops = {
@@ -4169,7 +4486,11 @@ static int dispc_runtime_resume(struct device *dev)
         if (REG_GET(DISPC_CONFIG, 2, 1) != OMAP_DSS_LOAD_FRAME_ONLY) {
                 _omap_dispc_initial_config();
 
+                dispc_errata_i734_wa();
+
                 dispc_restore_context();
+
+                dispc_restore_gamma_tables();
         }
 
         dispc.is_enabled = true;
diff --git a/drivers/gpu/drm/omapdrm/dss/dispc.h b/drivers/gpu/drm/omapdrm/dss/dispc.h
index 483744223dd1..bc1d8126ee87 100644
--- a/drivers/gpu/drm/omapdrm/dss/dispc.h
+++ b/drivers/gpu/drm/omapdrm/dss/dispc.h
@@ -42,6 +42,11 @@
 #define DISPC_MSTANDBY_CTRL             0x0858
 #define DISPC_GLOBAL_MFLAG_ATTRIBUTE    0x085C
 
+#define DISPC_GAMMA_TABLE0              0x0630
+#define DISPC_GAMMA_TABLE1              0x0634
+#define DISPC_GAMMA_TABLE2              0x0638
+#define DISPC_GAMMA_TABLE3              0x0850
+
 /* DISPC overlay registers */
 #define DISPC_OVL_BA0(n)                (DISPC_OVL_BASE(n) + \
                                         DISPC_BA0_OFFSET(n))
diff --git a/drivers/gpu/drm/omapdrm/dss/dispc_coefs.c b/drivers/gpu/drm/omapdrm/dss/dispc_coefs.c
index 038c15b04215..34fad2376f8d 100644
--- a/drivers/gpu/drm/omapdrm/dss/dispc_coefs.c
+++ b/drivers/gpu/drm/omapdrm/dss/dispc_coefs.c
@@ -18,8 +18,8 @@
  */
 
 #include <linux/kernel.h>
-#include <video/omapdss.h>
 
+#include "omapdss.h"
 #include "dispc.h"
 
 static const struct dispc_coef coef3_M8[8] = {
diff --git a/drivers/gpu/drm/omapdrm/dss/display.c b/drivers/gpu/drm/omapdrm/dss/display.c
index 9f3dd09b0a6c..8dcdd7cf9937 100644
--- a/drivers/gpu/drm/omapdrm/dss/display.c
+++ b/drivers/gpu/drm/omapdrm/dss/display.c
@@ -28,7 +28,7 @@
 #include <linux/platform_device.h>
 #include <linux/of.h>
 
-#include <video/omapdss.h>
+#include "omapdss.h"
 #include "dss.h"
 #include "dss_features.h"
 
diff --git a/drivers/gpu/drm/omapdrm/dss/dpi.c b/drivers/gpu/drm/omapdrm/dss/dpi.c
index 97ea60257884..b268295b76cf 100644
--- a/drivers/gpu/drm/omapdrm/dss/dpi.c
+++ b/drivers/gpu/drm/omapdrm/dss/dpi.c
@@ -34,17 +34,15 @@
 #include <linux/clk.h>
 #include <linux/component.h>
 
-#include <video/omapdss.h>
-
+#include "omapdss.h"
 #include "dss.h"
 #include "dss_features.h"
 
-#define HSDIV_DISPC     0
-
 struct dpi_data {
         struct platform_device *pdev;
 
         struct regulator *vdds_dsi_reg;
+        enum dss_clk_source clk_src;
         struct dss_pll *pll;
 
         struct mutex lock;
@@ -69,7 +67,7 @@ static struct dpi_data *dpi_get_data_from_pdev(struct platform_device *pdev)
         return dev_get_drvdata(&pdev->dev);
 }
 
-static struct dss_pll *dpi_get_pll(enum omap_channel channel)
+static enum dss_clk_source dpi_get_clk_src(enum omap_channel channel)
 {
         /*
          * XXX we can't currently use DSI PLL for DPI with OMAP3, as the DSI PLL
@@ -83,64 +81,51 @@ static struct dss_pll *dpi_get_pll(enum omap_channel channel)
         case OMAPDSS_VER_OMAP3630:
         case OMAPDSS_VER_AM35xx:
         case OMAPDSS_VER_AM43xx:
-                return NULL;
+                return DSS_CLK_SRC_FCK;
 
         case OMAPDSS_VER_OMAP4430_ES1:
         case OMAPDSS_VER_OMAP4430_ES2:
         case OMAPDSS_VER_OMAP4:
                 switch (channel) {
                 case OMAP_DSS_CHANNEL_LCD:
-                        return dss_pll_find("dsi0");
+                        return DSS_CLK_SRC_PLL1_1;
                 case OMAP_DSS_CHANNEL_LCD2:
-                        return dss_pll_find("dsi1");
+                        return DSS_CLK_SRC_PLL2_1;
                 default:
-                        return NULL;
+                        return DSS_CLK_SRC_FCK;
                 }
 
         case OMAPDSS_VER_OMAP5:
                 switch (channel) {
                 case OMAP_DSS_CHANNEL_LCD:
-                        return dss_pll_find("dsi0");
+                        return DSS_CLK_SRC_PLL1_1;
                 case OMAP_DSS_CHANNEL_LCD3:
-                        return dss_pll_find("dsi1");
+                        return DSS_CLK_SRC_PLL2_1;
+                case OMAP_DSS_CHANNEL_LCD2:
                 default:
-                        return NULL;
+                        return DSS_CLK_SRC_FCK;
                 }
 
         case OMAPDSS_VER_DRA7xx:
                 switch (channel) {
                 case OMAP_DSS_CHANNEL_LCD:
+                        return DSS_CLK_SRC_PLL1_1;
                 case OMAP_DSS_CHANNEL_LCD2:
-                        return dss_pll_find("video0");
+                        return DSS_CLK_SRC_PLL1_3;
                 case OMAP_DSS_CHANNEL_LCD3:
-                        return dss_pll_find("video1");
+                        return DSS_CLK_SRC_PLL2_1;
                 default:
-                        return NULL;
+                        return DSS_CLK_SRC_FCK;
                 }
 
         default:
-                return NULL;
-        }
-}
-
-static enum omap_dss_clk_source dpi_get_alt_clk_src(enum omap_channel channel)
-{
-        switch (channel) {
-        case OMAP_DSS_CHANNEL_LCD:
-                return OMAP_DSS_CLK_SRC_DSI_PLL_HSDIV_DISPC;
-        case OMAP_DSS_CHANNEL_LCD2:
-                return OMAP_DSS_CLK_SRC_DSI2_PLL_HSDIV_DISPC;
-        case OMAP_DSS_CHANNEL_LCD3:
-                return OMAP_DSS_CLK_SRC_DSI2_PLL_HSDIV_DISPC;
-        default:
-                /* this shouldn't happen */
-                WARN_ON(1);
-                return OMAP_DSS_CLK_SRC_FCK;
+                return DSS_CLK_SRC_FCK;
         }
 }
 
 struct dpi_clk_calc_ctx {
         struct dss_pll *pll;
+        unsigned clkout_idx;
 
         /* inputs */
 
@@ -148,7 +133,7 @@ struct dpi_clk_calc_ctx {
 
         /* outputs */
 
-        struct dss_pll_clock_info dsi_cinfo;
+        struct dss_pll_clock_info pll_cinfo;
         unsigned long fck;
         struct dispc_clock_info dispc_cinfo;
 };
@@ -193,8 +178,8 @@ static bool dpi_calc_hsdiv_cb(int m_dispc, unsigned long dispc,
         if (m_dispc > 1 && m_dispc % 2 != 0 && ctx->pck_min >= 100000000)
                 return false;
 
-        ctx->dsi_cinfo.mX[HSDIV_DISPC] = m_dispc;
-        ctx->dsi_cinfo.clkout[HSDIV_DISPC] = dispc;
+        ctx->pll_cinfo.mX[ctx->clkout_idx] = m_dispc;
+        ctx->pll_cinfo.clkout[ctx->clkout_idx] = dispc;
 
         return dispc_div_calc(dispc, ctx->pck_min, ctx->pck_max,
                         dpi_calc_dispc_cb, ctx);
@@ -207,12 +192,12 @@ static bool dpi_calc_pll_cb(int n, int m, unsigned long fint,
 {
         struct dpi_clk_calc_ctx *ctx = data;
 
-        ctx->dsi_cinfo.n = n;
-        ctx->dsi_cinfo.m = m;
-        ctx->dsi_cinfo.fint = fint;
-        ctx->dsi_cinfo.clkdco = clkdco;
+        ctx->pll_cinfo.n = n;
+        ctx->pll_cinfo.m = m;
+        ctx->pll_cinfo.fint = fint;
+        ctx->pll_cinfo.clkdco = clkdco;
 
-        return dss_pll_hsdiv_calc(ctx->pll, clkdco,
+        return dss_pll_hsdiv_calc_a(ctx->pll, clkdco,
                 ctx->pck_min, dss_feat_get_param_max(FEAT_PARAM_DSS_FCK),
                 dpi_calc_hsdiv_cb, ctx);
 }
@@ -227,25 +212,39 @@ static bool dpi_calc_dss_cb(unsigned long fck, void *data)
                         dpi_calc_dispc_cb, ctx);
 }
 
-static bool dpi_dsi_clk_calc(struct dpi_data *dpi, unsigned long pck,
+static bool dpi_pll_clk_calc(struct dpi_data *dpi, unsigned long pck,
                 struct dpi_clk_calc_ctx *ctx)
 {
         unsigned long clkin;
-        unsigned long pll_min, pll_max;
 
         memset(ctx, 0, sizeof(*ctx));
         ctx->pll = dpi->pll;
-        ctx->pck_min = pck - 1000;
-        ctx->pck_max = pck + 1000;
+        ctx->clkout_idx = dss_pll_get_clkout_idx_for_src(dpi->clk_src);
 
-        pll_min = 0;
-        pll_max = 0;
+        clkin = clk_get_rate(dpi->pll->clkin);
 
-        clkin = clk_get_rate(ctx->pll->clkin);
+        if (dpi->pll->hw->type == DSS_PLL_TYPE_A) {
+                unsigned long pll_min, pll_max;
 
-        return dss_pll_calc(ctx->pll, clkin,
-                        pll_min, pll_max,
-                        dpi_calc_pll_cb, ctx);
+                ctx->pck_min = pck - 1000;
+                ctx->pck_max = pck + 1000;
+
+                pll_min = 0;
+                pll_max = 0;
+
+                return dss_pll_calc_a(ctx->pll, clkin,
+                                pll_min, pll_max,
+                                dpi_calc_pll_cb, ctx);
+        } else { /* DSS_PLL_TYPE_B */
+                dss_pll_calc_b(dpi->pll, clkin, pck, &ctx->pll_cinfo);
+
+                ctx->dispc_cinfo.lck_div = 1;
+                ctx->dispc_cinfo.pck_div = 1;
+                ctx->dispc_cinfo.lck = ctx->pll_cinfo.clkout[0];
+                ctx->dispc_cinfo.pck = ctx->dispc_cinfo.lck;
+
+                return true;
+        }
 }
 
 static bool dpi_dss_clk_calc(unsigned long pck, struct dpi_clk_calc_ctx *ctx)
@@ -279,7 +278,7 @@ static bool dpi_dss_clk_calc(unsigned long pck, struct dpi_clk_calc_ctx *ctx)
 
 
 
-static int dpi_set_dsi_clk(struct dpi_data *dpi, enum omap_channel channel,
+static int dpi_set_pll_clk(struct dpi_data *dpi, enum omap_channel channel,
                 unsigned long pck_req, unsigned long *fck, int *lck_div,
                 int *pck_div)
 {
@@ -287,20 +286,19 @@ static int dpi_set_dsi_clk(struct dpi_data *dpi, enum omap_channel channel,
         int r;
         bool ok;
 
-        ok = dpi_dsi_clk_calc(dpi, pck_req, &ctx);
+        ok = dpi_pll_clk_calc(dpi, pck_req, &ctx);
         if (!ok)
                 return -EINVAL;
 
-        r = dss_pll_set_config(dpi->pll, &ctx.dsi_cinfo);
+        r = dss_pll_set_config(dpi->pll, &ctx.pll_cinfo);
         if (r)
                 return r;
 
-        dss_select_lcd_clk_source(channel,
-                        dpi_get_alt_clk_src(channel));
+        dss_select_lcd_clk_source(channel, dpi->clk_src);
 
         dpi->mgr_config.clock_info = ctx.dispc_cinfo;
 
-        *fck = ctx.dsi_cinfo.clkout[HSDIV_DISPC];
+        *fck = ctx.pll_cinfo.clkout[ctx.clkout_idx];
         *lck_div = ctx.dispc_cinfo.lck_div;
         *pck_div = ctx.dispc_cinfo.pck_div;
 
@@ -342,7 +340,7 @@ static int dpi_set_mode(struct dpi_data *dpi)
         int r = 0;
 
         if (dpi->pll)
-                r = dpi_set_dsi_clk(dpi, channel, t->pixelclock, &fck,
+                r = dpi_set_pll_clk(dpi, channel, t->pixelclock, &fck,
                                 &lck_div, &pck_div);
         else
                 r = dpi_set_dispc_clk(dpi, t->pixelclock, &fck,
@@ -419,7 +417,7 @@ static int dpi_display_enable(struct omap_dss_device *dssdev)
         if (dpi->pll) {
                 r = dss_pll_enable(dpi->pll);
                 if (r)
-                        goto err_dsi_pll_init;
+                        goto err_pll_init;
         }
 
         r = dpi_set_mode(dpi);
@@ -442,7 +440,7 @@ err_mgr_enable:
 err_set_mode:
         if (dpi->pll)
                 dss_pll_disable(dpi->pll);
-err_dsi_pll_init:
+err_pll_init:
 err_src_sel:
         dispc_runtime_put();
 err_get_dispc:
@@ -465,7 +463,7 @@ static void dpi_display_disable(struct omap_dss_device *dssdev)
         dss_mgr_disable(channel);
 
         if (dpi->pll) {
-                dss_select_lcd_clk_source(channel, OMAP_DSS_CLK_SRC_FCK);
+                dss_select_lcd_clk_source(channel, DSS_CLK_SRC_FCK);
                 dss_pll_disable(dpi->pll);
         }
 
@@ -524,11 +522,11 @@ static int dpi_check_timings(struct omap_dss_device *dssdev,
                 return -EINVAL;
 
         if (dpi->pll) {
-                ok = dpi_dsi_clk_calc(dpi, timings->pixelclock, &ctx);
+                ok = dpi_pll_clk_calc(dpi, timings->pixelclock, &ctx);
                 if (!ok)
                         return -EINVAL;
 
-                fck = ctx.dsi_cinfo.clkout[HSDIV_DISPC];
+                fck = ctx.pll_cinfo.clkout[ctx.clkout_idx];
         } else {
                 ok = dpi_dss_clk_calc(timings->pixelclock, &ctx);
                 if (!ok)
@@ -558,7 +556,7 @@ static void dpi_set_data_lines(struct omap_dss_device *dssdev, int data_lines)
         mutex_unlock(&dpi->lock);
 }
 
-static int dpi_verify_dsi_pll(struct dss_pll *pll)
+static int dpi_verify_pll(struct dss_pll *pll)
 {
         int r;
 
@@ -602,16 +600,14 @@ static void dpi_init_pll(struct dpi_data *dpi)
         if (dpi->pll)
                 return;
 
-        pll = dpi_get_pll(dpi->output.dispc_channel);
+        dpi->clk_src = dpi_get_clk_src(dpi->output.dispc_channel);
+
+        pll = dss_pll_find_by_src(dpi->clk_src);
         if (!pll)
                 return;
 
-        /* On DRA7 we need to set a mux to use the PLL */
-        if (omapdss_get_version() == OMAPDSS_VER_DRA7xx)
-                dss_ctrl_pll_set_control_mux(pll->id, dpi->output.dispc_channel);
-
-        if (dpi_verify_dsi_pll(pll)) {
-                DSSWARN("DSI PLL not operational\n");
+        if (dpi_verify_pll(pll)) {
+                DSSWARN("PLL not operational\n");
                 return;
         }
 
diff --git a/drivers/gpu/drm/omapdrm/dss/dsi.c b/drivers/gpu/drm/omapdrm/dss/dsi.c
index 56c43f355ce3..e1be5e795cd8 100644
--- a/drivers/gpu/drm/omapdrm/dss/dsi.c
+++ b/drivers/gpu/drm/omapdrm/dss/dsi.c
@@ -42,9 +42,9 @@
 #include <linux/of_platform.h>
 #include <linux/component.h>
 
-#include <video/omapdss.h>
 #include <video/mipi_display.h>
 
+#include "omapdss.h"
 #include "dss.h"
 #include "dss_features.h"
 
@@ -1261,7 +1261,7 @@ static unsigned long dsi_fclk_rate(struct platform_device *dsidev)
         unsigned long r;
         struct dsi_data *dsi = dsi_get_dsidrv_data(dsidev);
 
-        if (dss_get_dsi_clk_source(dsi->module_id) == OMAP_DSS_CLK_SRC_FCK) {
+        if (dss_get_dsi_clk_source(dsi->module_id) == DSS_CLK_SRC_FCK) {
                 /* DSI FCLK source is DSS_CLK_FCK */
                 r = clk_get_rate(dsi->dss_clk);
         } else {
@@ -1474,7 +1474,7 @@ static void dsi_dump_dsidev_clocks(struct platform_device *dsidev,
 {
         struct dsi_data *dsi = dsi_get_dsidrv_data(dsidev);
         struct dss_pll_clock_info *cinfo = &dsi->pll.cinfo;
-        enum omap_dss_clk_source dispc_clk_src, dsi_clk_src;
+        enum dss_clk_source dispc_clk_src, dsi_clk_src;
         int dsi_module = dsi->module_id;
         struct dss_pll *pll = &dsi->pll;
 
@@ -1494,28 +1494,27 @@ static void dsi_dump_dsidev_clocks(struct platform_device *dsidev,
                         cinfo->clkdco, cinfo->m);
 
         seq_printf(s,   "DSI_PLL_HSDIV_DISPC (%s)\t%-16lum_dispc %u\t(%s)\n",
-                        dss_feat_get_clk_source_name(dsi_module == 0 ?
-                                OMAP_DSS_CLK_SRC_DSI_PLL_HSDIV_DISPC :
-                                OMAP_DSS_CLK_SRC_DSI2_PLL_HSDIV_DISPC),
+                        dss_get_clk_source_name(dsi_module == 0 ?
+                                DSS_CLK_SRC_PLL1_1 :
+                                DSS_CLK_SRC_PLL2_1),
                         cinfo->clkout[HSDIV_DISPC],
                         cinfo->mX[HSDIV_DISPC],
-                        dispc_clk_src == OMAP_DSS_CLK_SRC_FCK ?
+                        dispc_clk_src == DSS_CLK_SRC_FCK ?
                         "off" : "on");
 
         seq_printf(s,   "DSI_PLL_HSDIV_DSI (%s)\t%-16lum_dsi %u\t(%s)\n",
-                        dss_feat_get_clk_source_name(dsi_module == 0 ?
-                                OMAP_DSS_CLK_SRC_DSI_PLL_HSDIV_DSI :
-                                OMAP_DSS_CLK_SRC_DSI2_PLL_HSDIV_DSI),
+                        dss_get_clk_source_name(dsi_module == 0 ?
+                                DSS_CLK_SRC_PLL1_2 :
+                                DSS_CLK_SRC_PLL2_2),
                         cinfo->clkout[HSDIV_DSI],
                         cinfo->mX[HSDIV_DSI],
-                        dsi_clk_src == OMAP_DSS_CLK_SRC_FCK ?
+                        dsi_clk_src == DSS_CLK_SRC_FCK ?
                         "off" : "on");
 
         seq_printf(s,   "- DSI%d -\n", dsi_module + 1);
 
-        seq_printf(s,   "dsi fclk source = %s (%s)\n",
-                        dss_get_generic_clk_source_name(dsi_clk_src),
-                        dss_feat_get_clk_source_name(dsi_clk_src));
+        seq_printf(s,   "dsi fclk source = %s\n",
+                        dss_get_clk_source_name(dsi_clk_src));
 
         seq_printf(s,   "DSI_FCLK\t%lu\n", dsi_fclk_rate(dsidev));
 
@@ -4101,8 +4100,8 @@ static int dsi_display_init_dispc(struct platform_device *dsidev,
         int r;
 
         dss_select_lcd_clk_source(channel, dsi->module_id == 0 ?
-                        OMAP_DSS_CLK_SRC_DSI_PLL_HSDIV_DISPC :
-                        OMAP_DSS_CLK_SRC_DSI2_PLL_HSDIV_DISPC);
+                        DSS_CLK_SRC_PLL1_1 :
+                        DSS_CLK_SRC_PLL2_1);
 
         if (dsi->mode == OMAP_DSS_DSI_CMD_MODE) {
                 r = dss_mgr_register_framedone_handler(channel,
@@ -4149,7 +4148,7 @@ err1:
                 dss_mgr_unregister_framedone_handler(channel,
                                 dsi_framedone_irq_callback, dsidev);
 err:
-        dss_select_lcd_clk_source(channel, OMAP_DSS_CLK_SRC_FCK);
+        dss_select_lcd_clk_source(channel, DSS_CLK_SRC_FCK);
         return r;
 }
 
@@ -4162,7 +4161,7 @@ static void dsi_display_uninit_dispc(struct platform_device *dsidev,
                 dss_mgr_unregister_framedone_handler(channel,
                                 dsi_framedone_irq_callback, dsidev);
 
-        dss_select_lcd_clk_source(channel, OMAP_DSS_CLK_SRC_FCK);
+        dss_select_lcd_clk_source(channel, DSS_CLK_SRC_FCK);
 }
 
 static int dsi_configure_dsi_clocks(struct platform_device *dsidev)
@@ -4196,8 +4195,8 @@ static int dsi_display_init_dsi(struct platform_device *dsidev)
                 goto err1;
 
         dss_select_dsi_clk_source(dsi->module_id, dsi->module_id == 0 ?
-                        OMAP_DSS_CLK_SRC_DSI_PLL_HSDIV_DSI :
-                        OMAP_DSS_CLK_SRC_DSI2_PLL_HSDIV_DSI);
+                        DSS_CLK_SRC_PLL1_2 :
+                        DSS_CLK_SRC_PLL2_2);
 
         DSSDBG("PLL OK\n");
 
@@ -4229,7 +4228,7 @@ static int dsi_display_init_dsi(struct platform_device *dsidev)
 err3:
         dsi_cio_uninit(dsidev);
 err2:
-        dss_select_dsi_clk_source(dsi->module_id, OMAP_DSS_CLK_SRC_FCK);
+        dss_select_dsi_clk_source(dsi->module_id, DSS_CLK_SRC_FCK);
 err1:
         dss_pll_disable(&dsi->pll);
 err0:
@@ -4251,7 +4250,7 @@ static void dsi_display_uninit_dsi(struct platform_device *dsidev,
         dsi_vc_enable(dsidev, 2, 0);
         dsi_vc_enable(dsidev, 3, 0);
 
-        dss_select_dsi_clk_source(dsi->module_id, OMAP_DSS_CLK_SRC_FCK);
+        dss_select_dsi_clk_source(dsi->module_id, DSS_CLK_SRC_FCK);
         dsi_cio_uninit(dsidev);
         dsi_pll_uninit(dsidev, disconnect_lanes);
 }
@@ -4452,7 +4451,7 @@ static bool dsi_cm_calc_pll_cb(int n, int m, unsigned long fint,
         ctx->dsi_cinfo.fint = fint;
         ctx->dsi_cinfo.clkdco = clkdco;
 
-        return dss_pll_hsdiv_calc(ctx->pll, clkdco, ctx->req_pck_min,
+        return dss_pll_hsdiv_calc_a(ctx->pll, clkdco, ctx->req_pck_min,
                         dss_feat_get_param_max(FEAT_PARAM_DSS_FCK),
                         dsi_cm_calc_hsdiv_cb, ctx);
 }
@@ -4491,7 +4490,7 @@ static bool dsi_cm_calc(struct dsi_data *dsi,
         pll_min = max(cfg->hs_clk_min * 4, txbyteclk * 4 * 4);
         pll_max = cfg->hs_clk_max * 4;
 
-        return dss_pll_calc(ctx->pll, clkin,
+        return dss_pll_calc_a(ctx->pll, clkin,
                         pll_min, pll_max,
                         dsi_cm_calc_pll_cb, ctx);
 }
@@ -4750,7 +4749,7 @@ static bool dsi_vm_calc_pll_cb(int n, int m, unsigned long fint,
         ctx->dsi_cinfo.fint = fint;
         ctx->dsi_cinfo.clkdco = clkdco;
 
-        return dss_pll_hsdiv_calc(ctx->pll, clkdco, ctx->req_pck_min,
+        return dss_pll_hsdiv_calc_a(ctx->pll, clkdco, ctx->req_pck_min,
                         dss_feat_get_param_max(FEAT_PARAM_DSS_FCK),
                         dsi_vm_calc_hsdiv_cb, ctx);
 }
@@ -4792,7 +4791,7 @@ static bool dsi_vm_calc(struct dsi_data *dsi,
                 pll_max = byteclk_max * 4 * 4;
         }
 
-        return dss_pll_calc(ctx->pll, clkin,
+        return dss_pll_calc_a(ctx->pll, clkin,
                         pll_min, pll_max,
                         dsi_vm_calc_pll_cb, ctx);
 }
@@ -5138,6 +5137,8 @@ static const struct dss_pll_ops dsi_pll_ops = {
 };
 
 static const struct dss_pll_hw dss_omap3_dsi_pll_hw = {
+        .type = DSS_PLL_TYPE_A,
+
         .n_max = (1 << 7) - 1,
         .m_max = (1 << 11) - 1,
         .mX_max = (1 << 4) - 1,
@@ -5163,6 +5164,8 @@ static const struct dss_pll_hw dss_omap3_dsi_pll_hw = {
 };
 
 static const struct dss_pll_hw dss_omap4_dsi_pll_hw = {
+        .type = DSS_PLL_TYPE_A,
+
         .n_max = (1 << 8) - 1,
         .m_max = (1 << 12) - 1,
         .mX_max = (1 << 5) - 1,
@@ -5188,6 +5191,8 @@ static const struct dss_pll_hw dss_omap4_dsi_pll_hw = {
 };
 
 static const struct dss_pll_hw dss_omap5_dsi_pll_hw = {
+        .type = DSS_PLL_TYPE_A,
+
         .n_max = (1 << 8) - 1,
         .m_max = (1 << 12) - 1,
         .mX_max = (1 << 5) - 1,
diff --git a/drivers/gpu/drm/omapdrm/dss/dss-of.c b/drivers/gpu/drm/omapdrm/dss/dss-of.c
index bf407b6ba15c..e256d879b25c 100644
--- a/drivers/gpu/drm/omapdrm/dss/dss-of.c
+++ b/drivers/gpu/drm/omapdrm/dss/dss-of.c
@@ -18,8 +18,7 @@
 #include <linux/of.h>
 #include <linux/seq_file.h>
 
-#include <video/omapdss.h>
-
+#include "omapdss.h"
 #include "dss.h"
 
 struct device_node *
@@ -126,15 +125,16 @@ u32 dss_of_port_get_port_number(struct device_node *port)
 
 static struct device_node *omapdss_of_get_remote_port(const struct device_node *node)
 {
-        struct device_node *np;
+        struct device_node *np, *np_parent;
 
         np = of_parse_phandle(node, "remote-endpoint", 0);
         if (!np)
                 return NULL;
 
-        np = of_get_next_parent(np);
+        np_parent = of_get_next_parent(np);
+        of_node_put(np);
 
-        return np;
+        return np_parent;
 }
 
 struct device_node *
diff --git a/drivers/gpu/drm/omapdrm/dss/dss.c b/drivers/gpu/drm/omapdrm/dss/dss.c
index 3303cfad4838..14887d5b02e5 100644
--- a/drivers/gpu/drm/omapdrm/dss/dss.c
+++ b/drivers/gpu/drm/omapdrm/dss/dss.c
@@ -42,8 +42,7 @@
 #include <linux/suspend.h>
 #include <linux/component.h>
 
-#include <video/omapdss.h>
-
+#include "omapdss.h"
 #include "dss.h"
 #include "dss_features.h"
 
@@ -76,6 +75,8 @@ struct dss_features {
         const enum omap_display_type *ports;
         int num_ports;
         int (*dpi_select_source)(int port, enum omap_channel channel);
+        int (*select_lcd_source)(enum omap_channel channel,
+                enum dss_clk_source clk_src);
 };
 
 static struct {
@@ -92,9 +93,9 @@ static struct {
         unsigned long   cache_prate;
         struct dispc_clock_info cache_dispc_cinfo;
 
-        enum omap_dss_clk_source dsi_clk_source[MAX_NUM_DSI];
-        enum omap_dss_clk_source dispc_clk_source;
-        enum omap_dss_clk_source lcd_clk_source[MAX_DSS_LCD_MANAGERS];
+        enum dss_clk_source dsi_clk_source[MAX_NUM_DSI];
+        enum dss_clk_source dispc_clk_source;
+        enum dss_clk_source lcd_clk_source[MAX_DSS_LCD_MANAGERS];
 
         bool            ctx_valid;
         u32             ctx[DSS_SZ_REGS / sizeof(u32)];
@@ -106,11 +107,14 @@ static struct {
 } dss;
 
 static const char * const dss_generic_clk_source_names[] = {
-        [OMAP_DSS_CLK_SRC_DSI_PLL_HSDIV_DISPC]  = "DSI_PLL_HSDIV_DISPC",
-        [OMAP_DSS_CLK_SRC_DSI_PLL_HSDIV_DSI]    = "DSI_PLL_HSDIV_DSI",
-        [OMAP_DSS_CLK_SRC_FCK]                  = "DSS_FCK",
-        [OMAP_DSS_CLK_SRC_DSI2_PLL_HSDIV_DISPC] = "DSI_PLL2_HSDIV_DISPC",
-        [OMAP_DSS_CLK_SRC_DSI2_PLL_HSDIV_DSI]   = "DSI_PLL2_HSDIV_DSI",
+        [DSS_CLK_SRC_FCK]       = "FCK",
+        [DSS_CLK_SRC_PLL1_1]    = "PLL1:1",
+        [DSS_CLK_SRC_PLL1_2]    = "PLL1:2",
+        [DSS_CLK_SRC_PLL1_3]    = "PLL1:3",
+        [DSS_CLK_SRC_PLL2_1]    = "PLL2:1",
+        [DSS_CLK_SRC_PLL2_2]    = "PLL2:2",
+        [DSS_CLK_SRC_PLL2_3]    = "PLL2:3",
+        [DSS_CLK_SRC_HDMI_PLL]  = "HDMI PLL",
 };
 
 static bool dss_initialized;
@@ -203,68 +207,70 @@ void dss_ctrl_pll_enable(enum dss_pll_id pll_id, bool enable)
                 1 << shift, val << shift);
 }
 
-void dss_ctrl_pll_set_control_mux(enum dss_pll_id pll_id,
+static int dss_ctrl_pll_set_control_mux(enum dss_clk_source clk_src,
         enum omap_channel channel)
 {
         unsigned shift, val;
 
         if (!dss.syscon_pll_ctrl)
-                return;
+                return -EINVAL;
 
         switch (channel) {
         case OMAP_DSS_CHANNEL_LCD:
                 shift = 3;
 
-                switch (pll_id) {
-                case DSS_PLL_VIDEO1:
+                switch (clk_src) {
+                case DSS_CLK_SRC_PLL1_1:
                         val = 0; break;
-                case DSS_PLL_HDMI:
+                case DSS_CLK_SRC_HDMI_PLL:
                         val = 1; break;
                 default:
                         DSSERR("error in PLL mux config for LCD\n");
-                        return;
+                        return -EINVAL;
                 }
 
                 break;
         case OMAP_DSS_CHANNEL_LCD2:
                 shift = 5;
 
-                switch (pll_id) {
-                case DSS_PLL_VIDEO1:
+                switch (clk_src) {
+                case DSS_CLK_SRC_PLL1_3:
                         val = 0; break;
-                case DSS_PLL_VIDEO2:
+                case DSS_CLK_SRC_PLL2_3:
                         val = 1; break;
-                case DSS_PLL_HDMI:
+                case DSS_CLK_SRC_HDMI_PLL:
                         val = 2; break;
                 default:
                         DSSERR("error in PLL mux config for LCD2\n");
-                        return;
+                        return -EINVAL;
                 }
 
                 break;
         case OMAP_DSS_CHANNEL_LCD3:
                 shift = 7;
 
-                switch (pll_id) {
-                case DSS_PLL_VIDEO1:
-                        val = 1; break;
-                case DSS_PLL_VIDEO2:
+                switch (clk_src) {
+                case DSS_CLK_SRC_PLL2_1:
                         val = 0; break;
-                case DSS_PLL_HDMI:
+                case DSS_CLK_SRC_PLL1_3:
+                        val = 1; break;
+                case DSS_CLK_SRC_HDMI_PLL:
                         val = 2; break;
                 default:
                         DSSERR("error in PLL mux config for LCD3\n");
-                        return;
+                        return -EINVAL;
                 }
 
                 break;
         default:
                 DSSERR("error in PLL mux config\n");
-                return;
+                return -EINVAL;
         }
 
         regmap_update_bits(dss.syscon_pll_ctrl, dss.syscon_pll_ctrl_offset,
                 0x3 << shift, val << shift);
+
+        return 0;
 }
 
 void dss_sdi_init(int datapairs)
@@ -354,14 +360,14 @@ void dss_sdi_disable(void)
         REG_FLD_MOD(DSS_PLL_CONTROL, 0, 18, 18); /* SDI_PLL_SYSRESET */
 }
 
-const char *dss_get_generic_clk_source_name(enum omap_dss_clk_source clk_src)
+const char *dss_get_clk_source_name(enum dss_clk_source clk_src)
 {
         return dss_generic_clk_source_names[clk_src];
 }
 
 void dss_dump_clocks(struct seq_file *s)
 {
-        const char *fclk_name, *fclk_real_name;
+        const char *fclk_name;
         unsigned long fclk_rate;
 
         if (dss_runtime_get())
@@ -369,12 +375,11 @@ void dss_dump_clocks(struct seq_file *s)
 
         seq_printf(s, "- DSS -\n");
 
-        fclk_name = dss_get_generic_clk_source_name(OMAP_DSS_CLK_SRC_FCK);
-        fclk_real_name = dss_feat_get_clk_source_name(OMAP_DSS_CLK_SRC_FCK);
+        fclk_name = dss_get_clk_source_name(DSS_CLK_SRC_FCK);
         fclk_rate = clk_get_rate(dss.dss_clk);
 
-        seq_printf(s, "%s (%s) = %lu\n",
-                        fclk_name, fclk_real_name,
+        seq_printf(s, "%s = %lu\n",
+                        fclk_name,
                         fclk_rate);
 
         dss_runtime_put();
@@ -403,19 +408,42 @@ static void dss_dump_regs(struct seq_file *s)
 #undef DUMPREG
 }
 
-static void dss_select_dispc_clk_source(enum omap_dss_clk_source clk_src)
+static int dss_get_channel_index(enum omap_channel channel)
+{
+        switch (channel) {
+        case OMAP_DSS_CHANNEL_LCD:
+                return 0;
+        case OMAP_DSS_CHANNEL_LCD2:
+                return 1;
+        case OMAP_DSS_CHANNEL_LCD3:
+                return 2;
+        default:
+                WARN_ON(1);
+                return 0;
+        }
+}
+
+static void dss_select_dispc_clk_source(enum dss_clk_source clk_src)
 {
         int b;
         u8 start, end;
 
+        /*
+         * We always use PRCM clock as the DISPC func clock, except on DSS3,
+         * where we don't have separate DISPC and LCD clock sources.
+         */
+        if (WARN_ON(dss_has_feature(FEAT_LCD_CLK_SRC) &&
+                clk_src != DSS_CLK_SRC_FCK))
+                return;
+
         switch (clk_src) {
-        case OMAP_DSS_CLK_SRC_FCK:
+        case DSS_CLK_SRC_FCK:
                 b = 0;
                 break;
-        case OMAP_DSS_CLK_SRC_DSI_PLL_HSDIV_DISPC:
+        case DSS_CLK_SRC_PLL1_1:
                 b = 1;
                 break;
-        case OMAP_DSS_CLK_SRC_DSI2_PLL_HSDIV_DISPC:
+        case DSS_CLK_SRC_PLL2_1:
                 b = 2;
                 break;
         default:
@@ -431,19 +459,19 @@ static void dss_select_dispc_clk_source(enum omap_dss_clk_source clk_src)
 }
 
 void dss_select_dsi_clk_source(int dsi_module,
-                enum omap_dss_clk_source clk_src)
+                enum dss_clk_source clk_src)
 {
         int b, pos;
 
         switch (clk_src) {
-        case OMAP_DSS_CLK_SRC_FCK:
+        case DSS_CLK_SRC_FCK:
                 b = 0;
                 break;
-        case OMAP_DSS_CLK_SRC_DSI_PLL_HSDIV_DSI:
+        case DSS_CLK_SRC_PLL1_2:
                 BUG_ON(dsi_module != 0);
                 b = 1;
                 break;
-        case OMAP_DSS_CLK_SRC_DSI2_PLL_HSDIV_DSI:
+        case DSS_CLK_SRC_PLL2_2:
                 BUG_ON(dsi_module != 1);
                 b = 1;
                 break;
@@ -458,59 +486,125 @@ void dss_select_dsi_clk_source(int dsi_module,
         dss.dsi_clk_source[dsi_module] = clk_src;
 }
 
+static int dss_lcd_clk_mux_dra7(enum omap_channel channel,
+        enum dss_clk_source clk_src)
+{
+        const u8 ctrl_bits[] = {
+                [OMAP_DSS_CHANNEL_LCD] = 0,
+                [OMAP_DSS_CHANNEL_LCD2] = 12,
+                [OMAP_DSS_CHANNEL_LCD3] = 19,
+        };
+
+        u8 ctrl_bit = ctrl_bits[channel];
+        int r;
+
+        if (clk_src == DSS_CLK_SRC_FCK) {
+                /* LCDx_CLK_SWITCH */
+                REG_FLD_MOD(DSS_CONTROL, 0, ctrl_bit, ctrl_bit);
+                return -EINVAL;
+        }
+
+        r = dss_ctrl_pll_set_control_mux(clk_src, channel);
+        if (r)
+                return r;
+
+        REG_FLD_MOD(DSS_CONTROL, 1, ctrl_bit, ctrl_bit);
+
+        return 0;
+}
+
+static int dss_lcd_clk_mux_omap5(enum omap_channel channel,
+        enum dss_clk_source clk_src)
+{
+        const u8 ctrl_bits[] = {
+                [OMAP_DSS_CHANNEL_LCD] = 0,
+                [OMAP_DSS_CHANNEL_LCD2] = 12,
+                [OMAP_DSS_CHANNEL_LCD3] = 19,
+        };
+        const enum dss_clk_source allowed_plls[] = {
+                [OMAP_DSS_CHANNEL_LCD] = DSS_CLK_SRC_PLL1_1,
+                [OMAP_DSS_CHANNEL_LCD2] = DSS_CLK_SRC_FCK,
+                [OMAP_DSS_CHANNEL_LCD3] = DSS_CLK_SRC_PLL2_1,
+        };
+
+        u8 ctrl_bit = ctrl_bits[channel];
+
+        if (clk_src == DSS_CLK_SRC_FCK) {
+                /* LCDx_CLK_SWITCH */
+                REG_FLD_MOD(DSS_CONTROL, 0, ctrl_bit, ctrl_bit);
+                return -EINVAL;
+        }
+
+        if (WARN_ON(allowed_plls[channel] != clk_src))
+                return -EINVAL;
+
+        REG_FLD_MOD(DSS_CONTROL, 1, ctrl_bit, ctrl_bit);
+
+        return 0;
+}
+
+static int dss_lcd_clk_mux_omap4(enum omap_channel channel,
+        enum dss_clk_source clk_src)
+{
+        const u8 ctrl_bits[] = {
+                [OMAP_DSS_CHANNEL_LCD] = 0,
+                [OMAP_DSS_CHANNEL_LCD2] = 12,
+        };
+        const enum dss_clk_source allowed_plls[] = {
+                [OMAP_DSS_CHANNEL_LCD] = DSS_CLK_SRC_PLL1_1,
+                [OMAP_DSS_CHANNEL_LCD2] = DSS_CLK_SRC_PLL2_1,
+        };
+
+        u8 ctrl_bit = ctrl_bits[channel];
+
+        if (clk_src == DSS_CLK_SRC_FCK) {
+                /* LCDx_CLK_SWITCH */
+                REG_FLD_MOD(DSS_CONTROL, 0, ctrl_bit, ctrl_bit);
+                return 0;
+        }
+
+        if (WARN_ON(allowed_plls[channel] != clk_src))
+                return -EINVAL;
+
+        REG_FLD_MOD(DSS_CONTROL, 1, ctrl_bit, ctrl_bit);
+
+        return 0;
+}
+
 void dss_select_lcd_clk_source(enum omap_channel channel,
-                enum omap_dss_clk_source clk_src)
+                enum dss_clk_source clk_src)
 {
-        int b, ix, pos;
+        int idx = dss_get_channel_index(channel);
+        int r;
 
         if (!dss_has_feature(FEAT_LCD_CLK_SRC)) {
                 dss_select_dispc_clk_source(clk_src);
+                dss.lcd_clk_source[idx] = clk_src;
                 return;
         }
 
-        switch (clk_src) {
-        case OMAP_DSS_CLK_SRC_FCK:
-                b = 0;
-                break;
-        case OMAP_DSS_CLK_SRC_DSI_PLL_HSDIV_DISPC:
-                BUG_ON(channel != OMAP_DSS_CHANNEL_LCD);
-                b = 1;
-                break;
-        case OMAP_DSS_CLK_SRC_DSI2_PLL_HSDIV_DISPC:
-                BUG_ON(channel != OMAP_DSS_CHANNEL_LCD2 &&
-                       channel != OMAP_DSS_CHANNEL_LCD3);
-                b = 1;
-                break;
-        default:
-                BUG();
+        r = dss.feat->select_lcd_source(channel, clk_src);
+        if (r)
                 return;
-        }
-
-        pos = channel == OMAP_DSS_CHANNEL_LCD ? 0 :
-             (channel == OMAP_DSS_CHANNEL_LCD2 ? 12 : 19);
-        REG_FLD_MOD(DSS_CONTROL, b, pos, pos);  /* LCDx_CLK_SWITCH */
 
-        ix = channel == OMAP_DSS_CHANNEL_LCD ? 0 :
-            (channel == OMAP_DSS_CHANNEL_LCD2 ? 1 : 2);
-        dss.lcd_clk_source[ix] = clk_src;
+        dss.lcd_clk_source[idx] = clk_src;
 }
 
-enum omap_dss_clk_source dss_get_dispc_clk_source(void)
+enum dss_clk_source dss_get_dispc_clk_source(void)
 {
         return dss.dispc_clk_source;
 }
 
-enum omap_dss_clk_source dss_get_dsi_clk_source(int dsi_module)
+enum dss_clk_source dss_get_dsi_clk_source(int dsi_module)
 {
         return dss.dsi_clk_source[dsi_module];
 }
 
-enum omap_dss_clk_source dss_get_lcd_clk_source(enum omap_channel channel)
+enum dss_clk_source dss_get_lcd_clk_source(enum omap_channel channel)
 {
         if (dss_has_feature(FEAT_LCD_CLK_SRC)) {
-                int ix = channel == OMAP_DSS_CHANNEL_LCD ? 0 :
-                        (channel == OMAP_DSS_CHANNEL_LCD2 ? 1 : 2);
-                return dss.lcd_clk_source[ix];
+                int idx = dss_get_channel_index(channel);
+                return dss.lcd_clk_source[idx];
         } else {
                 /* LCD_CLK source is the same as DISPC_FCLK source for
                  * OMAP2 and OMAP3 */
@@ -859,6 +953,7 @@ static const struct dss_features omap44xx_dss_feats = {
         .dpi_select_source      =       &dss_dpi_select_source_omap4,
         .ports                  =       omap2plus_ports,
         .num_ports              =       ARRAY_SIZE(omap2plus_ports),
+        .select_lcd_source      =       &dss_lcd_clk_mux_omap4,
 };
 
 static const struct dss_features omap54xx_dss_feats = {
@@ -868,6 +963,7 @@ static const struct dss_features omap54xx_dss_feats = {
         .dpi_select_source      =       &dss_dpi_select_source_omap5,
         .ports                  =       omap2plus_ports,
         .num_ports              =       ARRAY_SIZE(omap2plus_ports),
+        .select_lcd_source      =       &dss_lcd_clk_mux_omap5,
 };
 
 static const struct dss_features am43xx_dss_feats = {
@@ -886,6 +982,7 @@ static const struct dss_features dra7xx_dss_feats = {
         .dpi_select_source      =       &dss_dpi_select_source_dra7xx,
         .ports                  =       dra7xx_ports,
         .num_ports              =       ARRAY_SIZE(dra7xx_ports),
+        .select_lcd_source      =       &dss_lcd_clk_mux_dra7,
 };
 
 static int dss_init_features(struct platform_device *pdev)
@@ -1143,18 +1240,18 @@ static int dss_bind(struct device *dev)
         /* Select DPLL */
         REG_FLD_MOD(DSS_CONTROL, 0, 0, 0);
 
-        dss_select_dispc_clk_source(OMAP_DSS_CLK_SRC_FCK);
+        dss_select_dispc_clk_source(DSS_CLK_SRC_FCK);
 
 #ifdef CONFIG_OMAP2_DSS_VENC
         REG_FLD_MOD(DSS_CONTROL, 1, 4, 4);      /* venc dac demen */
         REG_FLD_MOD(DSS_CONTROL, 1, 3, 3);      /* venc clock 4x enable */
         REG_FLD_MOD(DSS_CONTROL, 0, 2, 2);      /* venc clock mode = normal */
 #endif
-        dss.dsi_clk_source[0] = OMAP_DSS_CLK_SRC_FCK;
-        dss.dsi_clk_source[1] = OMAP_DSS_CLK_SRC_FCK;
-        dss.dispc_clk_source = OMAP_DSS_CLK_SRC_FCK;
-        dss.lcd_clk_source[0] = OMAP_DSS_CLK_SRC_FCK;
-        dss.lcd_clk_source[1] = OMAP_DSS_CLK_SRC_FCK;
+        dss.dsi_clk_source[0] = DSS_CLK_SRC_FCK;
+        dss.dsi_clk_source[1] = DSS_CLK_SRC_FCK;
+        dss.dispc_clk_source = DSS_CLK_SRC_FCK;
+        dss.lcd_clk_source[0] = DSS_CLK_SRC_FCK;
+        dss.lcd_clk_source[1] = DSS_CLK_SRC_FCK;
 
         rev = dss_read_reg(DSS_REVISION);
         printk(KERN_INFO "OMAP DSS rev %d.%d\n",
diff --git a/drivers/gpu/drm/omapdrm/dss/dss.h b/drivers/gpu/drm/omapdrm/dss/dss.h
index 38e6ab50142d..4fd06dc41cb3 100644
--- a/drivers/gpu/drm/omapdrm/dss/dss.h
+++ b/drivers/gpu/drm/omapdrm/dss/dss.h
@@ -102,6 +102,20 @@ enum dss_writeback_channel {
         DSS_WB_LCD3_MGR =       7,
 };
 
+enum dss_clk_source {
+        DSS_CLK_SRC_FCK = 0,
+
+        DSS_CLK_SRC_PLL1_1,
+        DSS_CLK_SRC_PLL1_2,
+        DSS_CLK_SRC_PLL1_3,
+
+        DSS_CLK_SRC_PLL2_1,
+        DSS_CLK_SRC_PLL2_2,
+        DSS_CLK_SRC_PLL2_3,
+
+        DSS_CLK_SRC_HDMI_PLL,
+};
+
 enum dss_pll_id {
         DSS_PLL_DSI1,
         DSS_PLL_DSI2,
@@ -114,6 +128,11 @@ struct dss_pll;
 
 #define DSS_PLL_MAX_HSDIVS 4
 
+enum dss_pll_type {
+        DSS_PLL_TYPE_A,
+        DSS_PLL_TYPE_B,
+};
+
 /*
  * Type-A PLLs: clkout[]/mX[] refer to hsdiv outputs m4, m5, m6, m7.
  * Type-B PLLs: clkout[0] refers to m2.
@@ -140,6 +159,8 @@ struct dss_pll_ops {
 };
 
 struct dss_pll_hw {
+        enum dss_pll_type type;
+
         unsigned n_max;
         unsigned m_min;
         unsigned m_max;
@@ -227,7 +248,7 @@ unsigned long dss_get_dispc_clk_rate(void);
 int dss_dpi_select_source(int port, enum omap_channel channel);
 void dss_select_hdmi_venc_clk_source(enum dss_hdmi_venc_clk_source_select);
 enum dss_hdmi_venc_clk_source_select dss_get_hdmi_venc_clk_source(void);
-const char *dss_get_generic_clk_source_name(enum omap_dss_clk_source clk_src);
+const char *dss_get_clk_source_name(enum dss_clk_source clk_src);
 void dss_dump_clocks(struct seq_file *s);
 
 /* DSS VIDEO PLL */
@@ -244,20 +265,18 @@ void dss_debug_dump_clocks(struct seq_file *s);
 #endif
 
 void dss_ctrl_pll_enable(enum dss_pll_id pll_id, bool enable);
-void dss_ctrl_pll_set_control_mux(enum dss_pll_id pll_id,
-        enum omap_channel channel);
 
 void dss_sdi_init(int datapairs);
 int dss_sdi_enable(void);
 void dss_sdi_disable(void);
 
 void dss_select_dsi_clk_source(int dsi_module,
-                enum omap_dss_clk_source clk_src);
+                enum dss_clk_source clk_src);
 void dss_select_lcd_clk_source(enum omap_channel channel,
-                enum omap_dss_clk_source clk_src);
-enum omap_dss_clk_source dss_get_dispc_clk_source(void);
-enum omap_dss_clk_source dss_get_dsi_clk_source(int dsi_module);
-enum omap_dss_clk_source dss_get_lcd_clk_source(enum omap_channel channel);
+                enum dss_clk_source clk_src);
+enum dss_clk_source dss_get_dispc_clk_source(void);
+enum dss_clk_source dss_get_dsi_clk_source(int dsi_module);
+enum dss_clk_source dss_get_lcd_clk_source(enum omap_channel channel);
 
 void dss_set_venc_output(enum omap_dss_venc_type type);
 void dss_set_dac_pwrdn_bgz(bool enable);
@@ -409,17 +428,23 @@ typedef bool (*dss_hsdiv_calc_func)(int m_dispc, unsigned long dispc,
 int dss_pll_register(struct dss_pll *pll);
 void dss_pll_unregister(struct dss_pll *pll);
 struct dss_pll *dss_pll_find(const char *name);
+struct dss_pll *dss_pll_find_by_src(enum dss_clk_source src);
+unsigned dss_pll_get_clkout_idx_for_src(enum dss_clk_source src);
 int dss_pll_enable(struct dss_pll *pll);
 void dss_pll_disable(struct dss_pll *pll);
 int dss_pll_set_config(struct dss_pll *pll,
                 const struct dss_pll_clock_info *cinfo);
 
-bool dss_pll_hsdiv_calc(const struct dss_pll *pll, unsigned long clkdco,
+bool dss_pll_hsdiv_calc_a(const struct dss_pll *pll, unsigned long clkdco,
                 unsigned long out_min, unsigned long out_max,
                 dss_hsdiv_calc_func func, void *data);
-bool dss_pll_calc(const struct dss_pll *pll, unsigned long clkin,
+bool dss_pll_calc_a(const struct dss_pll *pll, unsigned long clkin,
                 unsigned long pll_min, unsigned long pll_max,
                 dss_pll_calc_func func, void *data);
+
+bool dss_pll_calc_b(const struct dss_pll *pll, unsigned long clkin,
+        unsigned long target_clkout, struct dss_pll_clock_info *cinfo);
+
 int dss_pll_write_config_type_a(struct dss_pll *pll,
                 const struct dss_pll_clock_info *cinfo);
 int dss_pll_write_config_type_b(struct dss_pll *pll,
diff --git a/drivers/gpu/drm/omapdrm/dss/dss_features.c b/drivers/gpu/drm/omapdrm/dss/dss_features.c
index c886a2927f73..ee5b93ce2763 100644
--- a/drivers/gpu/drm/omapdrm/dss/dss_features.c
+++ b/drivers/gpu/drm/omapdrm/dss/dss_features.c
@@ -23,8 +23,7 @@
 #include <linux/err.h>
 #include <linux/slab.h>
 
-#include <video/omapdss.h>
-
+#include "omapdss.h"
 #include "dss.h"
 #include "dss_features.h"
 
@@ -50,7 +49,6 @@ struct omap_dss_features {
         const enum omap_dss_output_id *supported_outputs;
         const enum omap_color_mode *supported_color_modes;
         const enum omap_overlay_caps *overlay_caps;
-        const char * const *clksrc_names;
         const struct dss_param_range *dss_params;
 
         const enum omap_dss_rotation_type supported_rotation_types;
@@ -389,34 +387,6 @@ static const enum omap_overlay_caps omap4_dss_overlay_caps[] = {
                 OMAP_DSS_OVL_CAP_POS | OMAP_DSS_OVL_CAP_REPLICATION,
 };
 
-static const char * const omap2_dss_clk_source_names[] = {
-        [OMAP_DSS_CLK_SRC_DSI_PLL_HSDIV_DISPC]  = "N/A",
-        [OMAP_DSS_CLK_SRC_DSI_PLL_HSDIV_DSI]    = "N/A",
-        [OMAP_DSS_CLK_SRC_FCK]                  = "DSS_FCLK1",
-};
-
-static const char * const omap3_dss_clk_source_names[] = {
-        [OMAP_DSS_CLK_SRC_DSI_PLL_HSDIV_DISPC]  = "DSI1_PLL_FCLK",
-        [OMAP_DSS_CLK_SRC_DSI_PLL_HSDIV_DSI]    = "DSI2_PLL_FCLK",
-        [OMAP_DSS_CLK_SRC_FCK]                  = "DSS1_ALWON_FCLK",
-};
-
-static const char * const omap4_dss_clk_source_names[] = {
-        [OMAP_DSS_CLK_SRC_DSI_PLL_HSDIV_DISPC]  = "PLL1_CLK1",
-        [OMAP_DSS_CLK_SRC_DSI_PLL_HSDIV_DSI]    = "PLL1_CLK2",
-        [OMAP_DSS_CLK_SRC_FCK]                  = "DSS_FCLK",
-        [OMAP_DSS_CLK_SRC_DSI2_PLL_HSDIV_DISPC] = "PLL2_CLK1",
-        [OMAP_DSS_CLK_SRC_DSI2_PLL_HSDIV_DSI]   = "PLL2_CLK2",
-};
-
-static const char * const omap5_dss_clk_source_names[] = {
-        [OMAP_DSS_CLK_SRC_DSI_PLL_HSDIV_DISPC]  = "DPLL_DSI1_A_CLK1",
-        [OMAP_DSS_CLK_SRC_DSI_PLL_HSDIV_DSI]    = "DPLL_DSI1_A_CLK2",
-        [OMAP_DSS_CLK_SRC_FCK]                  = "DSS_CLK",
-        [OMAP_DSS_CLK_SRC_DSI2_PLL_HSDIV_DISPC] = "DPLL_DSI1_C_CLK1",
-        [OMAP_DSS_CLK_SRC_DSI2_PLL_HSDIV_DSI]   = "DPLL_DSI1_C_CLK2",
-};
-
 static const struct dss_param_range omap2_dss_param_range[] = {
         [FEAT_PARAM_DSS_FCK]                    = { 0, 133000000 },
         [FEAT_PARAM_DSS_PCD]                    = { 2, 255 },
@@ -631,7 +601,6 @@ static const struct omap_dss_features omap2_dss_features = {
         .supported_outputs = omap2_dss_supported_outputs,
         .supported_color_modes = omap2_dss_supported_color_modes,
         .overlay_caps = omap2_dss_overlay_caps,
-        .clksrc_names = omap2_dss_clk_source_names,
         .dss_params = omap2_dss_param_range,
         .supported_rotation_types = OMAP_DSS_ROT_DMA | OMAP_DSS_ROT_VRFB,
         .buffer_size_unit = 1,
@@ -652,7 +621,6 @@ static const struct omap_dss_features omap3430_dss_features = {
         .supported_outputs = omap3430_dss_supported_outputs,
         .supported_color_modes = omap3_dss_supported_color_modes,
         .overlay_caps = omap3430_dss_overlay_caps,
-        .clksrc_names = omap3_dss_clk_source_names,
         .dss_params = omap3_dss_param_range,
         .supported_rotation_types = OMAP_DSS_ROT_DMA | OMAP_DSS_ROT_VRFB,
         .buffer_size_unit = 1,
@@ -676,7 +644,6 @@ static const struct omap_dss_features am35xx_dss_features = {
         .supported_outputs = omap3430_dss_supported_outputs,
         .supported_color_modes = omap3_dss_supported_color_modes,
         .overlay_caps = omap3430_dss_overlay_caps,
-        .clksrc_names = omap3_dss_clk_source_names,
         .dss_params = omap3_dss_param_range,
         .supported_rotation_types = OMAP_DSS_ROT_DMA | OMAP_DSS_ROT_VRFB,
         .buffer_size_unit = 1,
@@ -696,7 +663,6 @@ static const struct omap_dss_features am43xx_dss_features = {
         .supported_outputs = am43xx_dss_supported_outputs,
         .supported_color_modes = omap3_dss_supported_color_modes,
         .overlay_caps = omap3430_dss_overlay_caps,
-        .clksrc_names = omap2_dss_clk_source_names,
         .dss_params = am43xx_dss_param_range,
         .supported_rotation_types = OMAP_DSS_ROT_DMA,
         .buffer_size_unit = 1,
@@ -716,7 +682,6 @@ static const struct omap_dss_features omap3630_dss_features = {
         .supported_outputs = omap3630_dss_supported_outputs,
         .supported_color_modes = omap3_dss_supported_color_modes,
         .overlay_caps = omap3630_dss_overlay_caps,
-        .clksrc_names = omap3_dss_clk_source_names,
         .dss_params = omap3_dss_param_range,
         .supported_rotation_types = OMAP_DSS_ROT_DMA | OMAP_DSS_ROT_VRFB,
         .buffer_size_unit = 1,
@@ -738,7 +703,6 @@ static const struct omap_dss_features omap4430_es1_0_dss_features  = {
         .supported_outputs = omap4_dss_supported_outputs,
         .supported_color_modes = omap4_dss_supported_color_modes,
         .overlay_caps = omap4_dss_overlay_caps,
-        .clksrc_names = omap4_dss_clk_source_names,
         .dss_params = omap4_dss_param_range,
         .supported_rotation_types = OMAP_DSS_ROT_DMA | OMAP_DSS_ROT_TILER,
         .buffer_size_unit = 16,
@@ -759,7 +723,6 @@ static const struct omap_dss_features omap4430_es2_0_1_2_dss_features = {
         .supported_outputs = omap4_dss_supported_outputs,
         .supported_color_modes = omap4_dss_supported_color_modes,
         .overlay_caps = omap4_dss_overlay_caps,
-        .clksrc_names = omap4_dss_clk_source_names,
         .dss_params = omap4_dss_param_range,
         .supported_rotation_types = OMAP_DSS_ROT_DMA | OMAP_DSS_ROT_TILER,
         .buffer_size_unit = 16,
@@ -780,7 +743,6 @@ static const struct omap_dss_features omap4_dss_features = {
         .supported_outputs = omap4_dss_supported_outputs,
         .supported_color_modes = omap4_dss_supported_color_modes,
         .overlay_caps = omap4_dss_overlay_caps,
-        .clksrc_names = omap4_dss_clk_source_names,
         .dss_params = omap4_dss_param_range,
         .supported_rotation_types = OMAP_DSS_ROT_DMA | OMAP_DSS_ROT_TILER,
         .buffer_size_unit = 16,
@@ -801,7 +763,6 @@ static const struct omap_dss_features omap5_dss_features = {
         .supported_outputs = omap5_dss_supported_outputs,
         .supported_color_modes = omap4_dss_supported_color_modes,
         .overlay_caps = omap4_dss_overlay_caps,
-        .clksrc_names = omap5_dss_clk_source_names,
         .dss_params = omap5_dss_param_range,
         .supported_rotation_types = OMAP_DSS_ROT_DMA | OMAP_DSS_ROT_TILER,
         .buffer_size_unit = 16,
@@ -859,11 +820,6 @@ bool dss_feat_color_mode_supported(enum omap_plane plane,
                         color_mode;
 }
 
-const char *dss_feat_get_clk_source_name(enum omap_dss_clk_source id)
-{
-        return omap_current_dss_features->clksrc_names[id];
-}
-
 u32 dss_feat_get_buffer_size_unit(void)
 {
         return omap_current_dss_features->buffer_size_unit;
diff --git a/drivers/gpu/drm/omapdrm/dss/dss_features.h b/drivers/gpu/drm/omapdrm/dss/dss_features.h
index 3d67d39f192f..bb4b7f0e642b 100644
--- a/drivers/gpu/drm/omapdrm/dss/dss_features.h
+++ b/drivers/gpu/drm/omapdrm/dss/dss_features.h
@@ -91,7 +91,6 @@ unsigned long dss_feat_get_param_max(enum dss_range_param param);
 enum omap_overlay_caps dss_feat_get_overlay_caps(enum omap_plane plane);
 bool dss_feat_color_mode_supported(enum omap_plane plane,
                 enum omap_color_mode color_mode);
-const char *dss_feat_get_clk_source_name(enum omap_dss_clk_source id);
 
 u32 dss_feat_get_buffer_size_unit(void);        /* in bytes */
 u32 dss_feat_get_burst_size_unit(void);         /* in bytes */
diff --git a/drivers/gpu/drm/omapdrm/dss/hdmi.h b/drivers/gpu/drm/omapdrm/dss/hdmi.h
index 53616b02b613..63e711545865 100644
--- a/drivers/gpu/drm/omapdrm/dss/hdmi.h
+++ b/drivers/gpu/drm/omapdrm/dss/hdmi.h
@@ -23,8 +23,9 @@
 #include <linux/io.h>
 #include <linux/platform_device.h>
 #include <linux/hdmi.h>
-#include <video/omapdss.h>
+#include <sound/omap-hdmi-audio.h>
 
+#include "omapdss.h"
 #include "dss.h"
 
 /* HDMI Wrapper */
@@ -240,6 +241,7 @@ struct hdmi_pll_data {
 
         void __iomem *base;
 
+        struct platform_device *pdev;
         struct hdmi_wp_data *wp;
 };
 
@@ -306,8 +308,6 @@ phys_addr_t hdmi_wp_get_audio_dma_addr(struct hdmi_wp_data *wp);
 
 /* HDMI PLL funcs */
 void hdmi_pll_dump(struct hdmi_pll_data *pll, struct seq_file *s);
-void hdmi_pll_compute(struct hdmi_pll_data *pll,
-        unsigned long target_tmds, struct dss_pll_clock_info *pi);
 int hdmi_pll_init(struct platform_device *pdev, struct hdmi_pll_data *pll,
         struct hdmi_wp_data *wp);
 void hdmi_pll_uninit(struct hdmi_pll_data *hpll);
diff --git a/drivers/gpu/drm/omapdrm/dss/hdmi4.c b/drivers/gpu/drm/omapdrm/dss/hdmi4.c
index 4d46cdf7a037..cbd28dfdb86a 100644
--- a/drivers/gpu/drm/omapdrm/dss/hdmi4.c
+++ b/drivers/gpu/drm/omapdrm/dss/hdmi4.c
@@ -34,9 +34,9 @@
 #include <linux/regulator/consumer.h>
 #include <linux/component.h>
 #include <linux/of.h>
-#include <video/omapdss.h>
 #include <sound/omap-hdmi-audio.h>
 
+#include "omapdss.h"
 #include "hdmi4_core.h"
 #include "dss.h"
 #include "dss_features.h"
@@ -177,7 +177,11 @@ static int hdmi_power_on_full(struct omap_dss_device *dssdev)
         if (p->double_pixel)
                 pc *= 2;
 
-        hdmi_pll_compute(&hdmi.pll, pc, &hdmi_cinfo);
+        /* DSS_HDMI_TCLK is bitclk / 10 */
+        pc *= 10;
+
+        dss_pll_calc_b(&hdmi.pll.pll, clk_get_rate(hdmi.pll.pll.clkin),
+                pc, &hdmi_cinfo);
 
         r = dss_pll_enable(&hdmi.pll.pll);
         if (r) {
@@ -204,9 +208,6 @@ static int hdmi_power_on_full(struct omap_dss_device *dssdev)
 
         hdmi4_configure(&hdmi.core, &hdmi.wp, &hdmi.cfg);
 
-        /* bypass TV gamma table */
-        dispc_enable_gamma_table(0);
-
         /* tv size */
         dss_mgr_set_timings(channel, p);
 
diff --git a/drivers/gpu/drm/omapdrm/dss/hdmi5.c b/drivers/gpu/drm/omapdrm/dss/hdmi5.c
index 9255c0e1e4a7..0c0a5139a301 100644
--- a/drivers/gpu/drm/omapdrm/dss/hdmi5.c
+++ b/drivers/gpu/drm/omapdrm/dss/hdmi5.c
@@ -39,9 +39,9 @@
 #include <linux/regulator/consumer.h>
 #include <linux/component.h>
 #include <linux/of.h>
-#include <video/omapdss.h>
 #include <sound/omap-hdmi-audio.h>
 
+#include "omapdss.h"
 #include "hdmi5_core.h"
 #include "dss.h"
 #include "dss_features.h"
@@ -189,7 +189,11 @@ static int hdmi_power_on_full(struct omap_dss_device *dssdev)
         if (p->double_pixel)
                 pc *= 2;
 
-        hdmi_pll_compute(&hdmi.pll, pc, &hdmi_cinfo);
+        /* DSS_HDMI_TCLK is bitclk / 10 */
+        pc *= 10;
+
+        dss_pll_calc_b(&hdmi.pll.pll, clk_get_rate(hdmi.pll.pll.clkin),
+                pc, &hdmi_cinfo);
 
         /* disable and clear irqs */
         hdmi_wp_clear_irqenable(&hdmi.wp, 0xffffffff);
@@ -221,9 +225,6 @@ static int hdmi_power_on_full(struct omap_dss_device *dssdev)
 
         hdmi5_configure(&hdmi.core, &hdmi.wp, &hdmi.cfg);
 
-        /* bypass TV gamma table */
-        dispc_enable_gamma_table(0);
-
         /* tv size */
         dss_mgr_set_timings(channel, p);
 
diff --git a/drivers/gpu/drm/omapdrm/dss/hdmi_common.c b/drivers/gpu/drm/omapdrm/dss/hdmi_common.c
index 1b8fcc6c4ba1..4dfb67fe5f6d 100644
--- a/drivers/gpu/drm/omapdrm/dss/hdmi_common.c
+++ b/drivers/gpu/drm/omapdrm/dss/hdmi_common.c
@@ -4,8 +4,8 @@
 #include <linux/kernel.h>
 #include <linux/err.h>
 #include <linux/of.h>
-#include <video/omapdss.h>
 
+#include "omapdss.h"
 #include "hdmi.h"
 
 int hdmi_parse_lanes_of(struct platform_device *pdev, struct device_node *ep,
diff --git a/drivers/gpu/drm/omapdrm/dss/hdmi_phy.c b/drivers/gpu/drm/omapdrm/dss/hdmi_phy.c
index f98b750fc499..3ead47cccac5 100644
--- a/drivers/gpu/drm/omapdrm/dss/hdmi_phy.c
+++ b/drivers/gpu/drm/omapdrm/dss/hdmi_phy.c
@@ -14,8 +14,8 @@
 #include <linux/platform_device.h>
 #include <linux/slab.h>
 #include <linux/seq_file.h>
-#include <video/omapdss.h>
 
+#include "omapdss.h"
 #include "dss.h"
 #include "hdmi.h"
 
diff --git a/drivers/gpu/drm/omapdrm/dss/hdmi_pll.c b/drivers/gpu/drm/omapdrm/dss/hdmi_pll.c
index f1015e8b8267..b8bf6a9e5557 100644
--- a/drivers/gpu/drm/omapdrm/dss/hdmi_pll.c
+++ b/drivers/gpu/drm/omapdrm/dss/hdmi_pll.c
@@ -17,9 +17,9 @@
 #include <linux/platform_device.h>
 #include <linux/clk.h>
 #include <linux/seq_file.h>
+#include <linux/pm_runtime.h>
 
-#include <video/omapdss.h>
-
+#include "omapdss.h"
 #include "dss.h"
 #include "hdmi.h"
 
@@ -39,71 +39,14 @@ void hdmi_pll_dump(struct hdmi_pll_data *pll, struct seq_file *s)
         DUMPPLL(PLLCTRL_CFG4);
 }
 
-void hdmi_pll_compute(struct hdmi_pll_data *pll,
-        unsigned long target_tmds, struct dss_pll_clock_info *pi)
-{
-        unsigned long fint, clkdco, clkout;
-        unsigned long target_bitclk, target_clkdco;
-        unsigned long min_dco;
-        unsigned n, m, mf, m2, sd;
-        unsigned long clkin;
-        const struct dss_pll_hw *hw = pll->pll.hw;
-
-        clkin = clk_get_rate(pll->pll.clkin);
-
-        DSSDBG("clkin %lu, target tmds %lu\n", clkin, target_tmds);
-
-        target_bitclk = target_tmds * 10;
-
-        /* Fint */
-        n = DIV_ROUND_UP(clkin, hw->fint_max);
-        fint = clkin / n;
-
-        /* adjust m2 so that the clkdco will be high enough */
-        min_dco = roundup(hw->clkdco_min, fint);
-        m2 = DIV_ROUND_UP(min_dco, target_bitclk);
-        if (m2 == 0)
-                m2 = 1;
-
-        target_clkdco = target_bitclk * m2;
-        m = target_clkdco / fint;
-
-        clkdco = fint * m;
-
-        /* adjust clkdco with fractional mf */
-        if (WARN_ON(target_clkdco - clkdco > fint))
-                mf = 0;
-        else
-                mf = (u32)div_u64(262144ull * (target_clkdco - clkdco), fint);
-
-        if (mf > 0)
-                clkdco += (u32)div_u64((u64)mf * fint, 262144);
-
-        clkout = clkdco / m2;
-
-        /* sigma-delta */
-        sd = DIV_ROUND_UP(fint * m, 250000000);
-
-        DSSDBG("N = %u, M = %u, M.f = %u, M2 = %u, SD = %u\n",
-                n, m, mf, m2, sd);
-        DSSDBG("Fint %lu, clkdco %lu, clkout %lu\n", fint, clkdco, clkout);
-
-        pi->n = n;
-        pi->m = m;
-        pi->mf = mf;
-        pi->mX[0] = m2;
-        pi->sd = sd;
-
-        pi->fint = fint;
-        pi->clkdco = clkdco;
-        pi->clkout[0] = clkout;
-}
-
 static int hdmi_pll_enable(struct dss_pll *dsspll)
 {
         struct hdmi_pll_data *pll = container_of(dsspll, struct hdmi_pll_data, pll);
         struct hdmi_wp_data *wp = pll->wp;
-        u16 r = 0;
+        int r;
+
+        r = pm_runtime_get_sync(&pll->pdev->dev);
+        WARN_ON(r < 0);
 
         dss_ctrl_pll_enable(DSS_PLL_HDMI, true);
 
@@ -118,10 +61,14 @@ static void hdmi_pll_disable(struct dss_pll *dsspll)
 {
         struct hdmi_pll_data *pll = container_of(dsspll, struct hdmi_pll_data, pll);
         struct hdmi_wp_data *wp = pll->wp;
+        int r;
 
         hdmi_wp_set_pll_pwr(wp, HDMI_PLLPWRCMD_ALLOFF);
 
         dss_ctrl_pll_enable(DSS_PLL_HDMI, false);
+
+        r = pm_runtime_put_sync(&pll->pdev->dev);
+        WARN_ON(r < 0 && r != -ENOSYS);
 }
 
 static const struct dss_pll_ops dsi_pll_ops = {
@@ -131,6 +78,8 @@ static const struct dss_pll_ops dsi_pll_ops = {
 };
 
 static const struct dss_pll_hw dss_omap4_hdmi_pll_hw = {
+        .type = DSS_PLL_TYPE_B,
+
         .n_max = 255,
         .m_min = 20,
         .m_max = 4095,
@@ -154,6 +103,8 @@ static const struct dss_pll_hw dss_omap4_hdmi_pll_hw = {
 };
 
 static const struct dss_pll_hw dss_omap5_hdmi_pll_hw = {
+        .type = DSS_PLL_TYPE_B,
+
         .n_max = 255,
         .m_min = 20,
         .m_max = 2045,
@@ -225,6 +176,7 @@ int hdmi_pll_init(struct platform_device *pdev, struct hdmi_pll_data *pll,
         int r;
         struct resource *res;
 
+        pll->pdev = pdev;
         pll->wp = wp;
 
         res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "pll");
diff --git a/drivers/gpu/drm/omapdrm/dss/hdmi_wp.c b/drivers/gpu/drm/omapdrm/dss/hdmi_wp.c
index 055f62fca5dc..203694a52d18 100644
--- a/drivers/gpu/drm/omapdrm/dss/hdmi_wp.c
+++ b/drivers/gpu/drm/omapdrm/dss/hdmi_wp.c
@@ -15,8 +15,8 @@
 #include <linux/io.h>
 #include <linux/platform_device.h>
 #include <linux/seq_file.h>
-#include <video/omapdss.h>
 
+#include "omapdss.h"
 #include "dss.h"
 #include "hdmi.h"
 
diff --git a/drivers/gpu/drm/omapdrm/dss/omapdss.h b/drivers/gpu/drm/omapdrm/dss/omapdss.h
index d7e7c909bbc2..6eaf1adbd606 100644
--- a/drivers/gpu/drm/omapdrm/dss/omapdss.h
+++ b/drivers/gpu/drm/omapdrm/dss/omapdss.h
@@ -18,7 +18,872 @@
 #ifndef __OMAP_DRM_DSS_H
 #define __OMAP_DRM_DSS_H
 
-#include <video/omapdss.h>
+#include <linux/list.h>
+#include <linux/kobject.h>
+#include <linux/device.h>
+#include <linux/interrupt.h>
+#include <video/videomode.h>
+#include <linux/platform_data/omapdss.h>
+#include <uapi/drm/drm_mode.h>
+
+#define DISPC_IRQ_FRAMEDONE             (1 << 0)
+#define DISPC_IRQ_VSYNC                 (1 << 1)
+#define DISPC_IRQ_EVSYNC_EVEN           (1 << 2)
+#define DISPC_IRQ_EVSYNC_ODD            (1 << 3)
+#define DISPC_IRQ_ACBIAS_COUNT_STAT     (1 << 4)
+#define DISPC_IRQ_PROG_LINE_NUM         (1 << 5)
+#define DISPC_IRQ_GFX_FIFO_UNDERFLOW    (1 << 6)
+#define DISPC_IRQ_GFX_END_WIN           (1 << 7)
+#define DISPC_IRQ_PAL_GAMMA_MASK        (1 << 8)
+#define DISPC_IRQ_OCP_ERR               (1 << 9)
+#define DISPC_IRQ_VID1_FIFO_UNDERFLOW   (1 << 10)
+#define DISPC_IRQ_VID1_END_WIN          (1 << 11)
+#define DISPC_IRQ_VID2_FIFO_UNDERFLOW   (1 << 12)
+#define DISPC_IRQ_VID2_END_WIN          (1 << 13)
+#define DISPC_IRQ_SYNC_LOST             (1 << 14)
+#define DISPC_IRQ_SYNC_LOST_DIGIT       (1 << 15)
+#define DISPC_IRQ_WAKEUP                (1 << 16)
+#define DISPC_IRQ_SYNC_LOST2            (1 << 17)
+#define DISPC_IRQ_VSYNC2                (1 << 18)
+#define DISPC_IRQ_VID3_END_WIN          (1 << 19)
+#define DISPC_IRQ_VID3_FIFO_UNDERFLOW   (1 << 20)
+#define DISPC_IRQ_ACBIAS_COUNT_STAT2    (1 << 21)
+#define DISPC_IRQ_FRAMEDONE2            (1 << 22)
+#define DISPC_IRQ_FRAMEDONEWB           (1 << 23)
+#define DISPC_IRQ_FRAMEDONETV           (1 << 24)
+#define DISPC_IRQ_WBBUFFEROVERFLOW      (1 << 25)
+#define DISPC_IRQ_WBUNCOMPLETEERROR     (1 << 26)
+#define DISPC_IRQ_SYNC_LOST3            (1 << 27)
+#define DISPC_IRQ_VSYNC3                (1 << 28)
+#define DISPC_IRQ_ACBIAS_COUNT_STAT3    (1 << 29)
+#define DISPC_IRQ_FRAMEDONE3            (1 << 30)
+
+struct omap_dss_device;
+struct omap_overlay_manager;
+struct dss_lcd_mgr_config;
+struct snd_aes_iec958;
+struct snd_cea_861_aud_if;
+struct hdmi_avi_infoframe;
+
+enum omap_display_type {
+        OMAP_DISPLAY_TYPE_NONE          = 0,
+        OMAP_DISPLAY_TYPE_DPI           = 1 << 0,
+        OMAP_DISPLAY_TYPE_DBI           = 1 << 1,
+        OMAP_DISPLAY_TYPE_SDI           = 1 << 2,
+        OMAP_DISPLAY_TYPE_DSI           = 1 << 3,
+        OMAP_DISPLAY_TYPE_VENC          = 1 << 4,
+        OMAP_DISPLAY_TYPE_HDMI          = 1 << 5,
+        OMAP_DISPLAY_TYPE_DVI           = 1 << 6,
+};
+
+enum omap_plane {
+        OMAP_DSS_GFX    = 0,
+        OMAP_DSS_VIDEO1 = 1,
+        OMAP_DSS_VIDEO2 = 2,
+        OMAP_DSS_VIDEO3 = 3,
+        OMAP_DSS_WB     = 4,
+};
+
+enum omap_channel {
+        OMAP_DSS_CHANNEL_LCD    = 0,
+        OMAP_DSS_CHANNEL_DIGIT  = 1,
+        OMAP_DSS_CHANNEL_LCD2   = 2,
+        OMAP_DSS_CHANNEL_LCD3   = 3,
+        OMAP_DSS_CHANNEL_WB     = 4,
+};
+
+enum omap_color_mode {
+        OMAP_DSS_COLOR_CLUT1    = 1 << 0,  /* BITMAP 1 */
+        OMAP_DSS_COLOR_CLUT2    = 1 << 1,  /* BITMAP 2 */
+        OMAP_DSS_COLOR_CLUT4    = 1 << 2,  /* BITMAP 4 */
+        OMAP_DSS_COLOR_CLUT8    = 1 << 3,  /* BITMAP 8 */
+        OMAP_DSS_COLOR_RGB12U   = 1 << 4,  /* RGB12, 16-bit container */
+        OMAP_DSS_COLOR_ARGB16   = 1 << 5,  /* ARGB16 */
+        OMAP_DSS_COLOR_RGB16    = 1 << 6,  /* RGB16 */
+        OMAP_DSS_COLOR_RGB24U   = 1 << 7,  /* RGB24, 32-bit container */
+        OMAP_DSS_COLOR_RGB24P   = 1 << 8,  /* RGB24, 24-bit container */
+        OMAP_DSS_COLOR_YUV2     = 1 << 9,  /* YUV2 4:2:2 co-sited */
+        OMAP_DSS_COLOR_UYVY     = 1 << 10, /* UYVY 4:2:2 co-sited */
+        OMAP_DSS_COLOR_ARGB32   = 1 << 11, /* ARGB32 */
+        OMAP_DSS_COLOR_RGBA32   = 1 << 12, /* RGBA32 */
+        OMAP_DSS_COLOR_RGBX32   = 1 << 13, /* RGBx32 */
+        OMAP_DSS_COLOR_NV12             = 1 << 14, /* NV12 format: YUV 4:2:0 */
+        OMAP_DSS_COLOR_RGBA16           = 1 << 15, /* RGBA16 - 4444 */
+        OMAP_DSS_COLOR_RGBX16           = 1 << 16, /* RGBx16 - 4444 */
+        OMAP_DSS_COLOR_ARGB16_1555      = 1 << 17, /* ARGB16 - 1555 */
+        OMAP_DSS_COLOR_XRGB16_1555      = 1 << 18, /* xRGB16 - 1555 */
+};
+
+enum omap_dss_load_mode {
+        OMAP_DSS_LOAD_CLUT_AND_FRAME    = 0,
+        OMAP_DSS_LOAD_CLUT_ONLY         = 1,
+        OMAP_DSS_LOAD_FRAME_ONLY        = 2,
+        OMAP_DSS_LOAD_CLUT_ONCE_FRAME   = 3,
+};
+
+enum omap_dss_trans_key_type {
+        OMAP_DSS_COLOR_KEY_GFX_DST = 0,
+        OMAP_DSS_COLOR_KEY_VID_SRC = 1,
+};
+
+enum omap_rfbi_te_mode {
+        OMAP_DSS_RFBI_TE_MODE_1 = 1,
+        OMAP_DSS_RFBI_TE_MODE_2 = 2,
+};
+
+enum omap_dss_signal_level {
+        OMAPDSS_SIG_ACTIVE_LOW,
+        OMAPDSS_SIG_ACTIVE_HIGH,
+};
+
+enum omap_dss_signal_edge {
+        OMAPDSS_DRIVE_SIG_FALLING_EDGE,
+        OMAPDSS_DRIVE_SIG_RISING_EDGE,
+};
+
+enum omap_dss_venc_type {
+        OMAP_DSS_VENC_TYPE_COMPOSITE,
+        OMAP_DSS_VENC_TYPE_SVIDEO,
+};
+
+enum omap_dss_dsi_pixel_format {
+        OMAP_DSS_DSI_FMT_RGB888,
+        OMAP_DSS_DSI_FMT_RGB666,
+        OMAP_DSS_DSI_FMT_RGB666_PACKED,
+        OMAP_DSS_DSI_FMT_RGB565,
+};
+
+enum omap_dss_dsi_mode {
+        OMAP_DSS_DSI_CMD_MODE = 0,
+        OMAP_DSS_DSI_VIDEO_MODE,
+};
+
+enum omap_display_caps {
+        OMAP_DSS_DISPLAY_CAP_MANUAL_UPDATE      = 1 << 0,
+        OMAP_DSS_DISPLAY_CAP_TEAR_ELIM          = 1 << 1,
+};
+
+enum omap_dss_display_state {
+        OMAP_DSS_DISPLAY_DISABLED = 0,
+        OMAP_DSS_DISPLAY_ACTIVE,
+};
+
+enum omap_dss_rotation_type {
+        OMAP_DSS_ROT_DMA        = 1 << 0,
+        OMAP_DSS_ROT_VRFB       = 1 << 1,
+        OMAP_DSS_ROT_TILER      = 1 << 2,
+};
+
+/* clockwise rotation angle */
+enum omap_dss_rotation_angle {
+        OMAP_DSS_ROT_0   = 0,
+        OMAP_DSS_ROT_90  = 1,
+        OMAP_DSS_ROT_180 = 2,
+        OMAP_DSS_ROT_270 = 3,
+};
+
+enum omap_overlay_caps {
+        OMAP_DSS_OVL_CAP_SCALE = 1 << 0,
+        OMAP_DSS_OVL_CAP_GLOBAL_ALPHA = 1 << 1,
+        OMAP_DSS_OVL_CAP_PRE_MULT_ALPHA = 1 << 2,
+        OMAP_DSS_OVL_CAP_ZORDER = 1 << 3,
+        OMAP_DSS_OVL_CAP_POS = 1 << 4,
+        OMAP_DSS_OVL_CAP_REPLICATION = 1 << 5,
+};
+
+enum omap_overlay_manager_caps {
+        OMAP_DSS_DUMMY_VALUE, /* add a dummy value to prevent compiler error */
+};
+
+enum omap_dss_clk_source {
+        OMAP_DSS_CLK_SRC_FCK = 0,               /* OMAP2/3: DSS1_ALWON_FCLK
+                                                 * OMAP4: DSS_FCLK */
+        OMAP_DSS_CLK_SRC_DSI_PLL_HSDIV_DISPC,   /* OMAP3: DSI1_PLL_FCLK
+                                                 * OMAP4: PLL1_CLK1 */
+        OMAP_DSS_CLK_SRC_DSI_PLL_HSDIV_DSI,     /* OMAP3: DSI2_PLL_FCLK
+                                                 * OMAP4: PLL1_CLK2 */
+        OMAP_DSS_CLK_SRC_DSI2_PLL_HSDIV_DISPC,  /* OMAP4: PLL2_CLK1 */
+        OMAP_DSS_CLK_SRC_DSI2_PLL_HSDIV_DSI,    /* OMAP4: PLL2_CLK2 */
+};
+
+enum omap_hdmi_flags {
+        OMAP_HDMI_SDA_SCL_EXTERNAL_PULLUP = 1 << 0,
+};
+
+enum omap_dss_output_id {
+        OMAP_DSS_OUTPUT_DPI     = 1 << 0,
+        OMAP_DSS_OUTPUT_DBI     = 1 << 1,
+        OMAP_DSS_OUTPUT_SDI     = 1 << 2,
+        OMAP_DSS_OUTPUT_DSI1    = 1 << 3,
+        OMAP_DSS_OUTPUT_DSI2    = 1 << 4,
+        OMAP_DSS_OUTPUT_VENC    = 1 << 5,
+        OMAP_DSS_OUTPUT_HDMI    = 1 << 6,
+};
+
+/* RFBI */
+
+struct rfbi_timings {
+        int cs_on_time;
+        int cs_off_time;
+        int we_on_time;
+        int we_off_time;
+        int re_on_time;
+        int re_off_time;
+        int we_cycle_time;
+        int re_cycle_time;
+        int cs_pulse_width;
+        int access_time;
+
+        int clk_div;
+
+        u32 tim[5];             /* set by rfbi_convert_timings() */
+
+        int converted;
+};
+
+/* DSI */
+
+enum omap_dss_dsi_trans_mode {
+        /* Sync Pulses: both sync start and end packets sent */
+        OMAP_DSS_DSI_PULSE_MODE,
+        /* Sync Events: only sync start packets sent */
+        OMAP_DSS_DSI_EVENT_MODE,
+        /* Burst: only sync start packets sent, pixels are time compressed */
+        OMAP_DSS_DSI_BURST_MODE,
+};
+
+struct omap_dss_dsi_videomode_timings {
+        unsigned long hsclk;
+
+        unsigned ndl;
+        unsigned bitspp;
+
+        /* pixels */
+        u16 hact;
+        /* lines */
+        u16 vact;
+
+        /* DSI video mode blanking data */
+        /* Unit: byte clock cycles */
+        u16 hss;
+        u16 hsa;
+        u16 hse;
+        u16 hfp;
+        u16 hbp;
+        /* Unit: line clocks */
+        u16 vsa;
+        u16 vfp;
+        u16 vbp;
+
+        /* DSI blanking modes */
+        int blanking_mode;
+        int hsa_blanking_mode;
+        int hbp_blanking_mode;
+        int hfp_blanking_mode;
+
+        enum omap_dss_dsi_trans_mode trans_mode;
+
+        bool ddr_clk_always_on;
+        int window_sync;
+};
+
+struct omap_dss_dsi_config {
+        enum omap_dss_dsi_mode mode;
+        enum omap_dss_dsi_pixel_format pixel_format;
+        const struct omap_video_timings *timings;
+
+        unsigned long hs_clk_min, hs_clk_max;
+        unsigned long lp_clk_min, lp_clk_max;
+
+        bool ddr_clk_always_on;
+        enum omap_dss_dsi_trans_mode trans_mode;
+};
+
+struct omap_video_timings {
+        /* Unit: pixels */
+        u16 x_res;
+        /* Unit: pixels */
+        u16 y_res;
+        /* Unit: Hz */
+        u32 pixelclock;
+        /* Unit: pixel clocks */
+        u16 hsw;        /* Horizontal synchronization pulse width */
+        /* Unit: pixel clocks */
+        u16 hfp;        /* Horizontal front porch */
+        /* Unit: pixel clocks */
+        u16 hbp;        /* Horizontal back porch */
+        /* Unit: line clocks */
+        u16 vsw;        /* Vertical synchronization pulse width */
+        /* Unit: line clocks */
+        u16 vfp;        /* Vertical front porch */
+        /* Unit: line clocks */
+        u16 vbp;        /* Vertical back porch */
+
+        /* Vsync logic level */
+        enum omap_dss_signal_level vsync_level;
+        /* Hsync logic level */
+        enum omap_dss_signal_level hsync_level;
+        /* Interlaced or Progressive timings */
+        bool interlace;
+        /* Pixel clock edge to drive LCD data */
+        enum omap_dss_signal_edge data_pclk_edge;
+        /* Data enable logic level */
+        enum omap_dss_signal_level de_level;
+        /* Pixel clock edges to drive HSYNC and VSYNC signals */
+        enum omap_dss_signal_edge sync_pclk_edge;
+
+        bool double_pixel;
+};
+
+/* Hardcoded timings for tv modes. Venc only uses these to
+ * identify the mode, and does not actually use the configs
+ * itself. However, the configs should be something that
+ * a normal monitor can also show */
+extern const struct omap_video_timings omap_dss_pal_timings;
+extern const struct omap_video_timings omap_dss_ntsc_timings;
+
+struct omap_dss_cpr_coefs {
+        s16 rr, rg, rb;
+        s16 gr, gg, gb;
+        s16 br, bg, bb;
+};
+
+struct omap_overlay_info {
+        dma_addr_t paddr;
+        dma_addr_t p_uv_addr;  /* for NV12 format */
+        u16 screen_width;
+        u16 width;
+        u16 height;
+        enum omap_color_mode color_mode;
+        u8 rotation;
+        enum omap_dss_rotation_type rotation_type;
+        bool mirror;
+
+        u16 pos_x;
+        u16 pos_y;
+        u16 out_width;  /* if 0, out_width == width */
+        u16 out_height; /* if 0, out_height == height */
+        u8 global_alpha;
+        u8 pre_mult_alpha;
+        u8 zorder;
+};
+
+struct omap_overlay {
+        struct kobject kobj;
+        struct list_head list;
+
+        /* static fields */
+        const char *name;
+        enum omap_plane id;
+        enum omap_color_mode supported_modes;
+        enum omap_overlay_caps caps;
+
+        /* dynamic fields */
+        struct omap_overlay_manager *manager;
+
+        /*
+         * The following functions do not block:
+         *
+         * is_enabled
+         * set_overlay_info
+         * get_overlay_info
+         *
+         * The rest of the functions may block and cannot be called from
+         * interrupt context
+         */
+
+        int (*enable)(struct omap_overlay *ovl);
+        int (*disable)(struct omap_overlay *ovl);
+        bool (*is_enabled)(struct omap_overlay *ovl);
+
+        int (*set_manager)(struct omap_overlay *ovl,
+                struct omap_overlay_manager *mgr);
+        int (*unset_manager)(struct omap_overlay *ovl);
+
+        int (*set_overlay_info)(struct omap_overlay *ovl,
+                        struct omap_overlay_info *info);
+        void (*get_overlay_info)(struct omap_overlay *ovl,
+                        struct omap_overlay_info *info);
+
+        int (*wait_for_go)(struct omap_overlay *ovl);
+
+        struct omap_dss_device *(*get_device)(struct omap_overlay *ovl);
+};
+
+struct omap_overlay_manager_info {
+        u32 default_color;
+
+        enum omap_dss_trans_key_type trans_key_type;
+        u32 trans_key;
+        bool trans_enabled;
+
+        bool partial_alpha_enabled;
+
+        bool cpr_enable;
+        struct omap_dss_cpr_coefs cpr_coefs;
+};
+
+struct omap_overlay_manager {
+        struct kobject kobj;
+
+        /* static fields */
+        const char *name;
+        enum omap_channel id;
+        enum omap_overlay_manager_caps caps;
+        struct list_head overlays;
+        enum omap_display_type supported_displays;
+        enum omap_dss_output_id supported_outputs;
+
+        /* dynamic fields */
+        struct omap_dss_device *output;
+
+        /*
+         * The following functions do not block:
+         *
+         * set_manager_info
+         * get_manager_info
+         * apply
+         *
+         * The rest of the functions may block and cannot be called from
+         * interrupt context
+         */
+
+        int (*set_output)(struct omap_overlay_manager *mgr,
+                struct omap_dss_device *output);
+        int (*unset_output)(struct omap_overlay_manager *mgr);
+
+        int (*set_manager_info)(struct omap_overlay_manager *mgr,
+                        struct omap_overlay_manager_info *info);
+        void (*get_manager_info)(struct omap_overlay_manager *mgr,
+                        struct omap_overlay_manager_info *info);
+
+        int (*apply)(struct omap_overlay_manager *mgr);
+        int (*wait_for_go)(struct omap_overlay_manager *mgr);
+        int (*wait_for_vsync)(struct omap_overlay_manager *mgr);
+
+        struct omap_dss_device *(*get_device)(struct omap_overlay_manager *mgr);
+};
+
+/* 22 pins means 1 clk lane and 10 data lanes */
+#define OMAP_DSS_MAX_DSI_PINS 22
+
+struct omap_dsi_pin_config {
+        int num_pins;
+        /*
+         * pin numbers in the following order:
+         * clk+, clk-
+         * data1+, data1-
+         * data2+, data2-
+         * ...
+         */
+        int pins[OMAP_DSS_MAX_DSI_PINS];
+};
+
+struct omap_dss_writeback_info {
+        u32 paddr;
+        u32 p_uv_addr;
+        u16 buf_width;
+        u16 width;
+        u16 height;
+        enum omap_color_mode color_mode;
+        u8 rotation;
+        enum omap_dss_rotation_type rotation_type;
+        bool mirror;
+        u8 pre_mult_alpha;
+};
+
+struct omapdss_dpi_ops {
+        int (*connect)(struct omap_dss_device *dssdev,
+                        struct omap_dss_device *dst);
+        void (*disconnect)(struct omap_dss_device *dssdev,
+                        struct omap_dss_device *dst);
+
+        int (*enable)(struct omap_dss_device *dssdev);
+        void (*disable)(struct omap_dss_device *dssdev);
+
+        int (*check_timings)(struct omap_dss_device *dssdev,
+                        struct omap_video_timings *timings);
+        void (*set_timings)(struct omap_dss_device *dssdev,
+                        struct omap_video_timings *timings);
+        void (*get_timings)(struct omap_dss_device *dssdev,
+                        struct omap_video_timings *timings);
+
+        void (*set_data_lines)(struct omap_dss_device *dssdev, int data_lines);
+};
+
+struct omapdss_sdi_ops {
+        int (*connect)(struct omap_dss_device *dssdev,
+                        struct omap_dss_device *dst);
+        void (*disconnect)(struct omap_dss_device *dssdev,
+                        struct omap_dss_device *dst);
+
+        int (*enable)(struct omap_dss_device *dssdev);
+        void (*disable)(struct omap_dss_device *dssdev);
+
+        int (*check_timings)(struct omap_dss_device *dssdev,
+                        struct omap_video_timings *timings);
+        void (*set_timings)(struct omap_dss_device *dssdev,
+                        struct omap_video_timings *timings);
+        void (*get_timings)(struct omap_dss_device *dssdev,
+                        struct omap_video_timings *timings);
+
+        void (*set_datapairs)(struct omap_dss_device *dssdev, int datapairs);
+};
+
+struct omapdss_dvi_ops {
+        int (*connect)(struct omap_dss_device *dssdev,
+                        struct omap_dss_device *dst);
+        void (*disconnect)(struct omap_dss_device *dssdev,
+                        struct omap_dss_device *dst);
+
+        int (*enable)(struct omap_dss_device *dssdev);
+        void (*disable)(struct omap_dss_device *dssdev);
+
+        int (*check_timings)(struct omap_dss_device *dssdev,
+                        struct omap_video_timings *timings);
+        void (*set_timings)(struct omap_dss_device *dssdev,
+                        struct omap_video_timings *timings);
+        void (*get_timings)(struct omap_dss_device *dssdev,
+                        struct omap_video_timings *timings);
+};
+
+struct omapdss_atv_ops {
+        int (*connect)(struct omap_dss_device *dssdev,
+                        struct omap_dss_device *dst);
+        void (*disconnect)(struct omap_dss_device *dssdev,
+                        struct omap_dss_device *dst);
+
+        int (*enable)(struct omap_dss_device *dssdev);
+        void (*disable)(struct omap_dss_device *dssdev);
+
+        int (*check_timings)(struct omap_dss_device *dssdev,
+                        struct omap_video_timings *timings);
+        void (*set_timings)(struct omap_dss_device *dssdev,
+                        struct omap_video_timings *timings);
+        void (*get_timings)(struct omap_dss_device *dssdev,
+                        struct omap_video_timings *timings);
+
+        void (*set_type)(struct omap_dss_device *dssdev,
+                enum omap_dss_venc_type type);
+        void (*invert_vid_out_polarity)(struct omap_dss_device *dssdev,
+                bool invert_polarity);
+
+        int (*set_wss)(struct omap_dss_device *dssdev, u32 wss);
+        u32 (*get_wss)(struct omap_dss_device *dssdev);
+};
+
+struct omapdss_hdmi_ops {
+        int (*connect)(struct omap_dss_device *dssdev,
+                        struct omap_dss_device *dst);
+        void (*disconnect)(struct omap_dss_device *dssdev,
+                        struct omap_dss_device *dst);
+
+        int (*enable)(struct omap_dss_device *dssdev);
+        void (*disable)(struct omap_dss_device *dssdev);
+
+        int (*check_timings)(struct omap_dss_device *dssdev,
+                        struct omap_video_timings *timings);
+        void (*set_timings)(struct omap_dss_device *dssdev,
+                        struct omap_video_timings *timings);
+        void (*get_timings)(struct omap_dss_device *dssdev,
+                        struct omap_video_timings *timings);
+
+        int (*read_edid)(struct omap_dss_device *dssdev, u8 *buf, int len);
+        bool (*detect)(struct omap_dss_device *dssdev);
+
+        int (*set_hdmi_mode)(struct omap_dss_device *dssdev, bool hdmi_mode);
+        int (*set_infoframe)(struct omap_dss_device *dssdev,
+                const struct hdmi_avi_infoframe *avi);
+};
+
+struct omapdss_dsi_ops {
+        int (*connect)(struct omap_dss_device *dssdev,
+                        struct omap_dss_device *dst);
+        void (*disconnect)(struct omap_dss_device *dssdev,
+                        struct omap_dss_device *dst);
+
+        int (*enable)(struct omap_dss_device *dssdev);
+        void (*disable)(struct omap_dss_device *dssdev, bool disconnect_lanes,
+                        bool enter_ulps);
+
+        /* bus configuration */
+        int (*set_config)(struct omap_dss_device *dssdev,
+                        const struct omap_dss_dsi_config *cfg);
+        int (*configure_pins)(struct omap_dss_device *dssdev,
+                        const struct omap_dsi_pin_config *pin_cfg);
+
+        void (*enable_hs)(struct omap_dss_device *dssdev, int channel,
+                        bool enable);
+        int (*enable_te)(struct omap_dss_device *dssdev, bool enable);
+
+        int (*update)(struct omap_dss_device *dssdev, int channel,
+                        void (*callback)(int, void *), void *data);
+
+        void (*bus_lock)(struct omap_dss_device *dssdev);
+        void (*bus_unlock)(struct omap_dss_device *dssdev);
+
+        int (*enable_video_output)(struct omap_dss_device *dssdev, int channel);
+        void (*disable_video_output)(struct omap_dss_device *dssdev,
+                        int channel);
+
+        int (*request_vc)(struct omap_dss_device *dssdev, int *channel);
+        int (*set_vc_id)(struct omap_dss_device *dssdev, int channel,
+                        int vc_id);
+        void (*release_vc)(struct omap_dss_device *dssdev, int channel);
+
+        /* data transfer */
+        int (*dcs_write)(struct omap_dss_device *dssdev, int channel,
+                        u8 *data, int len);
+        int (*dcs_write_nosync)(struct omap_dss_device *dssdev, int channel,
+                        u8 *data, int len);
+        int (*dcs_read)(struct omap_dss_device *dssdev, int channel, u8 dcs_cmd,
+                        u8 *data, int len);
+
+        int (*gen_write)(struct omap_dss_device *dssdev, int channel,
+                        u8 *data, int len);
+        int (*gen_write_nosync)(struct omap_dss_device *dssdev, int channel,
+                        u8 *data, int len);
+        int (*gen_read)(struct omap_dss_device *dssdev, int channel,
+                        u8 *reqdata, int reqlen,
+                        u8 *data, int len);
+
+        int (*bta_sync)(struct omap_dss_device *dssdev, int channel);
+
+        int (*set_max_rx_packet_size)(struct omap_dss_device *dssdev,
+                        int channel, u16 plen);
+};
+
+struct omap_dss_device {
+        struct kobject kobj;
+        struct device *dev;
+
+        struct module *owner;
+
+        struct list_head panel_list;
+
+        /* alias in the form of "display%d" */
+        char alias[16];
+
+        enum omap_display_type type;
+        enum omap_display_type output_type;
+
+        union {
+                struct {
+                        u8 data_lines;
+                } dpi;
+
+                struct {
+                        u8 channel;
+                        u8 data_lines;
+                } rfbi;
+
+                struct {
+                        u8 datapairs;
+                } sdi;
+
+                struct {
+                        int module;
+                } dsi;
+
+                struct {
+                        enum omap_dss_venc_type type;
+                        bool invert_polarity;
+                } venc;
+        } phy;
+
+        struct {
+                struct omap_video_timings timings;
+
+                enum omap_dss_dsi_pixel_format dsi_pix_fmt;
+                enum omap_dss_dsi_mode dsi_mode;
+        } panel;
+
+        struct {
+                u8 pixel_size;
+                struct rfbi_timings rfbi_timings;
+        } ctrl;
+
+        const char *name;
+
+        /* used to match device to driver */
+        const char *driver_name;
+
+        void *data;
+
+        struct omap_dss_driver *driver;
+
+        union {
+                const struct omapdss_dpi_ops *dpi;
+                const struct omapdss_sdi_ops *sdi;
+                const struct omapdss_dvi_ops *dvi;
+                const struct omapdss_hdmi_ops *hdmi;
+                const struct omapdss_atv_ops *atv;
+                const struct omapdss_dsi_ops *dsi;
+        } ops;
+
+        /* helper variable for driver suspend/resume */
+        bool activate_after_resume;
+
+        enum omap_display_caps caps;
+
+        struct omap_dss_device *src;
+
+        enum omap_dss_display_state state;
+
+        /* OMAP DSS output specific fields */
+
+        struct list_head list;
+
+        /* DISPC channel for this output */
+        enum omap_channel dispc_channel;
+        bool dispc_channel_connected;
+
+        /* output instance */
+        enum omap_dss_output_id id;
+
+        /* the port number in the DT node */
+        int port_num;
+
+        /* dynamic fields */
+        struct omap_overlay_manager *manager;
+
+        struct omap_dss_device *dst;
+};
+
+struct omap_dss_driver {
+        int (*probe)(struct omap_dss_device *);
+        void (*remove)(struct omap_dss_device *);
+
+        int (*connect)(struct omap_dss_device *dssdev);
+        void (*disconnect)(struct omap_dss_device *dssdev);
+
+        int (*enable)(struct omap_dss_device *display);
+        void (*disable)(struct omap_dss_device *display);
+        int (*run_test)(struct omap_dss_device *display, int test);
+
+        int (*update)(struct omap_dss_device *dssdev,
+                               u16 x, u16 y, u16 w, u16 h);
+        int (*sync)(struct omap_dss_device *dssdev);
+
+        int (*enable_te)(struct omap_dss_device *dssdev, bool enable);
+        int (*get_te)(struct omap_dss_device *dssdev);
+
+        u8 (*get_rotate)(struct omap_dss_device *dssdev);
+        int (*set_rotate)(struct omap_dss_device *dssdev, u8 rotate);
+
+        bool (*get_mirror)(struct omap_dss_device *dssdev);
+        int (*set_mirror)(struct omap_dss_device *dssdev, bool enable);
+
+        int (*memory_read)(struct omap_dss_device *dssdev,
+                        void *buf, size_t size,
+                        u16 x, u16 y, u16 w, u16 h);
+
+        void (*get_resolution)(struct omap_dss_device *dssdev,
+                        u16 *xres, u16 *yres);
+        void (*get_dimensions)(struct omap_dss_device *dssdev,
+                        u32 *width, u32 *height);
+        int (*get_recommended_bpp)(struct omap_dss_device *dssdev);
+
+        int (*check_timings)(struct omap_dss_device *dssdev,
+                        struct omap_video_timings *timings);
+        void (*set_timings)(struct omap_dss_device *dssdev,
+                        struct omap_video_timings *timings);
+        void (*get_timings)(struct omap_dss_device *dssdev,
+                        struct omap_video_timings *timings);
+
+        int (*set_wss)(struct omap_dss_device *dssdev, u32 wss);
+        u32 (*get_wss)(struct omap_dss_device *dssdev);
+
+        int (*read_edid)(struct omap_dss_device *dssdev, u8 *buf, int len);
+        bool (*detect)(struct omap_dss_device *dssdev);
+
+        int (*set_hdmi_mode)(struct omap_dss_device *dssdev, bool hdmi_mode);
+        int (*set_hdmi_infoframe)(struct omap_dss_device *dssdev,
+                const struct hdmi_avi_infoframe *avi);
+};
+
+enum omapdss_version omapdss_get_version(void);
+bool omapdss_is_initialized(void);
+
+int omap_dss_register_driver(struct omap_dss_driver *);
+void omap_dss_unregister_driver(struct omap_dss_driver *);
+
+int omapdss_register_display(struct omap_dss_device *dssdev);
+void omapdss_unregister_display(struct omap_dss_device *dssdev);
+
+struct omap_dss_device *omap_dss_get_device(struct omap_dss_device *dssdev);
+void omap_dss_put_device(struct omap_dss_device *dssdev);
+#define for_each_dss_dev(d) while ((d = omap_dss_get_next_device(d)) != NULL)
+struct omap_dss_device *omap_dss_get_next_device(struct omap_dss_device *from);
+struct omap_dss_device *omap_dss_find_device(void *data,
+                int (*match)(struct omap_dss_device *dssdev, void *data));
+const char *omapdss_get_default_display_name(void);
+
+void videomode_to_omap_video_timings(const struct videomode *vm,
+                struct omap_video_timings *ovt);
+void omap_video_timings_to_videomode(const struct omap_video_timings *ovt,
+                struct videomode *vm);
+
+int dss_feat_get_num_mgrs(void);
+int dss_feat_get_num_ovls(void);
+enum omap_color_mode dss_feat_get_supported_color_modes(enum omap_plane plane);
+
+
+
+int omap_dss_get_num_overlay_managers(void);
+struct omap_overlay_manager *omap_dss_get_overlay_manager(int num);
+
+int omap_dss_get_num_overlays(void);
+struct omap_overlay *omap_dss_get_overlay(int num);
+
+int omapdss_register_output(struct omap_dss_device *output);
+void omapdss_unregister_output(struct omap_dss_device *output);
+struct omap_dss_device *omap_dss_get_output(enum omap_dss_output_id id);
+struct omap_dss_device *omap_dss_find_output(const char *name);
+struct omap_dss_device *omap_dss_find_output_by_port_node(struct device_node *port);
+int omapdss_output_set_device(struct omap_dss_device *out,
+                struct omap_dss_device *dssdev);
+int omapdss_output_unset_device(struct omap_dss_device *out);
+
+struct omap_dss_device *omapdss_find_output_from_display(struct omap_dss_device *dssdev);
+struct omap_overlay_manager *omapdss_find_mgr_from_display(struct omap_dss_device *dssdev);
+
+void omapdss_default_get_resolution(struct omap_dss_device *dssdev,
+                u16 *xres, u16 *yres);
+int omapdss_default_get_recommended_bpp(struct omap_dss_device *dssdev);
+void omapdss_default_get_timings(struct omap_dss_device *dssdev,
+                struct omap_video_timings *timings);
+
+typedef void (*omap_dispc_isr_t) (void *arg, u32 mask);
+int omap_dispc_register_isr(omap_dispc_isr_t isr, void *arg, u32 mask);
+int omap_dispc_unregister_isr(omap_dispc_isr_t isr, void *arg, u32 mask);
+
+int omapdss_compat_init(void);
+void omapdss_compat_uninit(void);
+
+static inline bool omapdss_device_is_connected(struct omap_dss_device *dssdev)
+{
+        return dssdev->src;
+}
+
+static inline bool omapdss_device_is_enabled(struct omap_dss_device *dssdev)
+{
+        return dssdev->state == OMAP_DSS_DISPLAY_ACTIVE;
+}
+
+struct device_node *
+omapdss_of_get_next_port(const struct device_node *parent,
+                         struct device_node *prev);
+
+struct device_node *
+omapdss_of_get_next_endpoint(const struct device_node *parent,
+                             struct device_node *prev);
+
+struct device_node *
+omapdss_of_get_first_endpoint(const struct device_node *parent);
+
+struct omap_dss_device *
+omapdss_of_find_source_for_first_ep(struct device_node *node);
 
 u32 dispc_read_irqstatus(void);
 void dispc_clear_irqstatus(u32 mask);
@@ -44,6 +909,10 @@ void dispc_mgr_set_timings(enum omap_channel channel,
                 const struct omap_video_timings *timings);
 void dispc_mgr_setup(enum omap_channel channel,
                 const struct omap_overlay_manager_info *info);
+u32 dispc_mgr_gamma_size(enum omap_channel channel);
+void dispc_mgr_set_gamma(enum omap_channel channel,
+                         const struct drm_color_lut *lut,
+                         unsigned int length);
 
 int dispc_ovl_enable(enum omap_plane plane, bool enable);
 bool dispc_ovl_enabled(enum omap_plane plane);
diff --git a/drivers/gpu/drm/omapdrm/dss/output.c b/drivers/gpu/drm/omapdrm/dss/output.c
index 829232ad8c81..24f859488201 100644
--- a/drivers/gpu/drm/omapdrm/dss/output.c
+++ b/drivers/gpu/drm/omapdrm/dss/output.c
@@ -21,8 +21,7 @@
 #include <linux/slab.h>
 #include <linux/of.h>
 
-#include <video/omapdss.h>
-
+#include "omapdss.h"
 #include "dss.h"
 
 static LIST_HEAD(output_list);
diff --git a/drivers/gpu/drm/omapdrm/dss/pll.c b/drivers/gpu/drm/omapdrm/dss/pll.c
index f974ddcd3b6e..0a76c89cdc2e 100644
--- a/drivers/gpu/drm/omapdrm/dss/pll.c
+++ b/drivers/gpu/drm/omapdrm/dss/pll.c
@@ -22,8 +22,7 @@
 #include <linux/regulator/consumer.h>
 #include <linux/sched.h>
 
-#include <video/omapdss.h>
-
+#include "omapdss.h"
 #include "dss.h"
 
 #define PLL_CONTROL                     0x0000
@@ -76,6 +75,59 @@ struct dss_pll *dss_pll_find(const char *name)
         return NULL;
 }
 
+struct dss_pll *dss_pll_find_by_src(enum dss_clk_source src)
+{
+        struct dss_pll *pll;
+
+        switch (src) {
+        default:
+        case DSS_CLK_SRC_FCK:
+                return NULL;
+
+        case DSS_CLK_SRC_HDMI_PLL:
+                return dss_pll_find("hdmi");
+
+        case DSS_CLK_SRC_PLL1_1:
+        case DSS_CLK_SRC_PLL1_2:
+        case DSS_CLK_SRC_PLL1_3:
+                pll = dss_pll_find("dsi0");
+                if (!pll)
+                        pll = dss_pll_find("video0");
+                return pll;
+
+        case DSS_CLK_SRC_PLL2_1:
+        case DSS_CLK_SRC_PLL2_2:
+        case DSS_CLK_SRC_PLL2_3:
+                pll = dss_pll_find("dsi1");
+                if (!pll)
+                        pll = dss_pll_find("video1");
+                return pll;
+        }
+}
+
+unsigned dss_pll_get_clkout_idx_for_src(enum dss_clk_source src)
+{
+        switch (src) {
+        case DSS_CLK_SRC_HDMI_PLL:
+                return 0;
+
+        case DSS_CLK_SRC_PLL1_1:
+        case DSS_CLK_SRC_PLL2_1:
+                return 0;
+
+        case DSS_CLK_SRC_PLL1_2:
+        case DSS_CLK_SRC_PLL2_2:
+                return 1;
+
+        case DSS_CLK_SRC_PLL1_3:
+        case DSS_CLK_SRC_PLL2_3:
+                return 2;
+
+        default:
+                return 0;
+        }
+}
+
 int dss_pll_enable(struct dss_pll *pll)
 {
         int r;
@@ -129,7 +181,7 @@ int dss_pll_set_config(struct dss_pll *pll, const struct dss_pll_clock_info *cin
         return 0;
 }
 
-bool dss_pll_hsdiv_calc(const struct dss_pll *pll, unsigned long clkdco,
+bool dss_pll_hsdiv_calc_a(const struct dss_pll *pll, unsigned long clkdco,
                 unsigned long out_min, unsigned long out_max,
                 dss_hsdiv_calc_func func, void *data)
 {
@@ -154,7 +206,11 @@ bool dss_pll_hsdiv_calc(const struct dss_pll *pll, unsigned long clkdco,
         return false;
 }
 
-bool dss_pll_calc(const struct dss_pll *pll, unsigned long clkin,
+/*
+ * clkdco = clkin / n * m * 2
+ * clkoutX = clkdco / mX
+ */
+bool dss_pll_calc_a(const struct dss_pll *pll, unsigned long clkin,
                 unsigned long pll_min, unsigned long pll_max,
                 dss_pll_calc_func func, void *data)
 {
@@ -195,6 +251,71 @@ bool dss_pll_calc(const struct dss_pll *pll, unsigned long clkin,
         return false;
 }
 
+/*
+ * This calculates a PLL config that will provide the target_clkout rate
+ * for clkout. Additionally clkdco rate will be the same as clkout rate
+ * when clkout rate is >= min_clkdco.
+ *
+ * clkdco = clkin / n * m + clkin / n * mf / 262144
+ * clkout = clkdco / m2
+ */
+bool dss_pll_calc_b(const struct dss_pll *pll, unsigned long clkin,
+        unsigned long target_clkout, struct dss_pll_clock_info *cinfo)
+{
+        unsigned long fint, clkdco, clkout;
+        unsigned long target_clkdco;
+        unsigned long min_dco;
+        unsigned n, m, mf, m2, sd;
+        const struct dss_pll_hw *hw = pll->hw;
+
+        DSSDBG("clkin %lu, target clkout %lu\n", clkin, target_clkout);
+
+        /* Fint */
+        n = DIV_ROUND_UP(clkin, hw->fint_max);
+        fint = clkin / n;
+
+        /* adjust m2 so that the clkdco will be high enough */
+        min_dco = roundup(hw->clkdco_min, fint);
+        m2 = DIV_ROUND_UP(min_dco, target_clkout);
+        if (m2 == 0)
+                m2 = 1;
+
+        target_clkdco = target_clkout * m2;
+        m = target_clkdco / fint;
+
+        clkdco = fint * m;
+
+        /* adjust clkdco with fractional mf */
+        if (WARN_ON(target_clkdco - clkdco > fint))
+                mf = 0;
+        else
+                mf = (u32)div_u64(262144ull * (target_clkdco - clkdco), fint);
+
+        if (mf > 0)
+                clkdco += (u32)div_u64((u64)mf * fint, 262144);
+
+        clkout = clkdco / m2;
+
+        /* sigma-delta */
+        sd = DIV_ROUND_UP(fint * m, 250000000);
+
+        DSSDBG("N = %u, M = %u, M.f = %u, M2 = %u, SD = %u\n",
+                n, m, mf, m2, sd);
+        DSSDBG("Fint %lu, clkdco %lu, clkout %lu\n", fint, clkdco, clkout);
+
+        cinfo->n = n;
+        cinfo->m = m;
+        cinfo->mf = mf;
+        cinfo->mX[0] = m2;
+        cinfo->sd = sd;
+
+        cinfo->fint = fint;
+        cinfo->clkdco = clkdco;
+        cinfo->clkout[0] = clkout;
+
+        return true;
+}
+
 static int wait_for_bit_change(void __iomem *reg, int bitnum, int value)
 {
         unsigned long timeout;
diff --git a/drivers/gpu/drm/omapdrm/dss/rfbi.c b/drivers/gpu/drm/omapdrm/dss/rfbi.c
index 3796576dfadf..cd53566d75eb 100644
--- a/drivers/gpu/drm/omapdrm/dss/rfbi.c
+++ b/drivers/gpu/drm/omapdrm/dss/rfbi.c
@@ -38,7 +38,7 @@
 #include <linux/pm_runtime.h>
 #include <linux/component.h>
 
-#include <video/omapdss.h>
+#include "omapdss.h"
 #include "dss.h"
 
 struct rfbi_reg { u16 idx; };
diff --git a/drivers/gpu/drm/omapdrm/dss/sdi.c b/drivers/gpu/drm/omapdrm/dss/sdi.c
index cd6d3bfb041d..0a96c321ce62 100644
--- a/drivers/gpu/drm/omapdrm/dss/sdi.c
+++ b/drivers/gpu/drm/omapdrm/dss/sdi.c
@@ -29,7 +29,7 @@
 #include <linux/of.h>
 #include <linux/component.h>
 
-#include <video/omapdss.h>
+#include "omapdss.h"
 #include "dss.h"
 
 static struct {
diff --git a/drivers/gpu/drm/omapdrm/dss/venc.c b/drivers/gpu/drm/omapdrm/dss/venc.c
index 08a2cc778ba9..6eedf2118708 100644
--- a/drivers/gpu/drm/omapdrm/dss/venc.c
+++ b/drivers/gpu/drm/omapdrm/dss/venc.c
@@ -37,8 +37,7 @@
 #include <linux/of.h>
 #include <linux/component.h>
 
-#include <video/omapdss.h>
-
+#include "omapdss.h"
 #include "dss.h"
 #include "dss_features.h"
 
diff --git a/drivers/gpu/drm/omapdrm/dss/video-pll.c b/drivers/gpu/drm/omapdrm/dss/video-pll.c
index b1ec59e42940..7429de928d4e 100644
--- a/drivers/gpu/drm/omapdrm/dss/video-pll.c
+++ b/drivers/gpu/drm/omapdrm/dss/video-pll.c
@@ -17,8 +17,7 @@
 #include <linux/platform_device.h>
 #include <linux/sched.h>
 
-#include <video/omapdss.h>
-
+#include "omapdss.h"
 #include "dss.h"
 #include "dss_features.h"
 
@@ -108,6 +107,8 @@ static const struct dss_pll_ops dss_pll_ops = {
 };
 
 static const struct dss_pll_hw dss_dra7_video_pll_hw = {
+        .type = DSS_PLL_TYPE_A,
+
         .n_max = (1 << 8) - 1,
         .m_max = (1 << 12) - 1,
         .mX_max = (1 << 5) - 1,
@@ -124,6 +125,10 @@ static const struct dss_pll_hw dss_dra7_video_pll_hw = {
         .mX_lsb[0] = 21,
         .mX_msb[1] = 30,
         .mX_lsb[1] = 26,
+        .mX_msb[2] = 4,
+        .mX_lsb[2] = 0,
+        .mX_msb[3] = 9,
+        .mX_lsb[3] = 5,
 
         .has_refsel = true,
 };
diff --git a/drivers/gpu/drm/omapdrm/omap_connector.c b/drivers/gpu/drm/omapdrm/omap_connector.c
index ce2d67b6a8c7..137fe690a0da 100644
--- a/drivers/gpu/drm/omapdrm/omap_connector.c
+++ b/drivers/gpu/drm/omapdrm/omap_connector.c
@@ -32,7 +32,6 @@
 struct omap_connector {
         struct drm_connector base;
         struct omap_dss_device *dssdev;
-        struct drm_encoder *encoder;
         bool hdmi_mode;
 };
 
@@ -256,13 +255,6 @@ static int omap_connector_mode_valid(struct drm_connector *connector,
         return ret;
 }
 
-struct drm_encoder *omap_connector_attached_encoder(
-                struct drm_connector *connector)
-{
-        struct omap_connector *omap_connector = to_omap_connector(connector);
-        return omap_connector->encoder;
-}
-
 static const struct drm_connector_funcs omap_connector_funcs = {
         .dpms = drm_atomic_helper_connector_dpms,
         .reset = drm_atomic_helper_connector_reset,
@@ -276,7 +268,6 @@ static const struct drm_connector_funcs omap_connector_funcs = {
 static const struct drm_connector_helper_funcs omap_connector_helper_funcs = {
         .get_modes = omap_connector_get_modes,
         .mode_valid = omap_connector_mode_valid,
-        .best_encoder = omap_connector_attached_encoder,
 };
 
 /* initialize connector */
@@ -296,7 +287,6 @@ struct drm_connector *omap_connector_init(struct drm_device *dev,
                 goto fail;
 
         omap_connector->dssdev = dssdev;
-        omap_connector->encoder = encoder;
 
         connector = &omap_connector->base;
 
diff --git a/drivers/gpu/drm/omapdrm/omap_crtc.c b/drivers/gpu/drm/omapdrm/omap_crtc.c
index 075f2bb44867..180f644e861e 100644
--- a/drivers/gpu/drm/omapdrm/omap_crtc.c
+++ b/drivers/gpu/drm/omapdrm/omap_crtc.c
@@ -372,6 +372,20 @@ static void omap_crtc_mode_set_nofb(struct drm_crtc *crtc)
         copy_timings_drm_to_omap(&omap_crtc->timings, mode);
 }
 
+static int omap_crtc_atomic_check(struct drm_crtc *crtc,
+                                struct drm_crtc_state *state)
+{
+        if (state->color_mgmt_changed && state->gamma_lut) {
+                uint length = state->gamma_lut->length /
+                        sizeof(struct drm_color_lut);
+
+                if (length < 2)
+                        return -EINVAL;
+        }
+
+        return 0;
+}
+
 static void omap_crtc_atomic_begin(struct drm_crtc *crtc,
                                   struct drm_crtc_state *old_crtc_state)
 {
@@ -384,6 +398,32 @@ static void omap_crtc_atomic_flush(struct drm_crtc *crtc,
 
         WARN_ON(omap_crtc->vblank_irq.registered);
 
+        if (crtc->state->color_mgmt_changed) {
+                struct drm_color_lut *lut = NULL;
+                uint length = 0;
+
+                if (crtc->state->gamma_lut) {
+                        lut = (struct drm_color_lut *)
+                                crtc->state->gamma_lut->data;
+                        length = crtc->state->gamma_lut->length /
+                                sizeof(*lut);
+                }
+                dispc_mgr_set_gamma(omap_crtc->channel, lut, length);
+        }
+
+        if (crtc->state->color_mgmt_changed) {
+                struct drm_color_lut *lut = NULL;
+                uint length = 0;
+
+                if (crtc->state->gamma_lut) {
+                        lut = (struct drm_color_lut *)
+                                crtc->state->gamma_lut->data;
+                        length = crtc->state->gamma_lut->length /
+                                sizeof(*lut);
+                }
+                dispc_mgr_set_gamma(omap_crtc->channel, lut, length);
+        }
+
         if (dispc_mgr_is_enabled(omap_crtc->channel)) {
 
                 DBG("%s: GO", omap_crtc->name);
@@ -460,6 +500,7 @@ static const struct drm_crtc_funcs omap_crtc_funcs = {
         .set_config = drm_atomic_helper_set_config,
         .destroy = omap_crtc_destroy,
         .page_flip = drm_atomic_helper_page_flip,
+        .gamma_set = drm_atomic_helper_legacy_gamma_set,
         .set_property = drm_atomic_helper_crtc_set_property,
         .atomic_duplicate_state = drm_atomic_helper_crtc_duplicate_state,
         .atomic_destroy_state = drm_atomic_helper_crtc_destroy_state,
@@ -471,6 +512,7 @@ static const struct drm_crtc_helper_funcs omap_crtc_helper_funcs = {
         .mode_set_nofb = omap_crtc_mode_set_nofb,
         .disable = omap_crtc_disable,
         .enable = omap_crtc_enable,
+        .atomic_check = omap_crtc_atomic_check,
         .atomic_begin = omap_crtc_atomic_begin,
         .atomic_flush = omap_crtc_atomic_flush,
 };
@@ -534,6 +576,20 @@ struct drm_crtc *omap_crtc_init(struct drm_device *dev,
 
         drm_crtc_helper_add(crtc, &omap_crtc_helper_funcs);
 
+        /* The dispc API adapts to what ever size, but the HW supports
+         * 256 element gamma table for LCDs and 1024 element table for
+         * OMAP_DSS_CHANNEL_DIGIT. X server assumes 256 element gamma
+         * tables so lets use that. Size of HW gamma table can be
+         * extracted with dispc_mgr_gamma_size(). If it returns 0
+         * gamma table is not supprted.
+         */
+        if (dispc_mgr_gamma_size(channel)) {
+                uint gamma_lut_size = 256;
+
+                drm_crtc_enable_color_mgmt(crtc, 0, false, gamma_lut_size);
+                drm_mode_crtc_set_gamma_size(crtc, gamma_lut_size);
+        }
+
         omap_plane_install_properties(crtc->primary, &crtc->base);
 
         omap_crtcs[channel] = omap_crtc;
diff --git a/drivers/gpu/drm/omapdrm/omap_drv.c b/drivers/gpu/drm/omapdrm/omap_drv.c
index d86f5479345b..26c6134eb744 100644
--- a/drivers/gpu/drm/omapdrm/omap_drv.c
+++ b/drivers/gpu/drm/omapdrm/omap_drv.c
@@ -142,8 +142,9 @@ static int omap_atomic_commit(struct drm_device *dev,
 {
         struct omap_drm_private *priv = dev->dev_private;
         struct omap_atomic_state_commit *commit;
-        unsigned int i;
-        int ret;
+        struct drm_crtc *crtc;
+        struct drm_crtc_state *crtc_state;
+        int i, ret;
 
         ret = drm_atomic_helper_prepare_planes(dev, state);
         if (ret)
@@ -163,10 +164,8 @@ static int omap_atomic_commit(struct drm_device *dev,
         /* Wait until all affected CRTCs have completed previous commits and
          * mark them as pending.
          */
-        for (i = 0; i < dev->mode_config.num_crtc; ++i) {
-                if (state->crtcs[i])
-                        commit->crtcs |= 1 << drm_crtc_index(state->crtcs[i]);
-        }
+        for_each_crtc_in_state(state, crtc, crtc_state, i)
+                commit->crtcs |= drm_crtc_mask(crtc);
 
         wait_event(priv->commit.wait, !omap_atomic_is_pending(priv, commit));
 
@@ -175,7 +174,7 @@ static int omap_atomic_commit(struct drm_device *dev,
         spin_unlock(&priv->commit.lock);
 
         /* Swap the state, this is the point of no return. */
-        drm_atomic_helper_swap_state(dev, state);
+        drm_atomic_helper_swap_state(state, true);
 
         if (nonblock)
                 schedule_work(&commit->work);
@@ -203,6 +202,8 @@ static int get_connector_type(struct omap_dss_device *dssdev)
                 return DRM_MODE_CONNECTOR_HDMIA;
         case OMAP_DISPLAY_TYPE_DVI:
                 return DRM_MODE_CONNECTOR_DVID;
+        case OMAP_DISPLAY_TYPE_DSI:
+                return DRM_MODE_CONNECTOR_DSI;
         default:
                 return DRM_MODE_CONNECTOR_Unknown;
         }
@@ -800,7 +801,6 @@ static struct drm_driver omap_drm_driver = {
         .unload = dev_unload,
         .open = dev_open,
         .lastclose = dev_lastclose,
-        .set_busid = drm_platform_set_busid,
         .get_vblank_counter = drm_vblank_no_hw_counter,
         .enable_vblank = omap_irq_enable_vblank,
         .disable_vblank = omap_irq_disable_vblank,
diff --git a/drivers/gpu/drm/omapdrm/omap_drv.h b/drivers/gpu/drm/omapdrm/omap_drv.h
index 3f823c368912..dcc30a98b9d4 100644
--- a/drivers/gpu/drm/omapdrm/omap_drv.h
+++ b/drivers/gpu/drm/omapdrm/omap_drv.h
@@ -24,7 +24,6 @@
 #include <linux/platform_data/omap_drm.h>
 #include <linux/types.h>
 #include <linux/wait.h>
-#include <video/omapdss.h>
 
 #include <drm/drmP.h>
 #include <drm/drm_crtc_helper.h>
@@ -183,7 +182,6 @@ struct drm_framebuffer *omap_framebuffer_create(struct drm_device *dev,
                 struct drm_file *file, const struct drm_mode_fb_cmd2 *mode_cmd);
 struct drm_framebuffer *omap_framebuffer_init(struct drm_device *dev,
                 const struct drm_mode_fb_cmd2 *mode_cmd, struct drm_gem_object **bos);
-struct drm_gem_object *omap_framebuffer_bo(struct drm_framebuffer *fb, int p);
 int omap_framebuffer_pin(struct drm_framebuffer *fb);
 void omap_framebuffer_unpin(struct drm_framebuffer *fb);
 void omap_framebuffer_update_scanout(struct drm_framebuffer *fb,
@@ -231,7 +229,6 @@ int omap_gem_rotated_paddr(struct drm_gem_object *obj, uint32_t orient,
                 int x, int y, dma_addr_t *paddr);
 uint64_t omap_gem_mmap_offset(struct drm_gem_object *obj);
 size_t omap_gem_mmap_size(struct drm_gem_object *obj);
-int omap_gem_tiled_size(struct drm_gem_object *obj, uint16_t *w, uint16_t *h);
 int omap_gem_tiled_stride(struct drm_gem_object *obj, uint32_t orient);
 
 struct dma_buf *omap_gem_prime_export(struct drm_device *dev,
@@ -239,17 +236,6 @@ struct dma_buf *omap_gem_prime_export(struct drm_device *dev,
 struct drm_gem_object *omap_gem_prime_import(struct drm_device *dev,
                 struct dma_buf *buffer);
 
-static inline int align_pitch(int pitch, int width, int bpp)
-{
-        int bytespp = (bpp + 7) / 8;
-        /* in case someone tries to feed us a completely bogus stride: */
-        pitch = max(pitch, width * bytespp);
-        /* PVR needs alignment to 8 pixels.. right now that is the most
-         * restrictive stride requirement..
-         */
-        return roundup(pitch, 8 * bytespp);
-}
-
 /* map crtc to vblank mask */
 uint32_t pipe2vbl(struct drm_crtc *crtc);
 struct omap_dss_device *omap_encoder_get_dssdev(struct drm_encoder *encoder);
diff --git a/drivers/gpu/drm/omapdrm/omap_fb.c b/drivers/gpu/drm/omapdrm/omap_fb.c
index f84570d1636c..31f5178c22c7 100644
--- a/drivers/gpu/drm/omapdrm/omap_fb.c
+++ b/drivers/gpu/drm/omapdrm/omap_fb.c
@@ -115,24 +115,16 @@ static void omap_framebuffer_destroy(struct drm_framebuffer *fb)
 
         for (i = 0; i < n; i++) {
                 struct plane *plane = &omap_fb->planes[i];
-                if (plane->bo)
-                        drm_gem_object_unreference_unlocked(plane->bo);
+
+                drm_gem_object_unreference_unlocked(plane->bo);
         }
 
         kfree(omap_fb);
 }
 
-static int omap_framebuffer_dirty(struct drm_framebuffer *fb,
-                struct drm_file *file_priv, unsigned flags, unsigned color,
-                struct drm_clip_rect *clips, unsigned num_clips)
-{
-        return 0;
-}
-
 static const struct drm_framebuffer_funcs omap_framebuffer_funcs = {
         .create_handle = omap_framebuffer_create_handle,
         .destroy = omap_framebuffer_destroy,
-        .dirty = omap_framebuffer_dirty,
 };
 
 static uint32_t get_linear_addr(struct plane *plane,
@@ -320,14 +312,6 @@ void omap_framebuffer_unpin(struct drm_framebuffer *fb)
         mutex_unlock(&omap_fb->lock);
 }
 
-struct drm_gem_object *omap_framebuffer_bo(struct drm_framebuffer *fb, int p)
-{
-        struct omap_framebuffer *omap_fb = to_omap_framebuffer(fb);
-        if (p >= drm_format_num_planes(fb->pixel_format))
-                return NULL;
-        return omap_fb->planes[p].bo;
-}
-
 /* iterate thru all the connectors, returning ones that are attached
  * to the same fb..
  */
diff --git a/drivers/gpu/drm/omapdrm/omap_fbdev.c b/drivers/gpu/drm/omapdrm/omap_fbdev.c
index 89da41ac64d2..adb10fbe918d 100644
--- a/drivers/gpu/drm/omapdrm/omap_fbdev.c
+++ b/drivers/gpu/drm/omapdrm/omap_fbdev.c
@@ -125,9 +125,8 @@ static int omap_fbdev_create(struct drm_fb_helper *helper,
         mode_cmd.width = sizes->surface_width;
         mode_cmd.height = sizes->surface_height;
 
-        mode_cmd.pitches[0] = align_pitch(
-                        mode_cmd.width * ((sizes->surface_bpp + 7) / 8),
-                        mode_cmd.width, sizes->surface_bpp);
+        mode_cmd.pitches[0] =
+                        DIV_ROUND_UP(mode_cmd.width * sizes->surface_bpp, 8);
 
         fbdev->ywrap_enabled = priv->has_dmm && ywrap_enabled;
         if (fbdev->ywrap_enabled) {
@@ -280,9 +279,6 @@ struct drm_fb_helper *omap_fbdev_init(struct drm_device *dev)
         if (ret)
                 goto fini;
 
-        /* disable all the possible outputs/crtcs before entering KMS mode */
-        drm_helper_disable_unused_functions(dev);
-
         ret = drm_fb_helper_initial_config(helper, 32);
         if (ret)
                 goto fini;
diff --git a/drivers/gpu/drm/omapdrm/omap_gem.c b/drivers/gpu/drm/omapdrm/omap_gem.c
index 0dbd0f03f9bd..505dee0db973 100644
--- a/drivers/gpu/drm/omapdrm/omap_gem.c
+++ b/drivers/gpu/drm/omapdrm/omap_gem.c
@@ -383,18 +383,6 @@ size_t omap_gem_mmap_size(struct drm_gem_object *obj)
         return size;
 }
 
-/* get tiled size, returns -EINVAL if not tiled buffer */
-int omap_gem_tiled_size(struct drm_gem_object *obj, uint16_t *w, uint16_t *h)
-{
-        struct omap_gem_object *omap_obj = to_omap_bo(obj);
-        if (omap_obj->flags & OMAP_BO_TILED) {
-                *w = omap_obj->width;
-                *h = omap_obj->height;
-                return 0;
-        }
-        return -EINVAL;
-}
-
 /* -----------------------------------------------------------------------------
  * Fault Handling
  */
@@ -661,7 +649,8 @@ int omap_gem_dumb_create(struct drm_file *file, struct drm_device *dev,
 {
         union omap_gem_size gsize;
 
-        args->pitch = align_pitch(0, args->width, args->bpp);
+        args->pitch = DIV_ROUND_UP(args->width * args->bpp, 8);
+
         args->size = PAGE_ALIGN(args->pitch * args->height);
 
         gsize = (union omap_gem_size){
diff --git a/drivers/gpu/drm/panel/panel-simple.c b/drivers/gpu/drm/panel/panel-simple.c
index 3a7bdf1c842b..85143d1b9b31 100644
--- a/drivers/gpu/drm/panel/panel-simple.c
+++ b/drivers/gpu/drm/panel/panel-simple.c
@@ -168,6 +168,7 @@ static int panel_simple_disable(struct drm_panel *panel)
 
         if (p->backlight) {
                 p->backlight->props.power = FB_BLANK_POWERDOWN;
+                p->backlight->props.state |= BL_CORE_FBBLANK;
                 backlight_update_status(p->backlight);
         }
 
@@ -235,6 +236,7 @@ static int panel_simple_enable(struct drm_panel *panel)
                 msleep(p->desc->delay.enable);
 
         if (p->backlight) {
+                p->backlight->props.state &= ~BL_CORE_FBBLANK;
                 p->backlight->props.power = FB_BLANK_UNBLANK;
                 backlight_update_status(p->backlight);
         }
@@ -964,8 +966,8 @@ static const struct panel_desc innolux_zj070na_01p = {
         .num_modes = 1,
         .bpc = 6,
         .size = {
-                .width = 1024,
-                .height = 600,
+                .width = 154,
+                .height = 90,
         },
 };
 
@@ -1017,6 +1019,51 @@ static const struct panel_desc lg_lb070wv8 = {
         .bus_format = MEDIA_BUS_FMT_RGB888_1X7X4_SPWG,
 };
 
+static const struct drm_display_mode lg_lp079qx1_sp0v_mode = {
+        .clock = 200000,
+        .hdisplay = 1536,
+        .hsync_start = 1536 + 12,
+        .hsync_end = 1536 + 12 + 16,
+        .htotal = 1536 + 12 + 16 + 48,
+        .vdisplay = 2048,
+        .vsync_start = 2048 + 8,
+        .vsync_end = 2048 + 8 + 4,
+        .vtotal = 2048 + 8 + 4 + 8,
+        .vrefresh = 60,
+        .flags = DRM_MODE_FLAG_NVSYNC | DRM_MODE_FLAG_NHSYNC,
+};
+
+static const struct panel_desc lg_lp079qx1_sp0v = {
+        .modes = &lg_lp079qx1_sp0v_mode,
+        .num_modes = 1,
+        .size = {
+                .width = 129,
+                .height = 171,
+        },
+};
+
+static const struct drm_display_mode lg_lp097qx1_spa1_mode = {
+        .clock = 205210,
+        .hdisplay = 2048,
+        .hsync_start = 2048 + 150,
+        .hsync_end = 2048 + 150 + 5,
+        .htotal = 2048 + 150 + 5 + 5,
+        .vdisplay = 1536,
+        .vsync_start = 1536 + 3,
+        .vsync_end = 1536 + 3 + 1,
+        .vtotal = 1536 + 3 + 1 + 9,
+        .vrefresh = 60,
+};
+
+static const struct panel_desc lg_lp097qx1_spa1 = {
+        .modes = &lg_lp097qx1_spa1_mode,
+        .num_modes = 1,
+        .size = {
+                .width = 208,
+                .height = 147,
+        },
+};
+
 static const struct drm_display_mode lg_lp120up1_mode = {
         .clock = 162300,
         .hdisplay = 1920,
@@ -1224,6 +1271,28 @@ static const struct panel_desc qd43003c0_40 = {
         .bus_format = MEDIA_BUS_FMT_RGB888_1X24,
 };
 
+static const struct drm_display_mode samsung_lsn122dl01_c01_mode = {
+        .clock = 271560,
+        .hdisplay = 2560,
+        .hsync_start = 2560 + 48,
+        .hsync_end = 2560 + 48 + 32,
+        .htotal = 2560 + 48 + 32 + 80,
+        .vdisplay = 1600,
+        .vsync_start = 1600 + 2,
+        .vsync_end = 1600 + 2 + 5,
+        .vtotal = 1600 + 2 + 5 + 57,
+        .vrefresh = 60,
+};
+
+static const struct panel_desc samsung_lsn122dl01_c01 = {
+        .modes = &samsung_lsn122dl01_c01_mode,
+        .num_modes = 1,
+        .size = {
+                .width = 263,
+                .height = 164,
+        },
+};
+
 static const struct drm_display_mode samsung_ltn101nt05_mode = {
         .clock = 54030,
         .hdisplay = 1024,
@@ -1242,8 +1311,8 @@ static const struct panel_desc samsung_ltn101nt05 = {
         .num_modes = 1,
         .bpc = 6,
         .size = {
-                .width = 1024,
-                .height = 600,
+                .width = 223,
+                .height = 125,
         },
 };
 
@@ -1270,6 +1339,53 @@ static const struct panel_desc samsung_ltn140at29_301 = {
         },
 };
 
+static const struct display_timing sharp_lq101k1ly04_timing = {
+        .pixelclock = { 60000000, 65000000, 80000000 },
+        .hactive = { 1280, 1280, 1280 },
+        .hfront_porch = { 20, 20, 20 },
+        .hback_porch = { 20, 20, 20 },
+        .hsync_len = { 10, 10, 10 },
+        .vactive = { 800, 800, 800 },
+        .vfront_porch = { 4, 4, 4 },
+        .vback_porch = { 4, 4, 4 },
+        .vsync_len = { 4, 4, 4 },
+        .flags = DISPLAY_FLAGS_PIXDATA_POSEDGE,
+};
+
+static const struct panel_desc sharp_lq101k1ly04 = {
+        .timings = &sharp_lq101k1ly04_timing,
+        .num_timings = 1,
+        .bpc = 8,
+        .size = {
+                .width = 217,
+                .height = 136,
+        },
+        .bus_format = MEDIA_BUS_FMT_RGB888_1X7X4_JEIDA,
+};
+
+static const struct drm_display_mode sharp_lq123p1jx31_mode = {
+        .clock = 252750,
+        .hdisplay = 2400,
+        .hsync_start = 2400 + 48,
+        .hsync_end = 2400 + 48 + 32,
+        .htotal = 2400 + 48 + 32 + 80,
+        .vdisplay = 1600,
+        .vsync_start = 1600 + 3,
+        .vsync_end = 1600 + 3 + 10,
+        .vtotal = 1600 + 3 + 10 + 33,
+        .vrefresh = 60,
+        .flags = DRM_MODE_FLAG_NVSYNC | DRM_MODE_FLAG_NHSYNC,
+};
+
+static const struct panel_desc sharp_lq123p1jx31 = {
+        .modes = &sharp_lq123p1jx31_mode,
+        .num_modes = 1,
+        .size = {
+                .width = 259,
+                .height = 173,
+        },
+};
+
 static const struct drm_display_mode shelly_sca07010_bfn_lnn_mode = {
         .clock = 33300,
         .hdisplay = 800,
@@ -1293,6 +1409,29 @@ static const struct panel_desc shelly_sca07010_bfn_lnn = {
         .bus_format = MEDIA_BUS_FMT_RGB666_1X18,
 };
 
+static const struct drm_display_mode starry_kr122ea0sra_mode = {
+        .clock = 147000,
+        .hdisplay = 1920,
+        .hsync_start = 1920 + 16,
+        .hsync_end = 1920 + 16 + 16,
+        .htotal = 1920 + 16 + 16 + 32,
+        .vdisplay = 1200,
+        .vsync_start = 1200 + 15,
+        .vsync_end = 1200 + 15 + 2,
+        .vtotal = 1200 + 15 + 2 + 18,
+        .vrefresh = 60,
+        .flags = DRM_MODE_FLAG_NVSYNC | DRM_MODE_FLAG_NHSYNC,
+};
+
+static const struct panel_desc starry_kr122ea0sra = {
+        .modes = &starry_kr122ea0sra_mode,
+        .num_modes = 1,
+        .size = {
+                .width = 263,
+                .height = 164,
+        },
+};
+
 static const struct drm_display_mode tpk_f07a_0102_mode = {
         .clock = 33260,
         .hdisplay = 800,
@@ -1457,6 +1596,12 @@ static const struct of_device_id platform_of_match[] = {
                 .compatible = "lg,lb070wv8",
                 .data = &lg_lb070wv8,
         }, {
+                .compatible = "lg,lp079qx1-sp0v",
+                .data = &lg_lp079qx1_sp0v,
+        }, {
+                .compatible = "lg,lp097qx1-spa1",
+                .data = &lg_lp097qx1_spa1,
+        }, {
                 .compatible = "lg,lp120up1",
                 .data = &lg_lp120up1,
         }, {
@@ -1481,15 +1626,27 @@ static const struct of_device_id platform_of_match[] = {
                 .compatible = "qiaodian,qd43003c0-40",
                 .data = &qd43003c0_40,
         }, {
+                .compatible = "samsung,lsn122dl01-c01",
+                .data = &samsung_lsn122dl01_c01,
+        }, {
                 .compatible = "samsung,ltn101nt05",
                 .data = &samsung_ltn101nt05,
         }, {
                 .compatible = "samsung,ltn140at29-301",
                 .data = &samsung_ltn140at29_301,
         }, {
+                .compatible = "sharp,lq101k1ly04",
+                .data = &sharp_lq101k1ly04,
+        }, {
+                .compatible = "sharp,lq123p1jx31",
+                .data = &sharp_lq123p1jx31,
+        }, {
                 .compatible = "shelly,sca07010-bfn-lnn",
                 .data = &shelly_sca07010_bfn_lnn,
         }, {
+                .compatible = "starry,kr122ea0sra",
+                .data = &starry_kr122ea0sra,
+        }, {
                 .compatible = "tpk,f07a-0102",
                 .data = &tpk_f07a_0102,
         }, {
@@ -1701,7 +1858,6 @@ static const struct panel_desc_dsi panasonic_vvx10f004b00 = {
         .lanes = 4,
 };
 
-
 static const struct of_device_id dsi_of_match[] = {
         {
                 .compatible = "auo,b080uan01",
diff --git a/drivers/gpu/drm/qxl/Kconfig b/drivers/gpu/drm/qxl/Kconfig
index 38c2bb72e456..da45b11b66b8 100644
--- a/drivers/gpu/drm/qxl/Kconfig
+++ b/drivers/gpu/drm/qxl/Kconfig
@@ -1,12 +1,7 @@
 config DRM_QXL
         tristate "QXL virtual GPU"
         depends on DRM && PCI
-        select FB_SYS_FILLRECT
-        select FB_SYS_COPYAREA
-        select FB_SYS_IMAGEBLIT
-        select FB_DEFERRED_IO
         select DRM_KMS_HELPER
-        select DRM_KMS_FB_HELPER
         select DRM_TTM
         select CRC32
         help
diff --git a/drivers/gpu/drm/qxl/qxl_cmd.c b/drivers/gpu/drm/qxl/qxl_cmd.c
index b5d4b41361bd..04270f5d110c 100644
--- a/drivers/gpu/drm/qxl/qxl_cmd.c
+++ b/drivers/gpu/drm/qxl/qxl_cmd.c
@@ -203,7 +203,7 @@ qxl_push_cursor_ring_release(struct qxl_device *qdev, struct qxl_release *releas
 bool qxl_queue_garbage_collect(struct qxl_device *qdev, bool flush)
 {
         if (!qxl_check_idle(qdev->release_ring)) {
-                queue_work(qdev->gc_queue, &qdev->gc_work);
+                schedule_work(&qdev->gc_work);
                 if (flush)
                         flush_work(&qdev->gc_work);
                 return true;
diff --git a/drivers/gpu/drm/qxl/qxl_display.c b/drivers/gpu/drm/qxl/qxl_display.c
index 8b5d54385892..3aef12742a53 100644
--- a/drivers/gpu/drm/qxl/qxl_display.c
+++ b/drivers/gpu/drm/qxl/qxl_display.c
@@ -221,7 +221,6 @@ static int qxl_crtc_page_flip(struct drm_crtc *crtc,
 {
         struct drm_device *dev = crtc->dev;
         struct qxl_device *qdev = dev->dev_private;
-        struct qxl_crtc *qcrtc = to_qxl_crtc(crtc);
         struct qxl_framebuffer *qfb_src = to_qxl_framebuffer(fb);
         struct qxl_framebuffer *qfb_old = to_qxl_framebuffer(crtc->primary->fb);
         struct qxl_bo *bo_old = gem_to_qxl_bo(qfb_old->obj);
@@ -252,14 +251,14 @@ static int qxl_crtc_page_flip(struct drm_crtc *crtc,
         qxl_draw_dirty_fb(qdev, qfb_src, bo, 0, 0,
                           &norect, one_clip_rect, inc);
 
-        drm_vblank_get(dev, qcrtc->index);
+        drm_crtc_vblank_get(crtc);
 
         if (event) {
                 spin_lock_irqsave(&dev->event_lock, flags);
-                drm_send_vblank_event(dev, qcrtc->index, event);
+                drm_crtc_send_vblank_event(crtc, event);
                 spin_unlock_irqrestore(&dev->event_lock, flags);
         }
-        drm_vblank_put(dev, qcrtc->index);
+        drm_crtc_vblank_put(crtc);
 
         ret = qxl_bo_reserve(bo, false);
         if (!ret) {
@@ -469,8 +468,7 @@ void qxl_user_framebuffer_destroy(struct drm_framebuffer *fb)
 {
         struct qxl_framebuffer *qxl_fb = to_qxl_framebuffer(fb);
 
-        if (qxl_fb->obj)
-                drm_gem_object_unreference_unlocked(qxl_fb->obj);
+        drm_gem_object_unreference_unlocked(qxl_fb->obj);
         drm_framebuffer_cleanup(fb);
         kfree(qxl_fb);
 }
@@ -730,7 +728,6 @@ static int qdev_crtc_init(struct drm_device *dev, int crtc_id)
 
         drm_crtc_init(dev, &qxl_crtc->base, &qxl_crtc_funcs);
         qxl_crtc->index = crtc_id;
-        drm_mode_crtc_set_gamma_size(&qxl_crtc->base, 256);
         drm_crtc_helper_add(&qxl_crtc->base, &qxl_crtc_helper_funcs);
         return 0;
 }
diff --git a/drivers/gpu/drm/qxl/qxl_draw.c b/drivers/gpu/drm/qxl/qxl_draw.c
index 56e1d633875e..ffe885395145 100644
--- a/drivers/gpu/drm/qxl/qxl_draw.c
+++ b/drivers/gpu/drm/qxl/qxl_draw.c
@@ -37,7 +37,6 @@ static int alloc_clips(struct qxl_device *qdev,
  * the qxl_clip_rects. This is *not* the same as the memory allocated
  * on the device, it is offset to qxl_clip_rects.chunk.data */
 static struct qxl_rect *drawable_set_clipping(struct qxl_device *qdev,
-                                              struct qxl_drawable *drawable,
                                               unsigned num_clips,
                                               struct qxl_bo *clips_bo)
 {
@@ -136,6 +135,8 @@ static int qxl_palette_create_1bit(struct qxl_bo *palette_bo,
                                  * correctly globaly, since that would require
                                  * tracking all of our palettes. */
         ret = qxl_bo_kmap(palette_bo, (void **)&pal);
+        if (ret)
+                return ret;
         pal->num_ents = 2;
         pal->unique = unique++;
         if (visual == FB_VISUAL_TRUECOLOR || visual == FB_VISUAL_DIRECTCOLOR) {
@@ -349,7 +350,7 @@ void qxl_draw_dirty_fb(struct qxl_device *qdev,
         if (ret)
                 goto out_release_backoff;
 
-        rects = drawable_set_clipping(qdev, drawable, num_clips, clips_bo);
+        rects = drawable_set_clipping(qdev, num_clips, clips_bo);
         if (!rects)
                 goto out_release_backoff;
 
diff --git a/drivers/gpu/drm/qxl/qxl_drv.c b/drivers/gpu/drm/qxl/qxl_drv.c
index dc9df5fe50ba..460bbceae297 100644
--- a/drivers/gpu/drm/qxl/qxl_drv.c
+++ b/drivers/gpu/drm/qxl/qxl_drv.c
@@ -256,7 +256,7 @@ static struct drm_driver qxl_driver = {
         .gem_prime_vmap = qxl_gem_prime_vmap,
         .gem_prime_vunmap = qxl_gem_prime_vunmap,
         .gem_prime_mmap = qxl_gem_prime_mmap,
-        .gem_free_object = qxl_gem_object_free,
+        .gem_free_object_unlocked = qxl_gem_object_free,
         .gem_open_object = qxl_gem_object_open,
         .gem_close_object = qxl_gem_object_close,
         .fops = &qxl_fops,
diff --git a/drivers/gpu/drm/qxl/qxl_drv.h b/drivers/gpu/drm/qxl/qxl_drv.h
index 3ad6604b34ce..8e633caa4078 100644
--- a/drivers/gpu/drm/qxl/qxl_drv.h
+++ b/drivers/gpu/drm/qxl/qxl_drv.h
@@ -321,7 +321,6 @@ struct qxl_device {
         struct qxl_bo *current_release_bo[3];
         int current_release_bo_offset[3];
 
-        struct workqueue_struct *gc_queue;
         struct work_struct gc_work;
 
         struct drm_property *hotplug_mode_update_property;
diff --git a/drivers/gpu/drm/qxl/qxl_fb.c b/drivers/gpu/drm/qxl/qxl_fb.c
index 5ea57f6320b8..df2657051afd 100644
--- a/drivers/gpu/drm/qxl/qxl_fb.c
+++ b/drivers/gpu/drm/qxl/qxl_fb.c
@@ -131,10 +131,6 @@ static int qxlfb_create_pinned_object(struct qxl_fbdev *qfbdev,
         int ret;
         int aligned_size, size;
         int height = mode_cmd->height;
-        int bpp;
-        int depth;
-
-        drm_fb_get_bpp_depth(mode_cmd->pixel_format, &bpp, &depth);
 
         size = mode_cmd->pitches[0] * height;
         aligned_size = ALIGN(size, PAGE_SIZE);
diff --git a/drivers/gpu/drm/qxl/qxl_kms.c b/drivers/gpu/drm/qxl/qxl_kms.c
index 2319800b7add..e642242728c0 100644
--- a/drivers/gpu/drm/qxl/qxl_kms.c
+++ b/drivers/gpu/drm/qxl/qxl_kms.c
@@ -258,7 +258,6 @@ static int qxl_device_init(struct qxl_device *qdev,
                  (unsigned long)qdev->surfaceram_size);
 
 
-        qdev->gc_queue = create_singlethread_workqueue("qxl_gc");
         INIT_WORK(&qdev->gc_work, qxl_gc_work);
 
         return 0;
@@ -270,10 +269,7 @@ static void qxl_device_fini(struct qxl_device *qdev)
                 qxl_bo_unref(&qdev->current_release_bo[0]);
         if (qdev->current_release_bo[1])
                 qxl_bo_unref(&qdev->current_release_bo[1]);
-        flush_workqueue(qdev->gc_queue);
-        destroy_workqueue(qdev->gc_queue);
-        qdev->gc_queue = NULL;
-
+        flush_work(&qdev->gc_work);
         qxl_ring_free(qdev->command_ring);
         qxl_ring_free(qdev->cursor_ring);
         qxl_ring_free(qdev->release_ring);
@@ -310,10 +306,6 @@ int qxl_driver_load(struct drm_device *dev, unsigned long flags)
         struct qxl_device *qdev;
         int r;
 
-        /* require kms */
-        if (!drm_core_check_feature(dev, DRIVER_MODESET))
-                return -ENODEV;
-
         qdev = kzalloc(sizeof(struct qxl_device), GFP_KERNEL);
         if (qdev == NULL)
                 return -ENOMEM;
diff --git a/drivers/gpu/drm/qxl/qxl_release.c b/drivers/gpu/drm/qxl/qxl_release.c
index 4efa8e261baf..f599cd073b72 100644
--- a/drivers/gpu/drm/qxl/qxl_release.c
+++ b/drivers/gpu/drm/qxl/qxl_release.c
@@ -96,7 +96,7 @@ retry:
                         return 0;
 
                 if (have_drawable_releases && sc > 300) {
-                        FENCE_WARN(fence, "failed to wait on release %d "
+                        FENCE_WARN(fence, "failed to wait on release %llu "
                                           "after spincount %d\n",
                                           fence->context & ~0xf0000000, sc);
                         goto signaled;
diff --git a/drivers/gpu/drm/qxl/qxl_ttm.c b/drivers/gpu/drm/qxl/qxl_ttm.c
index 0738d74c8d04..d50c9679e631 100644
--- a/drivers/gpu/drm/qxl/qxl_ttm.c
+++ b/drivers/gpu/drm/qxl/qxl_ttm.c
@@ -350,11 +350,19 @@ static int qxl_bo_move(struct ttm_buffer_object *bo,
                        struct ttm_mem_reg *new_mem)
 {
         struct ttm_mem_reg *old_mem = &bo->mem;
+        int ret;
+
+        ret = ttm_bo_wait(bo, interruptible, no_wait_gpu);
+        if (ret)
+                return ret;
+
+
         if (old_mem->mem_type == TTM_PL_SYSTEM && bo->ttm == NULL) {
                 qxl_move_null(bo, new_mem);
                 return 0;
         }
-        return ttm_bo_move_memcpy(bo, evict, no_wait_gpu, new_mem);
+        return ttm_bo_move_memcpy(bo, evict, interruptible,
+                                  no_wait_gpu, new_mem);
 }
 
 static void qxl_bo_move_notify(struct ttm_buffer_object *bo,
diff --git a/drivers/gpu/drm/radeon/atombios_crtc.c b/drivers/gpu/drm/radeon/atombios_crtc.c
index 259cd6e6d71c..a97abc8af657 100644
--- a/drivers/gpu/drm/radeon/atombios_crtc.c
+++ b/drivers/gpu/drm/radeon/atombios_crtc.c
@@ -276,14 +276,14 @@ void atombios_crtc_dpms(struct drm_crtc *crtc, int mode)
                         atombios_enable_crtc_memreq(crtc, ATOM_ENABLE);
                 atombios_blank_crtc(crtc, ATOM_DISABLE);
                 if (dev->num_crtcs > radeon_crtc->crtc_id)
-                        drm_vblank_on(dev, radeon_crtc->crtc_id);
+                        drm_crtc_vblank_on(crtc);
                 radeon_crtc_load_lut(crtc);
                 break;
         case DRM_MODE_DPMS_STANDBY:
         case DRM_MODE_DPMS_SUSPEND:
         case DRM_MODE_DPMS_OFF:
                 if (dev->num_crtcs > radeon_crtc->crtc_id)
-                        drm_vblank_off(dev, radeon_crtc->crtc_id);
+                        drm_crtc_vblank_off(crtc);
                 if (radeon_crtc->enabled)
                         atombios_blank_crtc(crtc, ATOM_ENABLE);
                 if (ASIC_IS_DCE3(rdev) && !ASIC_IS_DCE6(rdev))
diff --git a/drivers/gpu/drm/radeon/atombios_encoders.c b/drivers/gpu/drm/radeon/atombios_encoders.c
index 587cae4e73c9..56bb758f4e33 100644
--- a/drivers/gpu/drm/radeon/atombios_encoders.c
+++ b/drivers/gpu/drm/radeon/atombios_encoders.c
@@ -120,6 +120,7 @@ atombios_set_backlight_level(struct radeon_encoder *radeon_encoder, u8 level)
                 case ENCODER_OBJECT_ID_INTERNAL_KLDSCP_LVTMA:
                 case ENCODER_OBJECT_ID_INTERNAL_UNIPHY1:
                 case ENCODER_OBJECT_ID_INTERNAL_UNIPHY2:
+                case ENCODER_OBJECT_ID_INTERNAL_UNIPHY3:
                         if (dig->backlight_level == 0)
                                 atombios_dig_transmitter_setup(encoder, ATOM_TRANSMITTER_ACTION_LCD_BLOFF, 0, 0);
                         else {
diff --git a/drivers/gpu/drm/radeon/ci_dpm.c b/drivers/gpu/drm/radeon/ci_dpm.c
index 35e0fc3ae8a7..7ba450832e6b 100644
--- a/drivers/gpu/drm/radeon/ci_dpm.c
+++ b/drivers/gpu/drm/radeon/ci_dpm.c
@@ -3843,7 +3843,10 @@ static void ci_find_dpm_states_clocks_in_dpm_table(struct radeon_device *rdev,
         if (i >= sclk_table->count) {
                 pi->need_update_smu7_dpm_table |= DPMTABLE_OD_UPDATE_SCLK;
         } else {
-                /* XXX check display min clock requirements */
+                /* XXX The current code always reprogrammed the sclk levels,
+                 * but we don't currently handle disp sclk requirements
+                 * so just skip it.
+                 */
                 if (CISLAND_MINIMUM_ENGINE_CLOCK != CISLAND_MINIMUM_ENGINE_CLOCK)
                         pi->need_update_smu7_dpm_table |= DPMTABLE_UPDATE_SCLK;
         }
diff --git a/drivers/gpu/drm/radeon/cik.c b/drivers/gpu/drm/radeon/cik.c
index ba192a35c607..0c1b9ff433af 100644
--- a/drivers/gpu/drm/radeon/cik.c
+++ b/drivers/gpu/drm/radeon/cik.c
@@ -53,6 +53,7 @@ MODULE_FIRMWARE("radeon/bonaire_mc.bin");
 MODULE_FIRMWARE("radeon/bonaire_rlc.bin");
 MODULE_FIRMWARE("radeon/bonaire_sdma.bin");
 MODULE_FIRMWARE("radeon/bonaire_smc.bin");
+MODULE_FIRMWARE("radeon/bonaire_k_smc.bin");
 
 MODULE_FIRMWARE("radeon/HAWAII_pfp.bin");
 MODULE_FIRMWARE("radeon/HAWAII_me.bin");
@@ -72,6 +73,7 @@ MODULE_FIRMWARE("radeon/hawaii_mc.bin");
 MODULE_FIRMWARE("radeon/hawaii_rlc.bin");
 MODULE_FIRMWARE("radeon/hawaii_sdma.bin");
 MODULE_FIRMWARE("radeon/hawaii_smc.bin");
+MODULE_FIRMWARE("radeon/hawaii_k_smc.bin");
 
 MODULE_FIRMWARE("radeon/KAVERI_pfp.bin");
 MODULE_FIRMWARE("radeon/KAVERI_me.bin");
@@ -1990,12 +1992,17 @@ static int cik_init_microcode(struct radeon_device *rdev)
         int new_fw = 0;
         int err;
         int num_fw;
+        bool new_smc = false;
 
         DRM_DEBUG("\n");
 
         switch (rdev->family) {
         case CHIP_BONAIRE:
                 chip_name = "BONAIRE";
+                if ((rdev->pdev->revision == 0x80) ||
+                    (rdev->pdev->revision == 0x81) ||
+                    (rdev->pdev->device == 0x665f))
+                        new_smc = true;
                 new_chip_name = "bonaire";
                 pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
                 me_req_size = CIK_ME_UCODE_SIZE * 4;
@@ -2010,6 +2017,8 @@ static int cik_init_microcode(struct radeon_device *rdev)
                 break;
         case CHIP_HAWAII:
                 chip_name = "HAWAII";
+                if (rdev->pdev->revision == 0x80)
+                        new_smc = true;
                 new_chip_name = "hawaii";
                 pfp_req_size = CIK_PFP_UCODE_SIZE * 4;
                 me_req_size = CIK_ME_UCODE_SIZE * 4;
@@ -2259,7 +2268,10 @@ static int cik_init_microcode(struct radeon_device *rdev)
                         }
                 }
 
-                snprintf(fw_name, sizeof(fw_name), "radeon/%s_smc.bin", new_chip_name);
+                if (new_smc)
+                        snprintf(fw_name, sizeof(fw_name), "radeon/%s_k_smc.bin", new_chip_name);
+                else
+                        snprintf(fw_name, sizeof(fw_name), "radeon/%s_smc.bin", new_chip_name);
                 err = request_firmware(&rdev->smc_fw, fw_name, rdev->dev);
                 if (err) {
                         snprintf(fw_name, sizeof(fw_name), "radeon/%s_smc.bin", chip_name);
@@ -8354,7 +8366,8 @@ static int cik_startup(struct radeon_device *rdev)
                 }
         }
         rdev->rlc.cs_data = ci_cs_data;
-        rdev->rlc.cp_table_size = CP_ME_TABLE_SIZE * 5 * 4;
+        rdev->rlc.cp_table_size = ALIGN(CP_ME_TABLE_SIZE * 5 * 4, 2048); /* CP JT */
+        rdev->rlc.cp_table_size += 64 * 1024; /* GDS */
         r = sumo_rlc_init(rdev);
         if (r) {
                 DRM_ERROR("Failed to init rlc BOs!\n");
diff --git a/drivers/gpu/drm/radeon/evergreen_cs.c b/drivers/gpu/drm/radeon/evergreen_cs.c
index 0d3f744de35a..d960d3915408 100644
--- a/drivers/gpu/drm/radeon/evergreen_cs.c
+++ b/drivers/gpu/drm/radeon/evergreen_cs.c
@@ -2209,6 +2209,12 @@ static int evergreen_packet3_check(struct radeon_cs_parser *p,
                 }
                 break;
         }
+        case PACKET3_PFP_SYNC_ME:
+                if (pkt->count) {
+                        DRM_ERROR("bad PFP_SYNC_ME\n");
+                        return -EINVAL;
+                }
+                break;
         case PACKET3_SURFACE_SYNC:
                 if (pkt->count != 3) {
                         DRM_ERROR("bad SURFACE_SYNC\n");
@@ -3381,6 +3387,7 @@ static int evergreen_vm_packet3_check(struct radeon_device *rdev,
         case PACKET3_MPEG_INDEX:
         case PACKET3_WAIT_REG_MEM:
         case PACKET3_MEM_WRITE:
+        case PACKET3_PFP_SYNC_ME:
         case PACKET3_SURFACE_SYNC:
         case PACKET3_EVENT_WRITE:
         case PACKET3_EVENT_WRITE_EOP:
diff --git a/drivers/gpu/drm/radeon/evergreend.h b/drivers/gpu/drm/radeon/evergreend.h
index 0b174e14e9a6..c8e3d394cde7 100644
--- a/drivers/gpu/drm/radeon/evergreend.h
+++ b/drivers/gpu/drm/radeon/evergreend.h
@@ -1624,6 +1624,7 @@
                  */
 #              define PACKET3_CP_DMA_CMD_SAIC      (1 << 28)
 #              define PACKET3_CP_DMA_CMD_DAIC      (1 << 29)
+#define PACKET3_PFP_SYNC_ME                             0x42
 #define PACKET3_SURFACE_SYNC                            0x43
 #              define PACKET3_CB0_DEST_BASE_ENA    (1 << 6)
 #              define PACKET3_CB1_DEST_BASE_ENA    (1 << 7)
diff --git a/drivers/gpu/drm/radeon/radeon.h b/drivers/gpu/drm/radeon/radeon.h
index 80b24a495d6c..5633ee3eb46e 100644
--- a/drivers/gpu/drm/radeon/radeon.h
+++ b/drivers/gpu/drm/radeon/radeon.h
@@ -2386,7 +2386,7 @@ struct radeon_device {
         struct radeon_mman              mman;
         struct radeon_fence_driver      fence_drv[RADEON_NUM_RINGS];
         wait_queue_head_t               fence_queue;
-        unsigned                        fence_context;
+        u64                             fence_context;
         struct mutex                    ring_lock;
         struct radeon_ring              ring[RADEON_NUM_RINGS];
         bool                            ib_pool_ready;
diff --git a/drivers/gpu/drm/radeon/radeon_acpi.c b/drivers/gpu/drm/radeon/radeon_acpi.c
index 59acd0e5c2c6..31c9a92d6a1b 100644
--- a/drivers/gpu/drm/radeon/radeon_acpi.c
+++ b/drivers/gpu/drm/radeon/radeon_acpi.c
@@ -741,13 +741,6 @@ int radeon_acpi_init(struct radeon_device *rdev)
                 }
 
                 atif->encoder_for_bl = target;
-                if (!target) {
-                        /* Brightness change notification is enabled, but we
-                         * didn't find a backlight controller, this should
-                         * never happen.
-                         */
-                        DRM_ERROR("Cannot find a backlight controller\n");
-                }
         }
 
         if (atif->functions.sbios_requests && !atif->functions.system_params) {
diff --git a/drivers/gpu/drm/radeon/radeon_atombios.c b/drivers/gpu/drm/radeon/radeon_atombios.c
index f8097a0e7a79..5df3ec73021b 100644
--- a/drivers/gpu/drm/radeon/radeon_atombios.c
+++ b/drivers/gpu/drm/radeon/radeon_atombios.c
@@ -1155,7 +1155,7 @@ bool radeon_atom_get_clock_info(struct drm_device *dev)
                     le16_to_cpu(firmware_info->info.usReferenceClock);
                 p1pll->reference_div = 0;
 
-                if (crev < 2)
+                if ((frev < 2) && (crev < 2))
                         p1pll->pll_out_min =
                                 le16_to_cpu(firmware_info->info.usMinPixelClockPLL_Output);
                 else
@@ -1164,7 +1164,7 @@ bool radeon_atom_get_clock_info(struct drm_device *dev)
                 p1pll->pll_out_max =
                     le32_to_cpu(firmware_info->info.ulMaxPixelClockPLL_Output);
 
-                if (crev >= 4) {
+                if (((frev < 2) && (crev >= 4)) || (frev >= 2)) {
                         p1pll->lcd_pll_out_min =
                                 le16_to_cpu(firmware_info->info_14.usLcdMinPixelClockPLL_Output) * 100;
                         if (p1pll->lcd_pll_out_min == 0)
diff --git a/drivers/gpu/drm/radeon/radeon_atpx_handler.c b/drivers/gpu/drm/radeon/radeon_atpx_handler.c
index 95f4fea89302..6de342861202 100644
--- a/drivers/gpu/drm/radeon/radeon_atpx_handler.c
+++ b/drivers/gpu/drm/radeon/radeon_atpx_handler.c
@@ -10,6 +10,7 @@
 #include <linux/slab.h>
 #include <linux/acpi.h>
 #include <linux/pci.h>
+#include <linux/delay.h>
 
 #include "radeon_acpi.h"
 
@@ -27,6 +28,7 @@ struct radeon_atpx_functions {
 struct radeon_atpx {
         acpi_handle handle;
         struct radeon_atpx_functions functions;
+        bool is_hybrid;
 };
 
 static struct radeon_atpx_priv {
@@ -62,6 +64,14 @@ bool radeon_has_atpx(void) {
         return radeon_atpx_priv.atpx_detected;
 }
 
+bool radeon_has_atpx_dgpu_power_cntl(void) {
+        return radeon_atpx_priv.atpx.functions.power_cntl;
+}
+
+bool radeon_is_atpx_hybrid(void) {
+        return radeon_atpx_priv.atpx.is_hybrid;
+}
+
 /**
  * radeon_atpx_call - call an ATPX method
  *
@@ -141,18 +151,12 @@ static void radeon_atpx_parse_functions(struct radeon_atpx_functions *f, u32 mas
  */
 static int radeon_atpx_validate(struct radeon_atpx *atpx)
 {
-        /* make sure required functions are enabled */
-        /* dGPU power control is required */
-        if (atpx->functions.power_cntl == false) {
-                printk("ATPX dGPU power cntl not present, forcing\n");
-                atpx->functions.power_cntl = true;
-        }
+        u32 valid_bits = 0;
 
         if (atpx->functions.px_params) {
                 union acpi_object *info;
                 struct atpx_px_params output;
                 size_t size;
-                u32 valid_bits;
 
                 info = radeon_atpx_call(atpx->handle, ATPX_FUNCTION_GET_PX_PARAMETERS, NULL);
                 if (!info)
@@ -171,19 +175,42 @@ static int radeon_atpx_validate(struct radeon_atpx *atpx)
                 memcpy(&output, info->buffer.pointer, size);
 
                 valid_bits = output.flags & output.valid_flags;
-                /* if separate mux flag is set, mux controls are required */
-                if (valid_bits & ATPX_SEPARATE_MUX_FOR_I2C) {
-                        atpx->functions.i2c_mux_cntl = true;
-                        atpx->functions.disp_mux_cntl = true;
-                }
-                /* if any outputs are muxed, mux controls are required */
-                if (valid_bits & (ATPX_CRT1_RGB_SIGNAL_MUXED |
-                                  ATPX_TV_SIGNAL_MUXED |
-                                  ATPX_DFP_SIGNAL_MUXED))
-                        atpx->functions.disp_mux_cntl = true;
 
                 kfree(info);
         }
+
+        /* if separate mux flag is set, mux controls are required */
+        if (valid_bits & ATPX_SEPARATE_MUX_FOR_I2C) {
+                atpx->functions.i2c_mux_cntl = true;
+                atpx->functions.disp_mux_cntl = true;
+        }
+        /* if any outputs are muxed, mux controls are required */
+        if (valid_bits & (ATPX_CRT1_RGB_SIGNAL_MUXED |
+                          ATPX_TV_SIGNAL_MUXED |
+                          ATPX_DFP_SIGNAL_MUXED))
+                atpx->functions.disp_mux_cntl = true;
+
+        /* some bioses set these bits rather than flagging power_cntl as supported */
+        if (valid_bits & (ATPX_DYNAMIC_PX_SUPPORTED |
+                          ATPX_DYNAMIC_DGPU_POWER_OFF_SUPPORTED))
+                atpx->functions.power_cntl = true;
+
+        atpx->is_hybrid = false;
+        if (valid_bits & ATPX_MS_HYBRID_GFX_SUPPORTED) {
+                printk("ATPX Hybrid Graphics\n");
+#if 1
+                /* This is a temporary hack until the D3 cold support
+                 * makes it upstream.  The ATPX power_control method seems
+                 * to still work on even if the system should be using
+                 * the new standardized hybrid D3 cold ACPI interface.
+                 */
+                atpx->functions.power_cntl = true;
+#else
+                atpx->functions.power_cntl = false;
+#endif
+                atpx->is_hybrid = true;
+        }
+
         return 0;
 }
 
@@ -258,6 +285,10 @@ static int radeon_atpx_set_discrete_state(struct radeon_atpx *atpx, u8 state)
                 if (!info)
                         return -EIO;
                 kfree(info);
+
+                /* 200ms delay is required after off */
+                if (state == 0)
+                        msleep(200);
         }
         return 0;
 }
@@ -505,7 +536,6 @@ static int radeon_atpx_get_client_id(struct pci_dev *pdev)
 static const struct vga_switcheroo_handler radeon_atpx_handler = {
         .switchto = radeon_atpx_switchto,
         .power_state = radeon_atpx_power_state,
-        .init = radeon_atpx_init,
         .get_client_id = radeon_atpx_get_client_id,
 };
 
@@ -541,6 +571,7 @@ static bool radeon_atpx_detect(void)
                 printk(KERN_INFO "vga_switcheroo: detected switching method %s handle\n",
                        acpi_method_name);
                 radeon_atpx_priv.atpx_detected = true;
+                radeon_atpx_init();
                 return true;
         }
         return false;
diff --git a/drivers/gpu/drm/radeon/radeon_connectors.c b/drivers/gpu/drm/radeon/radeon_connectors.c
index 81a63d7f5cd9..b79f3b002471 100644
--- a/drivers/gpu/drm/radeon/radeon_connectors.c
+++ b/drivers/gpu/drm/radeon/radeon_connectors.c
@@ -2064,7 +2064,6 @@ radeon_add_atom_connector(struct drm_device *dev,
                                                            RADEON_OUTPUT_CSC_BYPASS);
                         /* no HPD on analog connectors */
                         radeon_connector->hpd.hpd = RADEON_HPD_NONE;
-                        connector->polled = DRM_CONNECTOR_POLL_CONNECT;
                         connector->interlace_allowed = true;
                         connector->doublescan_allowed = true;
                         break;
@@ -2314,8 +2313,10 @@ radeon_add_atom_connector(struct drm_device *dev,
         }
 
         if (radeon_connector->hpd.hpd == RADEON_HPD_NONE) {
-                if (i2c_bus->valid)
-                        connector->polled = DRM_CONNECTOR_POLL_CONNECT;
+                if (i2c_bus->valid) {
+                        connector->polled = DRM_CONNECTOR_POLL_CONNECT |
+                                            DRM_CONNECTOR_POLL_DISCONNECT;
+                }
         } else
                 connector->polled = DRM_CONNECTOR_POLL_HPD;
 
@@ -2391,7 +2392,6 @@ radeon_add_legacy_connector(struct drm_device *dev,
                                               1);
                 /* no HPD on analog connectors */
                 radeon_connector->hpd.hpd = RADEON_HPD_NONE;
-                connector->polled = DRM_CONNECTOR_POLL_CONNECT;
                 connector->interlace_allowed = true;
                 connector->doublescan_allowed = true;
                 break;
@@ -2476,10 +2476,13 @@ radeon_add_legacy_connector(struct drm_device *dev,
         }
 
         if (radeon_connector->hpd.hpd == RADEON_HPD_NONE) {
-                if (i2c_bus->valid)
-                        connector->polled = DRM_CONNECTOR_POLL_CONNECT;
+                if (i2c_bus->valid) {
+                        connector->polled = DRM_CONNECTOR_POLL_CONNECT |
+                                            DRM_CONNECTOR_POLL_DISCONNECT;
+                }
         } else
                 connector->polled = DRM_CONNECTOR_POLL_HPD;
+
         connector->display_info.subpixel_order = subpixel_order;
         drm_connector_register(connector);
 }
diff --git a/drivers/gpu/drm/radeon/radeon_device.c b/drivers/gpu/drm/radeon/radeon_device.c
index 21c44b2293bc..a00dd2f74527 100644
--- a/drivers/gpu/drm/radeon/radeon_device.c
+++ b/drivers/gpu/drm/radeon/radeon_device.c
@@ -30,6 +30,7 @@
 #include <drm/drmP.h>
 #include <drm/drm_crtc_helper.h>
 #include <drm/radeon_drm.h>
+#include <linux/pm_runtime.h>
 #include <linux/vgaarb.h>
 #include <linux/vga_switcheroo.h>
 #include <linux/efi.h>
@@ -1526,6 +1527,9 @@ int radeon_device_init(struct radeon_device *rdev,
         return 0;
 
 failed:
+        /* balance pm_runtime_get_sync() in radeon_driver_unload_kms() */
+        if (radeon_is_px(ddev))
+                pm_runtime_put_noidle(ddev->dev);
         if (runtime)
                 vga_switcheroo_fini_domain_pm_ops(rdev->dev);
         return r;
diff --git a/drivers/gpu/drm/radeon/radeon_display.c b/drivers/gpu/drm/radeon/radeon_display.c
index 6a41b4982647..c3206fb8f4cf 100644
--- a/drivers/gpu/drm/radeon/radeon_display.c
+++ b/drivers/gpu/drm/radeon/radeon_display.c
@@ -231,19 +231,21 @@ void radeon_crtc_fb_gamma_get(struct drm_crtc *crtc, u16 *red, u16 *green,
         *blue = radeon_crtc->lut_b[regno] << 6;
 }
 
-static void radeon_crtc_gamma_set(struct drm_crtc *crtc, u16 *red, u16 *green,
-                                  u16 *blue, uint32_t start, uint32_t size)
+static int radeon_crtc_gamma_set(struct drm_crtc *crtc, u16 *red, u16 *green,
+                                 u16 *blue, uint32_t size)
 {
         struct radeon_crtc *radeon_crtc = to_radeon_crtc(crtc);
-        int end = (start + size > 256) ? 256 : start + size, i;
+        int i;
 
         /* userspace palettes are always correct as is */
-        for (i = start; i < end; i++) {
+        for (i = 0; i < size; i++) {
                 radeon_crtc->lut_r[i] = red[i] >> 6;
                 radeon_crtc->lut_g[i] = green[i] >> 6;
                 radeon_crtc->lut_b[i] = blue[i] >> 6;
         }
         radeon_crtc_load_lut(crtc);
+
+        return 0;
 }
 
 static void radeon_crtc_destroy(struct drm_crtc *crtc)
@@ -381,7 +383,7 @@ void radeon_crtc_handle_flip(struct radeon_device *rdev, int crtc_id)
 
         spin_unlock_irqrestore(&rdev->ddev->event_lock, flags);
 
-        drm_vblank_put(rdev->ddev, radeon_crtc->crtc_id);
+        drm_crtc_vblank_put(&radeon_crtc->base);
         radeon_irq_kms_pflip_irq_put(rdev, work->crtc_id);
         queue_work(radeon_crtc->flip_queue, &work->unpin_work);
 }
@@ -598,7 +600,7 @@ static int radeon_crtc_page_flip(struct drm_crtc *crtc,
         }
         work->base = base;
 
-        r = drm_vblank_get(crtc->dev, radeon_crtc->crtc_id);
+        r = drm_crtc_vblank_get(crtc);
         if (r) {
                 DRM_ERROR("failed to get vblank before flip\n");
                 goto pflip_cleanup;
@@ -625,7 +627,7 @@ static int radeon_crtc_page_flip(struct drm_crtc *crtc,
         return 0;
 
 vblank_cleanup:
-        drm_vblank_put(crtc->dev, radeon_crtc->crtc_id);
+        drm_crtc_vblank_put(crtc);
 
 pflip_cleanup:
         if (unlikely(radeon_bo_reserve(new_rbo, false) != 0)) {
@@ -688,6 +690,7 @@ radeon_crtc_set_config(struct drm_mode_set *set)
         pm_runtime_put_autosuspend(dev->dev);
         return ret;
 }
+
 static const struct drm_crtc_funcs radeon_crtc_funcs = {
         .cursor_set2 = radeon_crtc_cursor_set2,
         .cursor_move = radeon_crtc_cursor_move,
@@ -711,7 +714,7 @@ static void radeon_crtc_init(struct drm_device *dev, int index)
 
         drm_mode_crtc_set_gamma_size(&radeon_crtc->base, 256);
         radeon_crtc->crtc_id = index;
-        radeon_crtc->flip_queue = create_singlethread_workqueue("radeon-crtc");
+        radeon_crtc->flip_queue = alloc_workqueue("radeon-crtc", WQ_HIGHPRI, 0);
         rdev->mode_info.crtcs[index] = radeon_crtc;
 
         if (rdev->family >= CHIP_BONAIRE) {
@@ -1321,9 +1324,7 @@ static void radeon_user_framebuffer_destroy(struct drm_framebuffer *fb)
 {
         struct radeon_framebuffer *radeon_fb = to_radeon_framebuffer(fb);
 
-        if (radeon_fb->obj) {
-                drm_gem_object_unreference_unlocked(radeon_fb->obj);
-        }
+        drm_gem_object_unreference_unlocked(radeon_fb->obj);
         drm_framebuffer_cleanup(fb);
         kfree(radeon_fb);
 }
@@ -1708,6 +1709,7 @@ void radeon_modeset_fini(struct radeon_device *rdev)
                 radeon_afmt_fini(rdev);
                 drm_kms_helper_poll_fini(rdev->ddev);
                 radeon_hpd_fini(rdev);
+                drm_crtc_force_disable_all(rdev->ddev);
                 drm_mode_config_cleanup(rdev->ddev);
                 rdev->mode_info.mode_config_initialized = false;
         }
diff --git a/drivers/gpu/drm/radeon/radeon_drv.c b/drivers/gpu/drm/radeon/radeon_drv.c
index b55aa740171f..c01a7c6abb49 100644
--- a/drivers/gpu/drm/radeon/radeon_drv.c
+++ b/drivers/gpu/drm/radeon/radeon_drv.c
@@ -34,11 +34,9 @@
 #include "radeon_drv.h"
 
 #include <drm/drm_pciids.h>
-#include <linux/apple-gmux.h>
 #include <linux/console.h>
 #include <linux/module.h>
 #include <linux/pm_runtime.h>
-#include <linux/vgaarb.h>
 #include <linux/vga_switcheroo.h>
 #include <drm/drm_gem.h>
 
@@ -95,9 +93,10 @@
  *   2.43.0 - RADEON_INFO_GPU_RESET_COUNTER
  *   2.44.0 - SET_APPEND_CNT packet3 support
  *   2.45.0 - Allow setting shader registers using DMA/COPY packet3 on SI
+ *   2.46.0 - Add PFP_SYNC_ME support on evergreen
  */
 #define KMS_DRIVER_MAJOR        2
-#define KMS_DRIVER_MINOR        45
+#define KMS_DRIVER_MINOR        46
 #define KMS_DRIVER_PATCHLEVEL   0
 int radeon_driver_load_kms(struct drm_device *dev, unsigned long flags);
 int radeon_driver_unload_kms(struct drm_device *dev);
@@ -164,9 +163,13 @@ void radeon_debugfs_cleanup(struct drm_minor *minor);
 #if defined(CONFIG_VGA_SWITCHEROO)
 void radeon_register_atpx_handler(void);
 void radeon_unregister_atpx_handler(void);
+bool radeon_has_atpx_dgpu_power_cntl(void);
+bool radeon_is_atpx_hybrid(void);
 #else
 static inline void radeon_register_atpx_handler(void) {}
 static inline void radeon_unregister_atpx_handler(void) {}
+static inline bool radeon_has_atpx_dgpu_power_cntl(void) { return false; }
+static inline bool radeon_is_atpx_hybrid(void) { return false; }
 #endif
 
 int radeon_no_wb;
@@ -340,13 +343,7 @@ static int radeon_pci_probe(struct pci_dev *pdev,
         if (ret == -EPROBE_DEFER)
                 return ret;
 
-        /*
-         * apple-gmux is needed on dual GPU MacBook Pro
-         * to probe the panel if we're the inactive GPU.
-         */
-        if (IS_ENABLED(CONFIG_VGA_ARB) && IS_ENABLED(CONFIG_VGA_SWITCHEROO) &&
-            apple_gmux_present() && pdev != vga_default_device() &&
-            !vga_switcheroo_handler_flags())
+        if (vga_switcheroo_client_probe_defer(pdev))
                 return -EPROBE_DEFER;
 
         /* Get rid of things like offb */
@@ -412,7 +409,10 @@ static int radeon_pmops_runtime_suspend(struct device *dev)
         pci_save_state(pdev);
         pci_disable_device(pdev);
         pci_ignore_hotplug(pdev);
-        pci_set_power_state(pdev, PCI_D3cold);
+        if (radeon_is_atpx_hybrid())
+                pci_set_power_state(pdev, PCI_D3cold);
+        else if (!radeon_has_atpx_dgpu_power_cntl())
+                pci_set_power_state(pdev, PCI_D3hot);
         drm_dev->switch_power_state = DRM_SWITCH_POWER_DYNAMIC_OFF;
 
         return 0;
@@ -429,7 +429,9 @@ static int radeon_pmops_runtime_resume(struct device *dev)
 
         drm_dev->switch_power_state = DRM_SWITCH_POWER_CHANGING;
 
-        pci_set_power_state(pdev, PCI_D0);
+        if (radeon_is_atpx_hybrid() ||
+            !radeon_has_atpx_dgpu_power_cntl())
+                pci_set_power_state(pdev, PCI_D0);
         pci_restore_state(pdev);
         ret = pci_enable_device(pdev);
         if (ret)
diff --git a/drivers/gpu/drm/radeon/radeon_kms.c b/drivers/gpu/drm/radeon/radeon_kms.c
index 414953c46a38..835563c1f0ed 100644
--- a/drivers/gpu/drm/radeon/radeon_kms.c
+++ b/drivers/gpu/drm/radeon/radeon_kms.c
@@ -63,7 +63,10 @@ int radeon_driver_unload_kms(struct drm_device *dev)
         if (rdev->rmmio == NULL)
                 goto done_free;
 
-        pm_runtime_get_sync(dev->dev);
+        if (radeon_is_px(dev)) {
+                pm_runtime_get_sync(dev->dev);
+                pm_runtime_forbid(dev->dev);
+        }
 
         radeon_kfd_device_fini(rdev);
 
diff --git a/drivers/gpu/drm/radeon/radeon_legacy_crtc.c b/drivers/gpu/drm/radeon/radeon_legacy_crtc.c
index 478d4099b0d0..d0de4022fff9 100644
--- a/drivers/gpu/drm/radeon/radeon_legacy_crtc.c
+++ b/drivers/gpu/drm/radeon/radeon_legacy_crtc.c
@@ -332,14 +332,14 @@ static void radeon_crtc_dpms(struct drm_crtc *crtc, int mode)
                         WREG32_P(RADEON_CRTC_EXT_CNTL, crtc_ext_cntl, ~(mask | crtc_ext_cntl));
                 }
                 if (dev->num_crtcs > radeon_crtc->crtc_id)
-                        drm_vblank_on(dev, radeon_crtc->crtc_id);
+                        drm_crtc_vblank_on(crtc);
                 radeon_crtc_load_lut(crtc);
                 break;
         case DRM_MODE_DPMS_STANDBY:
         case DRM_MODE_DPMS_SUSPEND:
         case DRM_MODE_DPMS_OFF:
                 if (dev->num_crtcs > radeon_crtc->crtc_id)
-                        drm_vblank_off(dev, radeon_crtc->crtc_id);
+                        drm_crtc_vblank_off(crtc);
                 if (radeon_crtc->crtc_id)
                         WREG32_P(RADEON_CRTC2_GEN_CNTL, mask, ~(RADEON_CRTC2_EN | mask));
                 else {
diff --git a/drivers/gpu/drm/radeon/radeon_pm.c b/drivers/gpu/drm/radeon/radeon_pm.c
index 38226d925a5b..4b6542538ff9 100644
--- a/drivers/gpu/drm/radeon/radeon_pm.c
+++ b/drivers/gpu/drm/radeon/radeon_pm.c
@@ -246,6 +246,7 @@ static void radeon_set_power_state(struct radeon_device *rdev)
 
 static void radeon_pm_set_clocks(struct radeon_device *rdev)
 {
+        struct drm_crtc *crtc;
         int i, r;
 
         /* no need to take locks, etc. if nothing's going to change */
@@ -274,26 +275,30 @@ static void radeon_pm_set_clocks(struct radeon_device *rdev)
         radeon_unmap_vram_bos(rdev);
 
         if (rdev->irq.installed) {
-                for (i = 0; i < rdev->num_crtc; i++) {
+                i = 0;
+                drm_for_each_crtc(crtc, rdev->ddev) {
                         if (rdev->pm.active_crtcs & (1 << i)) {
                                 /* This can fail if a modeset is in progress */
-                                if (drm_vblank_get(rdev->ddev, i) == 0)
+                                if (drm_crtc_vblank_get(crtc) == 0)
                                         rdev->pm.req_vblank |= (1 << i);
                                 else
                                         DRM_DEBUG_DRIVER("crtc %d no vblank, can glitch\n",
                                                          i);
                         }
+                        i++;
                 }
         }
 
         radeon_set_power_state(rdev);
 
         if (rdev->irq.installed) {
-                for (i = 0; i < rdev->num_crtc; i++) {
+                i = 0;
+                drm_for_each_crtc(crtc, rdev->ddev) {
                         if (rdev->pm.req_vblank & (1 << i)) {
                                 rdev->pm.req_vblank &= ~(1 << i);
-                                drm_vblank_put(rdev->ddev, i);
+                                drm_crtc_vblank_put(crtc);
                         }
+                        i++;
                 }
         }
 
diff --git a/drivers/gpu/drm/radeon/radeon_ttm.c b/drivers/gpu/drm/radeon/radeon_ttm.c
index 590b0377fbe2..ffdad81ef964 100644
--- a/drivers/gpu/drm/radeon/radeon_ttm.c
+++ b/drivers/gpu/drm/radeon/radeon_ttm.c
@@ -300,8 +300,7 @@ static int radeon_move_blit(struct ttm_buffer_object *bo,
         if (IS_ERR(fence))
                 return PTR_ERR(fence);
 
-        r = ttm_bo_move_accel_cleanup(bo, &fence->base,
-                                      evict, no_wait_gpu, new_mem);
+        r = ttm_bo_move_accel_cleanup(bo, &fence->base, evict, new_mem);
         radeon_fence_unref(&fence);
         return r;
 }
@@ -403,6 +402,10 @@ static int radeon_bo_move(struct ttm_buffer_object *bo,
         struct ttm_mem_reg *old_mem = &bo->mem;
         int r;
 
+        r = ttm_bo_wait(bo, interruptible, no_wait_gpu);
+        if (r)
+                return r;
+
         /* Can't move a pinned BO */
         rbo = container_of(bo, struct radeon_bo, tbo);
         if (WARN_ON_ONCE(rbo->pin_count > 0))
@@ -441,7 +444,8 @@ static int radeon_bo_move(struct ttm_buffer_object *bo,
 
         if (r) {
 memcpy:
-                r = ttm_bo_move_memcpy(bo, evict, no_wait_gpu, new_mem);
+                r = ttm_bo_move_memcpy(bo, evict, interruptible,
+                                       no_wait_gpu, new_mem);
                 if (r) {
                         return r;
                 }
diff --git a/drivers/gpu/drm/radeon/si.c b/drivers/gpu/drm/radeon/si.c
index b30e719dd56d..2523ca96c6c7 100644
--- a/drivers/gpu/drm/radeon/si.c
+++ b/drivers/gpu/drm/radeon/si.c
@@ -50,6 +50,7 @@ MODULE_FIRMWARE("radeon/tahiti_ce.bin");
 MODULE_FIRMWARE("radeon/tahiti_mc.bin");
 MODULE_FIRMWARE("radeon/tahiti_rlc.bin");
 MODULE_FIRMWARE("radeon/tahiti_smc.bin");
+MODULE_FIRMWARE("radeon/tahiti_k_smc.bin");
 
 MODULE_FIRMWARE("radeon/PITCAIRN_pfp.bin");
 MODULE_FIRMWARE("radeon/PITCAIRN_me.bin");
@@ -65,6 +66,7 @@ MODULE_FIRMWARE("radeon/pitcairn_ce.bin");
 MODULE_FIRMWARE("radeon/pitcairn_mc.bin");
 MODULE_FIRMWARE("radeon/pitcairn_rlc.bin");
 MODULE_FIRMWARE("radeon/pitcairn_smc.bin");
+MODULE_FIRMWARE("radeon/pitcairn_k_smc.bin");
 
 MODULE_FIRMWARE("radeon/VERDE_pfp.bin");
 MODULE_FIRMWARE("radeon/VERDE_me.bin");
@@ -80,6 +82,7 @@ MODULE_FIRMWARE("radeon/verde_ce.bin");
 MODULE_FIRMWARE("radeon/verde_mc.bin");
 MODULE_FIRMWARE("radeon/verde_rlc.bin");
 MODULE_FIRMWARE("radeon/verde_smc.bin");
+MODULE_FIRMWARE("radeon/verde_k_smc.bin");
 
 MODULE_FIRMWARE("radeon/OLAND_pfp.bin");
 MODULE_FIRMWARE("radeon/OLAND_me.bin");
@@ -95,6 +98,7 @@ MODULE_FIRMWARE("radeon/oland_ce.bin");
 MODULE_FIRMWARE("radeon/oland_mc.bin");
 MODULE_FIRMWARE("radeon/oland_rlc.bin");
 MODULE_FIRMWARE("radeon/oland_smc.bin");
+MODULE_FIRMWARE("radeon/oland_k_smc.bin");
 
 MODULE_FIRMWARE("radeon/HAINAN_pfp.bin");
 MODULE_FIRMWARE("radeon/HAINAN_me.bin");
@@ -110,6 +114,7 @@ MODULE_FIRMWARE("radeon/hainan_ce.bin");
 MODULE_FIRMWARE("radeon/hainan_mc.bin");
 MODULE_FIRMWARE("radeon/hainan_rlc.bin");
 MODULE_FIRMWARE("radeon/hainan_smc.bin");
+MODULE_FIRMWARE("radeon/hainan_k_smc.bin");
 
 static u32 si_get_cu_active_bitmap(struct radeon_device *rdev, u32 se, u32 sh);
 static void si_pcie_gen3_enable(struct radeon_device *rdev);
@@ -1653,12 +1658,16 @@ static int si_init_microcode(struct radeon_device *rdev)
         char fw_name[30];
         int err;
         int new_fw = 0;
+        bool new_smc = false;
 
         DRM_DEBUG("\n");
 
         switch (rdev->family) {
         case CHIP_TAHITI:
                 chip_name = "TAHITI";
+                /* XXX: figure out which Tahitis need the new ucode */
+                if (0)
+                        new_smc = true;
                 new_chip_name = "tahiti";
                 pfp_req_size = SI_PFP_UCODE_SIZE * 4;
                 me_req_size = SI_PM4_UCODE_SIZE * 4;
@@ -1670,6 +1679,13 @@ static int si_init_microcode(struct radeon_device *rdev)
                 break;
         case CHIP_PITCAIRN:
                 chip_name = "PITCAIRN";
+                if ((rdev->pdev->revision == 0x81) ||
+                    (rdev->pdev->device == 0x6810) ||
+                    (rdev->pdev->device == 0x6811) ||
+                    (rdev->pdev->device == 0x6816) ||
+                    (rdev->pdev->device == 0x6817) ||
+                    (rdev->pdev->device == 0x6806))
+                        new_smc = true;
                 new_chip_name = "pitcairn";
                 pfp_req_size = SI_PFP_UCODE_SIZE * 4;
                 me_req_size = SI_PM4_UCODE_SIZE * 4;
@@ -1681,6 +1697,16 @@ static int si_init_microcode(struct radeon_device *rdev)
                 break;
         case CHIP_VERDE:
                 chip_name = "VERDE";
+                if ((rdev->pdev->revision == 0x81) ||
+                    (rdev->pdev->revision == 0x83) ||
+                    (rdev->pdev->revision == 0x87) ||
+                    (rdev->pdev->device == 0x6820) ||
+                    (rdev->pdev->device == 0x6821) ||
+                    (rdev->pdev->device == 0x6822) ||
+                    (rdev->pdev->device == 0x6823) ||
+                    (rdev->pdev->device == 0x682A) ||
+                    (rdev->pdev->device == 0x682B))
+                        new_smc = true;
                 new_chip_name = "verde";
                 pfp_req_size = SI_PFP_UCODE_SIZE * 4;
                 me_req_size = SI_PM4_UCODE_SIZE * 4;
@@ -1692,6 +1718,13 @@ static int si_init_microcode(struct radeon_device *rdev)
                 break;
         case CHIP_OLAND:
                 chip_name = "OLAND";
+                if ((rdev->pdev->revision == 0xC7) ||
+                    (rdev->pdev->revision == 0x80) ||
+                    (rdev->pdev->revision == 0x81) ||
+                    (rdev->pdev->revision == 0x83) ||
+                    (rdev->pdev->device == 0x6604) ||
+                    (rdev->pdev->device == 0x6605))
+                        new_smc = true;
                 new_chip_name = "oland";
                 pfp_req_size = SI_PFP_UCODE_SIZE * 4;
                 me_req_size = SI_PM4_UCODE_SIZE * 4;
@@ -1702,6 +1735,13 @@ static int si_init_microcode(struct radeon_device *rdev)
                 break;
         case CHIP_HAINAN:
                 chip_name = "HAINAN";
+                if ((rdev->pdev->revision == 0x81) ||
+                    (rdev->pdev->revision == 0x83) ||
+                    (rdev->pdev->revision == 0xC3) ||
+                    (rdev->pdev->device == 0x6664) ||
+                    (rdev->pdev->device == 0x6665) ||
+                    (rdev->pdev->device == 0x6667))
+                        new_smc = true;
                 new_chip_name = "hainan";
                 pfp_req_size = SI_PFP_UCODE_SIZE * 4;
                 me_req_size = SI_PM4_UCODE_SIZE * 4;
@@ -1847,7 +1887,10 @@ static int si_init_microcode(struct radeon_device *rdev)
                 }
         }
 
-        snprintf(fw_name, sizeof(fw_name), "radeon/%s_smc.bin", new_chip_name);
+        if (new_smc)
+                snprintf(fw_name, sizeof(fw_name), "radeon/%s_k_smc.bin", new_chip_name);
+        else
+                snprintf(fw_name, sizeof(fw_name), "radeon/%s_smc.bin", new_chip_name);
         err = request_firmware(&rdev->smc_fw, fw_name, rdev->dev);
         if (err) {
                 snprintf(fw_name, sizeof(fw_name), "radeon/%s_smc.bin", chip_name);
diff --git a/drivers/gpu/drm/rcar-du/Kconfig b/drivers/gpu/drm/rcar-du/Kconfig
index 7fc3ca5ce6c7..4c2fd056dd6d 100644
--- a/drivers/gpu/drm/rcar-du/Kconfig
+++ b/drivers/gpu/drm/rcar-du/Kconfig
@@ -6,7 +6,6 @@ config DRM_RCAR_DU
         select DRM_KMS_HELPER
         select DRM_KMS_CMA_HELPER
         select DRM_GEM_CMA_HELPER
-        select DRM_KMS_FB_HELPER
         select VIDEOMODE_HELPERS
         help
           Choose this option if you have an R-Car chipset.
diff --git a/drivers/gpu/drm/rcar-du/Makefile b/drivers/gpu/drm/rcar-du/Makefile
index 827711e28226..d3b44651061a 100644
--- a/drivers/gpu/drm/rcar-du/Makefile
+++ b/drivers/gpu/drm/rcar-du/Makefile
@@ -7,8 +7,8 @@ rcar-du-drm-y := rcar_du_crtc.o \
                  rcar_du_plane.o \
                  rcar_du_vgacon.o
 
-rcar-du-drm-$(CONFIG_DRM_RCAR_HDMI)     += rcar_du_hdmicon.o \
-                                           rcar_du_hdmienc.o
+rcar-du-drm-$(CONFIG_DRM_RCAR_HDMI)     += rcar_du_hdmienc.o
+
 rcar-du-drm-$(CONFIG_DRM_RCAR_LVDS)     += rcar_du_lvdsenc.o
 
 rcar-du-drm-$(CONFIG_DRM_RCAR_VSP)      += rcar_du_vsp.o
diff --git a/drivers/gpu/drm/rcar-du/rcar_du_crtc.c b/drivers/gpu/drm/rcar-du/rcar_du_crtc.c
index 0d8bdda736f9..e39fcef2e033 100644
--- a/drivers/gpu/drm/rcar-du/rcar_du_crtc.c
+++ b/drivers/gpu/drm/rcar-du/rcar_du_crtc.c
@@ -552,7 +552,7 @@ static irqreturn_t rcar_du_crtc_irq(int irq, void *arg)
         rcar_du_crtc_write(rcrtc, DSRCR, status & DSRCR_MASK);
 
         if (status & DSSR_FRM) {
-                drm_handle_vblank(rcrtc->crtc.dev, rcrtc->index);
+                drm_crtc_handle_vblank(&rcrtc->crtc);
                 rcar_du_crtc_finish_page_flip(rcrtc);
                 ret = IRQ_HANDLED;
         }
diff --git a/drivers/gpu/drm/rcar-du/rcar_du_drv.c b/drivers/gpu/drm/rcar-du/rcar_du_drv.c
index fb9242d27883..899ef7a2a7b4 100644
--- a/drivers/gpu/drm/rcar-du/rcar_du_drv.c
+++ b/drivers/gpu/drm/rcar-du/rcar_du_drv.c
@@ -217,7 +217,7 @@ static struct drm_driver rcar_du_driver = {
         .get_vblank_counter     = drm_vblank_no_hw_counter,
         .enable_vblank          = rcar_du_enable_vblank,
         .disable_vblank         = rcar_du_disable_vblank,
-        .gem_free_object        = drm_gem_cma_free_object,
+        .gem_free_object_unlocked = drm_gem_cma_free_object,
         .gem_vm_ops             = &drm_gem_cma_vm_ops,
         .prime_handle_to_fd     = drm_gem_prime_handle_to_fd,
         .prime_fd_to_handle     = drm_gem_prime_fd_to_handle,
@@ -278,7 +278,6 @@ static int rcar_du_remove(struct platform_device *pdev)
         struct rcar_du_device *rcdu = platform_get_drvdata(pdev);
         struct drm_device *ddev = rcdu->ddev;
 
-        drm_connector_unregister_all(ddev);
         drm_dev_unregister(ddev);
 
         if (rcdu->fbdev)
@@ -320,8 +319,6 @@ static int rcar_du_probe(struct platform_device *pdev)
         if (!ddev)
                 return -ENOMEM;
 
-        drm_dev_set_unique(ddev, dev_name(&pdev->dev));
-
         rcdu->ddev = ddev;
         ddev->dev_private = rcdu;
 
@@ -339,15 +336,15 @@ static int rcar_du_probe(struct platform_device *pdev)
          * disabled for all CRTCs.
          */
         ret = drm_vblank_init(ddev, (1 << rcdu->info->num_crtcs) - 1);
-        if (ret < 0) {
-                dev_err(&pdev->dev, "failed to initialize vblank\n");
+        if (ret < 0)
                 goto error;
-        }
 
         /* DRM/KMS objects */
         ret = rcar_du_modeset_init(rcdu);
         if (ret < 0) {
-                dev_err(&pdev->dev, "failed to initialize DRM/KMS (%d)\n", ret);
+                if (ret != -EPROBE_DEFER)
+                        dev_err(&pdev->dev,
+                                "failed to initialize DRM/KMS (%d)\n", ret);
                 goto error;
         }
 
@@ -360,10 +357,6 @@ static int rcar_du_probe(struct platform_device *pdev)
         if (ret)
                 goto error;
 
-        ret = drm_connector_register_all(ddev);
-        if (ret < 0)
-                goto error;
-
         DRM_INFO("Device %s probed\n", dev_name(&pdev->dev));
 
         return 0;
diff --git a/drivers/gpu/drm/rcar-du/rcar_du_encoder.c b/drivers/gpu/drm/rcar-du/rcar_du_encoder.c
index 4e939e41f030..ab8645c57e2d 100644
--- a/drivers/gpu/drm/rcar-du/rcar_du_encoder.c
+++ b/drivers/gpu/drm/rcar-du/rcar_du_encoder.c
@@ -19,7 +19,6 @@
 
 #include "rcar_du_drv.h"
 #include "rcar_du_encoder.h"
-#include "rcar_du_hdmicon.h"
 #include "rcar_du_hdmienc.h"
 #include "rcar_du_kms.h"
 #include "rcar_du_lvdscon.h"
@@ -27,18 +26,6 @@
 #include "rcar_du_vgacon.h"
 
 /* -----------------------------------------------------------------------------
- * Common connector functions
- */
-
-struct drm_encoder *
-rcar_du_connector_best_encoder(struct drm_connector *connector)
-{
-        struct rcar_du_connector *rcon = to_rcar_connector(connector);
-
-        return rcar_encoder_to_drm_encoder(rcon->encoder);
-}
-
-/* -----------------------------------------------------------------------------
  * Encoder
  */
 
@@ -186,7 +173,7 @@ int rcar_du_encoder_init(struct rcar_du_device *rcdu,
                 break;
 
         case DRM_MODE_ENCODER_TMDS:
-                ret = rcar_du_hdmi_connector_init(rcdu, renc);
+                /* connector managed by the bridge driver */
                 break;
 
         default:
diff --git a/drivers/gpu/drm/rcar-du/rcar_du_encoder.h b/drivers/gpu/drm/rcar-du/rcar_du_encoder.h
index 719b6f2a031c..7fc10a9c34c3 100644
--- a/drivers/gpu/drm/rcar-du/rcar_du_encoder.h
+++ b/drivers/gpu/drm/rcar-du/rcar_du_encoder.h
@@ -15,7 +15,6 @@
 #define __RCAR_DU_ENCODER_H__
 
 #include <drm/drm_crtc.h>
-#include <drm/drm_encoder_slave.h>
 
 struct rcar_du_device;
 struct rcar_du_hdmienc;
@@ -30,16 +29,16 @@ enum rcar_du_encoder_type {
 };
 
 struct rcar_du_encoder {
-        struct drm_encoder_slave slave;
+        struct drm_encoder base;
         enum rcar_du_output output;
         struct rcar_du_hdmienc *hdmi;
         struct rcar_du_lvdsenc *lvds;
 };
 
 #define to_rcar_encoder(e) \
-        container_of(e, struct rcar_du_encoder, slave.base)
+        container_of(e, struct rcar_du_encoder, base)
 
-#define rcar_encoder_to_drm_encoder(e)  (&(e)->slave.base)
+#define rcar_encoder_to_drm_encoder(e)  (&(e)->base)
 
 struct rcar_du_connector {
         struct drm_connector connector;
@@ -49,9 +48,6 @@ struct rcar_du_connector {
 #define to_rcar_connector(c) \
         container_of(c, struct rcar_du_connector, connector)
 
-struct drm_encoder *
-rcar_du_connector_best_encoder(struct drm_connector *connector);
-
 int rcar_du_encoder_init(struct rcar_du_device *rcdu,
                          enum rcar_du_encoder_type type,
                          enum rcar_du_output output,
diff --git a/drivers/gpu/drm/rcar-du/rcar_du_hdmicon.c b/drivers/gpu/drm/rcar-du/rcar_du_hdmicon.c
deleted file mode 100644
index 6c927144b5c9..000000000000
--- a/drivers/gpu/drm/rcar-du/rcar_du_hdmicon.c
+++ /dev/null
@@ -1,117 +0,0 @@
-/*
- * R-Car Display Unit HDMI Connector
- *
- * Copyright (C) 2014 Renesas Electronics Corporation
- *
- * Contact: Laurent Pinchart (laurent.pinchart@ideasonboard.com)
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
- */
-
-#include <drm/drmP.h>
-#include <drm/drm_atomic_helper.h>
-#include <drm/drm_crtc.h>
-#include <drm/drm_crtc_helper.h>
-#include <drm/drm_encoder_slave.h>
-
-#include "rcar_du_drv.h"
-#include "rcar_du_encoder.h"
-#include "rcar_du_hdmicon.h"
-#include "rcar_du_kms.h"
-
-#define to_slave_funcs(e)       (to_rcar_encoder(e)->slave.slave_funcs)
-
-static int rcar_du_hdmi_connector_get_modes(struct drm_connector *connector)
-{
-        struct rcar_du_connector *con = to_rcar_connector(connector);
-        struct drm_encoder *encoder = rcar_encoder_to_drm_encoder(con->encoder);
-        const struct drm_encoder_slave_funcs *sfuncs = to_slave_funcs(encoder);
-
-        if (sfuncs->get_modes == NULL)
-                return 0;
-
-        return sfuncs->get_modes(encoder, connector);
-}
-
-static int rcar_du_hdmi_connector_mode_valid(struct drm_connector *connector,
-                                             struct drm_display_mode *mode)
-{
-        struct rcar_du_connector *con = to_rcar_connector(connector);
-        struct drm_encoder *encoder = rcar_encoder_to_drm_encoder(con->encoder);
-        const struct drm_encoder_slave_funcs *sfuncs = to_slave_funcs(encoder);
-
-        if (sfuncs->mode_valid == NULL)
-                return MODE_OK;
-
-        return sfuncs->mode_valid(encoder, mode);
-}
-
-static const struct drm_connector_helper_funcs connector_helper_funcs = {
-        .get_modes = rcar_du_hdmi_connector_get_modes,
-        .mode_valid = rcar_du_hdmi_connector_mode_valid,
-        .best_encoder = rcar_du_connector_best_encoder,
-};
-
-static enum drm_connector_status
-rcar_du_hdmi_connector_detect(struct drm_connector *connector, bool force)
-{
-        struct rcar_du_connector *con = to_rcar_connector(connector);
-        struct drm_encoder *encoder = rcar_encoder_to_drm_encoder(con->encoder);
-        const struct drm_encoder_slave_funcs *sfuncs = to_slave_funcs(encoder);
-
-        if (sfuncs->detect == NULL)
-                return connector_status_unknown;
-
-        return sfuncs->detect(encoder, connector);
-}
-
-static const struct drm_connector_funcs connector_funcs = {
-        .dpms = drm_atomic_helper_connector_dpms,
-        .reset = drm_atomic_helper_connector_reset,
-        .detect = rcar_du_hdmi_connector_detect,
-        .fill_modes = drm_helper_probe_single_connector_modes,
-        .destroy = drm_connector_cleanup,
-        .atomic_duplicate_state = drm_atomic_helper_connector_duplicate_state,
-        .atomic_destroy_state = drm_atomic_helper_connector_destroy_state,
-};
-
-int rcar_du_hdmi_connector_init(struct rcar_du_device *rcdu,
-                                struct rcar_du_encoder *renc)
-{
-        struct drm_encoder *encoder = rcar_encoder_to_drm_encoder(renc);
-        struct rcar_du_connector *rcon;
-        struct drm_connector *connector;
-        int ret;
-
-        rcon = devm_kzalloc(rcdu->dev, sizeof(*rcon), GFP_KERNEL);
-        if (rcon == NULL)
-                return -ENOMEM;
-
-        connector = &rcon->connector;
-        connector->display_info.width_mm = 0;
-        connector->display_info.height_mm = 0;
-        connector->interlace_allowed = true;
-        connector->polled = DRM_CONNECTOR_POLL_HPD;
-
-        ret = drm_connector_init(rcdu->ddev, connector, &connector_funcs,
-                                 DRM_MODE_CONNECTOR_HDMIA);
-        if (ret < 0)
-                return ret;
-
-        drm_connector_helper_add(connector, &connector_helper_funcs);
-
-        connector->dpms = DRM_MODE_DPMS_OFF;
-        drm_object_property_set_value(&connector->base,
-                rcdu->ddev->mode_config.dpms_property, DRM_MODE_DPMS_OFF);
-
-        ret = drm_mode_connector_attach_encoder(connector, encoder);
-        if (ret < 0)
-                return ret;
-
-        rcon->encoder = renc;
-
-        return 0;
-}
diff --git a/drivers/gpu/drm/rcar-du/rcar_du_hdmicon.h b/drivers/gpu/drm/rcar-du/rcar_du_hdmicon.h
deleted file mode 100644
index 87daa949227f..000000000000
--- a/drivers/gpu/drm/rcar-du/rcar_du_hdmicon.h
+++ /dev/null
@@ -1,31 +0,0 @@
-/*
- * R-Car Display Unit HDMI Connector
- *
- * Copyright (C) 2014 Renesas Electronics Corporation
- *
- * Contact: Laurent Pinchart (laurent.pinchart@ideasonboard.com)
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
- */
-
-#ifndef __RCAR_DU_HDMICON_H__
-#define __RCAR_DU_HDMICON_H__
-
-struct rcar_du_device;
-struct rcar_du_encoder;
-
-#if IS_ENABLED(CONFIG_DRM_RCAR_HDMI)
-int rcar_du_hdmi_connector_init(struct rcar_du_device *rcdu,
-                                struct rcar_du_encoder *renc);
-#else
-static inline int rcar_du_hdmi_connector_init(struct rcar_du_device *rcdu,
-                                              struct rcar_du_encoder *renc)
-{
-        return -ENOSYS;
-}
-#endif
-
-#endif /* __RCAR_DU_HDMICON_H__ */
diff --git a/drivers/gpu/drm/rcar-du/rcar_du_hdmienc.c b/drivers/gpu/drm/rcar-du/rcar_du_hdmienc.c
index 461662d231e2..4de3ff0dbebd 100644
--- a/drivers/gpu/drm/rcar-du/rcar_du_hdmienc.c
+++ b/drivers/gpu/drm/rcar-du/rcar_du_hdmienc.c
@@ -16,7 +16,6 @@
 #include <drm/drmP.h>
 #include <drm/drm_crtc.h>
 #include <drm/drm_crtc_helper.h>
-#include <drm/drm_encoder_slave.h>
 
 #include "rcar_du_drv.h"
 #include "rcar_du_encoder.h"
@@ -25,20 +24,14 @@
 
 struct rcar_du_hdmienc {
         struct rcar_du_encoder *renc;
-        struct device *dev;
         bool enabled;
 };
 
 #define to_rcar_hdmienc(e)      (to_rcar_encoder(e)->hdmi)
-#define to_slave_funcs(e)       (to_rcar_encoder(e)->slave.slave_funcs)
 
 static void rcar_du_hdmienc_disable(struct drm_encoder *encoder)
 {
         struct rcar_du_hdmienc *hdmienc = to_rcar_hdmienc(encoder);
-        const struct drm_encoder_slave_funcs *sfuncs = to_slave_funcs(encoder);
-
-        if (sfuncs->dpms)
-                sfuncs->dpms(encoder, DRM_MODE_DPMS_OFF);
 
         if (hdmienc->renc->lvds)
                 rcar_du_lvdsenc_enable(hdmienc->renc->lvds, encoder->crtc,
@@ -50,15 +43,11 @@ static void rcar_du_hdmienc_disable(struct drm_encoder *encoder)
 static void rcar_du_hdmienc_enable(struct drm_encoder *encoder)
 {
         struct rcar_du_hdmienc *hdmienc = to_rcar_hdmienc(encoder);
-        const struct drm_encoder_slave_funcs *sfuncs = to_slave_funcs(encoder);
 
         if (hdmienc->renc->lvds)
                 rcar_du_lvdsenc_enable(hdmienc->renc->lvds, encoder->crtc,
                                        true);
 
-        if (sfuncs->dpms)
-                sfuncs->dpms(encoder, DRM_MODE_DPMS_ON);
-
         hdmienc->enabled = true;
 }
 
@@ -67,29 +56,21 @@ static int rcar_du_hdmienc_atomic_check(struct drm_encoder *encoder,
                                         struct drm_connector_state *conn_state)
 {
         struct rcar_du_hdmienc *hdmienc = to_rcar_hdmienc(encoder);
-        const struct drm_encoder_slave_funcs *sfuncs = to_slave_funcs(encoder);
         struct drm_display_mode *adjusted_mode = &crtc_state->adjusted_mode;
-        const struct drm_display_mode *mode = &crtc_state->mode;
 
         if (hdmienc->renc->lvds)
                 rcar_du_lvdsenc_atomic_check(hdmienc->renc->lvds,
                                              adjusted_mode);
 
-        if (sfuncs->mode_fixup == NULL)
-                return 0;
-
-        return sfuncs->mode_fixup(encoder, mode, adjusted_mode) ? 0 : -EINVAL;
+        return 0;
 }
 
+
 static void rcar_du_hdmienc_mode_set(struct drm_encoder *encoder,
                                      struct drm_display_mode *mode,
                                      struct drm_display_mode *adjusted_mode)
 {
         struct rcar_du_hdmienc *hdmienc = to_rcar_hdmienc(encoder);
-        const struct drm_encoder_slave_funcs *sfuncs = to_slave_funcs(encoder);
-
-        if (sfuncs->mode_set)
-                sfuncs->mode_set(encoder, mode, adjusted_mode);
 
         rcar_du_crtc_route_output(encoder->crtc, hdmienc->renc->output);
 }
@@ -109,7 +90,6 @@ static void rcar_du_hdmienc_cleanup(struct drm_encoder *encoder)
                 rcar_du_hdmienc_disable(encoder);
 
         drm_encoder_cleanup(encoder);
-        put_device(hdmienc->dev);
 }
 
 static const struct drm_encoder_funcs encoder_funcs = {
@@ -120,8 +100,7 @@ int rcar_du_hdmienc_init(struct rcar_du_device *rcdu,
                          struct rcar_du_encoder *renc, struct device_node *np)
 {
         struct drm_encoder *encoder = rcar_encoder_to_drm_encoder(renc);
-        struct drm_i2c_encoder_driver *driver;
-        struct i2c_client *i2c_slave;
+        struct drm_bridge *bridge;
         struct rcar_du_hdmienc *hdmienc;
         int ret;
 
@@ -129,44 +108,29 @@ int rcar_du_hdmienc_init(struct rcar_du_device *rcdu,
         if (hdmienc == NULL)
                 return -ENOMEM;
 
-        /* Locate the slave I2C device and driver. */
-        i2c_slave = of_find_i2c_device_by_node(np);
-        if (!i2c_slave || !i2c_get_clientdata(i2c_slave)) {
-                dev_dbg(rcdu->dev,
-                        "can't get I2C slave for %s, deferring probe\n",
-                        of_node_full_name(np));
+        /* Locate drm bridge from the hdmi encoder DT node */
+        bridge = of_drm_find_bridge(np);
+        if (!bridge)
                 return -EPROBE_DEFER;
-        }
-
-        hdmienc->dev = &i2c_slave->dev;
-
-        if (hdmienc->dev->driver == NULL) {
-                dev_dbg(rcdu->dev,
-                        "I2C slave %s not probed yet, deferring probe\n",
-                        dev_name(hdmienc->dev));
-                ret = -EPROBE_DEFER;
-                goto error;
-        }
-
-        /* Initialize the slave encoder. */
-        driver = to_drm_i2c_encoder_driver(to_i2c_driver(hdmienc->dev->driver));
-        ret = driver->encoder_init(i2c_slave, rcdu->ddev, &renc->slave);
-        if (ret < 0)
-                goto error;
 
         ret = drm_encoder_init(rcdu->ddev, encoder, &encoder_funcs,
                                DRM_MODE_ENCODER_TMDS, NULL);
         if (ret < 0)
-                goto error;
+                return ret;
 
         drm_encoder_helper_add(encoder, &encoder_helper_funcs);
 
         renc->hdmi = hdmienc;
         hdmienc->renc = renc;
 
-        return 0;
+        /* Link drm_bridge to encoder */
+        bridge->encoder = encoder;
+
+        ret = drm_bridge_attach(rcdu->ddev, bridge);
+        if (ret) {
+                drm_encoder_cleanup(encoder);
+                return ret;
+        }
 
-error:
-        put_device(hdmienc->dev);
-        return ret;
+        return 0;
 }
diff --git a/drivers/gpu/drm/rcar-du/rcar_du_kms.c b/drivers/gpu/drm/rcar-du/rcar_du_kms.c
index e70a4f33d970..6bb032d8ac6b 100644
--- a/drivers/gpu/drm/rcar-du/rcar_du_kms.c
+++ b/drivers/gpu/drm/rcar-du/rcar_du_kms.c
@@ -288,6 +288,8 @@ static int rcar_du_atomic_commit(struct drm_device *dev,
 {
         struct rcar_du_device *rcdu = dev->dev_private;
         struct rcar_du_commit *commit;
+        struct drm_crtc *crtc;
+        struct drm_crtc_state *crtc_state;
         unsigned int i;
         int ret;
 
@@ -309,10 +311,8 @@ static int rcar_du_atomic_commit(struct drm_device *dev,
         /* Wait until all affected CRTCs have completed previous commits and
          * mark them as pending.
          */
-        for (i = 0; i < dev->mode_config.num_crtc; ++i) {
-                if (state->crtcs[i])
-                        commit->crtcs |= 1 << drm_crtc_index(state->crtcs[i]);
-        }
+        for_each_crtc_in_state(state, crtc, crtc_state, i)
+                commit->crtcs |= drm_crtc_mask(crtc);
 
         spin_lock(&rcdu->commit.wait.lock);
         ret = wait_event_interruptible_locked(rcdu->commit.wait,
@@ -327,7 +327,7 @@ static int rcar_du_atomic_commit(struct drm_device *dev,
         }
 
         /* Swap the state, this is the point of no return. */
-        drm_atomic_helper_swap_state(dev, state);
+        drm_atomic_helper_swap_state(state, true);
 
         if (nonblock)
                 schedule_work(&commit->work);
diff --git a/drivers/gpu/drm/rcar-du/rcar_du_lvdscon.c b/drivers/gpu/drm/rcar-du/rcar_du_lvdscon.c
index e905f5da7aaa..6afd0af312ba 100644
--- a/drivers/gpu/drm/rcar-du/rcar_du_lvdscon.c
+++ b/drivers/gpu/drm/rcar-du/rcar_du_lvdscon.c
@@ -59,7 +59,6 @@ static int rcar_du_lvds_connector_get_modes(struct drm_connector *connector)
 
 static const struct drm_connector_helper_funcs connector_helper_funcs = {
         .get_modes = rcar_du_lvds_connector_get_modes,
-        .best_encoder = rcar_du_connector_best_encoder,
 };
 
 static enum drm_connector_status
diff --git a/drivers/gpu/drm/rcar-du/rcar_du_plane.c b/drivers/gpu/drm/rcar-du/rcar_du_plane.c
index d445e67f78e1..bfe31ca870cc 100644
--- a/drivers/gpu/drm/rcar-du/rcar_du_plane.c
+++ b/drivers/gpu/drm/rcar-du/rcar_du_plane.c
@@ -140,18 +140,17 @@ int rcar_du_atomic_check_planes(struct drm_device *dev,
         bool needs_realloc = false;
         unsigned int groups = 0;
         unsigned int i;
+        struct drm_plane *drm_plane;
+        struct drm_plane_state *drm_plane_state;
 
         /* Check if hardware planes need to be reallocated. */
-        for (i = 0; i < dev->mode_config.num_total_plane; ++i) {
+        for_each_plane_in_state(state, drm_plane, drm_plane_state, i) {
                 struct rcar_du_plane_state *plane_state;
                 struct rcar_du_plane *plane;
                 unsigned int index;
 
-                if (!state->planes[i])
-                        continue;
-
-                plane = to_rcar_plane(state->planes[i]);
-                plane_state = to_rcar_plane_state(state->plane_states[i]);
+                plane = to_rcar_plane(drm_plane);
+                plane_state = to_rcar_plane_state(drm_plane_state);
 
                 dev_dbg(rcdu->dev, "%s: checking plane (%u,%tu)\n", __func__,
                         plane->group->index, plane - plane->group->planes);
@@ -247,18 +246,15 @@ int rcar_du_atomic_check_planes(struct drm_device *dev,
         }
 
         /* Reallocate hardware planes for each plane that needs it. */
-        for (i = 0; i < dev->mode_config.num_total_plane; ++i) {
+        for_each_plane_in_state(state, drm_plane, drm_plane_state, i) {
                 struct rcar_du_plane_state *plane_state;
                 struct rcar_du_plane *plane;
                 unsigned int crtc_planes;
                 unsigned int free;
                 int idx;
 
-                if (!state->planes[i])
-                        continue;
-
-                plane = to_rcar_plane(state->planes[i]);
-                plane_state = to_rcar_plane_state(state->plane_states[i]);
+                plane = to_rcar_plane(drm_plane);
+                plane_state = to_rcar_plane_state(drm_plane_state);
 
                 dev_dbg(rcdu->dev, "%s: allocating plane (%u,%tu)\n", __func__,
                         plane->group->index, plane - plane->group->planes);
diff --git a/drivers/gpu/drm/rcar-du/rcar_du_regs.h b/drivers/gpu/drm/rcar-du/rcar_du_regs.h
index d2f66068e52c..fedb0161e234 100644
--- a/drivers/gpu/drm/rcar-du/rcar_du_regs.h
+++ b/drivers/gpu/drm/rcar-du/rcar_du_regs.h
@@ -195,9 +195,10 @@
 #define DEFR6_ODPM12_DISP       (2 << 8)
 #define DEFR6_ODPM12_CDE        (3 << 8)
 #define DEFR6_ODPM12_MASK       (3 << 8)
-#define DEFR6_TCNE2             (1 << 6)
+#define DEFR6_TCNE1             (1 << 6)
+#define DEFR6_TCNE0             (1 << 4)
 #define DEFR6_MLOS1             (1 << 2)
-#define DEFR6_DEFAULT           (DEFR6_CODE | DEFR6_TCNE2)
+#define DEFR6_DEFAULT           (DEFR6_CODE | DEFR6_TCNE1)
 
 /* -----------------------------------------------------------------------------
  * R8A7790-only Control Registers
diff --git a/drivers/gpu/drm/rcar-du/rcar_du_vgacon.c b/drivers/gpu/drm/rcar-du/rcar_du_vgacon.c
index 9d7e5c99caf6..8d6125c1c0f9 100644
--- a/drivers/gpu/drm/rcar-du/rcar_du_vgacon.c
+++ b/drivers/gpu/drm/rcar-du/rcar_du_vgacon.c
@@ -28,7 +28,6 @@ static int rcar_du_vga_connector_get_modes(struct drm_connector *connector)
 
 static const struct drm_connector_helper_funcs connector_helper_funcs = {
         .get_modes = rcar_du_vga_connector_get_modes,
-        .best_encoder = rcar_du_connector_best_encoder,
 };
 
 static enum drm_connector_status
@@ -79,7 +78,5 @@ int rcar_du_vga_connector_init(struct rcar_du_device *rcdu,
         if (ret < 0)
                 return ret;
 
-        rcon->encoder = renc;
-
         return 0;
 }
diff --git a/drivers/gpu/drm/rockchip/Kconfig b/drivers/gpu/drm/rockchip/Kconfig
index d30bdc38a760..3c58669a06ce 100644
--- a/drivers/gpu/drm/rockchip/Kconfig
+++ b/drivers/gpu/drm/rockchip/Kconfig
@@ -2,12 +2,9 @@ config DRM_ROCKCHIP
         tristate "DRM Support for Rockchip"
         depends on DRM && ROCKCHIP_IOMMU
         depends on RESET_CONTROLLER
+        select DRM_GEM_CMA_HELPER
         select DRM_KMS_HELPER
-        select DRM_KMS_FB_HELPER
         select DRM_PANEL
-        select FB_CFB_FILLRECT
-        select FB_CFB_COPYAREA
-        select FB_CFB_IMAGEBLIT
         select VIDEOMODE_HELPERS
         help
           Choose this option if you have a Rockchip soc chipset.
diff --git a/drivers/gpu/drm/rockchip/analogix_dp-rockchip.c b/drivers/gpu/drm/rockchip/analogix_dp-rockchip.c
index 7f6a55cae27a..89aadbf465f8 100644
--- a/drivers/gpu/drm/rockchip/analogix_dp-rockchip.c
+++ b/drivers/gpu/drm/rockchip/analogix_dp-rockchip.c
@@ -14,6 +14,7 @@
 
 #include <linux/component.h>
 #include <linux/mfd/syscon.h>
+#include <linux/of_device.h>
 #include <linux/of_graph.h>
 #include <linux/regmap.h>
 #include <linux/reset.h>
@@ -33,13 +34,28 @@
 #include "rockchip_drm_drv.h"
 #include "rockchip_drm_vop.h"
 
+#define RK3288_GRF_SOC_CON6             0x25c
+#define RK3288_EDP_LCDC_SEL             BIT(5)
+#define RK3399_GRF_SOC_CON20            0x6250
+#define RK3399_EDP_LCDC_SEL             BIT(5)
+
+#define HIWORD_UPDATE(val, mask)        (val | (mask) << 16)
+
 #define to_dp(nm)       container_of(nm, struct rockchip_dp_device, nm)
 
-/* dp grf register offset */
-#define GRF_SOC_CON6                            0x025c
-#define GRF_EDP_LCD_SEL_MASK                    BIT(5)
-#define GRF_EDP_SEL_VOP_LIT                     BIT(5)
-#define GRF_EDP_SEL_VOP_BIG                     0
+/**
+ * struct rockchip_dp_chip_data - splite the grf setting of kind of chips
+ * @lcdsel_grf_reg: grf register offset of lcdc select
+ * @lcdsel_big: reg value of selecting vop big for eDP
+ * @lcdsel_lit: reg value of selecting vop little for eDP
+ * @chip_type: specific chip type
+ */
+struct rockchip_dp_chip_data {
+        u32     lcdsel_grf_reg;
+        u32     lcdsel_big;
+        u32     lcdsel_lit;
+        u32     chip_type;
+};
 
 struct rockchip_dp_device {
         struct drm_device        *drm_dev;
@@ -48,9 +64,12 @@ struct rockchip_dp_device {
         struct drm_display_mode  mode;
 
         struct clk               *pclk;
+        struct clk               *grfclk;
         struct regmap            *grf;
         struct reset_control     *rst;
 
+        const struct rockchip_dp_chip_data *data;
+
         struct analogix_dp_plat_data plat_data;
 };
 
@@ -77,6 +96,7 @@ static int rockchip_dp_poweron(struct analogix_dp_plat_data *plat_data)
         ret = rockchip_dp_pre_init(dp);
         if (ret < 0) {
                 dev_err(dp->dev, "failed to dp pre init %d\n", ret);
+                clk_disable_unprepare(dp->pclk);
                 return ret;
         }
 
@@ -92,6 +112,23 @@ static int rockchip_dp_powerdown(struct analogix_dp_plat_data *plat_data)
         return 0;
 }
 
+static int rockchip_dp_get_modes(struct analogix_dp_plat_data *plat_data,
+                                 struct drm_connector *connector)
+{
+        struct drm_display_info *di = &connector->display_info;
+        /* VOP couldn't output YUV video format for eDP rightly */
+        u32 mask = DRM_COLOR_FORMAT_YCRCB444 | DRM_COLOR_FORMAT_YCRCB422;
+
+        if ((di->color_formats & mask)) {
+                DRM_DEBUG_KMS("Swapping display color format from YUV to RGB\n");
+                di->color_formats &= ~mask;
+                di->color_formats |= DRM_COLOR_FORMAT_RGB444;
+                di->bpc = 8;
+        }
+
+        return 0;
+}
+
 static bool
 rockchip_dp_drm_encoder_mode_fixup(struct drm_encoder *encoder,
                                    const struct drm_display_mode *mode,
@@ -119,17 +156,23 @@ static void rockchip_dp_drm_encoder_enable(struct drm_encoder *encoder)
                 return;
 
         if (ret)
-                val = GRF_EDP_SEL_VOP_LIT | (GRF_EDP_LCD_SEL_MASK << 16);
+                val = dp->data->lcdsel_lit;
         else
-                val = GRF_EDP_SEL_VOP_BIG | (GRF_EDP_LCD_SEL_MASK << 16);
+                val = dp->data->lcdsel_big;
 
         dev_dbg(dp->dev, "vop %s output to dp\n", (ret) ? "LIT" : "BIG");
 
-        ret = regmap_write(dp->grf, GRF_SOC_CON6, val);
-        if (ret != 0) {
-                dev_err(dp->dev, "Could not write to GRF: %d\n", ret);
+        ret = clk_prepare_enable(dp->grfclk);
+        if (ret < 0) {
+                dev_err(dp->dev, "failed to enable grfclk %d\n", ret);
                 return;
         }
+
+        ret = regmap_write(dp->grf, dp->data->lcdsel_grf_reg, val);
+        if (ret != 0)
+                dev_err(dp->dev, "Could not write to GRF: %d\n", ret);
+
+        clk_disable_unprepare(dp->grfclk);
 }
 
 static void rockchip_dp_drm_encoder_nop(struct drm_encoder *encoder)
@@ -143,22 +186,29 @@ rockchip_dp_drm_encoder_atomic_check(struct drm_encoder *encoder,
                                       struct drm_connector_state *conn_state)
 {
         struct rockchip_crtc_state *s = to_rockchip_crtc_state(crtc_state);
+        struct rockchip_dp_device *dp = to_dp(encoder);
+        int ret;
 
         /*
-         * FIXME(Yakir): driver should configure the CRTC output video
-         * mode with the display information which indicated the monitor
-         * support colorimetry.
-         *
-         * But don't know why the CRTC driver seems could only output the
-         * RGBaaa rightly. For example, if connect the "innolux,n116bge"
-         * eDP screen, EDID would indicated that screen only accepted the
-         * 6bpc mode. But if I configure CRTC to RGB666 output, then eDP
-         * screen would show a blue picture (RGB888 show a green picture).
-         * But if I configure CTRC to RGBaaa, and eDP driver still keep
-         * RGB666 input video mode, then screen would works prefect.
+         * The hardware IC designed that VOP must output the RGB10 video
+         * format to eDP controller, and if eDP panel only support RGB8,
+         * then eDP controller should cut down the video data, not via VOP
+         * controller, that's why we need to hardcode the VOP output mode
+         * to RGA10 here.
          */
+
         s->output_mode = ROCKCHIP_OUT_MODE_AAAA;
         s->output_type = DRM_MODE_CONNECTOR_eDP;
+        if (dp->data->chip_type == RK3399_EDP) {
+                /*
+                 * For RK3399, VOP Lit must code the out mode to RGB888,
+                 * VOP Big must code the out mode to RGB10.
+                 */
+                ret = drm_of_encoder_active_endpoint_id(dp->dev->of_node,
+                                                        encoder);
+                if (ret > 0)
+                        s->output_mode = ROCKCHIP_OUT_MODE_P888;
+        }
 
         return 0;
 }
@@ -192,6 +242,16 @@ static int rockchip_dp_init(struct rockchip_dp_device *dp)
                 return PTR_ERR(dp->grf);
         }
 
+        dp->grfclk = devm_clk_get(dev, "grf");
+        if (PTR_ERR(dp->grfclk) == -ENOENT) {
+                dp->grfclk = NULL;
+        } else if (PTR_ERR(dp->grfclk) == -EPROBE_DEFER) {
+                return -EPROBE_DEFER;
+        } else if (IS_ERR(dp->grfclk)) {
+                dev_err(dev, "failed to get grf clock\n");
+                return PTR_ERR(dp->grfclk);
+        }
+
         dp->pclk = devm_clk_get(dev, "pclk");
         if (IS_ERR(dp->pclk)) {
                 dev_err(dev, "failed to get pclk property\n");
@@ -213,6 +273,7 @@ static int rockchip_dp_init(struct rockchip_dp_device *dp)
         ret = rockchip_dp_pre_init(dp);
         if (ret < 0) {
                 dev_err(dp->dev, "failed to pre init %d\n", ret);
+                clk_disable_unprepare(dp->pclk);
                 return ret;
         }
 
@@ -246,6 +307,7 @@ static int rockchip_dp_bind(struct device *dev, struct device *master,
                             void *data)
 {
         struct rockchip_dp_device *dp = dev_get_drvdata(dev);
+        const struct rockchip_dp_chip_data *dp_data;
         struct drm_device *drm_dev = data;
         int ret;
 
@@ -256,10 +318,15 @@ static int rockchip_dp_bind(struct device *dev, struct device *master,
          */
         dev_set_drvdata(dev, NULL);
 
+        dp_data = of_device_get_match_data(dev);
+        if (!dp_data)
+                return -ENODEV;
+
         ret = rockchip_dp_init(dp);
         if (ret < 0)
                 return ret;
 
+        dp->data = dp_data;
         dp->drm_dev = drm_dev;
 
         ret = rockchip_dp_drm_create_encoder(dp);
@@ -270,9 +337,10 @@ static int rockchip_dp_bind(struct device *dev, struct device *master,
 
         dp->plat_data.encoder = &dp->encoder;
 
-        dp->plat_data.dev_type = RK3288_DP;
+        dp->plat_data.dev_type = dp->data->chip_type;
         dp->plat_data.power_on = rockchip_dp_poweron;
         dp->plat_data.power_off = rockchip_dp_powerdown;
+        dp->plat_data.get_modes = rockchip_dp_get_modes;
 
         return analogix_dp_bind(dev, dp->drm_dev, &dp->plat_data);
 }
@@ -292,38 +360,33 @@ static int rockchip_dp_probe(struct platform_device *pdev)
 {
         struct device *dev = &pdev->dev;
         struct device_node *panel_node, *port, *endpoint;
+        struct drm_panel *panel = NULL;
         struct rockchip_dp_device *dp;
-        struct drm_panel *panel;
 
         port = of_graph_get_port_by_id(dev->of_node, 1);
-        if (!port) {
-                dev_err(dev, "can't find output port\n");
-                return -EINVAL;
-        }
-
-        endpoint = of_get_child_by_name(port, "endpoint");
-        of_node_put(port);
-        if (!endpoint) {
-                dev_err(dev, "no output endpoint found\n");
-                return -EINVAL;
-        }
-
-        panel_node = of_graph_get_remote_port_parent(endpoint);
-        of_node_put(endpoint);
-        if (!panel_node) {
-                dev_err(dev, "no output node found\n");
-                return -EINVAL;
-        }
-
-        panel = of_drm_find_panel(panel_node);
-        if (!panel) {
-                DRM_ERROR("failed to find panel\n");
+        if (port) {
+                endpoint = of_get_child_by_name(port, "endpoint");
+                of_node_put(port);
+                if (!endpoint) {
+                        dev_err(dev, "no output endpoint found\n");
+                        return -EINVAL;
+                }
+
+                panel_node = of_graph_get_remote_port_parent(endpoint);
+                of_node_put(endpoint);
+                if (!panel_node) {
+                        dev_err(dev, "no output node found\n");
+                        return -EINVAL;
+                }
+
+                panel = of_drm_find_panel(panel_node);
                 of_node_put(panel_node);
-                return -EPROBE_DEFER;
+                if (!panel) {
+                        DRM_ERROR("failed to find panel\n");
+                        return -EPROBE_DEFER;
+                }
         }
 
-        of_node_put(panel_node);
-
         dp = devm_kzalloc(dev, sizeof(*dp), GFP_KERNEL);
         if (!dp)
                 return -ENOMEM;
@@ -349,24 +412,30 @@ static int rockchip_dp_remove(struct platform_device *pdev)
         return 0;
 }
 
+static const struct dev_pm_ops rockchip_dp_pm_ops = {
 #ifdef CONFIG_PM_SLEEP
-static int rockchip_dp_suspend(struct device *dev)
-{
-        return analogix_dp_suspend(dev);
-}
-
-static int rockchip_dp_resume(struct device *dev)
-{
-        return analogix_dp_resume(dev);
-}
+        .suspend = analogix_dp_suspend,
+        .resume_early = analogix_dp_resume,
 #endif
+};
 
-static const struct dev_pm_ops rockchip_dp_pm_ops = {
-        SET_SYSTEM_SLEEP_PM_OPS(rockchip_dp_suspend, rockchip_dp_resume)
+static const struct rockchip_dp_chip_data rk3399_edp = {
+        .lcdsel_grf_reg = RK3399_GRF_SOC_CON20,
+        .lcdsel_big = HIWORD_UPDATE(0, RK3399_EDP_LCDC_SEL),
+        .lcdsel_lit = HIWORD_UPDATE(RK3399_EDP_LCDC_SEL, RK3399_EDP_LCDC_SEL),
+        .chip_type = RK3399_EDP,
+};
+
+static const struct rockchip_dp_chip_data rk3288_dp = {
+        .lcdsel_grf_reg = RK3288_GRF_SOC_CON6,
+        .lcdsel_big = HIWORD_UPDATE(0, RK3288_EDP_LCDC_SEL),
+        .lcdsel_lit = HIWORD_UPDATE(RK3288_EDP_LCDC_SEL, RK3288_EDP_LCDC_SEL),
+        .chip_type = RK3288_DP,
 };
 
 static const struct of_device_id rockchip_dp_dt_ids[] = {
-        {.compatible = "rockchip,rk3288-dp",},
+        {.compatible = "rockchip,rk3288-dp", .data = &rk3288_dp },
+        {.compatible = "rockchip,rk3399-edp", .data = &rk3399_edp },
         {}
 };
 MODULE_DEVICE_TABLE(of, rockchip_dp_dt_ids);
diff --git a/drivers/gpu/drm/rockchip/dw-mipi-dsi.c b/drivers/gpu/drm/rockchip/dw-mipi-dsi.c
index dedc65b40f36..ca22e5ee89ca 100644
--- a/drivers/gpu/drm/rockchip/dw-mipi-dsi.c
+++ b/drivers/gpu/drm/rockchip/dw-mipi-dsi.c
@@ -964,18 +964,9 @@ static enum drm_mode_status dw_mipi_dsi_mode_valid(
         return mode_status;
 }
 
-static struct drm_encoder *dw_mipi_dsi_connector_best_encoder(
-                                        struct drm_connector *connector)
-{
-        struct dw_mipi_dsi *dsi = con_to_dsi(connector);
-
-        return &dsi->encoder;
-}
-
 static struct drm_connector_helper_funcs dw_mipi_dsi_connector_helper_funcs = {
         .get_modes = dw_mipi_dsi_connector_get_modes,
         .mode_valid = dw_mipi_dsi_mode_valid,
-        .best_encoder = dw_mipi_dsi_connector_best_encoder,
 };
 
 static enum drm_connector_status
diff --git a/drivers/gpu/drm/rockchip/dw_hdmi-rockchip.c b/drivers/gpu/drm/rockchip/dw_hdmi-rockchip.c
index 801110f65a63..0665fb915579 100644
--- a/drivers/gpu/drm/rockchip/dw_hdmi-rockchip.c
+++ b/drivers/gpu/drm/rockchip/dw_hdmi-rockchip.c
@@ -15,7 +15,6 @@
 #include <drm/drmP.h>
 #include <drm/drm_crtc_helper.h>
 #include <drm/drm_edid.h>
-#include <drm/drm_encoder_slave.h>
 #include <drm/bridge/dw_hdmi.h>
 
 #include "rockchip_drm_drv.h"
diff --git a/drivers/gpu/drm/rockchip/inno_hdmi.c b/drivers/gpu/drm/rockchip/inno_hdmi.c
index f8b4feb60b25..006260de9dbd 100644
--- a/drivers/gpu/drm/rockchip/inno_hdmi.c
+++ b/drivers/gpu/drm/rockchip/inno_hdmi.c
@@ -579,14 +579,6 @@ inno_hdmi_connector_mode_valid(struct drm_connector *connector,
         return MODE_OK;
 }
 
-static struct drm_encoder *
-inno_hdmi_connector_best_encoder(struct drm_connector *connector)
-{
-        struct inno_hdmi *hdmi = to_inno_hdmi(connector);
-
-        return &hdmi->encoder;
-}
-
 static int
 inno_hdmi_probe_single_connector_modes(struct drm_connector *connector,
                                        uint32_t maxX, uint32_t maxY)
@@ -613,7 +605,6 @@ static struct drm_connector_funcs inno_hdmi_connector_funcs = {
 static struct drm_connector_helper_funcs inno_hdmi_connector_helper_funcs = {
         .get_modes = inno_hdmi_connector_get_modes,
         .mode_valid = inno_hdmi_connector_mode_valid,
-        .best_encoder = inno_hdmi_connector_best_encoder,
 };
 
 static int inno_hdmi_register(struct drm_device *drm, struct inno_hdmi *hdmi)
diff --git a/drivers/gpu/drm/rockchip/rockchip_drm_drv.c b/drivers/gpu/drm/rockchip/rockchip_drm_drv.c
index a409d1f703cb..a822d49a255a 100644
--- a/drivers/gpu/drm/rockchip/rockchip_drm_drv.c
+++ b/drivers/gpu/drm/rockchip/rockchip_drm_drv.c
@@ -19,11 +19,13 @@
 #include <drm/drmP.h>
 #include <drm/drm_crtc_helper.h>
 #include <drm/drm_fb_helper.h>
+#include <drm/drm_gem_cma_helper.h>
 #include <linux/dma-mapping.h>
 #include <linux/pm_runtime.h>
 #include <linux/module.h>
 #include <linux/of_graph.h>
 #include <linux/component.h>
+#include <linux/console.h>
 
 #include "rockchip_drm_drv.h"
 #include "rockchip_drm_fb.h"
@@ -37,6 +39,7 @@
 #define DRIVER_MINOR    0
 
 static bool is_support_iommu = true;
+static struct drm_driver rockchip_drm_driver;
 
 /*
  * Attach a (component) device to the shared drm dma mapping from master drm
@@ -76,7 +79,7 @@ int rockchip_register_crtc_funcs(struct drm_crtc *crtc,
         int pipe = drm_crtc_index(crtc);
         struct rockchip_drm_private *priv = crtc->dev->dev_private;
 
-        if (pipe > ROCKCHIP_MAX_CRTC)
+        if (pipe >= ROCKCHIP_MAX_CRTC)
                 return -EINVAL;
 
         priv->crtc_funcs[pipe] = crtc_funcs;
@@ -89,7 +92,7 @@ void rockchip_unregister_crtc_funcs(struct drm_crtc *crtc)
         int pipe = drm_crtc_index(crtc);
         struct rockchip_drm_private *priv = crtc->dev->dev_private;
 
-        if (pipe > ROCKCHIP_MAX_CRTC)
+        if (pipe >= ROCKCHIP_MAX_CRTC)
                 return;
 
         priv->crtc_funcs[pipe] = NULL;
@@ -132,20 +135,24 @@ static void rockchip_drm_crtc_disable_vblank(struct drm_device *dev,
                 priv->crtc_funcs[pipe]->disable_vblank(crtc);
 }
 
-static int rockchip_drm_load(struct drm_device *drm_dev, unsigned long flags)
+static int rockchip_drm_bind(struct device *dev)
 {
+        struct drm_device *drm_dev;
         struct rockchip_drm_private *private;
         struct dma_iommu_mapping *mapping = NULL;
-        struct device *dev = drm_dev->dev;
-        struct drm_connector *connector;
         int ret;
 
-        private = devm_kzalloc(drm_dev->dev, sizeof(*private), GFP_KERNEL);
-        if (!private)
+        drm_dev = drm_dev_alloc(&rockchip_drm_driver, dev);
+        if (!drm_dev)
                 return -ENOMEM;
 
-        mutex_init(&private->commit.lock);
-        INIT_WORK(&private->commit.work, rockchip_drm_atomic_work);
+        dev_set_drvdata(dev, drm_dev);
+
+        private = devm_kzalloc(drm_dev->dev, sizeof(*private), GFP_KERNEL);
+        if (!private) {
+                ret = -ENOMEM;
+                goto err_free;
+        }
 
         drm_dev->dev_private = private;
 
@@ -186,23 +193,6 @@ static int rockchip_drm_load(struct drm_device *drm_dev, unsigned long flags)
         if (ret)
                 goto err_detach_device;
 
-        /*
-         * All components are now added, we can publish the connector sysfs
-         * entries to userspace.  This will generate hotplug events and so
-         * userspace will expect to be able to access DRM at this point.
-         */
-        list_for_each_entry(connector, &drm_dev->mode_config.connector_list,
-                        head) {
-                ret = drm_connector_register(connector);
-                if (ret) {
-                        dev_err(drm_dev->dev,
-                                "[CONNECTOR:%d:%s] drm_connector_register failed: %d\n",
-                                connector->base.id,
-                                connector->name, ret);
-                        goto err_unbind;
-                }
-        }
-
         /* init kms poll for handling hpd */
         drm_kms_helper_poll_init(drm_dev);
 
@@ -222,14 +212,19 @@ static int rockchip_drm_load(struct drm_device *drm_dev, unsigned long flags)
         if (ret)
                 goto err_vblank_cleanup;
 
+        ret = drm_dev_register(drm_dev, 0);
+        if (ret)
+                goto err_fbdev_fini;
+
         if (is_support_iommu)
                 arm_iommu_release_mapping(mapping);
         return 0;
+err_fbdev_fini:
+        rockchip_drm_fbdev_fini(drm_dev);
 err_vblank_cleanup:
         drm_vblank_cleanup(drm_dev);
 err_kms_helper_poll_fini:
         drm_kms_helper_poll_fini(drm_dev);
-err_unbind:
         component_unbind_all(dev, drm_dev);
 err_detach_device:
         if (is_support_iommu)
@@ -240,12 +235,14 @@ err_release_mapping:
 err_config_cleanup:
         drm_mode_config_cleanup(drm_dev);
         drm_dev->dev_private = NULL;
+err_free:
+        drm_dev_unref(drm_dev);
         return ret;
 }
 
-static int rockchip_drm_unload(struct drm_device *drm_dev)
+static void rockchip_drm_unbind(struct device *dev)
 {
-        struct device *dev = drm_dev->dev;
+        struct drm_device *drm_dev = dev_get_drvdata(dev);
 
         rockchip_drm_fbdev_fini(drm_dev);
         drm_vblank_cleanup(drm_dev);
@@ -255,32 +252,12 @@ static int rockchip_drm_unload(struct drm_device *drm_dev)
                 arm_iommu_detach_device(dev);
         drm_mode_config_cleanup(drm_dev);
         drm_dev->dev_private = NULL;
-
-        return 0;
-}
-
-static void rockchip_drm_crtc_cancel_pending_vblank(struct drm_crtc *crtc,
-                                                    struct drm_file *file_priv)
-{
-        struct rockchip_drm_private *priv = crtc->dev->dev_private;
-        int pipe = drm_crtc_index(crtc);
-
-        if (pipe < ROCKCHIP_MAX_CRTC &&
-            priv->crtc_funcs[pipe] &&
-            priv->crtc_funcs[pipe]->cancel_pending_vblank)
-                priv->crtc_funcs[pipe]->cancel_pending_vblank(crtc, file_priv);
-}
-
-static void rockchip_drm_preclose(struct drm_device *dev,
-                                  struct drm_file *file_priv)
-{
-        struct drm_crtc *crtc;
-
-        list_for_each_entry(crtc, &dev->mode_config.crtc_list, head)
-                rockchip_drm_crtc_cancel_pending_vblank(crtc, file_priv);
+        drm_dev_unregister(drm_dev);
+        drm_dev_unref(drm_dev);
+        dev_set_drvdata(dev, NULL);
 }
 
-void rockchip_drm_lastclose(struct drm_device *dev)
+static void rockchip_drm_lastclose(struct drm_device *dev)
 {
         struct rockchip_drm_private *priv = dev->dev_private;
 
@@ -300,23 +277,15 @@ static const struct file_operations rockchip_drm_driver_fops = {
         .release = drm_release,
 };
 
-const struct vm_operations_struct rockchip_drm_vm_ops = {
-        .open = drm_gem_vm_open,
-        .close = drm_gem_vm_close,
-};
-
 static struct drm_driver rockchip_drm_driver = {
         .driver_features        = DRIVER_MODESET | DRIVER_GEM |
                                   DRIVER_PRIME | DRIVER_ATOMIC,
-        .load                   = rockchip_drm_load,
-        .unload                 = rockchip_drm_unload,
-        .preclose               = rockchip_drm_preclose,
         .lastclose              = rockchip_drm_lastclose,
         .get_vblank_counter     = drm_vblank_no_hw_counter,
         .enable_vblank          = rockchip_drm_crtc_enable_vblank,
         .disable_vblank         = rockchip_drm_crtc_disable_vblank,
-        .gem_vm_ops             = &rockchip_drm_vm_ops,
-        .gem_free_object        = rockchip_gem_free_object,
+        .gem_vm_ops             = &drm_gem_cma_vm_ops,
+        .gem_free_object_unlocked = rockchip_gem_free_object,
         .dumb_create            = rockchip_gem_dumb_create,
         .dumb_map_offset        = rockchip_gem_dumb_map_offset,
         .dumb_destroy           = drm_gem_dumb_destroy,
@@ -337,25 +306,38 @@ static struct drm_driver rockchip_drm_driver = {
 };
 
 #ifdef CONFIG_PM_SLEEP
-static int rockchip_drm_sys_suspend(struct device *dev)
+void rockchip_drm_fb_suspend(struct drm_device *drm)
 {
-        struct drm_device *drm = dev_get_drvdata(dev);
-        struct drm_connector *connector;
+        struct rockchip_drm_private *priv = drm->dev_private;
 
-        if (!drm)
-                return 0;
+        console_lock();
+        drm_fb_helper_set_suspend(&priv->fbdev_helper, 1);
+        console_unlock();
+}
 
-        drm_modeset_lock_all(drm);
-        list_for_each_entry(connector, &drm->mode_config.connector_list, head) {
-                int old_dpms = connector->dpms;
+void rockchip_drm_fb_resume(struct drm_device *drm)
+{
+        struct rockchip_drm_private *priv = drm->dev_private;
 
-                if (connector->funcs->dpms)
-                        connector->funcs->dpms(connector, DRM_MODE_DPMS_OFF);
+        console_lock();
+        drm_fb_helper_set_suspend(&priv->fbdev_helper, 0);
+        console_unlock();
+}
 
-                /* Set the old mode back to the connector for resume */
-                connector->dpms = old_dpms;
+static int rockchip_drm_sys_suspend(struct device *dev)
+{
+        struct drm_device *drm = dev_get_drvdata(dev);
+        struct rockchip_drm_private *priv = drm->dev_private;
+
+        drm_kms_helper_poll_disable(drm);
+        rockchip_drm_fb_suspend(drm);
+
+        priv->state = drm_atomic_helper_suspend(drm);
+        if (IS_ERR(priv->state)) {
+                rockchip_drm_fb_resume(drm);
+                drm_kms_helper_poll_enable(drm);
+                return PTR_ERR(priv->state);
         }
-        drm_modeset_unlock_all(drm);
 
         return 0;
 }
@@ -363,47 +345,11 @@ static int rockchip_drm_sys_suspend(struct device *dev)
 static int rockchip_drm_sys_resume(struct device *dev)
 {
         struct drm_device *drm = dev_get_drvdata(dev);
-        struct drm_connector *connector;
-        enum drm_connector_status status;
-        bool changed = false;
-
-        if (!drm)
-                return 0;
+        struct rockchip_drm_private *priv = drm->dev_private;
 
-        drm_modeset_lock_all(drm);
-        list_for_each_entry(connector, &drm->mode_config.connector_list, head) {
-                int desired_mode = connector->dpms;
-
-                /*
-                 * at suspend time, we save dpms to connector->dpms,
-                 * restore the old_dpms, and at current time, the connector
-                 * dpms status must be DRM_MODE_DPMS_OFF.
-                 */
-                connector->dpms = DRM_MODE_DPMS_OFF;
-
-                /*
-                 * If the connector has been disconnected during suspend,
-                 * disconnect it from the encoder and leave it off. We'll notify
-                 * userspace at the end.
-                 */
-                if (desired_mode == DRM_MODE_DPMS_ON) {
-                        status = connector->funcs->detect(connector, true);
-                        if (status == connector_status_disconnected) {
-                                connector->encoder = NULL;
-                                connector->status = status;
-                                changed = true;
-                                continue;
-                        }
-                }
-                if (connector->funcs->dpms)
-                        connector->funcs->dpms(connector, desired_mode);
-        }
-        drm_modeset_unlock_all(drm);
-
-        drm_helper_resume_force_mode(drm);
-
-        if (changed)
-                drm_kms_helper_hotplug_event(drm);
+        drm_atomic_helper_resume(drm, priv->state);
+        rockchip_drm_fb_resume(drm);
+        drm_kms_helper_poll_enable(drm);
 
         return 0;
 }
@@ -444,37 +390,6 @@ static void rockchip_add_endpoints(struct device *dev,
         }
 }
 
-static int rockchip_drm_bind(struct device *dev)
-{
-        struct drm_device *drm;
-        int ret;
-
-        drm = drm_dev_alloc(&rockchip_drm_driver, dev);
-        if (!drm)
-                return -ENOMEM;
-
-        ret = drm_dev_register(drm, 0);
-        if (ret)
-                goto err_free;
-
-        dev_set_drvdata(dev, drm);
-
-        return 0;
-
-err_free:
-        drm_dev_unref(drm);
-        return ret;
-}
-
-static void rockchip_drm_unbind(struct device *dev)
-{
-        struct drm_device *drm = dev_get_drvdata(dev);
-
-        drm_dev_unregister(drm);
-        drm_dev_unref(drm);
-        dev_set_drvdata(dev, NULL);
-}
-
 static const struct component_master_ops rockchip_drm_ops = {
         .bind = rockchip_drm_bind,
         .unbind = rockchip_drm_unbind,
@@ -518,6 +433,7 @@ static int rockchip_drm_platform_probe(struct platform_device *pdev)
                         is_support_iommu = false;
                 }
 
+                of_node_put(iommu);
                 component_match_add(dev, &match, compare_of, port->parent);
                 of_node_put(port);
         }
diff --git a/drivers/gpu/drm/rockchip/rockchip_drm_drv.h b/drivers/gpu/drm/rockchip/rockchip_drm_drv.h
index 56f43a364c7f..ea3932940061 100644
--- a/drivers/gpu/drm/rockchip/rockchip_drm_drv.h
+++ b/drivers/gpu/drm/rockchip/rockchip_drm_drv.h
@@ -40,14 +40,6 @@ struct rockchip_crtc_funcs {
         int (*enable_vblank)(struct drm_crtc *crtc);
         void (*disable_vblank)(struct drm_crtc *crtc);
         void (*wait_for_update)(struct drm_crtc *crtc);
-        void (*cancel_pending_vblank)(struct drm_crtc *crtc, struct drm_file *file_priv);
-};
-
-struct rockchip_atomic_commit {
-        struct work_struct      work;
-        struct drm_atomic_state *state;
-        struct drm_device *dev;
-        struct mutex lock;
 };
 
 struct rockchip_crtc_state {
@@ -68,11 +60,9 @@ struct rockchip_drm_private {
         struct drm_fb_helper fbdev_helper;
         struct drm_gem_object *fbdev_bo;
         const struct rockchip_crtc_funcs *crtc_funcs[ROCKCHIP_MAX_CRTC];
-
-        struct rockchip_atomic_commit commit;
+        struct drm_atomic_state *state;
 };
 
-void rockchip_drm_atomic_work(struct work_struct *work);
 int rockchip_register_crtc_funcs(struct drm_crtc *crtc,
                                  const struct rockchip_crtc_funcs *crtc_funcs);
 void rockchip_unregister_crtc_funcs(struct drm_crtc *crtc);
diff --git a/drivers/gpu/drm/rockchip/rockchip_drm_fb.c b/drivers/gpu/drm/rockchip/rockchip_drm_fb.c
index 755cfdba61cd..55c52734c52d 100644
--- a/drivers/gpu/drm/rockchip/rockchip_drm_fb.c
+++ b/drivers/gpu/drm/rockchip/rockchip_drm_fb.c
@@ -20,6 +20,7 @@
 #include <drm/drm_crtc_helper.h>
 
 #include "rockchip_drm_drv.h"
+#include "rockchip_drm_fb.h"
 #include "rockchip_drm_gem.h"
 
 #define to_rockchip_fb(x) container_of(x, struct rockchip_drm_fb, fb)
@@ -43,14 +44,10 @@ struct drm_gem_object *rockchip_fb_get_gem_obj(struct drm_framebuffer *fb,
 static void rockchip_drm_fb_destroy(struct drm_framebuffer *fb)
 {
         struct rockchip_drm_fb *rockchip_fb = to_rockchip_fb(fb);
-        struct drm_gem_object *obj;
         int i;
 
-        for (i = 0; i < ROCKCHIP_MAX_FB_BUFFER; i++) {
-                obj = rockchip_fb->obj[i];
-                if (obj)
-                        drm_gem_object_unreference_unlocked(obj);
-        }
+        for (i = 0; i < ROCKCHIP_MAX_FB_BUFFER; i++)
+                drm_gem_object_unreference_unlocked(rockchip_fb->obj[i]);
 
         drm_framebuffer_cleanup(fb);
         kfree(rockchip_fb);
@@ -228,87 +225,32 @@ rockchip_atomic_wait_for_complete(struct drm_device *dev, struct drm_atomic_stat
 }
 
 static void
-rockchip_atomic_commit_complete(struct rockchip_atomic_commit *commit)
+rockchip_atomic_commit_tail(struct drm_atomic_state *state)
 {
-        struct drm_atomic_state *state = commit->state;
-        struct drm_device *dev = commit->dev;
+        struct drm_device *dev = state->dev;
 
-        /*
-         * TODO: do fence wait here.
-         */
-
-        /*
-         * Rockchip crtc support runtime PM, can't update display planes
-         * when crtc is disabled.
-         *
-         * drm_atomic_helper_commit comments detail that:
-         *     For drivers supporting runtime PM the recommended sequence is
-         *
-         *     drm_atomic_helper_commit_modeset_disables(dev, state);
-         *
-         *     drm_atomic_helper_commit_modeset_enables(dev, state);
-         *
-         *     drm_atomic_helper_commit_planes(dev, state, true);
-         *
-         * See the kerneldoc entries for these three functions for more details.
-         */
         drm_atomic_helper_commit_modeset_disables(dev, state);
 
         drm_atomic_helper_commit_modeset_enables(dev, state);
 
         drm_atomic_helper_commit_planes(dev, state, true);
 
+        drm_atomic_helper_commit_hw_done(state);
+
         rockchip_atomic_wait_for_complete(dev, state);
 
         drm_atomic_helper_cleanup_planes(dev, state);
-
-        drm_atomic_state_free(state);
-}
-
-void rockchip_drm_atomic_work(struct work_struct *work)
-{
-        struct rockchip_atomic_commit *commit = container_of(work,
-                                        struct rockchip_atomic_commit, work);
-
-        rockchip_atomic_commit_complete(commit);
 }
 
-int rockchip_drm_atomic_commit(struct drm_device *dev,
-                               struct drm_atomic_state *state,
-                               bool nonblock)
-{
-        struct rockchip_drm_private *private = dev->dev_private;
-        struct rockchip_atomic_commit *commit = &private->commit;
-        int ret;
-
-        ret = drm_atomic_helper_prepare_planes(dev, state);
-        if (ret)
-                return ret;
-
-        /* serialize outstanding nonblocking commits */
-        mutex_lock(&commit->lock);
-        flush_work(&commit->work);
-
-        drm_atomic_helper_swap_state(dev, state);
-
-        commit->dev = dev;
-        commit->state = state;
-
-        if (nonblock)
-                schedule_work(&commit->work);
-        else
-                rockchip_atomic_commit_complete(commit);
-
-        mutex_unlock(&commit->lock);
-
-        return 0;
-}
+static struct drm_mode_config_helper_funcs rockchip_mode_config_helpers = {
+        .atomic_commit_tail = rockchip_atomic_commit_tail,
+};
 
 static const struct drm_mode_config_funcs rockchip_drm_mode_config_funcs = {
         .fb_create = rockchip_user_fb_create,
         .output_poll_changed = rockchip_drm_output_poll_changed,
         .atomic_check = drm_atomic_helper_check,
-        .atomic_commit = rockchip_drm_atomic_commit,
+        .atomic_commit = drm_atomic_helper_commit,
 };
 
 struct drm_framebuffer *
@@ -339,4 +281,5 @@ void rockchip_drm_mode_config_init(struct drm_device *dev)
         dev->mode_config.max_height = 4096;
 
         dev->mode_config.funcs = &rockchip_drm_mode_config_funcs;
+        dev->mode_config.helper_private = &rockchip_mode_config_helpers;
 }
diff --git a/drivers/gpu/drm/rockchip/rockchip_drm_fbdev.c b/drivers/gpu/drm/rockchip/rockchip_drm_fbdev.c
index f261512bb4a0..207e01de6e32 100644
--- a/drivers/gpu/drm/rockchip/rockchip_drm_fbdev.c
+++ b/drivers/gpu/drm/rockchip/rockchip_drm_fbdev.c
@@ -108,7 +108,7 @@ static int rockchip_drm_fbdev_create(struct drm_fb_helper *helper,
         fbi->screen_size = rk_obj->base.size;
         fbi->fix.smem_len = rk_obj->base.size;
 
-        DRM_DEBUG_KMS("FB [%dx%d]-%d kvaddr=%p offset=%ld size=%d\n",
+        DRM_DEBUG_KMS("FB [%dx%d]-%d kvaddr=%p offset=%ld size=%zu\n",
                       fb->width, fb->height, fb->depth, rk_obj->kvaddr,
                       offset, size);
 
@@ -156,9 +156,6 @@ int rockchip_drm_fbdev_init(struct drm_device *dev)
                 goto err_drm_fb_helper_fini;
         }
 
-        /* disable all the possible outputs/crtcs before entering KMS mode */
-        drm_helper_disable_unused_functions(dev);
-
         ret = drm_fb_helper_initial_config(helper, PREFERRED_BPP);
         if (ret < 0) {
                 dev_err(dev->dev, "Failed to set initial hw config - %d.\n",
diff --git a/drivers/gpu/drm/rockchip/rockchip_drm_gem.c b/drivers/gpu/drm/rockchip/rockchip_drm_gem.c
index 9c2d8a894093..059e902f872d 100644
--- a/drivers/gpu/drm/rockchip/rockchip_drm_gem.c
+++ b/drivers/gpu/drm/rockchip/rockchip_drm_gem.c
@@ -38,7 +38,7 @@ static int rockchip_gem_alloc_buf(struct rockchip_gem_object *rk_obj,
                                          &rk_obj->dma_addr, GFP_KERNEL,
                                          &rk_obj->dma_attrs);
         if (!rk_obj->kvaddr) {
-                DRM_ERROR("failed to allocate %#x byte dma buffer", obj->size);
+                DRM_ERROR("failed to allocate %zu byte dma buffer", obj->size);
                 return -ENOMEM;
         }
 
diff --git a/drivers/gpu/drm/rockchip/rockchip_drm_vop.c b/drivers/gpu/drm/rockchip/rockchip_drm_vop.c
index 1c4d5b5a70a2..91305eb7d312 100644
--- a/drivers/gpu/drm/rockchip/rockchip_drm_vop.c
+++ b/drivers/gpu/drm/rockchip/rockchip_drm_vop.c
@@ -98,7 +98,9 @@ struct vop_win {
         const struct vop_win_data *data;
         struct vop *vop;
 
-        struct vop_plane_state state;
+        /* protected by dev->event_lock */
+        bool enable;
+        dma_addr_t yrgb_mst;
 };
 
 struct vop {
@@ -112,6 +114,8 @@ struct vop {
         bool vsync_work_pending;
         struct completion dsp_hold_completion;
         struct completion wait_update_complete;
+
+        /* protected by dev->event_lock */
         struct drm_pending_vblank_event *event;
 
         const struct vop_data *data;
@@ -324,9 +328,9 @@ static void scl_vop_cal_scl_fac(struct vop *vop, const struct vop_win_data *win,
                             scl_cal_scale2(src_h, dst_h));
                 if (is_yuv) {
                         VOP_SCL_SET(vop, win, scale_cbcr_x,
-                                    scl_cal_scale2(src_w, dst_w));
+                                    scl_cal_scale2(cbcr_src_w, dst_w));
                         VOP_SCL_SET(vop, win, scale_cbcr_y,
-                                    scl_cal_scale2(src_h, dst_h));
+                                    scl_cal_scale2(cbcr_src_h, dst_h));
                 }
                 return;
         }
@@ -431,9 +435,6 @@ static void vop_enable(struct drm_crtc *crtc)
         struct vop *vop = to_vop(crtc);
         int ret;
 
-        if (vop->is_enabled)
-                return;
-
         ret = pm_runtime_get_sync(vop->dev);
         if (ret < 0) {
                 dev_err(vop->dev, "failed to get pm runtime: %d\n", ret);
@@ -501,8 +502,7 @@ static void vop_crtc_disable(struct drm_crtc *crtc)
         struct vop *vop = to_vop(crtc);
         int i;
 
-        if (!vop->is_enabled)
-                return;
+        WARN_ON(vop->event);
 
         /*
          * We need to make sure that all windows are disabled before we
@@ -553,6 +553,14 @@ static void vop_crtc_disable(struct drm_crtc *crtc)
         clk_disable(vop->aclk);
         clk_disable(vop->hclk);
         pm_runtime_put(vop->dev);
+
+        if (crtc->state->event && !crtc->state->active) {
+                spin_lock_irq(&crtc->dev->event_lock);
+                drm_crtc_send_vblank_event(crtc, crtc->state->event);
+                spin_unlock_irq(&crtc->dev->event_lock);
+
+                crtc->state->event = NULL;
+        }
 }
 
 static void vop_plane_destroy(struct drm_plane *plane)
@@ -618,6 +626,7 @@ static int vop_plane_atomic_check(struct drm_plane *plane,
 
         ret = drm_plane_helper_check_update(plane, crtc, state->fb,
                                             src, dest, &clip,
+                                            state->rotation,
                                             min_scale,
                                             max_scale,
                                             true, true, &visible);
@@ -658,6 +667,11 @@ static void vop_plane_atomic_disable(struct drm_plane *plane,
         if (!old_state->crtc)
                 return;
 
+        spin_lock_irq(&plane->dev->event_lock);
+        vop_win->enable = false;
+        vop_win->yrgb_mst = 0;
+        spin_unlock_irq(&plane->dev->event_lock);
+
         spin_lock(&vop->reg_lock);
 
         VOP_WIN_SET(vop, win, enable, 0);
@@ -692,7 +706,7 @@ static void vop_plane_atomic_update(struct drm_plane *plane,
         /*
          * can't update plane when vop is disabled.
          */
-        if (!crtc)
+        if (WARN_ON(!crtc))
                 return;
 
         if (WARN_ON(!vop->is_enabled))
@@ -721,6 +735,11 @@ static void vop_plane_atomic_update(struct drm_plane *plane,
         offset += (src->y1 >> 16) * fb->pitches[0];
         vop_plane_state->yrgb_mst = rk_obj->dma_addr + offset + fb->offsets[0];
 
+        spin_lock_irq(&plane->dev->event_lock);
+        vop_win->enable = true;
+        vop_win->yrgb_mst = vop_plane_state->yrgb_mst;
+        spin_unlock_irq(&plane->dev->event_lock);
+
         spin_lock(&vop->reg_lock);
 
         VOP_WIN_SET(vop, win, format, vop_plane_state->format);
@@ -779,7 +798,7 @@ static const struct drm_plane_helper_funcs plane_helper_funcs = {
         .atomic_disable = vop_plane_atomic_disable,
 };
 
-void vop_atomic_plane_reset(struct drm_plane *plane)
+static void vop_atomic_plane_reset(struct drm_plane *plane)
 {
         struct vop_plane_state *vop_plane_state =
                                         to_vop_plane_state(plane->state);
@@ -796,7 +815,7 @@ void vop_atomic_plane_reset(struct drm_plane *plane)
         plane->state->plane = plane;
 }
 
-struct drm_plane_state *
+static struct drm_plane_state *
 vop_atomic_plane_duplicate_state(struct drm_plane *plane)
 {
         struct vop_plane_state *old_vop_plane_state;
@@ -876,30 +895,10 @@ static void vop_crtc_wait_for_update(struct drm_crtc *crtc)
         WARN_ON(!wait_for_completion_timeout(&vop->wait_update_complete, 100));
 }
 
-static void vop_crtc_cancel_pending_vblank(struct drm_crtc *crtc,
-                                           struct drm_file *file_priv)
-{
-        struct drm_device *drm = crtc->dev;
-        struct vop *vop = to_vop(crtc);
-        struct drm_pending_vblank_event *e;
-        unsigned long flags;
-
-        spin_lock_irqsave(&drm->event_lock, flags);
-        e = vop->event;
-        if (e && e->base.file_priv == file_priv) {
-                vop->event = NULL;
-
-                e->base.destroy(&e->base);
-                file_priv->event_space += sizeof(e->event);
-        }
-        spin_unlock_irqrestore(&drm->event_lock, flags);
-}
-
 static const struct rockchip_crtc_funcs private_crtc_funcs = {
         .enable_vblank = vop_crtc_enable_vblank,
         .disable_vblank = vop_crtc_disable_vblank,
         .wait_for_update = vop_crtc_wait_for_update,
-        .cancel_pending_vblank = vop_crtc_cancel_pending_vblank,
 };
 
 static bool vop_crtc_mode_fixup(struct drm_crtc *crtc,
@@ -931,6 +930,8 @@ static void vop_crtc_enable(struct drm_crtc *crtc)
         u16 vact_end = vact_st + vdisplay;
         uint32_t val;
 
+        WARN_ON(vop->event);
+
         vop_enable(crtc);
         /*
          * If dclk rate is zero, mean that scanout is stop,
@@ -1027,12 +1028,15 @@ static void vop_crtc_atomic_begin(struct drm_crtc *crtc,
 {
         struct vop *vop = to_vop(crtc);
 
+        spin_lock_irq(&crtc->dev->event_lock);
         if (crtc->state->event) {
                 WARN_ON(drm_crtc_vblank_get(crtc) != 0);
+                WARN_ON(vop->event);
 
                 vop->event = crtc->state->event;
                 crtc->state->event = NULL;
         }
+        spin_unlock_irq(&crtc->dev->event_lock);
 }
 
 static const struct drm_crtc_helper_funcs vop_crtc_helper_funcs = {
@@ -1048,6 +1052,17 @@ static void vop_crtc_destroy(struct drm_crtc *crtc)
         drm_crtc_cleanup(crtc);
 }
 
+static void vop_crtc_reset(struct drm_crtc *crtc)
+{
+        if (crtc->state)
+                __drm_atomic_helper_crtc_destroy_state(crtc->state);
+        kfree(crtc->state);
+
+        crtc->state = kzalloc(sizeof(struct rockchip_crtc_state), GFP_KERNEL);
+        if (crtc->state)
+                crtc->state->crtc = crtc;
+}
+
 static struct drm_crtc_state *vop_crtc_duplicate_state(struct drm_crtc *crtc)
 {
         struct rockchip_crtc_state *rockchip_state;
@@ -1073,23 +1088,21 @@ static const struct drm_crtc_funcs vop_crtc_funcs = {
         .set_config = drm_atomic_helper_set_config,
         .page_flip = drm_atomic_helper_page_flip,
         .destroy = vop_crtc_destroy,
-        .reset = drm_atomic_helper_crtc_reset,
+        .reset = vop_crtc_reset,
         .atomic_duplicate_state = vop_crtc_duplicate_state,
         .atomic_destroy_state = vop_crtc_destroy_state,
 };
 
 static bool vop_win_pending_is_complete(struct vop_win *vop_win)
 {
-        struct drm_plane *plane = &vop_win->base;
-        struct vop_plane_state *state = to_vop_plane_state(plane->state);
         dma_addr_t yrgb_mst;
 
-        if (!state->enable)
+        if (!vop_win->enable)
                 return VOP_WIN_GET(vop_win->vop, vop_win->data, enable) == 0;
 
         yrgb_mst = VOP_WIN_GET_YRGBADDR(vop_win->vop, vop_win->data);
 
-        return yrgb_mst == state->yrgb_mst;
+        return yrgb_mst == vop_win->yrgb_mst;
 }
 
 static void vop_handle_vblank(struct vop *vop)
@@ -1104,15 +1117,16 @@ static void vop_handle_vblank(struct vop *vop)
                         return;
         }
 
+        spin_lock_irqsave(&drm->event_lock, flags);
         if (vop->event) {
-                spin_lock_irqsave(&drm->event_lock, flags);
 
                 drm_crtc_send_vblank_event(crtc, vop->event);
                 drm_crtc_vblank_put(crtc);
                 vop->event = NULL;
 
-                spin_unlock_irqrestore(&drm->event_lock, flags);
         }
+        spin_unlock_irqrestore(&drm->event_lock, flags);
+
         if (!completion_done(&vop->wait_update_complete))
                 complete(&vop->wait_update_complete);
 }
diff --git a/drivers/gpu/drm/rockchip/rockchip_vop_reg.c b/drivers/gpu/drm/rockchip/rockchip_vop_reg.c
index 3166b46a5893..919992cdc97e 100644
--- a/drivers/gpu/drm/rockchip/rockchip_vop_reg.c
+++ b/drivers/gpu/drm/rockchip/rockchip_vop_reg.c
@@ -190,7 +190,7 @@ static const struct vop_data rk3288_vop = {
         .win_size = ARRAY_SIZE(rk3288_vop_win_data),
 };
 
-static const struct vop_scl_regs rk3066_win_scl = {
+static const struct vop_scl_regs rk3036_win_scl = {
         .scale_yrgb_x = VOP_REG(RK3036_WIN0_SCL_FACTOR_YRGB, 0xffff, 0x0),
         .scale_yrgb_y = VOP_REG(RK3036_WIN0_SCL_FACTOR_YRGB, 0xffff, 16),
         .scale_cbcr_x = VOP_REG(RK3036_WIN0_SCL_FACTOR_CBR, 0xffff, 0x0),
@@ -198,7 +198,7 @@ static const struct vop_scl_regs rk3066_win_scl = {
 };
 
 static const struct vop_win_phy rk3036_win0_data = {
-        .scl = &rk3066_win_scl,
+        .scl = &rk3036_win_scl,
         .data_formats = formats_win_full,
         .nformats = ARRAY_SIZE(formats_win_full),
         .enable = VOP_REG(RK3036_SYS_CTRL, 0x1, 0),
@@ -210,6 +210,7 @@ static const struct vop_win_phy rk3036_win0_data = {
         .yrgb_mst = VOP_REG(RK3036_WIN0_YRGB_MST, 0xffffffff, 0),
         .uv_mst = VOP_REG(RK3036_WIN0_CBR_MST, 0xffffffff, 0),
         .yrgb_vir = VOP_REG(RK3036_WIN0_VIR, 0xffff, 0),
+        .uv_vir = VOP_REG(RK3036_WIN0_VIR, 0x1fff, 16),
 };
 
 static const struct vop_win_phy rk3036_win1_data = {
@@ -299,7 +300,7 @@ static int vop_remove(struct platform_device *pdev)
         return 0;
 }
 
-struct platform_driver vop_platform_driver = {
+static struct platform_driver vop_platform_driver = {
         .probe = vop_probe,
         .remove = vop_remove,
         .driver = {
diff --git a/drivers/gpu/drm/shmobile/Kconfig b/drivers/gpu/drm/shmobile/Kconfig
index 8d17d00ddb4b..c987c826daa3 100644
--- a/drivers/gpu/drm/shmobile/Kconfig
+++ b/drivers/gpu/drm/shmobile/Kconfig
@@ -6,7 +6,6 @@ config DRM_SHMOBILE
         select BACKLIGHT_CLASS_DEVICE
         select BACKLIGHT_LCD_SUPPORT
         select DRM_KMS_HELPER
-        select DRM_KMS_FB_HELPER
         select DRM_KMS_CMA_HELPER
         select DRM_GEM_CMA_HELPER
         help
diff --git a/drivers/gpu/drm/shmobile/shmob_drm_crtc.c b/drivers/gpu/drm/shmobile/shmob_drm_crtc.c
index 1e154fc779d5..6547b1db460a 100644
--- a/drivers/gpu/drm/shmobile/shmob_drm_crtc.c
+++ b/drivers/gpu/drm/shmobile/shmob_drm_crtc.c
@@ -441,7 +441,7 @@ void shmob_drm_crtc_finish_page_flip(struct shmob_drm_crtc *scrtc)
         scrtc->event = NULL;
         if (event) {
                 drm_crtc_send_vblank_event(&scrtc->crtc, event);
-                drm_vblank_put(dev, 0);
+                drm_crtc_vblank_put(&scrtc->crtc);
         }
         spin_unlock_irqrestore(&dev->event_lock, flags);
 }
@@ -467,7 +467,7 @@ static int shmob_drm_crtc_page_flip(struct drm_crtc *crtc,
 
         if (event) {
                 event->pipe = 0;
-                drm_vblank_get(dev, 0);
+                drm_crtc_vblank_get(&scrtc->crtc);
                 spin_lock_irqsave(&dev->event_lock, flags);
                 scrtc->event = event;
                 spin_unlock_irqrestore(&dev->event_lock, flags);
diff --git a/drivers/gpu/drm/shmobile/shmob_drm_drv.c b/drivers/gpu/drm/shmobile/shmob_drm_drv.c
index 7700ff172079..f0492603ea88 100644
--- a/drivers/gpu/drm/shmobile/shmob_drm_drv.c
+++ b/drivers/gpu/drm/shmobile/shmob_drm_drv.c
@@ -259,12 +259,11 @@ static struct drm_driver shmob_drm_driver = {
                                 | DRIVER_PRIME,
         .load                   = shmob_drm_load,
         .unload                 = shmob_drm_unload,
-        .set_busid              = drm_platform_set_busid,
         .irq_handler            = shmob_drm_irq,
         .get_vblank_counter     = drm_vblank_no_hw_counter,
         .enable_vblank          = shmob_drm_enable_vblank,
         .disable_vblank         = shmob_drm_disable_vblank,
-        .gem_free_object        = drm_gem_cma_free_object,
+        .gem_free_object_unlocked = drm_gem_cma_free_object,
         .gem_vm_ops             = &drm_gem_cma_vm_ops,
         .prime_handle_to_fd     = drm_gem_prime_handle_to_fd,
         .prime_fd_to_handle     = drm_gem_prime_fd_to_handle,
diff --git a/drivers/gpu/drm/sis/sis_mm.c b/drivers/gpu/drm/sis/sis_mm.c
index 93ad8a5704d1..03defda77766 100644
--- a/drivers/gpu/drm/sis/sis_mm.c
+++ b/drivers/gpu/drm/sis/sis_mm.c
@@ -316,7 +316,7 @@ void sis_reclaim_buffers_locked(struct drm_device *dev,
         struct sis_file_private *file_priv = file->driver_priv;
         struct sis_memblock *entry, *next;
 
-        if (!(file->minor->master && file->master->lock.hw_lock))
+        if (!(dev->master && file->master->lock.hw_lock))
                 return;
 
         drm_legacy_idlelock_take(&file->master->lock);
diff --git a/drivers/gpu/drm/sti/Kconfig b/drivers/gpu/drm/sti/Kconfig
index 5ad43a1bb260..494ab257f77c 100644
--- a/drivers/gpu/drm/sti/Kconfig
+++ b/drivers/gpu/drm/sti/Kconfig
@@ -7,5 +7,6 @@ config DRM_STI
         select DRM_KMS_CMA_HELPER
         select DRM_PANEL
         select FW_LOADER
+        select SND_SOC_HDMI_CODEC if SND_SOC
         help
           Choose this option to enable DRM on STM stiH41x chipset
diff --git a/drivers/gpu/drm/sti/sti_awg_utils.c b/drivers/gpu/drm/sti/sti_awg_utils.c
index a516eb869f6f..2da7d6866d5d 100644
--- a/drivers/gpu/drm/sti/sti_awg_utils.c
+++ b/drivers/gpu/drm/sti/sti_awg_utils.c
@@ -6,6 +6,8 @@
 
 #include "sti_awg_utils.h"
 
+#define AWG_DELAY (-5)
+
 #define AWG_OPCODE_OFFSET 10
 #define AWG_MAX_ARG       0x3ff
 
@@ -125,7 +127,7 @@ static int awg_generate_line_signal(
                 val = timing->blanking_level;
                 ret |= awg_generate_instr(RPLSET, val, 0, 0, fwparams);
 
-                val = timing->trailing_pixels - 1;
+                val = timing->trailing_pixels - 1 + AWG_DELAY;
                 ret |= awg_generate_instr(SKIP, val, 0, 0, fwparams);
         }
 
diff --git a/drivers/gpu/drm/sti/sti_compositor.c b/drivers/gpu/drm/sti/sti_compositor.c
index 3d2fa3ab33df..134201ecc6fd 100644
--- a/drivers/gpu/drm/sti/sti_compositor.c
+++ b/drivers/gpu/drm/sti/sti_compositor.c
@@ -55,6 +55,26 @@ struct sti_compositor_data stih416_compositor_data = {
         },
 };
 
+int sti_compositor_debufs_init(struct sti_compositor *compo,
+                               struct drm_minor *minor)
+{
+        int ret = 0, i;
+
+        for (i = 0; compo->vid[i]; i++) {
+                ret = vid_debugfs_init(compo->vid[i], minor);
+                if (ret)
+                        return ret;
+        }
+
+        for (i = 0; compo->mixer[i]; i++) {
+                ret = sti_mixer_debugfs_init(compo->mixer[i], minor);
+                if (ret)
+                        return ret;
+        }
+
+        return 0;
+}
+
 static int sti_compositor_bind(struct device *dev,
                                struct device *master,
                                void *data)
@@ -234,12 +254,12 @@ static int sti_compositor_probe(struct platform_device *pdev)
         }
 
         /* Get reset resources */
-        compo->rst_main = devm_reset_control_get(dev, "compo-main");
+        compo->rst_main = devm_reset_control_get_shared(dev, "compo-main");
         /* Take compo main out of reset */
         if (!IS_ERR(compo->rst_main))
                 reset_control_deassert(compo->rst_main);
 
-        compo->rst_aux = devm_reset_control_get(dev, "compo-aux");
+        compo->rst_aux = devm_reset_control_get_shared(dev, "compo-aux");
         /* Take compo aux out of reset */
         if (!IS_ERR(compo->rst_aux))
                 reset_control_deassert(compo->rst_aux);
@@ -247,10 +267,12 @@ static int sti_compositor_probe(struct platform_device *pdev)
         vtg_np = of_parse_phandle(pdev->dev.of_node, "st,vtg", 0);
         if (vtg_np)
                 compo->vtg_main = of_vtg_find(vtg_np);
+        of_node_put(vtg_np);
 
         vtg_np = of_parse_phandle(pdev->dev.of_node, "st,vtg", 1);
         if (vtg_np)
                 compo->vtg_aux = of_vtg_find(vtg_np);
+        of_node_put(vtg_np);
 
         platform_set_drvdata(pdev, compo);
 
diff --git a/drivers/gpu/drm/sti/sti_compositor.h b/drivers/gpu/drm/sti/sti_compositor.h
index 1a4a73dab11e..24444ef42a98 100644
--- a/drivers/gpu/drm/sti/sti_compositor.h
+++ b/drivers/gpu/drm/sti/sti_compositor.h
@@ -81,4 +81,7 @@ struct sti_compositor {
         struct notifier_block vtg_vblank_nb;
 };
 
+int sti_compositor_debufs_init(struct sti_compositor *compo,
+                               struct drm_minor *minor);
+
 #endif
diff --git a/drivers/gpu/drm/sti/sti_crtc.c b/drivers/gpu/drm/sti/sti_crtc.c
index e04deedabd4a..c7d734dc3cf4 100644
--- a/drivers/gpu/drm/sti/sti_crtc.c
+++ b/drivers/gpu/drm/sti/sti_crtc.c
@@ -23,22 +23,11 @@
 static void sti_crtc_enable(struct drm_crtc *crtc)
 {
         struct sti_mixer *mixer = to_sti_mixer(crtc);
-        struct device *dev = mixer->dev;
-        struct sti_compositor *compo = dev_get_drvdata(dev);
 
         DRM_DEBUG_DRIVER("\n");
 
         mixer->status = STI_MIXER_READY;
 
-        /* Prepare and enable the compo IP clock */
-        if (mixer->id == STI_MIXER_MAIN) {
-                if (clk_prepare_enable(compo->clk_compo_main))
-                        DRM_INFO("Failed to prepare/enable compo_main clk\n");
-        } else {
-                if (clk_prepare_enable(compo->clk_compo_aux))
-                        DRM_INFO("Failed to prepare/enable compo_aux clk\n");
-        }
-
         drm_crtc_vblank_on(crtc);
 }
 
@@ -57,9 +46,8 @@ sti_crtc_mode_set(struct drm_crtc *crtc, struct drm_display_mode *mode)
         struct sti_mixer *mixer = to_sti_mixer(crtc);
         struct device *dev = mixer->dev;
         struct sti_compositor *compo = dev_get_drvdata(dev);
-        struct clk *clk;
+        struct clk *compo_clk, *pix_clk;
         int rate = mode->clock * 1000;
-        int res;
 
         DRM_DEBUG_KMS("CRTC:%d (%s) mode:%d (%s)\n",
                       crtc->base.id, sti_mixer_to_str(mixer),
@@ -74,32 +62,46 @@ sti_crtc_mode_set(struct drm_crtc *crtc, struct drm_display_mode *mode)
                       mode->vsync_start, mode->vsync_end,
                       mode->vtotal, mode->type, mode->flags);
 
-        /* Set rate and prepare/enable pixel clock */
-        if (mixer->id == STI_MIXER_MAIN)
-                clk = compo->clk_pix_main;
-        else
-                clk = compo->clk_pix_aux;
+        if (mixer->id == STI_MIXER_MAIN) {
+                compo_clk = compo->clk_compo_main;
+                pix_clk = compo->clk_pix_main;
+        } else {
+                compo_clk = compo->clk_compo_aux;
+                pix_clk = compo->clk_pix_aux;
+        }
+
+        /* Prepare and enable the compo IP clock */
+        if (clk_prepare_enable(compo_clk)) {
+                DRM_INFO("Failed to prepare/enable compositor clk\n");
+                goto compo_error;
+        }
 
-        res = clk_set_rate(clk, rate);
-        if (res < 0) {
+        /* Set rate and prepare/enable pixel clock */
+        if (clk_set_rate(pix_clk, rate) < 0) {
                 DRM_ERROR("Cannot set rate (%dHz) for pix clk\n", rate);
-                return -EINVAL;
+                goto pix_error;
         }
-        if (clk_prepare_enable(clk)) {
+        if (clk_prepare_enable(pix_clk)) {
                 DRM_ERROR("Failed to prepare/enable pix clk\n");
-                return -EINVAL;
+                goto pix_error;
         }
 
         sti_vtg_set_config(mixer->id == STI_MIXER_MAIN ?
                         compo->vtg_main : compo->vtg_aux, &crtc->mode);
 
-        res = sti_mixer_active_video_area(mixer, &crtc->mode);
-        if (res) {
+        if (sti_mixer_active_video_area(mixer, &crtc->mode)) {
                 DRM_ERROR("Can't set active video area\n");
-                return -EINVAL;
+                goto mixer_error;
         }
 
-        return res;
+        return 0;
+
+mixer_error:
+        clk_disable_unprepare(pix_clk);
+pix_error:
+        clk_disable_unprepare(compo_clk);
+compo_error:
+        return -EINVAL;
 }
 
 static void sti_crtc_disable(struct drm_crtc *crtc)
@@ -130,7 +132,6 @@ static void sti_crtc_disable(struct drm_crtc *crtc)
 static void
 sti_crtc_mode_set_nofb(struct drm_crtc *crtc)
 {
-        sti_crtc_enable(crtc);
         sti_crtc_mode_set(crtc, &crtc->state->adjusted_mode);
 }
 
@@ -221,9 +222,7 @@ static void sti_crtc_atomic_flush(struct drm_crtc *crtc,
 static const struct drm_crtc_helper_funcs sti_crtc_helper_funcs = {
         .enable = sti_crtc_enable,
         .disable = sti_crtc_disabling,
-        .mode_set = drm_helper_crtc_mode_set,
         .mode_set_nofb = sti_crtc_mode_set_nofb,
-        .mode_set_base = drm_helper_crtc_mode_set_base,
         .atomic_begin = sti_crtc_atomic_begin,
         .atomic_flush = sti_crtc_atomic_flush,
 };
@@ -331,6 +330,17 @@ void sti_crtc_disable_vblank(struct drm_device *drm_dev, unsigned int pipe)
         }
 }
 
+static int sti_crtc_late_register(struct drm_crtc *crtc)
+{
+        struct sti_mixer *mixer = to_sti_mixer(crtc);
+        struct sti_compositor *compo = dev_get_drvdata(mixer->dev);
+
+        if (drm_crtc_index(crtc) == 0)
+                return sti_compositor_debufs_init(compo, crtc->dev->primary);
+
+        return 0;
+}
+
 static const struct drm_crtc_funcs sti_crtc_funcs = {
         .set_config = drm_atomic_helper_set_config,
         .page_flip = drm_atomic_helper_page_flip,
@@ -339,6 +349,7 @@ static const struct drm_crtc_funcs sti_crtc_funcs = {
         .reset = drm_atomic_helper_crtc_reset,
         .atomic_duplicate_state = drm_atomic_helper_crtc_duplicate_state,
         .atomic_destroy_state = drm_atomic_helper_crtc_destroy_state,
+        .late_register = sti_crtc_late_register,
 };
 
 bool sti_crtc_is_main(struct drm_crtc *crtc)
diff --git a/drivers/gpu/drm/sti/sti_cursor.c b/drivers/gpu/drm/sti/sti_cursor.c
index 4e990299735c..a263bbba4119 100644
--- a/drivers/gpu/drm/sti/sti_cursor.c
+++ b/drivers/gpu/drm/sti/sti_cursor.c
@@ -105,12 +105,6 @@ static int cursor_dbg_show(struct seq_file *s, void *data)
 {
         struct drm_info_node *node = s->private;
         struct sti_cursor *cursor = (struct sti_cursor *)node->info_ent->data;
-        struct drm_device *dev = node->minor->dev;
-        int ret;
-
-        ret = mutex_lock_interruptible(&dev->struct_mutex);
-        if (ret)
-                return ret;
 
         seq_printf(s, "%s: (vaddr = 0x%p)",
                    sti_plane_to_str(&cursor->plane), cursor->regs);
@@ -129,7 +123,6 @@ static int cursor_dbg_show(struct seq_file *s, void *data)
         DBGFS_DUMP(CUR_AWE);
         seq_puts(s, "\n");
 
-        mutex_unlock(&dev->struct_mutex);
         return 0;
 }
 
@@ -336,6 +329,33 @@ static const struct drm_plane_helper_funcs sti_cursor_helpers_funcs = {
         .atomic_disable = sti_cursor_atomic_disable,
 };
 
+static void sti_cursor_destroy(struct drm_plane *drm_plane)
+{
+        DRM_DEBUG_DRIVER("\n");
+
+        drm_plane_helper_disable(drm_plane);
+        drm_plane_cleanup(drm_plane);
+}
+
+static int sti_cursor_late_register(struct drm_plane *drm_plane)
+{
+        struct sti_plane *plane = to_sti_plane(drm_plane);
+        struct sti_cursor *cursor = to_sti_cursor(plane);
+
+        return cursor_debugfs_init(cursor, drm_plane->dev->primary);
+}
+
+struct drm_plane_funcs sti_cursor_plane_helpers_funcs = {
+        .update_plane = drm_atomic_helper_update_plane,
+        .disable_plane = drm_atomic_helper_disable_plane,
+        .destroy = sti_cursor_destroy,
+        .set_property = sti_plane_set_property,
+        .reset = drm_atomic_helper_plane_reset,
+        .atomic_duplicate_state = drm_atomic_helper_plane_duplicate_state,
+        .atomic_destroy_state = drm_atomic_helper_plane_destroy_state,
+        .late_register = sti_cursor_late_register,
+};
+
 struct drm_plane *sti_cursor_create(struct drm_device *drm_dev,
                                     struct device *dev, int desc,
                                     void __iomem *baseaddr,
@@ -370,7 +390,7 @@ struct drm_plane *sti_cursor_create(struct drm_device *drm_dev,
 
         res = drm_universal_plane_init(drm_dev, &cursor->plane.drm_plane,
                                        possible_crtcs,
-                                       &sti_plane_helpers_funcs,
+                                       &sti_cursor_plane_helpers_funcs,
                                        cursor_supported_formats,
                                        ARRAY_SIZE(cursor_supported_formats),
                                        DRM_PLANE_TYPE_CURSOR, NULL);
@@ -384,9 +404,6 @@ struct drm_plane *sti_cursor_create(struct drm_device *drm_dev,
 
         sti_plane_init_property(&cursor->plane, DRM_PLANE_TYPE_CURSOR);
 
-        if (cursor_debugfs_init(cursor, drm_dev->primary))
-                DRM_ERROR("CURSOR debugfs setup failed\n");
-
         return &cursor->plane.drm_plane;
 
 err_plane:
diff --git a/drivers/gpu/drm/sti/sti_drv.c b/drivers/gpu/drm/sti/sti_drv.c
index 872495e72294..96bd3d08b2d4 100644
--- a/drivers/gpu/drm/sti/sti_drv.c
+++ b/drivers/gpu/drm/sti/sti_drv.c
@@ -72,11 +72,6 @@ static int sti_drm_fps_dbg_show(struct seq_file *s, void *data)
         struct drm_info_node *node = s->private;
         struct drm_device *dev = node->minor->dev;
         struct drm_plane *p;
-        int ret;
-
-        ret = mutex_lock_interruptible(&dev->struct_mutex);
-        if (ret)
-                return ret;
 
         list_for_each_entry(p, &dev->mode_config.plane_list, head) {
                 struct sti_plane *plane = to_sti_plane(p);
@@ -86,7 +81,6 @@ static int sti_drm_fps_dbg_show(struct seq_file *s, void *data)
                            plane->fps_info.fips_str);
         }
 
-        mutex_unlock(&dev->struct_mutex);
         return 0;
 }
 
@@ -221,7 +215,7 @@ static int sti_atomic_commit(struct drm_device *drm,
          * the software side now.
          */
 
-        drm_atomic_helper_swap_state(drm, state);
+        drm_atomic_helper_swap_state(state, true);
 
         if (nonblock)
                 sti_atomic_schedule(private, state);
@@ -232,8 +226,28 @@ static int sti_atomic_commit(struct drm_device *drm,
         return 0;
 }
 
+static void sti_output_poll_changed(struct drm_device *ddev)
+{
+        struct sti_private *private = ddev->dev_private;
+
+        if (!ddev->mode_config.num_connector)
+                return;
+
+        if (private->fbdev) {
+                drm_fbdev_cma_hotplug_event(private->fbdev);
+                return;
+        }
+
+        private->fbdev = drm_fbdev_cma_init(ddev, 32,
+                                            ddev->mode_config.num_crtc,
+                                            ddev->mode_config.num_connector);
+        if (IS_ERR(private->fbdev))
+                private->fbdev = NULL;
+}
+
 static const struct drm_mode_config_funcs sti_mode_config_funcs = {
         .fb_create = drm_fb_cma_create,
+        .output_poll_changed = sti_output_poll_changed,
         .atomic_check = drm_atomic_helper_check,
         .atomic_commit = sti_atomic_commit,
 };
@@ -254,45 +268,6 @@ static void sti_mode_config_init(struct drm_device *dev)
         dev->mode_config.funcs = &sti_mode_config_funcs;
 }
 
-static int sti_load(struct drm_device *dev, unsigned long flags)
-{
-        struct sti_private *private;
-        int ret;
-
-        private = kzalloc(sizeof(*private), GFP_KERNEL);
-        if (!private) {
-                DRM_ERROR("Failed to allocate private\n");
-                return -ENOMEM;
-        }
-        dev->dev_private = (void *)private;
-        private->drm_dev = dev;
-
-        mutex_init(&private->commit.lock);
-        INIT_WORK(&private->commit.work, sti_atomic_work);
-
-        drm_mode_config_init(dev);
-        drm_kms_helper_poll_init(dev);
-
-        sti_mode_config_init(dev);
-
-        ret = component_bind_all(dev->dev, dev);
-        if (ret) {
-                drm_kms_helper_poll_fini(dev);
-                drm_mode_config_cleanup(dev);
-                kfree(private);
-                return ret;
-        }
-
-        drm_mode_config_reset(dev);
-
-        drm_helper_disable_unused_functions(dev);
-        drm_fbdev_cma_init(dev, 32,
-                           dev->mode_config.num_crtc,
-                           dev->mode_config.num_connector);
-
-        return 0;
-}
-
 static const struct file_operations sti_driver_fops = {
         .owner = THIS_MODULE,
         .open = drm_open,
@@ -309,8 +284,7 @@ static const struct file_operations sti_driver_fops = {
 static struct drm_driver sti_driver = {
         .driver_features = DRIVER_HAVE_IRQ | DRIVER_MODESET |
             DRIVER_GEM | DRIVER_PRIME | DRIVER_ATOMIC,
-        .load = sti_load,
-        .gem_free_object = drm_gem_cma_free_object,
+        .gem_free_object_unlocked = drm_gem_cma_free_object,
         .gem_vm_ops = &drm_gem_cma_vm_ops,
         .dumb_create = drm_gem_cma_dumb_create,
         .dumb_map_offset = drm_gem_cma_dumb_map_offset,
@@ -346,14 +320,88 @@ static int compare_of(struct device *dev, void *data)
         return dev->of_node == data;
 }
 
+static int sti_init(struct drm_device *ddev)
+{
+        struct sti_private *private;
+
+        private = kzalloc(sizeof(*private), GFP_KERNEL);
+        if (!private)
+                return -ENOMEM;
+
+        ddev->dev_private = (void *)private;
+        dev_set_drvdata(ddev->dev, ddev);
+        private->drm_dev = ddev;
+
+        mutex_init(&private->commit.lock);
+        INIT_WORK(&private->commit.work, sti_atomic_work);
+
+        drm_mode_config_init(ddev);
+
+        sti_mode_config_init(ddev);
+
+        drm_kms_helper_poll_init(ddev);
+
+        return 0;
+}
+
+static void sti_cleanup(struct drm_device *ddev)
+{
+        struct sti_private *private = ddev->dev_private;
+
+        if (private->fbdev) {
+                drm_fbdev_cma_fini(private->fbdev);
+                private->fbdev = NULL;
+        }
+
+        drm_kms_helper_poll_fini(ddev);
+        drm_vblank_cleanup(ddev);
+        kfree(private);
+        ddev->dev_private = NULL;
+}
+
 static int sti_bind(struct device *dev)
 {
-        return drm_platform_init(&sti_driver, to_platform_device(dev));
+        struct drm_device *ddev;
+        int ret;
+
+        ddev = drm_dev_alloc(&sti_driver, dev);
+        if (!ddev)
+                return -ENOMEM;
+
+        ddev->platformdev = to_platform_device(dev);
+
+        ret = sti_init(ddev);
+        if (ret)
+                goto err_drm_dev_unref;
+
+        ret = component_bind_all(ddev->dev, ddev);
+        if (ret)
+                goto err_cleanup;
+
+        ret = drm_dev_register(ddev, 0);
+        if (ret)
+                goto err_register;
+
+        drm_mode_config_reset(ddev);
+
+        return 0;
+
+err_register:
+        drm_mode_config_cleanup(ddev);
+err_cleanup:
+        sti_cleanup(ddev);
+err_drm_dev_unref:
+        drm_dev_unref(ddev);
+        return ret;
 }
 
 static void sti_unbind(struct device *dev)
 {
-        drm_put_dev(dev_get_drvdata(dev));
+        struct drm_device *ddev = dev_get_drvdata(dev);
+
+        drm_dev_unregister(ddev);
+        sti_cleanup(ddev);
+        drm_dev_unref(ddev);
 }
 
 static const struct component_master_ops sti_ops = {
diff --git a/drivers/gpu/drm/sti/sti_drv.h b/drivers/gpu/drm/sti/sti_drv.h
index 30ddc20841c3..78ebe5e30f53 100644
--- a/drivers/gpu/drm/sti/sti_drv.h
+++ b/drivers/gpu/drm/sti/sti_drv.h
@@ -24,6 +24,7 @@ struct sti_private {
         struct sti_compositor *compo;
         struct drm_property *plane_zorder_property;
         struct drm_device *drm_dev;
+        struct drm_fbdev_cma *fbdev;
 
         struct {
                 struct drm_atomic_state *state;
diff --git a/drivers/gpu/drm/sti/sti_dvo.c b/drivers/gpu/drm/sti/sti_dvo.c
index 25f76632002c..00881eb4536e 100644
--- a/drivers/gpu/drm/sti/sti_dvo.c
+++ b/drivers/gpu/drm/sti/sti_dvo.c
@@ -177,12 +177,6 @@ static int dvo_dbg_show(struct seq_file *s, void *data)
 {
         struct drm_info_node *node = s->private;
         struct sti_dvo *dvo = (struct sti_dvo *)node->info_ent->data;
-        struct drm_device *dev = node->minor->dev;
-        int ret;
-
-        ret = mutex_lock_interruptible(&dev->struct_mutex);
-        if (ret)
-                return ret;
 
         seq_printf(s, "DVO: (vaddr = 0x%p)", dvo->regs);
         DBGFS_DUMP(DVO_AWG_DIGSYNC_CTRL);
@@ -193,7 +187,6 @@ static int dvo_dbg_show(struct seq_file *s, void *data)
         dvo_dbg_awg_microcode(s, dvo->regs + DVO_DIGSYNC_INSTR_I);
         seq_puts(s, "\n");
 
-        mutex_unlock(&dev->struct_mutex);
         return 0;
 }
 
@@ -384,20 +377,10 @@ static int sti_dvo_connector_mode_valid(struct drm_connector *connector,
         return MODE_OK;
 }
 
-struct drm_encoder *sti_dvo_best_encoder(struct drm_connector *connector)
-{
-        struct sti_dvo_connector *dvo_connector
-                = to_sti_dvo_connector(connector);
-
-        /* Best encoder is the one associated during connector creation */
-        return dvo_connector->encoder;
-}
-
 static const
 struct drm_connector_helper_funcs sti_dvo_connector_helper_funcs = {
         .get_modes = sti_dvo_connector_get_modes,
         .mode_valid = sti_dvo_connector_mode_valid,
-        .best_encoder = sti_dvo_best_encoder,
 };
 
 static enum drm_connector_status
@@ -421,24 +404,29 @@ sti_dvo_connector_detect(struct drm_connector *connector, bool force)
         return connector_status_disconnected;
 }
 
-static void sti_dvo_connector_destroy(struct drm_connector *connector)
+static int sti_dvo_late_register(struct drm_connector *connector)
 {
         struct sti_dvo_connector *dvo_connector
                 = to_sti_dvo_connector(connector);
+        struct sti_dvo *dvo = dvo_connector->dvo;
 
-        drm_connector_unregister(connector);
-        drm_connector_cleanup(connector);
-        kfree(dvo_connector);
+        if (dvo_debugfs_init(dvo, dvo->drm_dev->primary)) {
+                DRM_ERROR("DVO debugfs setup failed\n");
+                return -EINVAL;
+        }
+
+        return 0;
 }
 
 static const struct drm_connector_funcs sti_dvo_connector_funcs = {
         .dpms = drm_atomic_helper_connector_dpms,
         .fill_modes = drm_helper_probe_single_connector_modes,
         .detect = sti_dvo_connector_detect,
-        .destroy = sti_dvo_connector_destroy,
+        .destroy = drm_connector_cleanup,
         .reset = drm_atomic_helper_connector_reset,
         .atomic_duplicate_state = drm_atomic_helper_connector_duplicate_state,
         .atomic_destroy_state = drm_atomic_helper_connector_destroy_state,
+        .late_register = sti_dvo_late_register,
 };
 
 static struct drm_encoder *sti_dvo_find_encoder(struct drm_device *dev)
@@ -509,26 +497,16 @@ static int sti_dvo_bind(struct device *dev, struct device *master, void *data)
         drm_connector_helper_add(drm_connector,
                                  &sti_dvo_connector_helper_funcs);
 
-        err = drm_connector_register(drm_connector);
-        if (err)
-                goto err_connector;
-
         err = drm_mode_connector_attach_encoder(drm_connector, encoder);
         if (err) {
                 DRM_ERROR("Failed to attach a connector to a encoder\n");
                 goto err_sysfs;
         }
 
-        if (dvo_debugfs_init(dvo, drm_dev->primary))
-                DRM_ERROR("DVO debugfs setup failed\n");
-
         return 0;
 
 err_sysfs:
-        drm_connector_unregister(drm_connector);
-err_connector:
         drm_bridge_remove(bridge);
-        drm_connector_cleanup(drm_connector);
         return -EINVAL;
 }
 
@@ -602,6 +580,7 @@ static int sti_dvo_probe(struct platform_device *pdev)
         dvo->panel_node = of_parse_phandle(np, "sti,panel", 0);
         if (!dvo->panel_node)
                 DRM_ERROR("No panel associated to the dvo output\n");
+        of_node_put(dvo->panel_node);
 
         platform_set_drvdata(pdev, dvo);
 
diff --git a/drivers/gpu/drm/sti/sti_gdp.c b/drivers/gpu/drm/sti/sti_gdp.c
index ff33c38da197..bf63086a3dc8 100644
--- a/drivers/gpu/drm/sti/sti_gdp.c
+++ b/drivers/gpu/drm/sti/sti_gdp.c
@@ -208,14 +208,8 @@ static int gdp_dbg_show(struct seq_file *s, void *data)
 {
         struct drm_info_node *node = s->private;
         struct sti_gdp *gdp = (struct sti_gdp *)node->info_ent->data;
-        struct drm_device *dev = node->minor->dev;
         struct drm_plane *drm_plane = &gdp->plane.drm_plane;
         struct drm_crtc *crtc = drm_plane->crtc;
-        int ret;
-
-        ret = mutex_lock_interruptible(&dev->struct_mutex);
-        if (ret)
-                return ret;
 
         seq_printf(s, "%s: (vaddr = 0x%p)",
                    sti_plane_to_str(&gdp->plane), gdp->regs);
@@ -248,7 +242,6 @@ static int gdp_dbg_show(struct seq_file *s, void *data)
                 seq_printf(s, "  Connected to DRM CRTC #%d (%s)\n",
                            crtc->base.id, sti_mixer_to_str(to_sti_mixer(crtc)));
 
-        mutex_unlock(&dev->struct_mutex);
         return 0;
 }
 
@@ -279,13 +272,7 @@ static int gdp_node_dbg_show(struct seq_file *s, void *arg)
 {
         struct drm_info_node *node = s->private;
         struct sti_gdp *gdp = (struct sti_gdp *)node->info_ent->data;
-        struct drm_device *dev = node->minor->dev;
         unsigned int b;
-        int ret;
-
-        ret = mutex_lock_interruptible(&dev->struct_mutex);
-        if (ret)
-                return ret;
 
         for (b = 0; b < GDP_NODE_NB_BANK; b++) {
                 seq_printf(s, "\n%s[%d].top", sti_plane_to_str(&gdp->plane), b);
@@ -294,7 +281,6 @@ static int gdp_node_dbg_show(struct seq_file *s, void *arg)
                 gdp_node_dump_node(s, gdp->node_list[b].btm_field);
         }
 
-        mutex_unlock(&dev->struct_mutex);
         return 0;
 }
 
@@ -880,6 +866,33 @@ static const struct drm_plane_helper_funcs sti_gdp_helpers_funcs = {
         .atomic_disable = sti_gdp_atomic_disable,
 };
 
+static void sti_gdp_destroy(struct drm_plane *drm_plane)
+{
+        DRM_DEBUG_DRIVER("\n");
+
+        drm_plane_helper_disable(drm_plane);
+        drm_plane_cleanup(drm_plane);
+}
+
+static int sti_gdp_late_register(struct drm_plane *drm_plane)
+{
+        struct sti_plane *plane = to_sti_plane(drm_plane);
+        struct sti_gdp *gdp = to_sti_gdp(plane);
+
+        return gdp_debugfs_init(gdp, drm_plane->dev->primary);
+}
+
+struct drm_plane_funcs sti_gdp_plane_helpers_funcs = {
+        .update_plane = drm_atomic_helper_update_plane,
+        .disable_plane = drm_atomic_helper_disable_plane,
+        .destroy = sti_gdp_destroy,
+        .set_property = sti_plane_set_property,
+        .reset = drm_atomic_helper_plane_reset,
+        .atomic_duplicate_state = drm_atomic_helper_plane_duplicate_state,
+        .atomic_destroy_state = drm_atomic_helper_plane_destroy_state,
+        .late_register = sti_gdp_late_register,
+};
+
 struct drm_plane *sti_gdp_create(struct drm_device *drm_dev,
                                  struct device *dev, int desc,
                                  void __iomem *baseaddr,
@@ -906,7 +919,7 @@ struct drm_plane *sti_gdp_create(struct drm_device *drm_dev,
 
         res = drm_universal_plane_init(drm_dev, &gdp->plane.drm_plane,
                                        possible_crtcs,
-                                       &sti_plane_helpers_funcs,
+                                       &sti_gdp_plane_helpers_funcs,
                                        gdp_supported_formats,
                                        ARRAY_SIZE(gdp_supported_formats),
                                        type, NULL);
@@ -919,9 +932,6 @@ struct drm_plane *sti_gdp_create(struct drm_device *drm_dev,
 
         sti_plane_init_property(&gdp->plane, type);
 
-        if (gdp_debugfs_init(gdp, drm_dev->primary))
-                DRM_ERROR("GDP debugfs setup failed\n");
-
         return &gdp->plane.drm_plane;
 
 err:
diff --git a/drivers/gpu/drm/sti/sti_hda.c b/drivers/gpu/drm/sti/sti_hda.c
index f7d3464cdf09..8505569f75de 100644
--- a/drivers/gpu/drm/sti/sti_hda.c
+++ b/drivers/gpu/drm/sti/sti_hda.c
@@ -376,12 +376,6 @@ static int hda_dbg_show(struct seq_file *s, void *data)
 {
         struct drm_info_node *node = s->private;
         struct sti_hda *hda = (struct sti_hda *)node->info_ent->data;
-        struct drm_device *dev = node->minor->dev;
-        int ret;
-
-        ret = mutex_lock_interruptible(&dev->struct_mutex);
-        if (ret)
-                return ret;
 
         seq_printf(s, "HD Analog: (vaddr = 0x%p)", hda->regs);
         DBGFS_DUMP(HDA_ANA_CFG);
@@ -397,7 +391,6 @@ static int hda_dbg_show(struct seq_file *s, void *data)
                 hda_dbg_video_dacs_ctrl(s, hda->video_dacs_ctrl);
         seq_puts(s, "\n");
 
-        mutex_unlock(&dev->struct_mutex);
         return 0;
 }
 
@@ -676,20 +669,10 @@ static int sti_hda_connector_mode_valid(struct drm_connector *connector,
         return MODE_OK;
 }
 
-struct drm_encoder *sti_hda_best_encoder(struct drm_connector *connector)
-{
-        struct sti_hda_connector *hda_connector
-                = to_sti_hda_connector(connector);
-
-        /* Best encoder is the one associated during connector creation */
-        return hda_connector->encoder;
-}
-
 static const
 struct drm_connector_helper_funcs sti_hda_connector_helper_funcs = {
         .get_modes = sti_hda_connector_get_modes,
         .mode_valid = sti_hda_connector_mode_valid,
-        .best_encoder = sti_hda_best_encoder,
 };
 
 static enum drm_connector_status
@@ -698,24 +681,29 @@ sti_hda_connector_detect(struct drm_connector *connector, bool force)
         return connector_status_connected;
 }
 
-static void sti_hda_connector_destroy(struct drm_connector *connector)
+static int sti_hda_late_register(struct drm_connector *connector)
 {
         struct sti_hda_connector *hda_connector
                 = to_sti_hda_connector(connector);
+        struct sti_hda *hda = hda_connector->hda;
+
+        if (hda_debugfs_init(hda, hda->drm_dev->primary)) {
+                DRM_ERROR("HDA debugfs setup failed\n");
+                return -EINVAL;
+        }
 
-        drm_connector_unregister(connector);
-        drm_connector_cleanup(connector);
-        kfree(hda_connector);
+        return 0;
 }
 
 static const struct drm_connector_funcs sti_hda_connector_funcs = {
         .dpms = drm_atomic_helper_connector_dpms,
         .fill_modes = drm_helper_probe_single_connector_modes,
         .detect = sti_hda_connector_detect,
-        .destroy = sti_hda_connector_destroy,
+        .destroy = drm_connector_cleanup,
         .reset = drm_atomic_helper_connector_reset,
         .atomic_duplicate_state = drm_atomic_helper_connector_duplicate_state,
         .atomic_destroy_state = drm_atomic_helper_connector_destroy_state,
+        .late_register = sti_hda_late_register,
 };
 
 static struct drm_encoder *sti_hda_find_encoder(struct drm_device *dev)
@@ -773,10 +761,6 @@ static int sti_hda_bind(struct device *dev, struct device *master, void *data)
         drm_connector_helper_add(drm_connector,
                         &sti_hda_connector_helper_funcs);
 
-        err = drm_connector_register(drm_connector);
-        if (err)
-                goto err_connector;
-
         err = drm_mode_connector_attach_encoder(drm_connector, encoder);
         if (err) {
                 DRM_ERROR("Failed to attach a connector to a encoder\n");
@@ -786,15 +770,10 @@ static int sti_hda_bind(struct device *dev, struct device *master, void *data)
         /* force to disable hd dacs at startup */
         hda_enable_hd_dacs(hda, false);
 
-        if (hda_debugfs_init(hda, drm_dev->primary))
-                DRM_ERROR("HDA debugfs setup failed\n");
-
         return 0;
 
 err_sysfs:
-        drm_connector_unregister(drm_connector);
-err_connector:
-        drm_connector_cleanup(drm_connector);
+        drm_bridge_remove(bridge);
         return -EINVAL;
 }
 
diff --git a/drivers/gpu/drm/sti/sti_hdmi.c b/drivers/gpu/drm/sti/sti_hdmi.c
index 6ef0715bd5b9..fedc17f98d9b 100644
--- a/drivers/gpu/drm/sti/sti_hdmi.c
+++ b/drivers/gpu/drm/sti/sti_hdmi.c
@@ -18,6 +18,8 @@
 #include <drm/drm_crtc_helper.h>
 #include <drm/drm_edid.h>
 
+#include <sound/hdmi-codec.h>
+
 #include "sti_hdmi.h"
 #include "sti_hdmi_tx3g4c28phy.h"
 #include "sti_hdmi_tx3g0c55phy.h"
@@ -35,6 +37,8 @@
 #define HDMI_DFLT_CHL0_DAT              0x0110
 #define HDMI_DFLT_CHL1_DAT              0x0114
 #define HDMI_DFLT_CHL2_DAT              0x0118
+#define HDMI_AUDIO_CFG                  0x0200
+#define HDMI_SPDIF_FIFO_STATUS          0x0204
 #define HDMI_SW_DI_1_HEAD_WORD          0x0210
 #define HDMI_SW_DI_1_PKT_WORD0          0x0214
 #define HDMI_SW_DI_1_PKT_WORD1          0x0218
@@ -44,6 +48,9 @@
 #define HDMI_SW_DI_1_PKT_WORD5          0x0228
 #define HDMI_SW_DI_1_PKT_WORD6          0x022C
 #define HDMI_SW_DI_CFG                  0x0230
+#define HDMI_SAMPLE_FLAT_MASK           0x0244
+#define HDMI_AUDN                       0x0400
+#define HDMI_AUD_CTS                    0x0404
 #define HDMI_SW_DI_2_HEAD_WORD          0x0600
 #define HDMI_SW_DI_2_PKT_WORD0          0x0604
 #define HDMI_SW_DI_2_PKT_WORD1          0x0608
@@ -103,6 +110,7 @@
 #define HDMI_INT_DLL_LCK                BIT(5)
 #define HDMI_INT_NEW_FRAME              BIT(6)
 #define HDMI_INT_GENCTRL_PKT            BIT(7)
+#define HDMI_INT_AUDIO_FIFO_XRUN        BIT(8)
 #define HDMI_INT_SINK_TERM_PRESENT      BIT(11)
 
 #define HDMI_DEFAULT_INT (HDMI_INT_SINK_TERM_PRESENT \
@@ -111,6 +119,7 @@
                         | HDMI_INT_GLOBAL)
 
 #define HDMI_WORKING_INT (HDMI_INT_SINK_TERM_PRESENT \
+                        | HDMI_INT_AUDIO_FIFO_XRUN \
                         | HDMI_INT_GENCTRL_PKT \
                         | HDMI_INT_NEW_FRAME \
                         | HDMI_INT_DLL_LCK \
@@ -121,6 +130,27 @@
 
 #define HDMI_STA_SW_RST                 BIT(1)
 
+#define HDMI_AUD_CFG_8CH                BIT(0)
+#define HDMI_AUD_CFG_SPDIF_DIV_2        BIT(1)
+#define HDMI_AUD_CFG_SPDIF_DIV_3        BIT(2)
+#define HDMI_AUD_CFG_SPDIF_CLK_DIV_4    (BIT(1) | BIT(2))
+#define HDMI_AUD_CFG_CTS_CLK_256FS      BIT(12)
+#define HDMI_AUD_CFG_DTS_INVALID        BIT(16)
+#define HDMI_AUD_CFG_ONE_BIT_INVALID    (BIT(18) | BIT(19) | BIT(20) |  BIT(21))
+#define HDMI_AUD_CFG_CH12_VALID BIT(28)
+#define HDMI_AUD_CFG_CH34_VALID BIT(29)
+#define HDMI_AUD_CFG_CH56_VALID BIT(30)
+#define HDMI_AUD_CFG_CH78_VALID BIT(31)
+
+/* sample flat mask */
+#define HDMI_SAMPLE_FLAT_NO      0
+#define HDMI_SAMPLE_FLAT_SP0 BIT(0)
+#define HDMI_SAMPLE_FLAT_SP1 BIT(1)
+#define HDMI_SAMPLE_FLAT_SP2 BIT(2)
+#define HDMI_SAMPLE_FLAT_SP3 BIT(3)
+#define HDMI_SAMPLE_FLAT_ALL (HDMI_SAMPLE_FLAT_SP0 | HDMI_SAMPLE_FLAT_SP1 |\
+                              HDMI_SAMPLE_FLAT_SP2 | HDMI_SAMPLE_FLAT_SP3)
+
 #define HDMI_INFOFRAME_HEADER_TYPE(x)    (((x) & 0xff) <<  0)
 #define HDMI_INFOFRAME_HEADER_VERSION(x) (((x) & 0xff) <<  8)
 #define HDMI_INFOFRAME_HEADER_LEN(x)     (((x) & 0x0f) << 16)
@@ -171,6 +201,10 @@ static irqreturn_t hdmi_irq_thread(int irq, void *arg)
                 wake_up_interruptible(&hdmi->wait_event);
         }
 
+        /* Audio FIFO underrun IRQ */
+        if (hdmi->irq_status & HDMI_INT_AUDIO_FIFO_XRUN)
+                DRM_INFO("Warning: audio FIFO underrun occurs!");
+
         return IRQ_HANDLED;
 }
 
@@ -441,26 +475,29 @@ static int hdmi_avi_infoframe_config(struct sti_hdmi *hdmi)
  */
 static int hdmi_audio_infoframe_config(struct sti_hdmi *hdmi)
 {
-        struct hdmi_audio_infoframe infofame;
+        struct hdmi_audio_params *audio = &hdmi->audio;
         u8 buffer[HDMI_INFOFRAME_SIZE(AUDIO)];
-        int ret;
-
-        ret = hdmi_audio_infoframe_init(&infofame);
-        if (ret < 0) {
-                DRM_ERROR("failed to setup audio infoframe: %d\n", ret);
-                return ret;
-        }
-
-        infofame.channels = 2;
-
-        ret = hdmi_audio_infoframe_pack(&infofame, buffer, sizeof(buffer));
-        if (ret < 0) {
-                DRM_ERROR("failed to pack audio infoframe: %d\n", ret);
-                return ret;
+        int ret, val;
+
+        DRM_DEBUG_DRIVER("enter %s, AIF %s\n", __func__,
+                         audio->enabled ? "enable" : "disable");
+        if (audio->enabled) {
+                /* set audio parameters stored*/
+                ret = hdmi_audio_infoframe_pack(&audio->cea, buffer,
+                                                sizeof(buffer));
+                if (ret < 0) {
+                        DRM_ERROR("failed to pack audio infoframe: %d\n", ret);
+                        return ret;
+                }
+                hdmi_infoframe_write_infopack(hdmi, buffer, ret);
+        } else {
+                /*disable audio info frame transmission */
+                val = hdmi_read(hdmi, HDMI_SW_DI_CFG);
+                val &= ~HDMI_IFRAME_CFG_DI_N(HDMI_IFRAME_MASK,
+                                             HDMI_IFRAME_SLOT_AUDIO);
+                hdmi_write(hdmi, val, HDMI_SW_DI_CFG);
         }
 
-        hdmi_infoframe_write_infopack(hdmi, buffer, ret);
-
         return 0;
 }
 
@@ -628,12 +665,6 @@ static int hdmi_dbg_show(struct seq_file *s, void *data)
 {
         struct drm_info_node *node = s->private;
         struct sti_hdmi *hdmi = (struct sti_hdmi *)node->info_ent->data;
-        struct drm_device *dev = node->minor->dev;
-        int ret;
-
-        ret = mutex_lock_interruptible(&dev->struct_mutex);
-        if (ret)
-                return ret;
 
         seq_printf(s, "HDMI: (vaddr = 0x%p)", hdmi->regs);
         DBGFS_DUMP("\n", HDMI_CFG);
@@ -656,6 +687,10 @@ static int hdmi_dbg_show(struct seq_file *s, void *data)
         DBGFS_DUMP("", HDMI_SW_DI_CFG);
         hdmi_dbg_sw_di_cfg(s, hdmi_read(hdmi, HDMI_SW_DI_CFG));
 
+        DBGFS_DUMP("\n", HDMI_AUDIO_CFG);
+        DBGFS_DUMP("\n", HDMI_SPDIF_FIFO_STATUS);
+        DBGFS_DUMP("\n", HDMI_AUDN);
+
         seq_printf(s, "\n AVI Infoframe (Data Island slot N=%d):",
                    HDMI_IFRAME_SLOT_AVI);
         DBGFS_DUMP_DI(HDMI_SW_DI_N_HEAD_WORD, HDMI_IFRAME_SLOT_AVI);
@@ -690,7 +725,6 @@ static int hdmi_dbg_show(struct seq_file *s, void *data)
         DBGFS_DUMP_DI(HDMI_SW_DI_N_PKT_WORD6, HDMI_IFRAME_SLOT_VENDOR);
         seq_puts(s, "\n");
 
-        mutex_unlock(&dev->struct_mutex);
         return 0;
 }
 
@@ -861,6 +895,7 @@ static int sti_hdmi_connector_get_modes(struct drm_connector *connector)
 
         count = drm_add_edid_modes(connector, edid);
         drm_mode_connector_update_edid_property(connector, edid);
+        drm_edid_to_eld(connector, edid);
 
         kfree(edid);
         return count;
@@ -897,20 +932,10 @@ static int sti_hdmi_connector_mode_valid(struct drm_connector *connector,
         return MODE_OK;
 }
 
-struct drm_encoder *sti_hdmi_best_encoder(struct drm_connector *connector)
-{
-        struct sti_hdmi_connector *hdmi_connector
-                = to_sti_hdmi_connector(connector);
-
-        /* Best encoder is the one associated during connector creation */
-        return hdmi_connector->encoder;
-}
-
 static const
 struct drm_connector_helper_funcs sti_hdmi_connector_helper_funcs = {
         .get_modes = sti_hdmi_connector_get_modes,
         .mode_valid = sti_hdmi_connector_mode_valid,
-        .best_encoder = sti_hdmi_best_encoder,
 };
 
 /* get detection status of display device */
@@ -932,16 +957,6 @@ sti_hdmi_connector_detect(struct drm_connector *connector, bool force)
         return connector_status_disconnected;
 }
 
-static void sti_hdmi_connector_destroy(struct drm_connector *connector)
-{
-        struct sti_hdmi_connector *hdmi_connector
-                = to_sti_hdmi_connector(connector);
-
-        drm_connector_unregister(connector);
-        drm_connector_cleanup(connector);
-        kfree(hdmi_connector);
-}
-
 static void sti_hdmi_connector_init_property(struct drm_device *drm_dev,
                                              struct drm_connector *connector)
 {
@@ -1024,17 +1039,31 @@ sti_hdmi_connector_get_property(struct drm_connector *connector,
         return -EINVAL;
 }
 
+static int sti_hdmi_late_register(struct drm_connector *connector)
+{
+        struct sti_hdmi_connector *hdmi_connector
+                = to_sti_hdmi_connector(connector);
+        struct sti_hdmi *hdmi = hdmi_connector->hdmi;
+
+        if (hdmi_debugfs_init(hdmi, hdmi->drm_dev->primary)) {
+                DRM_ERROR("HDMI debugfs setup failed\n");
+                return -EINVAL;
+        }
+
+        return 0;
+}
+
 static const struct drm_connector_funcs sti_hdmi_connector_funcs = {
-        .dpms = drm_atomic_helper_connector_dpms,
         .fill_modes = drm_helper_probe_single_connector_modes,
         .detect = sti_hdmi_connector_detect,
-        .destroy = sti_hdmi_connector_destroy,
+        .destroy = drm_connector_cleanup,
         .reset = drm_atomic_helper_connector_reset,
         .set_property = drm_atomic_helper_connector_set_property,
         .atomic_set_property = sti_hdmi_connector_set_property,
         .atomic_get_property = sti_hdmi_connector_get_property,
         .atomic_duplicate_state = drm_atomic_helper_connector_duplicate_state,
         .atomic_destroy_state = drm_atomic_helper_connector_destroy_state,
+        .late_register = sti_hdmi_late_register,
 };
 
 static struct drm_encoder *sti_hdmi_find_encoder(struct drm_device *dev)
@@ -1049,6 +1078,207 @@ static struct drm_encoder *sti_hdmi_find_encoder(struct drm_device *dev)
         return NULL;
 }
 
+/**
+ * sti_hdmi_audio_get_non_coherent_n() - get N parameter for non-coherent
+ * clocks. None-coherent clocks means that audio and TMDS clocks have not the
+ * same source (drifts between clocks). In this case assumption is that CTS is
+ * automatically calculated by hardware.
+ *
+ * @audio_fs: audio frame clock frequency in Hz
+ *
+ * Values computed are based on table described in HDMI specification 1.4b
+ *
+ * Returns n value.
+ */
+static int sti_hdmi_audio_get_non_coherent_n(unsigned int audio_fs)
+{
+        unsigned int n;
+
+        switch (audio_fs) {
+        case 32000:
+                n = 4096;
+                break;
+        case 44100:
+                n = 6272;
+                break;
+        case 48000:
+                n = 6144;
+                break;
+        case 88200:
+                n = 6272 * 2;
+                break;
+        case 96000:
+                n = 6144 * 2;
+                break;
+        case 176400:
+                n = 6272 * 4;
+                break;
+        case 192000:
+                n = 6144 * 4;
+                break;
+        default:
+                /* Not pre-defined, recommended value: 128 * fs / 1000 */
+                n = (audio_fs * 128) / 1000;
+        }
+
+        return n;
+}
+
+static int hdmi_audio_configure(struct sti_hdmi *hdmi,
+                                struct hdmi_audio_params *params)
+{
+        int audio_cfg, n;
+        struct hdmi_audio_infoframe *info = &params->cea;
+
+        DRM_DEBUG_DRIVER("\n");
+
+        if (!hdmi->enabled)
+                return 0;
+
+        /* update N parameter */
+        n = sti_hdmi_audio_get_non_coherent_n(params->sample_rate);
+
+        DRM_DEBUG_DRIVER("Audio rate = %d Hz, TMDS clock = %d Hz, n = %d\n",
+                         params->sample_rate, hdmi->mode.clock * 1000, n);
+        hdmi_write(hdmi, n, HDMI_AUDN);
+
+        /* update HDMI registers according to configuration */
+        audio_cfg = HDMI_AUD_CFG_SPDIF_DIV_2 | HDMI_AUD_CFG_DTS_INVALID |
+                    HDMI_AUD_CFG_ONE_BIT_INVALID;
+
+        switch (info->channels) {
+        case 8:
+                audio_cfg |= HDMI_AUD_CFG_CH78_VALID;
+        case 6:
+                audio_cfg |= HDMI_AUD_CFG_CH56_VALID;
+        case 4:
+                audio_cfg |= HDMI_AUD_CFG_CH34_VALID | HDMI_AUD_CFG_8CH;
+        case 2:
+                audio_cfg |= HDMI_AUD_CFG_CH12_VALID;
+                break;
+        default:
+                DRM_ERROR("ERROR: Unsupported number of channels (%d)!\n",
+                          info->channels);
+                return -EINVAL;
+        }
+
+        hdmi_write(hdmi, audio_cfg, HDMI_AUDIO_CFG);
+
+        hdmi->audio = *params;
+
+        return hdmi_audio_infoframe_config(hdmi);
+}
+
+static void hdmi_audio_shutdown(struct device *dev, void *data)
+{
+        struct sti_hdmi *hdmi = dev_get_drvdata(dev);
+        int audio_cfg;
+
+        DRM_DEBUG_DRIVER("\n");
+
+        /* disable audio */
+        audio_cfg = HDMI_AUD_CFG_SPDIF_DIV_2 | HDMI_AUD_CFG_DTS_INVALID |
+                    HDMI_AUD_CFG_ONE_BIT_INVALID;
+        hdmi_write(hdmi, audio_cfg, HDMI_AUDIO_CFG);
+
+        hdmi->audio.enabled = 0;
+        hdmi_audio_infoframe_config(hdmi);
+}
+
+static int hdmi_audio_hw_params(struct device *dev,
+                                void *data,
+                                struct hdmi_codec_daifmt *daifmt,
+                                struct hdmi_codec_params *params)
+{
+        struct sti_hdmi *hdmi = dev_get_drvdata(dev);
+        int ret;
+        struct hdmi_audio_params audio = {
+                .sample_width = params->sample_width,
+                .sample_rate = params->sample_rate,
+                .cea = params->cea,
+        };
+
+        DRM_DEBUG_DRIVER("\n");
+
+        if (!hdmi->enabled)
+                return 0;
+
+        if ((daifmt->fmt != HDMI_I2S) || daifmt->bit_clk_inv ||
+            daifmt->frame_clk_inv || daifmt->bit_clk_master ||
+            daifmt->frame_clk_master) {
+                dev_err(dev, "%s: Bad flags %d %d %d %d\n", __func__,
+                        daifmt->bit_clk_inv, daifmt->frame_clk_inv,
+                        daifmt->bit_clk_master,
+                        daifmt->frame_clk_master);
+                return -EINVAL;
+        }
+
+        audio.enabled = 1;
+
+        ret = hdmi_audio_configure(hdmi, &audio);
+        if (ret < 0)
+                return ret;
+
+        return 0;
+}
+
+static int hdmi_audio_digital_mute(struct device *dev, void *data, bool enable)
+{
+        struct sti_hdmi *hdmi = dev_get_drvdata(dev);
+
+        DRM_DEBUG_DRIVER("%s\n", enable ? "enable" : "disable");
+
+        if (enable)
+                hdmi_write(hdmi, HDMI_SAMPLE_FLAT_ALL, HDMI_SAMPLE_FLAT_MASK);
+        else
+                hdmi_write(hdmi, HDMI_SAMPLE_FLAT_NO, HDMI_SAMPLE_FLAT_MASK);
+
+        return 0;
+}
+
+static int hdmi_audio_get_eld(struct device *dev, void *data, uint8_t *buf, size_t len)
+{
+        struct sti_hdmi *hdmi = dev_get_drvdata(dev);
+        struct drm_connector *connector = hdmi->drm_connector;
+
+        DRM_DEBUG_DRIVER("\n");
+        memcpy(buf, connector->eld, min(sizeof(connector->eld), len));
+
+        return 0;
+}
+
+static const struct hdmi_codec_ops audio_codec_ops = {
+        .hw_params = hdmi_audio_hw_params,
+        .audio_shutdown = hdmi_audio_shutdown,
+        .digital_mute = hdmi_audio_digital_mute,
+        .get_eld = hdmi_audio_get_eld,
+};
+
+static int sti_hdmi_register_audio_driver(struct device *dev,
+                                          struct sti_hdmi *hdmi)
+{
+        struct hdmi_codec_pdata codec_data = {
+                .ops = &audio_codec_ops,
+                .max_i2s_channels = 8,
+                .i2s = 1,
+        };
+
+        DRM_DEBUG_DRIVER("\n");
+
+        hdmi->audio.enabled = 0;
+
+        hdmi->audio_pdev = platform_device_register_data(
+                dev, HDMI_CODEC_DRV_NAME, PLATFORM_DEVID_AUTO,
+                &codec_data, sizeof(codec_data));
+
+        if (IS_ERR(hdmi->audio_pdev))
+                return PTR_ERR(hdmi->audio_pdev);
+
+        DRM_INFO("%s Driver bound %s\n", HDMI_CODEC_DRV_NAME, dev_name(dev));
+
+        return 0;
+}
+
 static int sti_hdmi_bind(struct device *dev, struct device *master, void *data)
 {
         struct sti_hdmi *hdmi = dev_get_drvdata(dev);
@@ -1095,9 +1325,7 @@ static int sti_hdmi_bind(struct device *dev, struct device *master, void *data)
         /* initialise property */
         sti_hdmi_connector_init_property(drm_dev, drm_connector);
 
-        err = drm_connector_register(drm_connector);
-        if (err)
-                goto err_connector;
+        hdmi->drm_connector = drm_connector;
 
         err = drm_mode_connector_attach_encoder(drm_connector, encoder);
         if (err) {
@@ -1105,19 +1333,27 @@ static int sti_hdmi_bind(struct device *dev, struct device *master, void *data)
                 goto err_sysfs;
         }
 
+        err = sti_hdmi_register_audio_driver(dev, hdmi);
+        if (err) {
+                DRM_ERROR("Failed to attach an audio codec\n");
+                goto err_sysfs;
+        }
+
+        /* Initialize audio infoframe */
+        err = hdmi_audio_infoframe_init(&hdmi->audio.cea);
+        if (err) {
+                DRM_ERROR("Failed to init audio infoframe\n");
+                goto err_sysfs;
+        }
+
         /* Enable default interrupts */
         hdmi_write(hdmi, HDMI_DEFAULT_INT, HDMI_INT_EN);
 
-        if (hdmi_debugfs_init(hdmi, drm_dev->primary))
-                DRM_ERROR("HDMI debugfs setup failed\n");
-
         return 0;
 
 err_sysfs:
-        drm_connector_unregister(drm_connector);
-err_connector:
-        drm_connector_cleanup(drm_connector);
-
+        drm_bridge_remove(bridge);
+        hdmi->drm_connector = NULL;
         return -EINVAL;
 }
 
@@ -1267,6 +1503,8 @@ static int sti_hdmi_remove(struct platform_device *pdev)
         struct sti_hdmi *hdmi = dev_get_drvdata(&pdev->dev);
 
         i2c_put_adapter(hdmi->ddc_adapt);
+        if (hdmi->audio_pdev)
+                platform_device_unregister(hdmi->audio_pdev);
         component_del(&pdev->dev, &sti_hdmi_ops);
 
         return 0;
diff --git a/drivers/gpu/drm/sti/sti_hdmi.h b/drivers/gpu/drm/sti/sti_hdmi.h
index ef3a94583bbd..119bc3582ac7 100644
--- a/drivers/gpu/drm/sti/sti_hdmi.h
+++ b/drivers/gpu/drm/sti/sti_hdmi.h
@@ -23,6 +23,13 @@ struct hdmi_phy_ops {
         void (*stop)(struct sti_hdmi *hdmi);
 };
 
+struct hdmi_audio_params {
+        bool enabled;
+        unsigned int sample_width;
+        unsigned int sample_rate;
+        struct hdmi_audio_infoframe cea;
+};
+
 /* values for the framing mode property */
 enum sti_hdmi_modes {
         HDMI_MODE_HDMI,
@@ -67,6 +74,9 @@ static const struct drm_prop_enum_list colorspace_mode_names[] = {
  * @ddc_adapt: i2c ddc adapter
  * @colorspace: current colorspace selected
  * @hdmi_mode: select framing for HDMI or DVI
+ * @audio_pdev: ASoC hdmi-codec platform device
+ * @audio: hdmi audio parameters.
+ * @drm_connector: hdmi connector
  */
 struct sti_hdmi {
         struct device dev;
@@ -89,6 +99,9 @@ struct sti_hdmi {
         struct i2c_adapter *ddc_adapt;
         enum hdmi_colorspace colorspace;
         enum sti_hdmi_modes hdmi_mode;
+        struct platform_device *audio_pdev;
+        struct hdmi_audio_params audio;
+        struct drm_connector *drm_connector;
 };
 
 u32 hdmi_read(struct sti_hdmi *hdmi, int offset);
diff --git a/drivers/gpu/drm/sti/sti_hqvdp.c b/drivers/gpu/drm/sti/sti_hqvdp.c
index 1edec29b9e45..b03232247966 100644
--- a/drivers/gpu/drm/sti/sti_hqvdp.c
+++ b/drivers/gpu/drm/sti/sti_hqvdp.c
@@ -555,14 +555,8 @@ static int hqvdp_dbg_show(struct seq_file *s, void *data)
 {
         struct drm_info_node *node = s->private;
         struct sti_hqvdp *hqvdp = (struct sti_hqvdp *)node->info_ent->data;
-        struct drm_device *dev = node->minor->dev;
         int cmd, cmd_offset, infoxp70;
         void *virt;
-        int ret;
-
-        ret = mutex_lock_interruptible(&dev->struct_mutex);
-        if (ret)
-                return ret;
 
         seq_printf(s, "%s: (vaddr = 0x%p)",
                    sti_plane_to_str(&hqvdp->plane), hqvdp->regs);
@@ -630,7 +624,6 @@ static int hqvdp_dbg_show(struct seq_file *s, void *data)
 
         seq_puts(s, "\n");
 
-        mutex_unlock(&dev->struct_mutex);
         return 0;
 }
 
@@ -1241,6 +1234,33 @@ static const struct drm_plane_helper_funcs sti_hqvdp_helpers_funcs = {
         .atomic_disable = sti_hqvdp_atomic_disable,
 };
 
+static void sti_hqvdp_destroy(struct drm_plane *drm_plane)
+{
+        DRM_DEBUG_DRIVER("\n");
+
+        drm_plane_helper_disable(drm_plane);
+        drm_plane_cleanup(drm_plane);
+}
+
+static int sti_hqvdp_late_register(struct drm_plane *drm_plane)
+{
+        struct sti_plane *plane = to_sti_plane(drm_plane);
+        struct sti_hqvdp *hqvdp = to_sti_hqvdp(plane);
+
+        return hqvdp_debugfs_init(hqvdp, drm_plane->dev->primary);
+}
+
+struct drm_plane_funcs sti_hqvdp_plane_helpers_funcs = {
+        .update_plane = drm_atomic_helper_update_plane,
+        .disable_plane = drm_atomic_helper_disable_plane,
+        .destroy = sti_hqvdp_destroy,
+        .set_property = sti_plane_set_property,
+        .reset = drm_atomic_helper_plane_reset,
+        .atomic_duplicate_state = drm_atomic_helper_plane_duplicate_state,
+        .atomic_destroy_state = drm_atomic_helper_plane_destroy_state,
+        .late_register = sti_hqvdp_late_register,
+};
+
 static struct drm_plane *sti_hqvdp_create(struct drm_device *drm_dev,
                                           struct device *dev, int desc)
 {
@@ -1253,7 +1273,7 @@ static struct drm_plane *sti_hqvdp_create(struct drm_device *drm_dev,
         sti_hqvdp_init(hqvdp);
 
         res = drm_universal_plane_init(drm_dev, &hqvdp->plane.drm_plane, 1,
-                                       &sti_plane_helpers_funcs,
+                                       &sti_hqvdp_plane_helpers_funcs,
                                        hqvdp_supported_formats,
                                        ARRAY_SIZE(hqvdp_supported_formats),
                                        DRM_PLANE_TYPE_OVERLAY, NULL);
@@ -1266,9 +1286,6 @@ static struct drm_plane *sti_hqvdp_create(struct drm_device *drm_dev,
 
         sti_plane_init_property(&hqvdp->plane, DRM_PLANE_TYPE_OVERLAY);
 
-        if (hqvdp_debugfs_init(hqvdp, drm_dev->primary))
-                DRM_ERROR("HQVDP debugfs setup failed\n");
-
         return &hqvdp->plane.drm_plane;
 }
 
@@ -1346,6 +1363,7 @@ static int sti_hqvdp_probe(struct platform_device *pdev)
         vtg_np = of_parse_phandle(pdev->dev.of_node, "st,vtg", 0);
         if (vtg_np)
                 hqvdp->vtg = of_vtg_find(vtg_np);
+        of_node_put(vtg_np);
 
         platform_set_drvdata(pdev, hqvdp);
 
diff --git a/drivers/gpu/drm/sti/sti_mixer.c b/drivers/gpu/drm/sti/sti_mixer.c
index aed7801b51f7..1885c7ab5a8b 100644
--- a/drivers/gpu/drm/sti/sti_mixer.c
+++ b/drivers/gpu/drm/sti/sti_mixer.c
@@ -151,12 +151,6 @@ static int mixer_dbg_show(struct seq_file *s, void *arg)
 {
         struct drm_info_node *node = s->private;
         struct sti_mixer *mixer = (struct sti_mixer *)node->info_ent->data;
-        struct drm_device *dev = node->minor->dev;
-        int ret;
-
-        ret = mutex_lock_interruptible(&dev->struct_mutex);
-        if (ret)
-                return ret;
 
         seq_printf(s, "%s: (vaddr = 0x%p)",
                    sti_mixer_to_str(mixer), mixer->regs);
@@ -176,7 +170,6 @@ static int mixer_dbg_show(struct seq_file *s, void *arg)
         mixer_dbg_mxn(s, mixer->regs + GAM_MIXER_MX0);
         seq_puts(s, "\n");
 
-        mutex_unlock(&dev->struct_mutex);
         return 0;
 }
 
@@ -188,7 +181,7 @@ static struct drm_info_list mixer1_debugfs_files[] = {
         { "mixer_aux", mixer_dbg_show, 0, NULL },
 };
 
-static int mixer_debugfs_init(struct sti_mixer *mixer, struct drm_minor *minor)
+int sti_mixer_debugfs_init(struct sti_mixer *mixer, struct drm_minor *minor)
 {
         unsigned int i;
         struct drm_info_list *mixer_debugfs_files;
@@ -400,8 +393,5 @@ struct sti_mixer *sti_mixer_create(struct device *dev,
         DRM_DEBUG_DRIVER("%s created. Regs=%p\n",
                          sti_mixer_to_str(mixer), mixer->regs);
 
-        if (mixer_debugfs_init(mixer, drm_dev->primary))
-                DRM_ERROR("MIXER debugfs setup failed\n");
-
         return mixer;
 }
diff --git a/drivers/gpu/drm/sti/sti_mixer.h b/drivers/gpu/drm/sti/sti_mixer.h
index 6f35fc086873..830a3c42d886 100644
--- a/drivers/gpu/drm/sti/sti_mixer.h
+++ b/drivers/gpu/drm/sti/sti_mixer.h
@@ -55,6 +55,8 @@ int sti_mixer_active_video_area(struct sti_mixer *mixer,
 
 void sti_mixer_set_background_status(struct sti_mixer *mixer, bool enable);
 
+int sti_mixer_debugfs_init(struct sti_mixer *mixer, struct drm_minor *minor);
+
 /* depth in Cross-bar control = z order */
 #define GAM_MIXER_NB_DEPTH_LEVEL 6
 
diff --git a/drivers/gpu/drm/sti/sti_plane.c b/drivers/gpu/drm/sti/sti_plane.c
index f10c98d3f012..0cf3335ef37c 100644
--- a/drivers/gpu/drm/sti/sti_plane.c
+++ b/drivers/gpu/drm/sti/sti_plane.c
@@ -45,25 +45,15 @@ const char *sti_plane_to_str(struct sti_plane *plane)
 
 #define STI_FPS_INTERVAL_MS     3000
 
-static int sti_plane_timespec_ms_diff(struct timespec lhs, struct timespec rhs)
-{
-        struct timespec tmp_ts = timespec_sub(lhs, rhs);
-        u64 tmp_ns = (u64)timespec_to_ns(&tmp_ts);
-
-        do_div(tmp_ns, NSEC_PER_MSEC);
-
-        return (u32)tmp_ns;
-}
-
 void sti_plane_update_fps(struct sti_plane *plane,
                           bool new_frame,
                           bool new_field)
 {
-        struct timespec now;
+        ktime_t now;
         struct sti_fps_info *fps;
         int fpks, fipks, ms_since_last, num_frames, num_fields;
 
-        getrawmonotonic(&now);
+        now = ktime_get();
 
         /* Compute number of frame updates */
         fps = &plane->fps_info;
@@ -76,7 +66,7 @@ void sti_plane_update_fps(struct sti_plane *plane,
                 return;
 
         fps->curr_frame_counter++;
-        ms_since_last = sti_plane_timespec_ms_diff(now, fps->last_timestamp);
+        ms_since_last = ktime_to_ms(ktime_sub(now, fps->last_timestamp));
         num_frames = fps->curr_frame_counter - fps->last_frame_counter;
 
         if (num_frames <= 0  || ms_since_last < STI_FPS_INTERVAL_MS)
@@ -106,17 +96,9 @@ void sti_plane_update_fps(struct sti_plane *plane,
                          plane->fps_info.fips_str);
 }
 
-static void sti_plane_destroy(struct drm_plane *drm_plane)
-{
-        DRM_DEBUG_DRIVER("\n");
-
-        drm_plane_helper_disable(drm_plane);
-        drm_plane_cleanup(drm_plane);
-}
-
-static int sti_plane_set_property(struct drm_plane *drm_plane,
-                                  struct drm_property *property,
-                                  uint64_t val)
+int sti_plane_set_property(struct drm_plane *drm_plane,
+                           struct drm_property *property,
+                           uint64_t val)
 {
         struct drm_device *dev = drm_plane->dev;
         struct sti_private *private = dev->dev_private;
@@ -170,13 +152,3 @@ void sti_plane_init_property(struct sti_plane *plane,
                          plane->drm_plane.base.id,
                          sti_plane_to_str(plane), plane->zorder);
 }
-
-struct drm_plane_funcs sti_plane_helpers_funcs = {
-        .update_plane = drm_atomic_helper_update_plane,
-        .disable_plane = drm_atomic_helper_disable_plane,
-        .destroy = sti_plane_destroy,
-        .set_property = sti_plane_set_property,
-        .reset = drm_atomic_helper_plane_reset,
-        .atomic_duplicate_state = drm_atomic_helper_plane_duplicate_state,
-        .atomic_destroy_state = drm_atomic_helper_plane_destroy_state,
-};
diff --git a/drivers/gpu/drm/sti/sti_plane.h b/drivers/gpu/drm/sti/sti_plane.h
index c50a3b9f5d37..e0ea1dd3bb88 100644
--- a/drivers/gpu/drm/sti/sti_plane.h
+++ b/drivers/gpu/drm/sti/sti_plane.h
@@ -11,8 +11,6 @@
 #include <drm/drm_atomic_helper.h>
 #include <drm/drm_plane_helper.h>
 
-extern struct drm_plane_funcs sti_plane_helpers_funcs;
-
 #define to_sti_plane(x) container_of(x, struct sti_plane, drm_plane)
 
 #define STI_PLANE_TYPE_SHIFT 8
@@ -57,7 +55,7 @@ struct sti_fps_info {
         unsigned int last_frame_counter;
         unsigned int curr_field_counter;
         unsigned int last_field_counter;
-        struct timespec last_timestamp;
+        ktime_t      last_timestamp;
         char fps_str[FPS_LENGTH];
         char fips_str[FPS_LENGTH];
 };
@@ -83,6 +81,11 @@ const char *sti_plane_to_str(struct sti_plane *plane);
 void sti_plane_update_fps(struct sti_plane *plane,
                           bool new_frame,
                           bool new_field);
+
+int sti_plane_set_property(struct drm_plane *drm_plane,
+                           struct drm_property *property,
+                           uint64_t val);
+
 void sti_plane_init_property(struct sti_plane *plane,
                              enum drm_plane_type type);
 #endif
diff --git a/drivers/gpu/drm/sti/sti_tvout.c b/drivers/gpu/drm/sti/sti_tvout.c
index f983db5a59da..e25995b35715 100644
--- a/drivers/gpu/drm/sti/sti_tvout.c
+++ b/drivers/gpu/drm/sti/sti_tvout.c
@@ -112,6 +112,7 @@ struct sti_tvout {
         struct drm_encoder *hdmi;
         struct drm_encoder *hda;
         struct drm_encoder *dvo;
+        bool debugfs_registered;
 };
 
 struct sti_tvout_encoder {
@@ -515,13 +516,7 @@ static int tvout_dbg_show(struct seq_file *s, void *data)
 {
         struct drm_info_node *node = s->private;
         struct sti_tvout *tvout = (struct sti_tvout *)node->info_ent->data;
-        struct drm_device *dev = node->minor->dev;
         struct drm_crtc *crtc;
-        int ret;
-
-        ret = mutex_lock_interruptible(&dev->struct_mutex);
-        if (ret)
-                return ret;
 
         seq_printf(s, "TVOUT: (vaddr = 0x%p)", tvout->regs);
 
@@ -587,7 +582,6 @@ static int tvout_dbg_show(struct seq_file *s, void *data)
         DBGFS_DUMP(TVO_AUX_IN_VID_FORMAT);
         seq_puts(s, "\n");
 
-        mutex_unlock(&dev->struct_mutex);
         return 0;
 }
 
@@ -632,8 +626,37 @@ static void sti_tvout_encoder_destroy(struct drm_encoder *encoder)
         kfree(sti_encoder);
 }
 
+static int sti_tvout_late_register(struct drm_encoder *encoder)
+{
+        struct sti_tvout *tvout = to_sti_tvout(encoder);
+        int ret;
+
+        if (tvout->debugfs_registered)
+                return 0;
+
+        ret = tvout_debugfs_init(tvout, encoder->dev->primary);
+        if (ret)
+                return ret;
+
+        tvout->debugfs_registered = true;
+        return 0;
+}
+
+static void sti_tvout_early_unregister(struct drm_encoder *encoder)
+{
+        struct sti_tvout *tvout = to_sti_tvout(encoder);
+
+        if (!tvout->debugfs_registered)
+                return;
+
+        tvout_debugfs_exit(tvout, encoder->dev->primary);
+        tvout->debugfs_registered = false;
+}
+
 static const struct drm_encoder_funcs sti_tvout_encoder_funcs = {
         .destroy = sti_tvout_encoder_destroy,
+        .late_register = sti_tvout_late_register,
+        .early_unregister = sti_tvout_early_unregister,
 };
 
 static void sti_dvo_encoder_enable(struct drm_encoder *encoder)
@@ -820,9 +843,6 @@ static int sti_tvout_bind(struct device *dev, struct device *master, void *data)
 
         sti_tvout_create_encoders(drm_dev, tvout);
 
-        if (tvout_debugfs_init(tvout, drm_dev->primary))
-                DRM_ERROR("TVOUT debugfs setup failed\n");
-
         return 0;
 }
 
@@ -830,11 +850,8 @@ static void sti_tvout_unbind(struct device *dev, struct device *master,
         void *data)
 {
         struct sti_tvout *tvout = dev_get_drvdata(dev);
-        struct drm_device *drm_dev = data;
 
         sti_tvout_destroy_encoders(tvout);
-
-        tvout_debugfs_exit(tvout, drm_dev->primary);
 }
 
 static const struct component_ops sti_tvout_ops = {
diff --git a/drivers/gpu/drm/sti/sti_vid.c b/drivers/gpu/drm/sti/sti_vid.c
index 523ed19f5ac6..47634a0251fc 100644
--- a/drivers/gpu/drm/sti/sti_vid.c
+++ b/drivers/gpu/drm/sti/sti_vid.c
@@ -92,12 +92,6 @@ static int vid_dbg_show(struct seq_file *s, void *arg)
 {
         struct drm_info_node *node = s->private;
         struct sti_vid *vid = (struct sti_vid *)node->info_ent->data;
-        struct drm_device *dev = node->minor->dev;
-        int ret;
-
-        ret = mutex_lock_interruptible(&dev->struct_mutex);
-        if (ret)
-                return ret;
 
         seq_printf(s, "VID: (vaddr= 0x%p)", vid->regs);
 
@@ -122,7 +116,6 @@ static int vid_dbg_show(struct seq_file *s, void *arg)
         DBGFS_DUMP(VID_CSAT);
         seq_puts(s, "\n");
 
-        mutex_unlock(&dev->struct_mutex);
         return 0;
 }
 
@@ -130,7 +123,7 @@ static struct drm_info_list vid_debugfs_files[] = {
         { "vid", vid_dbg_show, 0, NULL },
 };
 
-static int vid_debugfs_init(struct sti_vid *vid, struct drm_minor *minor)
+int vid_debugfs_init(struct sti_vid *vid, struct drm_minor *minor)
 {
         unsigned int i;
 
@@ -227,8 +220,5 @@ struct sti_vid *sti_vid_create(struct device *dev, struct drm_device *drm_dev,
 
         sti_vid_init(vid);
 
-        if (vid_debugfs_init(vid, drm_dev->primary))
-                DRM_ERROR("VID debugfs setup failed\n");
-
         return vid;
 }
diff --git a/drivers/gpu/drm/sti/sti_vid.h b/drivers/gpu/drm/sti/sti_vid.h
index 6c842344f3d8..fdc90f922a05 100644
--- a/drivers/gpu/drm/sti/sti_vid.h
+++ b/drivers/gpu/drm/sti/sti_vid.h
@@ -26,4 +26,6 @@ void sti_vid_disable(struct sti_vid *vid);
 struct sti_vid *sti_vid_create(struct device *dev, struct drm_device *drm_dev,
                                int id, void __iomem *baseaddr);
 
+int vid_debugfs_init(struct sti_vid *vid, struct drm_minor *minor);
+
 #endif
diff --git a/drivers/gpu/drm/sti/sti_vtg.c b/drivers/gpu/drm/sti/sti_vtg.c
index 6bf4ce466d20..0bdc385eec17 100644
--- a/drivers/gpu/drm/sti/sti_vtg.c
+++ b/drivers/gpu/drm/sti/sti_vtg.c
@@ -65,7 +65,7 @@
 #define HDMI_DELAY          (5)
 
 /* Delay introduced by the DVO in nb of pixel */
-#define DVO_DELAY           (2)
+#define DVO_DELAY           (7)
 
 /* delay introduced by the Arbitrary Waveform Generator in nb of pixels */
 #define AWG_DELAY_HD        (-9)
@@ -432,6 +432,7 @@ static int vtg_probe(struct platform_device *pdev)
         np = of_parse_phandle(pdev->dev.of_node, "st,slave", 0);
         if (np) {
                 vtg->slave = of_vtg_find(np);
+                of_node_put(np);
 
                 if (!vtg->slave)
                         return -EPROBE_DEFER;
diff --git a/drivers/gpu/drm/sun4i/sun4i_crtc.c b/drivers/gpu/drm/sun4i/sun4i_crtc.c
index 41cacecbea9a..4a192210574f 100644
--- a/drivers/gpu/drm/sun4i/sun4i_crtc.c
+++ b/drivers/gpu/drm/sun4i/sun4i_crtc.c
@@ -51,10 +51,22 @@ static void sun4i_crtc_atomic_flush(struct drm_crtc *crtc,
 {
         struct sun4i_crtc *scrtc = drm_crtc_to_sun4i_crtc(crtc);
         struct sun4i_drv *drv = scrtc->drv;
+        struct drm_pending_vblank_event *event = crtc->state->event;
 
         DRM_DEBUG_DRIVER("Committing plane changes\n");
 
         sun4i_backend_commit(drv->backend);
+
+        if (event) {
+                crtc->state->event = NULL;
+
+                spin_lock_irq(&crtc->dev->event_lock);
+                if (drm_crtc_vblank_get(crtc) == 0)
+                        drm_crtc_arm_vblank_event(crtc, event);
+                else
+                        drm_crtc_send_vblank_event(crtc, event);
+                spin_unlock_irq(&crtc->dev->event_lock);
+        }
 }
 
 static void sun4i_crtc_disable(struct drm_crtc *crtc)
diff --git a/drivers/gpu/drm/sun4i/sun4i_drv.c b/drivers/gpu/drm/sun4i/sun4i_drv.c
index 937394cbc241..7092daaf6c43 100644
--- a/drivers/gpu/drm/sun4i/sun4i_drv.c
+++ b/drivers/gpu/drm/sun4i/sun4i_drv.c
@@ -75,7 +75,7 @@ static struct drm_driver sun4i_drv_driver = {
         .dumb_create            = drm_gem_cma_dumb_create,
         .dumb_destroy           = drm_gem_dumb_destroy,
         .dumb_map_offset        = drm_gem_cma_dumb_map_offset,
-        .gem_free_object        = drm_gem_cma_free_object,
+        .gem_free_object_unlocked = drm_gem_cma_free_object,
         .gem_vm_ops             = &drm_gem_cma_vm_ops,
 
         /* PRIME Operations */
@@ -123,10 +123,6 @@ static int sun4i_drv_bind(struct device *dev)
         if (!drm)
                 return -ENOMEM;
 
-        ret = drm_dev_set_unique(drm, dev_name(drm->dev));
-        if (ret)
-                goto free_drm;
-
         drv = devm_kzalloc(dev, sizeof(*drv), GFP_KERNEL);
         if (!drv) {
                 ret = -ENOMEM;
@@ -178,14 +174,8 @@ static int sun4i_drv_bind(struct device *dev)
         if (ret)
                 goto free_drm;
 
-        ret = drm_connector_register_all(drm);
-        if (ret)
-                goto unregister_drm;
-
         return 0;
 
-unregister_drm:
-        drm_dev_unregister(drm);
 free_drm:
         drm_dev_unref(drm);
         return ret;
@@ -195,7 +185,6 @@ static void sun4i_drv_unbind(struct device *dev)
 {
         struct drm_device *drm = dev_get_drvdata(dev);
 
-        drm_connector_unregister_all(drm);
         drm_dev_unregister(drm);
         drm_kms_helper_poll_fini(drm);
         sun4i_framebuffer_free(drm);
diff --git a/drivers/gpu/drm/sun4i/sun4i_framebuffer.c b/drivers/gpu/drm/sun4i/sun4i_framebuffer.c
index a0b30c216a5b..70688febd7ac 100644
--- a/drivers/gpu/drm/sun4i/sun4i_framebuffer.c
+++ b/drivers/gpu/drm/sun4i/sun4i_framebuffer.c
@@ -20,8 +20,7 @@ static void sun4i_de_output_poll_changed(struct drm_device *drm)
 {
         struct sun4i_drv *drv = drm->dev_private;
 
-        if (drv->fbdev)
-                drm_fbdev_cma_hotplug_event(drv->fbdev);
+        drm_fbdev_cma_hotplug_event(drv->fbdev);
 }
 
 static const struct drm_mode_config_funcs sun4i_de_mode_config_funcs = {
diff --git a/drivers/gpu/drm/sun4i/sun4i_rgb.c b/drivers/gpu/drm/sun4i/sun4i_rgb.c
index aaffe9e64ffb..f5bbac6efb4c 100644
--- a/drivers/gpu/drm/sun4i/sun4i_rgb.c
+++ b/drivers/gpu/drm/sun4i/sun4i_rgb.c
@@ -104,19 +104,9 @@ static int sun4i_rgb_mode_valid(struct drm_connector *connector,
         return MODE_OK;
 }
 
-static struct drm_encoder *
-sun4i_rgb_best_encoder(struct drm_connector *connector)
-{
-        struct sun4i_rgb *rgb =
-                drm_connector_to_sun4i_rgb(connector);
-
-        return &rgb->encoder;
-}
-
 static struct drm_connector_helper_funcs sun4i_rgb_con_helper_funcs = {
         .get_modes      = sun4i_rgb_get_modes,
         .mode_valid     = sun4i_rgb_mode_valid,
-        .best_encoder   = sun4i_rgb_best_encoder,
 };
 
 static enum drm_connector_status
diff --git a/drivers/gpu/drm/sun4i/sun4i_tv.c b/drivers/gpu/drm/sun4i/sun4i_tv.c
index bc047f923508..b84147896294 100644
--- a/drivers/gpu/drm/sun4i/sun4i_tv.c
+++ b/drivers/gpu/drm/sun4i/sun4i_tv.c
@@ -526,18 +526,9 @@ static int sun4i_tv_comp_mode_valid(struct drm_connector *connector,
         return MODE_OK;
 }
 
-static struct drm_encoder *
-sun4i_tv_comp_best_encoder(struct drm_connector *connector)
-{
-        struct sun4i_tv *tv = drm_connector_to_sun4i_tv(connector);
-
-        return &tv->encoder;
-}
-
 static struct drm_connector_helper_funcs sun4i_tv_comp_connector_helper_funcs = {
         .get_modes      = sun4i_tv_comp_get_modes,
         .mode_valid     = sun4i_tv_comp_mode_valid,
-        .best_encoder   = sun4i_tv_comp_best_encoder,
 };
 
 static enum drm_connector_status
diff --git a/drivers/gpu/drm/tegra/dc.c b/drivers/gpu/drm/tegra/dc.c
index 39940f5b7c91..8495bd01b544 100644
--- a/drivers/gpu/drm/tegra/dc.c
+++ b/drivers/gpu/drm/tegra/dc.c
@@ -10,6 +10,7 @@
 #include <linux/clk.h>
 #include <linux/debugfs.h>
 #include <linux/iommu.h>
+#include <linux/pm_runtime.h>
 #include <linux/reset.h>
 
 #include <soc/tegra/pmc.h>
@@ -1216,6 +1217,8 @@ static void tegra_crtc_disable(struct drm_crtc *crtc)
 
         tegra_dc_stats_reset(&dc->stats);
         drm_crtc_vblank_off(crtc);
+
+        pm_runtime_put_sync(dc->dev);
 }
 
 static void tegra_crtc_enable(struct drm_crtc *crtc)
@@ -1225,6 +1228,48 @@ static void tegra_crtc_enable(struct drm_crtc *crtc)
         struct tegra_dc *dc = to_tegra_dc(crtc);
         u32 value;
 
+        pm_runtime_get_sync(dc->dev);
+
+        /* initialize display controller */
+        if (dc->syncpt) {
+                u32 syncpt = host1x_syncpt_id(dc->syncpt);
+
+                value = SYNCPT_CNTRL_NO_STALL;
+                tegra_dc_writel(dc, value, DC_CMD_GENERAL_INCR_SYNCPT_CNTRL);
+
+                value = SYNCPT_VSYNC_ENABLE | syncpt;
+                tegra_dc_writel(dc, value, DC_CMD_CONT_SYNCPT_VSYNC);
+        }
+
+        value = WIN_A_UF_INT | WIN_B_UF_INT | WIN_C_UF_INT |
+                WIN_A_OF_INT | WIN_B_OF_INT | WIN_C_OF_INT;
+        tegra_dc_writel(dc, value, DC_CMD_INT_TYPE);
+
+        value = WIN_A_UF_INT | WIN_B_UF_INT | WIN_C_UF_INT |
+                WIN_A_OF_INT | WIN_B_OF_INT | WIN_C_OF_INT;
+        tegra_dc_writel(dc, value, DC_CMD_INT_POLARITY);
+
+        /* initialize timer */
+        value = CURSOR_THRESHOLD(0) | WINDOW_A_THRESHOLD(0x20) |
+                WINDOW_B_THRESHOLD(0x20) | WINDOW_C_THRESHOLD(0x20);
+        tegra_dc_writel(dc, value, DC_DISP_DISP_MEM_HIGH_PRIORITY);
+
+        value = CURSOR_THRESHOLD(0) | WINDOW_A_THRESHOLD(1) |
+                WINDOW_B_THRESHOLD(1) | WINDOW_C_THRESHOLD(1);
+        tegra_dc_writel(dc, value, DC_DISP_DISP_MEM_HIGH_PRIORITY_TIMER);
+
+        value = VBLANK_INT | WIN_A_UF_INT | WIN_B_UF_INT | WIN_C_UF_INT |
+                WIN_A_OF_INT | WIN_B_OF_INT | WIN_C_OF_INT;
+        tegra_dc_writel(dc, value, DC_CMD_INT_ENABLE);
+
+        value = WIN_A_UF_INT | WIN_B_UF_INT | WIN_C_UF_INT |
+                WIN_A_OF_INT | WIN_B_OF_INT | WIN_C_OF_INT;
+        tegra_dc_writel(dc, value, DC_CMD_INT_MASK);
+
+        if (dc->soc->supports_border_color)
+                tegra_dc_writel(dc, 0, DC_DISP_BORDER_COLOR);
+
+        /* apply PLL and pixel clock changes */
         tegra_dc_commit_state(dc, state);
 
         /* program display mode */
@@ -1685,7 +1730,6 @@ static int tegra_dc_init(struct host1x_client *client)
         struct tegra_drm *tegra = drm->dev_private;
         struct drm_plane *primary = NULL;
         struct drm_plane *cursor = NULL;
-        u32 value;
         int err;
 
         dc->syncpt = host1x_syncpt_request(dc->dev, flags);
@@ -1755,47 +1799,6 @@ static int tegra_dc_init(struct host1x_client *client)
                 goto cleanup;
         }
 
-        /* initialize display controller */
-        if (dc->syncpt) {
-                u32 syncpt = host1x_syncpt_id(dc->syncpt);
-
-                value = SYNCPT_CNTRL_NO_STALL;
-                tegra_dc_writel(dc, value, DC_CMD_GENERAL_INCR_SYNCPT_CNTRL);
-
-                value = SYNCPT_VSYNC_ENABLE | syncpt;
-                tegra_dc_writel(dc, value, DC_CMD_CONT_SYNCPT_VSYNC);
-        }
-
-        value = WIN_A_UF_INT | WIN_B_UF_INT | WIN_C_UF_INT |
-                WIN_A_OF_INT | WIN_B_OF_INT | WIN_C_OF_INT;
-        tegra_dc_writel(dc, value, DC_CMD_INT_TYPE);
-
-        value = WIN_A_UF_INT | WIN_B_UF_INT | WIN_C_UF_INT |
-                WIN_A_OF_INT | WIN_B_OF_INT | WIN_C_OF_INT;
-        tegra_dc_writel(dc, value, DC_CMD_INT_POLARITY);
-
-        /* initialize timer */
-        value = CURSOR_THRESHOLD(0) | WINDOW_A_THRESHOLD(0x20) |
-                WINDOW_B_THRESHOLD(0x20) | WINDOW_C_THRESHOLD(0x20);
-        tegra_dc_writel(dc, value, DC_DISP_DISP_MEM_HIGH_PRIORITY);
-
-        value = CURSOR_THRESHOLD(0) | WINDOW_A_THRESHOLD(1) |
-                WINDOW_B_THRESHOLD(1) | WINDOW_C_THRESHOLD(1);
-        tegra_dc_writel(dc, value, DC_DISP_DISP_MEM_HIGH_PRIORITY_TIMER);
-
-        value = VBLANK_INT | WIN_A_UF_INT | WIN_B_UF_INT | WIN_C_UF_INT |
-                WIN_A_OF_INT | WIN_B_OF_INT | WIN_C_OF_INT;
-        tegra_dc_writel(dc, value, DC_CMD_INT_ENABLE);
-
-        value = WIN_A_UF_INT | WIN_B_UF_INT | WIN_C_UF_INT |
-                WIN_A_OF_INT | WIN_B_OF_INT | WIN_C_OF_INT;
-        tegra_dc_writel(dc, value, DC_CMD_INT_MASK);
-
-        if (dc->soc->supports_border_color)
-                tegra_dc_writel(dc, 0, DC_DISP_BORDER_COLOR);
-
-        tegra_dc_stats_reset(&dc->stats);
-
         return 0;
 
 cleanup:
@@ -1987,33 +1990,15 @@ static int tegra_dc_probe(struct platform_device *pdev)
                 return PTR_ERR(dc->rst);
         }
 
+        reset_control_assert(dc->rst);
+
         if (dc->soc->has_powergate) {
                 if (dc->pipe == 0)
                         dc->powergate = TEGRA_POWERGATE_DIS;
                 else
                         dc->powergate = TEGRA_POWERGATE_DISB;
 
-                err = tegra_powergate_sequence_power_up(dc->powergate, dc->clk,
-                                                        dc->rst);
-                if (err < 0) {
-                        dev_err(&pdev->dev, "failed to power partition: %d\n",
-                                err);
-                        return err;
-                }
-        } else {
-                err = clk_prepare_enable(dc->clk);
-                if (err < 0) {
-                        dev_err(&pdev->dev, "failed to enable clock: %d\n",
-                                err);
-                        return err;
-                }
-
-                err = reset_control_deassert(dc->rst);
-                if (err < 0) {
-                        dev_err(&pdev->dev, "failed to deassert reset: %d\n",
-                                err);
-                        return err;
-                }
+                tegra_powergate_power_off(dc->powergate);
         }
 
         regs = platform_get_resource(pdev, IORESOURCE_MEM, 0);
@@ -2027,16 +2012,19 @@ static int tegra_dc_probe(struct platform_device *pdev)
                 return -ENXIO;
         }
 
-        INIT_LIST_HEAD(&dc->client.list);
-        dc->client.ops = &dc_client_ops;
-        dc->client.dev = &pdev->dev;
-
         err = tegra_dc_rgb_probe(dc);
         if (err < 0 && err != -ENODEV) {
                 dev_err(&pdev->dev, "failed to probe RGB output: %d\n", err);
                 return err;
         }
 
+        platform_set_drvdata(pdev, dc);
+        pm_runtime_enable(&pdev->dev);
+
+        INIT_LIST_HEAD(&dc->client.list);
+        dc->client.ops = &dc_client_ops;
+        dc->client.dev = &pdev->dev;
+
         err = host1x_client_register(&dc->client);
         if (err < 0) {
                 dev_err(&pdev->dev, "failed to register host1x client: %d\n",
@@ -2044,8 +2032,6 @@ static int tegra_dc_probe(struct platform_device *pdev)
                 return err;
         }
 
-        platform_set_drvdata(pdev, dc);
-
         return 0;
 }
 
@@ -2067,7 +2053,22 @@ static int tegra_dc_remove(struct platform_device *pdev)
                 return err;
         }
 
-        reset_control_assert(dc->rst);
+        pm_runtime_disable(&pdev->dev);
+
+        return 0;
+}
+
+#ifdef CONFIG_PM
+static int tegra_dc_suspend(struct device *dev)
+{
+        struct tegra_dc *dc = dev_get_drvdata(dev);
+        int err;
+
+        err = reset_control_assert(dc->rst);
+        if (err < 0) {
+                dev_err(dev, "failed to assert reset: %d\n", err);
+                return err;
+        }
 
         if (dc->soc->has_powergate)
                 tegra_powergate_power_off(dc->powergate);
@@ -2077,10 +2078,45 @@ static int tegra_dc_remove(struct platform_device *pdev)
         return 0;
 }
 
+static int tegra_dc_resume(struct device *dev)
+{
+        struct tegra_dc *dc = dev_get_drvdata(dev);
+        int err;
+
+        if (dc->soc->has_powergate) {
+                err = tegra_powergate_sequence_power_up(dc->powergate, dc->clk,
+                                                        dc->rst);
+                if (err < 0) {
+                        dev_err(dev, "failed to power partition: %d\n", err);
+                        return err;
+                }
+        } else {
+                err = clk_prepare_enable(dc->clk);
+                if (err < 0) {
+                        dev_err(dev, "failed to enable clock: %d\n", err);
+                        return err;
+                }
+
+                err = reset_control_deassert(dc->rst);
+                if (err < 0) {
+                        dev_err(dev, "failed to deassert reset: %d\n", err);
+                        return err;
+                }
+        }
+
+        return 0;
+}
+#endif
+
+static const struct dev_pm_ops tegra_dc_pm_ops = {
+        SET_RUNTIME_PM_OPS(tegra_dc_suspend, tegra_dc_resume, NULL)
+};
+
 struct platform_driver tegra_dc_driver = {
         .driver = {
                 .name = "tegra-dc",
                 .of_match_table = tegra_dc_of_match,
+                .pm = &tegra_dc_pm_ops,
         },
         .probe = tegra_dc_probe,
         .remove = tegra_dc_remove,
diff --git a/drivers/gpu/drm/tegra/dpaux.c b/drivers/gpu/drm/tegra/dpaux.c
index b24a0f14821a..059f409556d5 100644
--- a/drivers/gpu/drm/tegra/dpaux.c
+++ b/drivers/gpu/drm/tegra/dpaux.c
@@ -12,6 +12,9 @@
 #include <linux/interrupt.h>
 #include <linux/io.h>
 #include <linux/of_gpio.h>
+#include <linux/pinctrl/pinconf-generic.h>
+#include <linux/pinctrl/pinctrl.h>
+#include <linux/pinctrl/pinmux.h>
 #include <linux/platform_device.h>
 #include <linux/reset.h>
 #include <linux/regulator/consumer.h>
@@ -44,6 +47,11 @@ struct tegra_dpaux {
         struct completion complete;
         struct work_struct work;
         struct list_head list;
+
+#ifdef CONFIG_GENERIC_PINCONF
+        struct pinctrl_dev *pinctrl;
+        struct pinctrl_desc desc;
+#endif
 };
 
 static inline struct tegra_dpaux *to_dpaux(struct drm_dp_aux *aux)
@@ -267,6 +275,148 @@ static irqreturn_t tegra_dpaux_irq(int irq, void *data)
         return ret;
 }
 
+enum tegra_dpaux_functions {
+        DPAUX_PADCTL_FUNC_AUX,
+        DPAUX_PADCTL_FUNC_I2C,
+        DPAUX_PADCTL_FUNC_OFF,
+};
+
+static void tegra_dpaux_pad_power_down(struct tegra_dpaux *dpaux)
+{
+        u32 value = tegra_dpaux_readl(dpaux, DPAUX_HYBRID_SPARE);
+
+        value |= DPAUX_HYBRID_SPARE_PAD_POWER_DOWN;
+
+        tegra_dpaux_writel(dpaux, value, DPAUX_HYBRID_SPARE);
+}
+
+static void tegra_dpaux_pad_power_up(struct tegra_dpaux *dpaux)
+{
+        u32 value = tegra_dpaux_readl(dpaux, DPAUX_HYBRID_SPARE);
+
+        value &= ~DPAUX_HYBRID_SPARE_PAD_POWER_DOWN;
+
+        tegra_dpaux_writel(dpaux, value, DPAUX_HYBRID_SPARE);
+}
+
+static int tegra_dpaux_pad_config(struct tegra_dpaux *dpaux, unsigned function)
+{
+        u32 value;
+
+        switch (function) {
+        case DPAUX_PADCTL_FUNC_AUX:
+                value = DPAUX_HYBRID_PADCTL_AUX_CMH(2) |
+                        DPAUX_HYBRID_PADCTL_AUX_DRVZ(4) |
+                        DPAUX_HYBRID_PADCTL_AUX_DRVI(0x18) |
+                        DPAUX_HYBRID_PADCTL_AUX_INPUT_RCV |
+                        DPAUX_HYBRID_PADCTL_MODE_AUX;
+                break;
+
+        case DPAUX_PADCTL_FUNC_I2C:
+                value = DPAUX_HYBRID_PADCTL_I2C_SDA_INPUT_RCV |
+                        DPAUX_HYBRID_PADCTL_I2C_SCL_INPUT_RCV |
+                        DPAUX_HYBRID_PADCTL_MODE_I2C;
+                break;
+
+        case DPAUX_PADCTL_FUNC_OFF:
+                tegra_dpaux_pad_power_down(dpaux);
+                return 0;
+
+        default:
+                return -ENOTSUPP;
+        }
+
+        tegra_dpaux_writel(dpaux, value, DPAUX_HYBRID_PADCTL);
+        tegra_dpaux_pad_power_up(dpaux);
+
+        return 0;
+}
+
+#ifdef CONFIG_GENERIC_PINCONF
+static const struct pinctrl_pin_desc tegra_dpaux_pins[] = {
+        PINCTRL_PIN(0, "DP_AUX_CHx_P"),
+        PINCTRL_PIN(1, "DP_AUX_CHx_N"),
+};
+
+static const unsigned tegra_dpaux_pin_numbers[] = { 0, 1 };
+
+static const char * const tegra_dpaux_groups[] = {
+        "dpaux-io",
+};
+
+static const char * const tegra_dpaux_functions[] = {
+        "aux",
+        "i2c",
+        "off",
+};
+
+static int tegra_dpaux_get_groups_count(struct pinctrl_dev *pinctrl)
+{
+        return ARRAY_SIZE(tegra_dpaux_groups);
+}
+
+static const char *tegra_dpaux_get_group_name(struct pinctrl_dev *pinctrl,
+                                              unsigned int group)
+{
+        return tegra_dpaux_groups[group];
+}
+
+static int tegra_dpaux_get_group_pins(struct pinctrl_dev *pinctrl,
+                                      unsigned group, const unsigned **pins,
+                                      unsigned *num_pins)
+{
+        *pins = tegra_dpaux_pin_numbers;
+        *num_pins = ARRAY_SIZE(tegra_dpaux_pin_numbers);
+
+        return 0;
+}
+
+static const struct pinctrl_ops tegra_dpaux_pinctrl_ops = {
+        .get_groups_count = tegra_dpaux_get_groups_count,
+        .get_group_name = tegra_dpaux_get_group_name,
+        .get_group_pins = tegra_dpaux_get_group_pins,
+        .dt_node_to_map = pinconf_generic_dt_node_to_map_group,
+        .dt_free_map = pinconf_generic_dt_free_map,
+};
+
+static int tegra_dpaux_get_functions_count(struct pinctrl_dev *pinctrl)
+{
+        return ARRAY_SIZE(tegra_dpaux_functions);
+}
+
+static const char *tegra_dpaux_get_function_name(struct pinctrl_dev *pinctrl,
+                                                 unsigned int function)
+{
+        return tegra_dpaux_functions[function];
+}
+
+static int tegra_dpaux_get_function_groups(struct pinctrl_dev *pinctrl,
+                                           unsigned int function,
+                                           const char * const **groups,
+                                           unsigned * const num_groups)
+{
+        *num_groups = ARRAY_SIZE(tegra_dpaux_groups);
+        *groups = tegra_dpaux_groups;
+
+        return 0;
+}
+
+static int tegra_dpaux_set_mux(struct pinctrl_dev *pinctrl,
+                               unsigned int function, unsigned int group)
+{
+        struct tegra_dpaux *dpaux = pinctrl_dev_get_drvdata(pinctrl);
+
+        return tegra_dpaux_pad_config(dpaux, function);
+}
+
+static const struct pinmux_ops tegra_dpaux_pinmux_ops = {
+        .get_functions_count = tegra_dpaux_get_functions_count,
+        .get_function_name = tegra_dpaux_get_function_name,
+        .get_function_groups = tegra_dpaux_get_function_groups,
+        .set_mux = tegra_dpaux_set_mux,
+};
+#endif
+
 static int tegra_dpaux_probe(struct platform_device *pdev)
 {
         struct tegra_dpaux *dpaux;
@@ -294,11 +444,14 @@ static int tegra_dpaux_probe(struct platform_device *pdev)
                 return -ENXIO;
         }
 
-        dpaux->rst = devm_reset_control_get(&pdev->dev, "dpaux");
-        if (IS_ERR(dpaux->rst)) {
-                dev_err(&pdev->dev, "failed to get reset control: %ld\n",
-                        PTR_ERR(dpaux->rst));
-                return PTR_ERR(dpaux->rst);
+        if (!pdev->dev.pm_domain) {
+                dpaux->rst = devm_reset_control_get(&pdev->dev, "dpaux");
+                if (IS_ERR(dpaux->rst)) {
+                        dev_err(&pdev->dev,
+                                "failed to get reset control: %ld\n",
+                                PTR_ERR(dpaux->rst));
+                        return PTR_ERR(dpaux->rst);
+                }
         }
 
         dpaux->clk = devm_clk_get(&pdev->dev, NULL);
@@ -315,34 +468,37 @@ static int tegra_dpaux_probe(struct platform_device *pdev)
                 return err;
         }
 
-        reset_control_deassert(dpaux->rst);
+        if (dpaux->rst)
+                reset_control_deassert(dpaux->rst);
 
         dpaux->clk_parent = devm_clk_get(&pdev->dev, "parent");
         if (IS_ERR(dpaux->clk_parent)) {
                 dev_err(&pdev->dev, "failed to get parent clock: %ld\n",
                         PTR_ERR(dpaux->clk_parent));
-                return PTR_ERR(dpaux->clk_parent);
+                err = PTR_ERR(dpaux->clk_parent);
+                goto assert_reset;
         }
 
         err = clk_prepare_enable(dpaux->clk_parent);
         if (err < 0) {
                 dev_err(&pdev->dev, "failed to enable parent clock: %d\n",
                         err);
-                return err;
+                goto assert_reset;
         }
 
         err = clk_set_rate(dpaux->clk_parent, 270000000);
         if (err < 0) {
                 dev_err(&pdev->dev, "failed to set clock to 270 MHz: %d\n",
                         err);
-                return err;
+                goto disable_parent_clk;
         }
 
         dpaux->vdd = devm_regulator_get(&pdev->dev, "vdd");
         if (IS_ERR(dpaux->vdd)) {
                 dev_err(&pdev->dev, "failed to get VDD supply: %ld\n",
                         PTR_ERR(dpaux->vdd));
-                return PTR_ERR(dpaux->vdd);
+                err = PTR_ERR(dpaux->vdd);
+                goto disable_parent_clk;
         }
 
         err = devm_request_irq(dpaux->dev, dpaux->irq, tegra_dpaux_irq, 0,
@@ -350,7 +506,7 @@ static int tegra_dpaux_probe(struct platform_device *pdev)
         if (err < 0) {
                 dev_err(dpaux->dev, "failed to request IRQ#%u: %d\n",
                         dpaux->irq, err);
-                return err;
+                goto disable_parent_clk;
         }
 
         disable_irq(dpaux->irq);
@@ -360,7 +516,7 @@ static int tegra_dpaux_probe(struct platform_device *pdev)
 
         err = drm_dp_aux_register(&dpaux->aux);
         if (err < 0)
-                return err;
+                goto disable_parent_clk;
 
         /*
          * Assume that by default the DPAUX/I2C pads will be used for HDMI,
@@ -370,16 +526,24 @@ static int tegra_dpaux_probe(struct platform_device *pdev)
          * is no possibility to perform the I2C mode configuration in the
          * HDMI path.
          */
-        value = tegra_dpaux_readl(dpaux, DPAUX_HYBRID_SPARE);
-        value &= ~DPAUX_HYBRID_SPARE_PAD_POWER_DOWN;
-        tegra_dpaux_writel(dpaux, value, DPAUX_HYBRID_SPARE);
-
-        value = tegra_dpaux_readl(dpaux, DPAUX_HYBRID_PADCTL);
-        value = DPAUX_HYBRID_PADCTL_I2C_SDA_INPUT_RCV |
-                DPAUX_HYBRID_PADCTL_I2C_SCL_INPUT_RCV |
-                DPAUX_HYBRID_PADCTL_MODE_I2C;
-        tegra_dpaux_writel(dpaux, value, DPAUX_HYBRID_PADCTL);
+        err = tegra_dpaux_pad_config(dpaux, DPAUX_HYBRID_PADCTL_MODE_I2C);
+        if (err < 0)
+                return err;
 
+#ifdef CONFIG_GENERIC_PINCONF
+        dpaux->desc.name = dev_name(&pdev->dev);
+        dpaux->desc.pins = tegra_dpaux_pins;
+        dpaux->desc.npins = ARRAY_SIZE(tegra_dpaux_pins);
+        dpaux->desc.pctlops = &tegra_dpaux_pinctrl_ops;
+        dpaux->desc.pmxops = &tegra_dpaux_pinmux_ops;
+        dpaux->desc.owner = THIS_MODULE;
+
+        dpaux->pinctrl = devm_pinctrl_register(&pdev->dev, &dpaux->desc, dpaux);
+        if (!dpaux->pinctrl) {
+                dev_err(&pdev->dev, "failed to register pincontrol\n");
+                return -ENODEV;
+        }
+#endif
         /* enable and clear all interrupts */
         value = DPAUX_INTR_AUX_DONE | DPAUX_INTR_IRQ_EVENT |
                 DPAUX_INTR_UNPLUG_EVENT | DPAUX_INTR_PLUG_EVENT;
@@ -393,17 +557,24 @@ static int tegra_dpaux_probe(struct platform_device *pdev)
         platform_set_drvdata(pdev, dpaux);
 
         return 0;
+
+disable_parent_clk:
+        clk_disable_unprepare(dpaux->clk_parent);
+assert_reset:
+        if (dpaux->rst)
+                reset_control_assert(dpaux->rst);
+
+        clk_disable_unprepare(dpaux->clk);
+
+        return err;
 }
 
 static int tegra_dpaux_remove(struct platform_device *pdev)
 {
         struct tegra_dpaux *dpaux = platform_get_drvdata(pdev);
-        u32 value;
 
         /* make sure pads are powered down when not in use */
-        value = tegra_dpaux_readl(dpaux, DPAUX_HYBRID_SPARE);
-        value |= DPAUX_HYBRID_SPARE_PAD_POWER_DOWN;
-        tegra_dpaux_writel(dpaux, value, DPAUX_HYBRID_SPARE);
+        tegra_dpaux_pad_power_down(dpaux);
 
         drm_dp_aux_unregister(&dpaux->aux);
 
@@ -414,7 +585,10 @@ static int tegra_dpaux_remove(struct platform_device *pdev)
         cancel_work_sync(&dpaux->work);
 
         clk_disable_unprepare(dpaux->clk_parent);
-        reset_control_assert(dpaux->rst);
+
+        if (dpaux->rst)
+                reset_control_assert(dpaux->rst);
+
         clk_disable_unprepare(dpaux->clk);
 
         return 0;
@@ -528,30 +702,15 @@ enum drm_connector_status drm_dp_aux_detect(struct drm_dp_aux *aux)
 int drm_dp_aux_enable(struct drm_dp_aux *aux)
 {
         struct tegra_dpaux *dpaux = to_dpaux(aux);
-        u32 value;
-
-        value = DPAUX_HYBRID_PADCTL_AUX_CMH(2) |
-                DPAUX_HYBRID_PADCTL_AUX_DRVZ(4) |
-                DPAUX_HYBRID_PADCTL_AUX_DRVI(0x18) |
-                DPAUX_HYBRID_PADCTL_AUX_INPUT_RCV |
-                DPAUX_HYBRID_PADCTL_MODE_AUX;
-        tegra_dpaux_writel(dpaux, value, DPAUX_HYBRID_PADCTL);
-
-        value = tegra_dpaux_readl(dpaux, DPAUX_HYBRID_SPARE);
-        value &= ~DPAUX_HYBRID_SPARE_PAD_POWER_DOWN;
-        tegra_dpaux_writel(dpaux, value, DPAUX_HYBRID_SPARE);
 
-        return 0;
+        return tegra_dpaux_pad_config(dpaux, DPAUX_PADCTL_FUNC_AUX);
 }
 
 int drm_dp_aux_disable(struct drm_dp_aux *aux)
 {
         struct tegra_dpaux *dpaux = to_dpaux(aux);
-        u32 value;
 
-        value = tegra_dpaux_readl(dpaux, DPAUX_HYBRID_SPARE);
-        value |= DPAUX_HYBRID_SPARE_PAD_POWER_DOWN;
-        tegra_dpaux_writel(dpaux, value, DPAUX_HYBRID_SPARE);
+        tegra_dpaux_pad_power_down(dpaux);
 
         return 0;
 }
diff --git a/drivers/gpu/drm/tegra/drm.c b/drivers/gpu/drm/tegra/drm.c
index b59c3bf0df44..755264d9db22 100644
--- a/drivers/gpu/drm/tegra/drm.c
+++ b/drivers/gpu/drm/tegra/drm.c
@@ -56,8 +56,8 @@ static void tegra_atomic_complete(struct tegra_drm *tegra,
          */
 
         drm_atomic_helper_commit_modeset_disables(drm, state);
-        drm_atomic_helper_commit_planes(drm, state, false);
         drm_atomic_helper_commit_modeset_enables(drm, state);
+        drm_atomic_helper_commit_planes(drm, state, true);
 
         drm_atomic_helper_wait_for_vblanks(drm, state);
 
@@ -93,7 +93,7 @@ static int tegra_atomic_commit(struct drm_device *drm,
          * the software side now.
          */
 
-        drm_atomic_helper_swap_state(drm, state);
+        drm_atomic_helper_swap_state(state, true);
 
         if (nonblock)
                 tegra_atomic_schedule(tegra, state);
diff --git a/drivers/gpu/drm/tegra/drm.h b/drivers/gpu/drm/tegra/drm.h
index f52d6cb24ff5..0ddcce1b420d 100644
--- a/drivers/gpu/drm/tegra/drm.h
+++ b/drivers/gpu/drm/tegra/drm.h
@@ -239,8 +239,6 @@ int tegra_output_init(struct drm_device *drm, struct tegra_output *output);
 void tegra_output_exit(struct tegra_output *output);
 
 int tegra_output_connector_get_modes(struct drm_connector *connector);
-struct drm_encoder *
-tegra_output_connector_best_encoder(struct drm_connector *connector);
 enum drm_connector_status
 tegra_output_connector_detect(struct drm_connector *connector, bool force);
 void tegra_output_connector_destroy(struct drm_connector *connector);
diff --git a/drivers/gpu/drm/tegra/dsi.c b/drivers/gpu/drm/tegra/dsi.c
index d1239ebc190f..3d228ad90e0f 100644
--- a/drivers/gpu/drm/tegra/dsi.c
+++ b/drivers/gpu/drm/tegra/dsi.c
@@ -13,6 +13,7 @@
 #include <linux/of.h>
 #include <linux/of_platform.h>
 #include <linux/platform_device.h>
+#include <linux/pm_runtime.h>
 #include <linux/reset.h>
 
 #include <linux/regulator/consumer.h>
@@ -677,6 +678,45 @@ static void tegra_dsi_ganged_disable(struct tegra_dsi *dsi)
         tegra_dsi_writel(dsi, 0, DSI_GANGED_MODE_CONTROL);
 }
 
+static int tegra_dsi_pad_enable(struct tegra_dsi *dsi)
+{
+        u32 value;
+
+        value = DSI_PAD_CONTROL_VS1_PULLDN(0) | DSI_PAD_CONTROL_VS1_PDIO(0);
+        tegra_dsi_writel(dsi, value, DSI_PAD_CONTROL_0);
+
+        return 0;
+}
+
+static int tegra_dsi_pad_calibrate(struct tegra_dsi *dsi)
+{
+        u32 value;
+
+        /*
+         * XXX Is this still needed? The module reset is deasserted right
+         * before this function is called.
+         */
+        tegra_dsi_writel(dsi, 0, DSI_PAD_CONTROL_0);
+        tegra_dsi_writel(dsi, 0, DSI_PAD_CONTROL_1);
+        tegra_dsi_writel(dsi, 0, DSI_PAD_CONTROL_2);
+        tegra_dsi_writel(dsi, 0, DSI_PAD_CONTROL_3);
+        tegra_dsi_writel(dsi, 0, DSI_PAD_CONTROL_4);
+
+        /* start calibration */
+        tegra_dsi_pad_enable(dsi);
+
+        value = DSI_PAD_SLEW_UP(0x7) | DSI_PAD_SLEW_DN(0x7) |
+                DSI_PAD_LP_UP(0x1) | DSI_PAD_LP_DN(0x1) |
+                DSI_PAD_OUT_CLK(0x0);
+        tegra_dsi_writel(dsi, value, DSI_PAD_CONTROL_2);
+
+        value = DSI_PAD_PREEMP_PD_CLK(0x3) | DSI_PAD_PREEMP_PU_CLK(0x3) |
+                DSI_PAD_PREEMP_PD(0x03) | DSI_PAD_PREEMP_PU(0x3);
+        tegra_dsi_writel(dsi, value, DSI_PAD_CONTROL_3);
+
+        return tegra_mipi_calibrate(dsi->mipi);
+}
+
 static void tegra_dsi_set_timeout(struct tegra_dsi *dsi, unsigned long bclk,
                                   unsigned int vrefresh)
 {
@@ -794,7 +834,6 @@ tegra_dsi_connector_mode_valid(struct drm_connector *connector,
 static const struct drm_connector_helper_funcs tegra_dsi_connector_helper_funcs = {
         .get_modes = tegra_output_connector_get_modes,
         .mode_valid = tegra_dsi_connector_mode_valid,
-        .best_encoder = tegra_output_connector_best_encoder,
 };
 
 static const struct drm_encoder_funcs tegra_dsi_encoder_funcs = {
@@ -837,7 +876,7 @@ static void tegra_dsi_encoder_disable(struct drm_encoder *encoder)
 
         tegra_dsi_disable(dsi);
 
-        return;
+        pm_runtime_put(dsi->dev);
 }
 
 static void tegra_dsi_encoder_enable(struct drm_encoder *encoder)
@@ -848,6 +887,13 @@ static void tegra_dsi_encoder_enable(struct drm_encoder *encoder)
         struct tegra_dsi *dsi = to_dsi(output);
         struct tegra_dsi_state *state;
         u32 value;
+        int err;
+
+        pm_runtime_get_sync(dsi->dev);
+
+        err = tegra_dsi_pad_calibrate(dsi);
+        if (err < 0)
+                dev_err(dsi->dev, "MIPI calibration failed: %d\n", err);
 
         state = tegra_dsi_get_state(dsi);
 
@@ -876,8 +922,6 @@ static void tegra_dsi_encoder_enable(struct drm_encoder *encoder)
 
         if (output->panel)
                 drm_panel_enable(output->panel);
-
-        return;
 }
 
 static int
@@ -967,55 +1011,12 @@ static const struct drm_encoder_helper_funcs tegra_dsi_encoder_helper_funcs = {
         .atomic_check = tegra_dsi_encoder_atomic_check,
 };
 
-static int tegra_dsi_pad_enable(struct tegra_dsi *dsi)
-{
-        u32 value;
-
-        value = DSI_PAD_CONTROL_VS1_PULLDN(0) | DSI_PAD_CONTROL_VS1_PDIO(0);
-        tegra_dsi_writel(dsi, value, DSI_PAD_CONTROL_0);
-
-        return 0;
-}
-
-static int tegra_dsi_pad_calibrate(struct tegra_dsi *dsi)
-{
-        u32 value;
-
-        tegra_dsi_writel(dsi, 0, DSI_PAD_CONTROL_0);
-        tegra_dsi_writel(dsi, 0, DSI_PAD_CONTROL_1);
-        tegra_dsi_writel(dsi, 0, DSI_PAD_CONTROL_2);
-        tegra_dsi_writel(dsi, 0, DSI_PAD_CONTROL_3);
-        tegra_dsi_writel(dsi, 0, DSI_PAD_CONTROL_4);
-
-        /* start calibration */
-        tegra_dsi_pad_enable(dsi);
-
-        value = DSI_PAD_SLEW_UP(0x7) | DSI_PAD_SLEW_DN(0x7) |
-                DSI_PAD_LP_UP(0x1) | DSI_PAD_LP_DN(0x1) |
-                DSI_PAD_OUT_CLK(0x0);
-        tegra_dsi_writel(dsi, value, DSI_PAD_CONTROL_2);
-
-        value = DSI_PAD_PREEMP_PD_CLK(0x3) | DSI_PAD_PREEMP_PU_CLK(0x3) |
-                DSI_PAD_PREEMP_PD(0x03) | DSI_PAD_PREEMP_PU(0x3);
-        tegra_dsi_writel(dsi, value, DSI_PAD_CONTROL_3);
-
-        return tegra_mipi_calibrate(dsi->mipi);
-}
-
 static int tegra_dsi_init(struct host1x_client *client)
 {
         struct drm_device *drm = dev_get_drvdata(client->parent);
         struct tegra_dsi *dsi = host1x_client_to_dsi(client);
         int err;
 
-        reset_control_deassert(dsi->rst);
-
-        err = tegra_dsi_pad_calibrate(dsi);
-        if (err < 0) {
-                dev_err(dsi->dev, "MIPI calibration failed: %d\n", err);
-                goto reset;
-        }
-
         /* Gangsters must not register their own outputs. */
         if (!dsi->master) {
                 dsi->output.dev = client->dev;
@@ -1038,12 +1039,9 @@ static int tegra_dsi_init(struct host1x_client *client)
                 drm_connector_register(&dsi->output.connector);
 
                 err = tegra_output_init(drm, &dsi->output);
-                if (err < 0) {
-                        dev_err(client->dev,
-                                "failed to initialize output: %d\n",
+                if (err < 0)
+                        dev_err(dsi->dev, "failed to initialize output: %d\n",
                                 err);
-                        goto reset;
-                }
 
                 dsi->output.encoder.possible_crtcs = 0x3;
         }
@@ -1055,10 +1053,6 @@ static int tegra_dsi_init(struct host1x_client *client)
         }
 
         return 0;
-
-reset:
-        reset_control_assert(dsi->rst);
-        return err;
 }
 
 static int tegra_dsi_exit(struct host1x_client *client)
@@ -1070,7 +1064,7 @@ static int tegra_dsi_exit(struct host1x_client *client)
         if (IS_ENABLED(CONFIG_DEBUG_FS))
                 tegra_dsi_debugfs_exit(dsi);
 
-        reset_control_assert(dsi->rst);
+        regulator_disable(dsi->vdd);
 
         return 0;
 }
@@ -1494,74 +1488,50 @@ static int tegra_dsi_probe(struct platform_device *pdev)
         dsi->format = MIPI_DSI_FMT_RGB888;
         dsi->lanes = 4;
 
-        dsi->rst = devm_reset_control_get(&pdev->dev, "dsi");
-        if (IS_ERR(dsi->rst))
-                return PTR_ERR(dsi->rst);
+        if (!pdev->dev.pm_domain) {
+                dsi->rst = devm_reset_control_get(&pdev->dev, "dsi");
+                if (IS_ERR(dsi->rst))
+                        return PTR_ERR(dsi->rst);
+        }
 
         dsi->clk = devm_clk_get(&pdev->dev, NULL);
         if (IS_ERR(dsi->clk)) {
                 dev_err(&pdev->dev, "cannot get DSI clock\n");
-                err = PTR_ERR(dsi->clk);
-                goto reset;
-        }
-
-        err = clk_prepare_enable(dsi->clk);
-        if (err < 0) {
-                dev_err(&pdev->dev, "cannot enable DSI clock\n");
-                goto reset;
+                return PTR_ERR(dsi->clk);
         }
 
         dsi->clk_lp = devm_clk_get(&pdev->dev, "lp");
         if (IS_ERR(dsi->clk_lp)) {
                 dev_err(&pdev->dev, "cannot get low-power clock\n");
-                err = PTR_ERR(dsi->clk_lp);
-                goto disable_clk;
-        }
-
-        err = clk_prepare_enable(dsi->clk_lp);
-        if (err < 0) {
-                dev_err(&pdev->dev, "cannot enable low-power clock\n");
-                goto disable_clk;
+                return PTR_ERR(dsi->clk_lp);
         }
 
         dsi->clk_parent = devm_clk_get(&pdev->dev, "parent");
         if (IS_ERR(dsi->clk_parent)) {
                 dev_err(&pdev->dev, "cannot get parent clock\n");
-                err = PTR_ERR(dsi->clk_parent);
-                goto disable_clk_lp;
+                return PTR_ERR(dsi->clk_parent);
         }
 
         dsi->vdd = devm_regulator_get(&pdev->dev, "avdd-dsi-csi");
         if (IS_ERR(dsi->vdd)) {
                 dev_err(&pdev->dev, "cannot get VDD supply\n");
-                err = PTR_ERR(dsi->vdd);
-                goto disable_clk_lp;
-        }
-
-        err = regulator_enable(dsi->vdd);
-        if (err < 0) {
-                dev_err(&pdev->dev, "cannot enable VDD supply\n");
-                goto disable_clk_lp;
+                return PTR_ERR(dsi->vdd);
         }
 
         err = tegra_dsi_setup_clocks(dsi);
         if (err < 0) {
                 dev_err(&pdev->dev, "cannot setup clocks\n");
-                goto disable_vdd;
+                return err;
         }
 
         regs = platform_get_resource(pdev, IORESOURCE_MEM, 0);
         dsi->regs = devm_ioremap_resource(&pdev->dev, regs);
-        if (IS_ERR(dsi->regs)) {
-                err = PTR_ERR(dsi->regs);
-                goto disable_vdd;
-        }
+        if (IS_ERR(dsi->regs))
+                return PTR_ERR(dsi->regs);
 
         dsi->mipi = tegra_mipi_request(&pdev->dev);
-        if (IS_ERR(dsi->mipi)) {
-                err = PTR_ERR(dsi->mipi);
-                goto disable_vdd;
-        }
+        if (IS_ERR(dsi->mipi))
+                return PTR_ERR(dsi->mipi);
 
         dsi->host.ops = &tegra_dsi_host_ops;
         dsi->host.dev = &pdev->dev;
@@ -1572,6 +1542,9 @@ static int tegra_dsi_probe(struct platform_device *pdev)
                 goto mipi_free;
         }
 
+        platform_set_drvdata(pdev, dsi);
+        pm_runtime_enable(&pdev->dev);
+
         INIT_LIST_HEAD(&dsi->client.list);
         dsi->client.ops = &dsi_client_ops;
         dsi->client.dev = &pdev->dev;
@@ -1583,22 +1556,12 @@ static int tegra_dsi_probe(struct platform_device *pdev)
                 goto unregister;
         }
 
-        platform_set_drvdata(pdev, dsi);
-
         return 0;
 
 unregister:
         mipi_dsi_host_unregister(&dsi->host);
 mipi_free:
         tegra_mipi_free(dsi->mipi);
-disable_vdd:
-        regulator_disable(dsi->vdd);
-disable_clk_lp:
-        clk_disable_unprepare(dsi->clk_lp);
-disable_clk:
-        clk_disable_unprepare(dsi->clk);
-reset:
-        reset_control_assert(dsi->rst);
         return err;
 }
 
@@ -1607,6 +1570,8 @@ static int tegra_dsi_remove(struct platform_device *pdev)
         struct tegra_dsi *dsi = platform_get_drvdata(pdev);
         int err;
 
+        pm_runtime_disable(&pdev->dev);
+
         err = host1x_client_unregister(&dsi->client);
         if (err < 0) {
                 dev_err(&pdev->dev, "failed to unregister host1x client: %d\n",
@@ -1619,14 +1584,82 @@ static int tegra_dsi_remove(struct platform_device *pdev)
         mipi_dsi_host_unregister(&dsi->host);
         tegra_mipi_free(dsi->mipi);
 
-        regulator_disable(dsi->vdd);
+        return 0;
+}
+
+#ifdef CONFIG_PM
+static int tegra_dsi_suspend(struct device *dev)
+{
+        struct tegra_dsi *dsi = dev_get_drvdata(dev);
+        int err;
+
+        if (dsi->rst) {
+                err = reset_control_assert(dsi->rst);
+                if (err < 0) {
+                        dev_err(dev, "failed to assert reset: %d\n", err);
+                        return err;
+                }
+        }
+
+        usleep_range(1000, 2000);
+
         clk_disable_unprepare(dsi->clk_lp);
         clk_disable_unprepare(dsi->clk);
-        reset_control_assert(dsi->rst);
+
+        regulator_disable(dsi->vdd);
 
         return 0;
 }
 
+static int tegra_dsi_resume(struct device *dev)
+{
+        struct tegra_dsi *dsi = dev_get_drvdata(dev);
+        int err;
+
+        err = regulator_enable(dsi->vdd);
+        if (err < 0) {
+                dev_err(dsi->dev, "failed to enable VDD supply: %d\n", err);
+                return err;
+        }
+
+        err = clk_prepare_enable(dsi->clk);
+        if (err < 0) {
+                dev_err(dev, "cannot enable DSI clock: %d\n", err);
+                goto disable_vdd;
+        }
+
+        err = clk_prepare_enable(dsi->clk_lp);
+        if (err < 0) {
+                dev_err(dev, "cannot enable low-power clock: %d\n", err);
+                goto disable_clk;
+        }
+
+        usleep_range(1000, 2000);
+
+        if (dsi->rst) {
+                err = reset_control_deassert(dsi->rst);
+                if (err < 0) {
+                        dev_err(dev, "cannot assert reset: %d\n", err);
+                        goto disable_clk_lp;
+                }
+        }
+
+        return 0;
+
+disable_clk_lp:
+        clk_disable_unprepare(dsi->clk_lp);
+disable_clk:
+        clk_disable_unprepare(dsi->clk);
+disable_vdd:
+        regulator_disable(dsi->vdd);
+        return err;
+}
+#endif
+
+static const struct dev_pm_ops tegra_dsi_pm_ops = {
+        SET_RUNTIME_PM_OPS(tegra_dsi_suspend, tegra_dsi_resume, NULL)
+};
+
 static const struct of_device_id tegra_dsi_of_match[] = {
         { .compatible = "nvidia,tegra210-dsi", },
         { .compatible = "nvidia,tegra132-dsi", },
@@ -1640,6 +1673,7 @@ struct platform_driver tegra_dsi_driver = {
         .driver = {
                 .name = "tegra-dsi",
                 .of_match_table = tegra_dsi_of_match,
+                .pm = &tegra_dsi_pm_ops,
         },
         .probe = tegra_dsi_probe,
         .remove = tegra_dsi_remove,
diff --git a/drivers/gpu/drm/tegra/fb.c b/drivers/gpu/drm/tegra/fb.c
index 1b12aa7a715e..e6d71fa4028e 100644
--- a/drivers/gpu/drm/tegra/fb.c
+++ b/drivers/gpu/drm/tegra/fb.c
@@ -68,7 +68,7 @@ static void tegra_fb_destroy(struct drm_framebuffer *framebuffer)
                 struct tegra_bo *bo = fb->planes[i];
 
                 if (bo) {
-                        if (bo->pages && bo->vaddr)
+                        if (bo->pages)
                                 vunmap(bo->vaddr);
 
                         drm_gem_object_unreference_unlocked(&bo->gem);
diff --git a/drivers/gpu/drm/tegra/hdmi.c b/drivers/gpu/drm/tegra/hdmi.c
index b7ef4929e347..cda0491ed6bf 100644
--- a/drivers/gpu/drm/tegra/hdmi.c
+++ b/drivers/gpu/drm/tegra/hdmi.c
@@ -11,6 +11,7 @@
 #include <linux/debugfs.h>
 #include <linux/gpio.h>
 #include <linux/hdmi.h>
+#include <linux/pm_runtime.h>
 #include <linux/regulator/consumer.h>
 #include <linux/reset.h>
 
@@ -18,10 +19,14 @@
 #include <drm/drm_crtc.h>
 #include <drm/drm_crtc_helper.h>
 
+#include <sound/hda_verbs.h>
+
 #include "hdmi.h"
 #include "drm.h"
 #include "dc.h"
 
+#define HDMI_ELD_BUFFER_SIZE 96
+
 struct tmds_config {
         unsigned int pclk;
         u32 pll0;
@@ -39,6 +44,8 @@ struct tegra_hdmi_config {
         u32 fuse_override_value;
 
         bool has_sor_io_peak_current;
+        bool has_hda;
+        bool has_hbr;
 };
 
 struct tegra_hdmi {
@@ -60,7 +67,10 @@ struct tegra_hdmi {
         const struct tegra_hdmi_config *config;
 
         unsigned int audio_source;
-        unsigned int audio_freq;
+        unsigned int audio_sample_rate;
+        unsigned int audio_channels;
+
+        unsigned int pixel_clock;
         bool stereo;
         bool dvi;
 
@@ -402,11 +412,11 @@ static const struct tmds_config tegra124_tmds_config[] = {
 };
 
 static const struct tegra_hdmi_audio_config *
-tegra_hdmi_get_audio_config(unsigned int audio_freq, unsigned int pclk)
+tegra_hdmi_get_audio_config(unsigned int sample_rate, unsigned int pclk)
 {
         const struct tegra_hdmi_audio_config *table;
 
-        switch (audio_freq) {
+        switch (sample_rate) {
         case 32000:
                 table = tegra_hdmi_audio_32k;
                 break;
@@ -476,44 +486,114 @@ static void tegra_hdmi_setup_audio_fs_tables(struct tegra_hdmi *hdmi)
         }
 }
 
-static int tegra_hdmi_setup_audio(struct tegra_hdmi *hdmi, unsigned int pclk)
+static void tegra_hdmi_write_aval(struct tegra_hdmi *hdmi, u32 value)
+{
+        static const struct {
+                unsigned int sample_rate;
+                unsigned int offset;
+        } regs[] = {
+                {  32000, HDMI_NV_PDISP_SOR_AUDIO_AVAL_0320 },
+                {  44100, HDMI_NV_PDISP_SOR_AUDIO_AVAL_0441 },
+                {  48000, HDMI_NV_PDISP_SOR_AUDIO_AVAL_0480 },
+                {  88200, HDMI_NV_PDISP_SOR_AUDIO_AVAL_0882 },
+                {  96000, HDMI_NV_PDISP_SOR_AUDIO_AVAL_0960 },
+                { 176400, HDMI_NV_PDISP_SOR_AUDIO_AVAL_1764 },
+                { 192000, HDMI_NV_PDISP_SOR_AUDIO_AVAL_1920 },
+        };
+        unsigned int i;
+
+        for (i = 0; i < ARRAY_SIZE(regs); i++) {
+                if (regs[i].sample_rate == hdmi->audio_sample_rate) {
+                        tegra_hdmi_writel(hdmi, value, regs[i].offset);
+                        break;
+                }
+        }
+}
+
+static int tegra_hdmi_setup_audio(struct tegra_hdmi *hdmi)
 {
-        struct device_node *node = hdmi->dev->of_node;
         const struct tegra_hdmi_audio_config *config;
-        unsigned int offset = 0;
-        u32 value;
+        u32 source, value;
 
         switch (hdmi->audio_source) {
         case HDA:
-                value = AUDIO_CNTRL0_SOURCE_SELECT_HDAL;
+                if (hdmi->config->has_hda)
+                        source = SOR_AUDIO_CNTRL0_SOURCE_SELECT_HDAL;
+                else
+                        return -EINVAL;
+
                 break;
 
         case SPDIF:
-                value = AUDIO_CNTRL0_SOURCE_SELECT_SPDIF;
+                if (hdmi->config->has_hda)
+                        source = SOR_AUDIO_CNTRL0_SOURCE_SELECT_SPDIF;
+                else
+                        source = AUDIO_CNTRL0_SOURCE_SELECT_SPDIF;
                 break;
 
         default:
-                value = AUDIO_CNTRL0_SOURCE_SELECT_AUTO;
+                if (hdmi->config->has_hda)
+                        source = SOR_AUDIO_CNTRL0_SOURCE_SELECT_AUTO;
+                else
+                        source = AUDIO_CNTRL0_SOURCE_SELECT_AUTO;
                 break;
         }
 
-        if (of_device_is_compatible(node, "nvidia,tegra30-hdmi")) {
-                value |= AUDIO_CNTRL0_ERROR_TOLERANCE(6) |
-                         AUDIO_CNTRL0_FRAMES_PER_BLOCK(0xc0);
-                tegra_hdmi_writel(hdmi, value, HDMI_NV_PDISP_AUDIO_CNTRL0);
-        } else {
-                value |= AUDIO_CNTRL0_INJECT_NULLSMPL;
+        /*
+         * Tegra30 and later use a slightly modified version of the register
+         * layout to accomodate for changes related to supporting HDA as the
+         * audio input source for HDMI. The source select field has moved to
+         * the SOR_AUDIO_CNTRL0 register, but the error tolerance and frames
+         * per block fields remain in the AUDIO_CNTRL0 register.
+         */
+        if (hdmi->config->has_hda) {
+                /*
+                 * Inject null samples into the audio FIFO for every frame in
+                 * which the codec did not receive any samples. This applies
+                 * to stereo LPCM only.
+                 *
+                 * XXX: This seems to be a remnant of MCP days when this was
+                 * used to work around issues with monitors not being able to
+                 * play back system startup sounds early. It is possibly not
+                 * needed on Linux at all.
+                 */
+                if (hdmi->audio_channels == 2)
+                        value = SOR_AUDIO_CNTRL0_INJECT_NULLSMPL;
+                else
+                        value = 0;
+
+                value |= source;
+
                 tegra_hdmi_writel(hdmi, value, HDMI_NV_PDISP_SOR_AUDIO_CNTRL0);
+        }
 
-                value = AUDIO_CNTRL0_ERROR_TOLERANCE(6) |
-                        AUDIO_CNTRL0_FRAMES_PER_BLOCK(0xc0);
-                tegra_hdmi_writel(hdmi, value, HDMI_NV_PDISP_AUDIO_CNTRL0);
+        /*
+         * On Tegra20, HDA is not a supported audio source and the source
+         * select field is part of the AUDIO_CNTRL0 register.
+         */
+        value = AUDIO_CNTRL0_FRAMES_PER_BLOCK(0xc0) |
+                AUDIO_CNTRL0_ERROR_TOLERANCE(6);
+
+        if (!hdmi->config->has_hda)
+                value |= source;
+
+        tegra_hdmi_writel(hdmi, value, HDMI_NV_PDISP_AUDIO_CNTRL0);
+
+        /*
+         * Advertise support for High Bit-Rate on Tegra114 and later.
+         */
+        if (hdmi->config->has_hbr) {
+                value = tegra_hdmi_readl(hdmi, HDMI_NV_PDISP_SOR_AUDIO_SPARE0);
+                value |= SOR_AUDIO_SPARE0_HBR_ENABLE;
+                tegra_hdmi_writel(hdmi, value, HDMI_NV_PDISP_SOR_AUDIO_SPARE0);
         }
 
-        config = tegra_hdmi_get_audio_config(hdmi->audio_freq, pclk);
+        config = tegra_hdmi_get_audio_config(hdmi->audio_sample_rate,
+                                             hdmi->pixel_clock);
         if (!config) {
-                dev_err(hdmi->dev, "cannot set audio to %u at %u pclk\n",
-                        hdmi->audio_freq, pclk);
+                dev_err(hdmi->dev,
+                        "cannot set audio to %u Hz at %u Hz pixel clock\n",
+                        hdmi->audio_sample_rate, hdmi->pixel_clock);
                 return -EINVAL;
         }
 
@@ -526,8 +606,8 @@ static int tegra_hdmi_setup_audio(struct tegra_hdmi *hdmi, unsigned int pclk)
         tegra_hdmi_writel(hdmi, ACR_SUBPACK_N(config->n) | ACR_ENABLE,
                           HDMI_NV_PDISP_HDMI_ACR_0441_SUBPACK_HIGH);
 
-        value = ACR_SUBPACK_CTS(config->cts);
-        tegra_hdmi_writel(hdmi, value, HDMI_NV_PDISP_HDMI_ACR_0441_SUBPACK_LOW);
+        tegra_hdmi_writel(hdmi, ACR_SUBPACK_CTS(config->cts),
+                          HDMI_NV_PDISP_HDMI_ACR_0441_SUBPACK_LOW);
 
         value = SPARE_HW_CTS | SPARE_FORCE_SW_CTS | SPARE_CTS_RESET_VAL(1);
         tegra_hdmi_writel(hdmi, value, HDMI_NV_PDISP_HDMI_SPARE);
@@ -536,43 +616,53 @@ static int tegra_hdmi_setup_audio(struct tegra_hdmi *hdmi, unsigned int pclk)
         value &= ~AUDIO_N_RESETF;
         tegra_hdmi_writel(hdmi, value, HDMI_NV_PDISP_AUDIO_N);
 
-        if (of_device_is_compatible(node, "nvidia,tegra30-hdmi")) {
-                switch (hdmi->audio_freq) {
-                case 32000:
-                        offset = HDMI_NV_PDISP_SOR_AUDIO_AVAL_0320;
-                        break;
+        if (hdmi->config->has_hda)
+                tegra_hdmi_write_aval(hdmi, config->aval);
 
-                case 44100:
-                        offset = HDMI_NV_PDISP_SOR_AUDIO_AVAL_0441;
-                        break;
+        tegra_hdmi_setup_audio_fs_tables(hdmi);
 
-                case 48000:
-                        offset = HDMI_NV_PDISP_SOR_AUDIO_AVAL_0480;
-                        break;
+        return 0;
+}
 
-                case 88200:
-                        offset = HDMI_NV_PDISP_SOR_AUDIO_AVAL_0882;
-                        break;
+static void tegra_hdmi_disable_audio(struct tegra_hdmi *hdmi)
+{
+        u32 value;
 
-                case 96000:
-                        offset = HDMI_NV_PDISP_SOR_AUDIO_AVAL_0960;
-                        break;
+        value = tegra_hdmi_readl(hdmi, HDMI_NV_PDISP_HDMI_GENERIC_CTRL);
+        value &= ~GENERIC_CTRL_AUDIO;
+        tegra_hdmi_writel(hdmi, value, HDMI_NV_PDISP_HDMI_GENERIC_CTRL);
+}
 
-                case 176400:
-                        offset = HDMI_NV_PDISP_SOR_AUDIO_AVAL_1764;
-                        break;
+static void tegra_hdmi_enable_audio(struct tegra_hdmi *hdmi)
+{
+        u32 value;
 
-                case 192000:
-                        offset = HDMI_NV_PDISP_SOR_AUDIO_AVAL_1920;
-                        break;
-                }
+        value = tegra_hdmi_readl(hdmi, HDMI_NV_PDISP_HDMI_GENERIC_CTRL);
+        value |= GENERIC_CTRL_AUDIO;
+        tegra_hdmi_writel(hdmi, value, HDMI_NV_PDISP_HDMI_GENERIC_CTRL);
+}
 
-                tegra_hdmi_writel(hdmi, config->aval, offset);
-        }
+static void tegra_hdmi_write_eld(struct tegra_hdmi *hdmi)
+{
+        size_t length = drm_eld_size(hdmi->output.connector.eld), i;
+        u32 value;
 
-        tegra_hdmi_setup_audio_fs_tables(hdmi);
+        for (i = 0; i < length; i++)
+                tegra_hdmi_writel(hdmi, i << 8 | hdmi->output.connector.eld[i],
+                                  HDMI_NV_PDISP_SOR_AUDIO_HDA_ELD_BUFWR);
 
-        return 0;
+        /*
+         * The HDA codec will always report an ELD buffer size of 96 bytes and
+         * the HDA codec driver will check that each byte read from the buffer
+         * is valid. Therefore every byte must be written, even if no 96 bytes
+         * were parsed from EDID.
+         */
+        for (i = length; i < HDMI_ELD_BUFFER_SIZE; i++)
+                tegra_hdmi_writel(hdmi, i << 8 | 0,
+                                  HDMI_NV_PDISP_SOR_AUDIO_HDA_ELD_BUFWR);
+
+        value = SOR_AUDIO_HDA_PRESENSE_VALID | SOR_AUDIO_HDA_PRESENSE_PRESENT;
+        tegra_hdmi_writel(hdmi, value, HDMI_NV_PDISP_SOR_AUDIO_HDA_PRESENSE);
 }
 
 static inline u32 tegra_hdmi_subpack(const u8 *ptr, size_t size)
@@ -644,12 +734,6 @@ static void tegra_hdmi_setup_avi_infoframe(struct tegra_hdmi *hdmi,
         u8 buffer[17];
         ssize_t err;
 
-        if (hdmi->dvi) {
-                tegra_hdmi_writel(hdmi, 0,
-                                  HDMI_NV_PDISP_HDMI_AVI_INFOFRAME_CTRL);
-                return;
-        }
-
         err = drm_hdmi_avi_infoframe_from_display_mode(&frame, mode);
         if (err < 0) {
                 dev_err(hdmi->dev, "failed to setup AVI infoframe: %zd\n", err);
@@ -663,9 +747,24 @@ static void tegra_hdmi_setup_avi_infoframe(struct tegra_hdmi *hdmi,
         }
 
         tegra_hdmi_write_infopack(hdmi, buffer, err);
+}
+
+static void tegra_hdmi_disable_avi_infoframe(struct tegra_hdmi *hdmi)
+{
+        u32 value;
 
-        tegra_hdmi_writel(hdmi, INFOFRAME_CTRL_ENABLE,
-                          HDMI_NV_PDISP_HDMI_AVI_INFOFRAME_CTRL);
+        value = tegra_hdmi_readl(hdmi, HDMI_NV_PDISP_HDMI_AVI_INFOFRAME_CTRL);
+        value &= ~INFOFRAME_CTRL_ENABLE;
+        tegra_hdmi_writel(hdmi, value, HDMI_NV_PDISP_HDMI_AVI_INFOFRAME_CTRL);
+}
+
+static void tegra_hdmi_enable_avi_infoframe(struct tegra_hdmi *hdmi)
+{
+        u32 value;
+
+        value = tegra_hdmi_readl(hdmi, HDMI_NV_PDISP_HDMI_AVI_INFOFRAME_CTRL);
+        value |= INFOFRAME_CTRL_ENABLE;
+        tegra_hdmi_writel(hdmi, value, HDMI_NV_PDISP_HDMI_AVI_INFOFRAME_CTRL);
 }
 
 static void tegra_hdmi_setup_audio_infoframe(struct tegra_hdmi *hdmi)
@@ -674,12 +773,6 @@ static void tegra_hdmi_setup_audio_infoframe(struct tegra_hdmi *hdmi)
         u8 buffer[14];
         ssize_t err;
 
-        if (hdmi->dvi) {
-                tegra_hdmi_writel(hdmi, 0,
-                                  HDMI_NV_PDISP_HDMI_AUDIO_INFOFRAME_CTRL);
-                return;
-        }
-
         err = hdmi_audio_infoframe_init(&frame);
         if (err < 0) {
                 dev_err(hdmi->dev, "failed to setup audio infoframe: %zd\n",
@@ -687,7 +780,7 @@ static void tegra_hdmi_setup_audio_infoframe(struct tegra_hdmi *hdmi)
                 return;
         }
 
-        frame.channels = 2;
+        frame.channels = hdmi->audio_channels;
 
         err = hdmi_audio_infoframe_pack(&frame, buffer, sizeof(buffer));
         if (err < 0) {
@@ -703,9 +796,24 @@ static void tegra_hdmi_setup_audio_infoframe(struct tegra_hdmi *hdmi)
          * bytes can be programmed.
          */
         tegra_hdmi_write_infopack(hdmi, buffer, min_t(size_t, 10, err));
+}
 
-        tegra_hdmi_writel(hdmi, INFOFRAME_CTRL_ENABLE,
-                          HDMI_NV_PDISP_HDMI_AUDIO_INFOFRAME_CTRL);
+static void tegra_hdmi_disable_audio_infoframe(struct tegra_hdmi *hdmi)
+{
+        u32 value;
+
+        value = tegra_hdmi_readl(hdmi, HDMI_NV_PDISP_HDMI_AUDIO_INFOFRAME_CTRL);
+        value &= ~INFOFRAME_CTRL_ENABLE;
+        tegra_hdmi_writel(hdmi, value, HDMI_NV_PDISP_HDMI_AUDIO_INFOFRAME_CTRL);
+}
+
+static void tegra_hdmi_enable_audio_infoframe(struct tegra_hdmi *hdmi)
+{
+        u32 value;
+
+        value = tegra_hdmi_readl(hdmi, HDMI_NV_PDISP_HDMI_AUDIO_INFOFRAME_CTRL);
+        value |= INFOFRAME_CTRL_ENABLE;
+        tegra_hdmi_writel(hdmi, value, HDMI_NV_PDISP_HDMI_AUDIO_INFOFRAME_CTRL);
 }
 
 static void tegra_hdmi_setup_stereo_infoframe(struct tegra_hdmi *hdmi)
@@ -713,14 +821,6 @@ static void tegra_hdmi_setup_stereo_infoframe(struct tegra_hdmi *hdmi)
         struct hdmi_vendor_infoframe frame;
         u8 buffer[10];
         ssize_t err;
-        u32 value;
-
-        if (!hdmi->stereo) {
-                value = tegra_hdmi_readl(hdmi, HDMI_NV_PDISP_HDMI_GENERIC_CTRL);
-                value &= ~GENERIC_CTRL_ENABLE;
-                tegra_hdmi_writel(hdmi, value, HDMI_NV_PDISP_HDMI_GENERIC_CTRL);
-                return;
-        }
 
         hdmi_vendor_infoframe_init(&frame);
         frame.s3d_struct = HDMI_3D_STRUCTURE_FRAME_PACKING;
@@ -733,6 +833,20 @@ static void tegra_hdmi_setup_stereo_infoframe(struct tegra_hdmi *hdmi)
         }
 
         tegra_hdmi_write_infopack(hdmi, buffer, err);
+}
+
+static void tegra_hdmi_disable_stereo_infoframe(struct tegra_hdmi *hdmi)
+{
+        u32 value;
+
+        value = tegra_hdmi_readl(hdmi, HDMI_NV_PDISP_HDMI_GENERIC_CTRL);
+        value &= ~GENERIC_CTRL_ENABLE;
+        tegra_hdmi_writel(hdmi, value, HDMI_NV_PDISP_HDMI_GENERIC_CTRL);
+}
+
+static void tegra_hdmi_enable_stereo_infoframe(struct tegra_hdmi *hdmi)
+{
+        u32 value;
 
         value = tegra_hdmi_readl(hdmi, HDMI_NV_PDISP_HDMI_GENERIC_CTRL);
         value |= GENERIC_CTRL_ENABLE;
@@ -772,10 +886,25 @@ static bool tegra_output_is_hdmi(struct tegra_output *output)
         return drm_detect_hdmi_monitor(edid);
 }
 
+static enum drm_connector_status
+tegra_hdmi_connector_detect(struct drm_connector *connector, bool force)
+{
+        struct tegra_output *output = connector_to_output(connector);
+        struct tegra_hdmi *hdmi = to_hdmi(output);
+        enum drm_connector_status status;
+
+        status = tegra_output_connector_detect(connector, force);
+        if (status == connector_status_connected)
+                return status;
+
+        tegra_hdmi_writel(hdmi, 0, HDMI_NV_PDISP_SOR_AUDIO_HDA_PRESENSE);
+        return status;
+}
+
 static const struct drm_connector_funcs tegra_hdmi_connector_funcs = {
         .dpms = drm_atomic_helper_connector_dpms,
         .reset = drm_atomic_helper_connector_reset,
-        .detect = tegra_output_connector_detect,
+        .detect = tegra_hdmi_connector_detect,
         .fill_modes = drm_helper_probe_single_connector_modes,
         .destroy = tegra_output_connector_destroy,
         .atomic_duplicate_state = drm_atomic_helper_connector_duplicate_state,
@@ -806,7 +935,6 @@ static const struct drm_connector_helper_funcs
 tegra_hdmi_connector_helper_funcs = {
         .get_modes = tegra_output_connector_get_modes,
         .mode_valid = tegra_hdmi_connector_mode_valid,
-        .best_encoder = tegra_output_connector_best_encoder,
 };
 
 static const struct drm_encoder_funcs tegra_hdmi_encoder_funcs = {
@@ -815,7 +943,9 @@ static const struct drm_encoder_funcs tegra_hdmi_encoder_funcs = {
 
 static void tegra_hdmi_encoder_disable(struct drm_encoder *encoder)
 {
+        struct tegra_output *output = encoder_to_output(encoder);
         struct tegra_dc *dc = to_tegra_dc(encoder->crtc);
+        struct tegra_hdmi *hdmi = to_hdmi(output);
         u32 value;
 
         /*
@@ -829,6 +959,20 @@ static void tegra_hdmi_encoder_disable(struct drm_encoder *encoder)
 
                 tegra_dc_commit(dc);
         }
+
+        if (!hdmi->dvi) {
+                if (hdmi->stereo)
+                        tegra_hdmi_disable_stereo_infoframe(hdmi);
+
+                tegra_hdmi_disable_audio_infoframe(hdmi);
+                tegra_hdmi_disable_avi_infoframe(hdmi);
+                tegra_hdmi_disable_audio(hdmi);
+        }
+
+        tegra_hdmi_writel(hdmi, 0, HDMI_NV_PDISP_INT_ENABLE);
+        tegra_hdmi_writel(hdmi, 0, HDMI_NV_PDISP_INT_MASK);
+
+        pm_runtime_put(hdmi->dev);
 }
 
 static void tegra_hdmi_encoder_enable(struct drm_encoder *encoder)
@@ -837,21 +981,28 @@ static void tegra_hdmi_encoder_enable(struct drm_encoder *encoder)
         unsigned int h_sync_width, h_front_porch, h_back_porch, i, rekey;
         struct tegra_output *output = encoder_to_output(encoder);
         struct tegra_dc *dc = to_tegra_dc(encoder->crtc);
-        struct device_node *node = output->dev->of_node;
         struct tegra_hdmi *hdmi = to_hdmi(output);
-        unsigned int pulse_start, div82, pclk;
+        unsigned int pulse_start, div82;
         int retries = 1000;
         u32 value;
         int err;
 
-        hdmi->dvi = !tegra_output_is_hdmi(output);
+        pm_runtime_get_sync(hdmi->dev);
 
-        pclk = mode->clock * 1000;
+        /*
+         * Enable and unmask the HDA codec SCRATCH0 register interrupt. This
+         * is used for interoperability between the HDA codec driver and the
+         * HDMI driver.
+         */
+        tegra_hdmi_writel(hdmi, INT_CODEC_SCRATCH0, HDMI_NV_PDISP_INT_ENABLE);
+        tegra_hdmi_writel(hdmi, INT_CODEC_SCRATCH0, HDMI_NV_PDISP_INT_MASK);
+
+        hdmi->pixel_clock = mode->clock * 1000;
         h_sync_width = mode->hsync_end - mode->hsync_start;
         h_back_porch = mode->htotal - mode->hsync_end;
         h_front_porch = mode->hsync_start - mode->hdisplay;
 
-        err = clk_set_rate(hdmi->clk, pclk);
+        err = clk_set_rate(hdmi->clk, hdmi->pixel_clock);
         if (err < 0) {
                 dev_err(hdmi->dev, "failed to set HDMI clock frequency: %d\n",
                         err);
@@ -910,17 +1061,15 @@ static void tegra_hdmi_encoder_enable(struct drm_encoder *encoder)
         value = SOR_REFCLK_DIV_INT(div82 >> 2) | SOR_REFCLK_DIV_FRAC(div82);
         tegra_hdmi_writel(hdmi, value, HDMI_NV_PDISP_SOR_REFCLK);
 
+        hdmi->dvi = !tegra_output_is_hdmi(output);
         if (!hdmi->dvi) {
-                err = tegra_hdmi_setup_audio(hdmi, pclk);
+                err = tegra_hdmi_setup_audio(hdmi);
                 if (err < 0)
                         hdmi->dvi = true;
         }
 
-        if (of_device_is_compatible(node, "nvidia,tegra20-hdmi")) {
-                /*
-                 * TODO: add ELD support
-                 */
-        }
+        if (hdmi->config->has_hda)
+                tegra_hdmi_write_eld(hdmi);
 
         rekey = HDMI_REKEY_DEFAULT;
         value = HDMI_CTRL_REKEY(rekey);
@@ -932,20 +1081,17 @@ static void tegra_hdmi_encoder_enable(struct drm_encoder *encoder)
 
         tegra_hdmi_writel(hdmi, value, HDMI_NV_PDISP_HDMI_CTRL);
 
-        if (hdmi->dvi)
-                tegra_hdmi_writel(hdmi, 0x0,
-                                  HDMI_NV_PDISP_HDMI_GENERIC_CTRL);
-        else
-                tegra_hdmi_writel(hdmi, GENERIC_CTRL_AUDIO,
-                                  HDMI_NV_PDISP_HDMI_GENERIC_CTRL);
+        if (!hdmi->dvi) {
+                tegra_hdmi_setup_avi_infoframe(hdmi, mode);
+                tegra_hdmi_setup_audio_infoframe(hdmi);
 
-        tegra_hdmi_setup_avi_infoframe(hdmi, mode);
-        tegra_hdmi_setup_audio_infoframe(hdmi);
-        tegra_hdmi_setup_stereo_infoframe(hdmi);
+                if (hdmi->stereo)
+                        tegra_hdmi_setup_stereo_infoframe(hdmi);
+        }
 
         /* TMDS CONFIG */
         for (i = 0; i < hdmi->config->num_tmds; i++) {
-                if (pclk <= hdmi->config->tmds[i].pclk) {
+                if (hdmi->pixel_clock <= hdmi->config->tmds[i].pclk) {
                         tegra_hdmi_setup_tmds(hdmi, &hdmi->config->tmds[i]);
                         break;
                 }
@@ -1032,6 +1178,15 @@ static void tegra_hdmi_encoder_enable(struct drm_encoder *encoder)
 
         tegra_dc_commit(dc);
 
+        if (!hdmi->dvi) {
+                tegra_hdmi_enable_avi_infoframe(hdmi);
+                tegra_hdmi_enable_audio_infoframe(hdmi);
+                tegra_hdmi_enable_audio(hdmi);
+
+                if (hdmi->stereo)
+                        tegra_hdmi_enable_stereo_infoframe(hdmi);
+        }
+
         /* TODO: add HDCP support */
 }
 
@@ -1236,8 +1391,14 @@ static int tegra_hdmi_show_regs(struct seq_file *s, void *data)
         DUMP_REG(HDMI_NV_PDISP_KEY_HDCP_KEY_TRIG);
         DUMP_REG(HDMI_NV_PDISP_KEY_SKEY_INDEX);
         DUMP_REG(HDMI_NV_PDISP_SOR_AUDIO_CNTRL0);
+        DUMP_REG(HDMI_NV_PDISP_SOR_AUDIO_SPARE0);
+        DUMP_REG(HDMI_NV_PDISP_SOR_AUDIO_HDA_CODEC_SCRATCH0);
+        DUMP_REG(HDMI_NV_PDISP_SOR_AUDIO_HDA_CODEC_SCRATCH1);
         DUMP_REG(HDMI_NV_PDISP_SOR_AUDIO_HDA_ELD_BUFWR);
         DUMP_REG(HDMI_NV_PDISP_SOR_AUDIO_HDA_PRESENSE);
+        DUMP_REG(HDMI_NV_PDISP_INT_STATUS);
+        DUMP_REG(HDMI_NV_PDISP_INT_MASK);
+        DUMP_REG(HDMI_NV_PDISP_INT_ENABLE);
         DUMP_REG(HDMI_NV_PDISP_SOR_IO_PEAK_CURRENT);
 
 #undef DUMP_REG
@@ -1361,14 +1522,6 @@ static int tegra_hdmi_init(struct host1x_client *client)
                 return err;
         }
 
-        err = clk_prepare_enable(hdmi->clk);
-        if (err < 0) {
-                dev_err(hdmi->dev, "failed to enable clock: %d\n", err);
-                return err;
-        }
-
-        reset_control_deassert(hdmi->rst);
-
         return 0;
 }
 
@@ -1378,9 +1531,6 @@ static int tegra_hdmi_exit(struct host1x_client *client)
 
         tegra_output_exit(&hdmi->output);
 
-        reset_control_assert(hdmi->rst);
-        clk_disable_unprepare(hdmi->clk);
-
         regulator_disable(hdmi->vdd);
         regulator_disable(hdmi->pll);
         regulator_disable(hdmi->hdmi);
@@ -1402,6 +1552,8 @@ static const struct tegra_hdmi_config tegra20_hdmi_config = {
         .fuse_override_offset = HDMI_NV_PDISP_SOR_LANE_DRIVE_CURRENT,
         .fuse_override_value = 1 << 31,
         .has_sor_io_peak_current = false,
+        .has_hda = false,
+        .has_hbr = false,
 };
 
 static const struct tegra_hdmi_config tegra30_hdmi_config = {
@@ -1410,6 +1562,8 @@ static const struct tegra_hdmi_config tegra30_hdmi_config = {
         .fuse_override_offset = HDMI_NV_PDISP_SOR_LANE_DRIVE_CURRENT,
         .fuse_override_value = 1 << 31,
         .has_sor_io_peak_current = false,
+        .has_hda = true,
+        .has_hbr = false,
 };
 
 static const struct tegra_hdmi_config tegra114_hdmi_config = {
@@ -1418,6 +1572,8 @@ static const struct tegra_hdmi_config tegra114_hdmi_config = {
         .fuse_override_offset = HDMI_NV_PDISP_SOR_PAD_CTLS0,
         .fuse_override_value = 1 << 31,
         .has_sor_io_peak_current = true,
+        .has_hda = true,
+        .has_hbr = true,
 };
 
 static const struct tegra_hdmi_config tegra124_hdmi_config = {
@@ -1426,6 +1582,8 @@ static const struct tegra_hdmi_config tegra124_hdmi_config = {
         .fuse_override_offset = HDMI_NV_PDISP_SOR_PAD_CTLS0,
         .fuse_override_value = 1 << 31,
         .has_sor_io_peak_current = true,
+        .has_hda = true,
+        .has_hbr = true,
 };
 
 static const struct of_device_id tegra_hdmi_of_match[] = {
@@ -1437,6 +1595,67 @@ static const struct of_device_id tegra_hdmi_of_match[] = {
 };
 MODULE_DEVICE_TABLE(of, tegra_hdmi_of_match);
 
+static void hda_format_parse(unsigned int format, unsigned int *rate,
+                             unsigned int *channels)
+{
+        unsigned int mul, div;
+
+        if (format & AC_FMT_BASE_44K)
+                *rate = 44100;
+        else
+                *rate = 48000;
+
+        mul = (format & AC_FMT_MULT_MASK) >> AC_FMT_MULT_SHIFT;
+        div = (format & AC_FMT_DIV_MASK) >> AC_FMT_DIV_SHIFT;
+
+        *rate = *rate * (mul + 1) / (div + 1);
+
+        *channels = (format & AC_FMT_CHAN_MASK) >> AC_FMT_CHAN_SHIFT;
+}
+
+static irqreturn_t tegra_hdmi_irq(int irq, void *data)
+{
+        struct tegra_hdmi *hdmi = data;
+        u32 value;
+        int err;
+
+        value = tegra_hdmi_readl(hdmi, HDMI_NV_PDISP_INT_STATUS);
+        tegra_hdmi_writel(hdmi, value, HDMI_NV_PDISP_INT_STATUS);
+
+        if (value & INT_CODEC_SCRATCH0) {
+                unsigned int format;
+                u32 value;
+
+                value = tegra_hdmi_readl(hdmi, HDMI_NV_PDISP_SOR_AUDIO_HDA_CODEC_SCRATCH0);
+
+                if (value & SOR_AUDIO_HDA_CODEC_SCRATCH0_VALID) {
+                        unsigned int sample_rate, channels;
+
+                        format = value & SOR_AUDIO_HDA_CODEC_SCRATCH0_FMT_MASK;
+
+                        hda_format_parse(format, &sample_rate, &channels);
+
+                        hdmi->audio_sample_rate = sample_rate;
+                        hdmi->audio_channels = channels;
+
+                        err = tegra_hdmi_setup_audio(hdmi);
+                        if (err < 0) {
+                                tegra_hdmi_disable_audio_infoframe(hdmi);
+                                tegra_hdmi_disable_audio(hdmi);
+                        } else {
+                                tegra_hdmi_setup_audio_infoframe(hdmi);
+                                tegra_hdmi_enable_audio_infoframe(hdmi);
+                                tegra_hdmi_enable_audio(hdmi);
+                        }
+                } else {
+                        tegra_hdmi_disable_audio_infoframe(hdmi);
+                        tegra_hdmi_disable_audio(hdmi);
+                }
+        }
+
+        return IRQ_HANDLED;
+}
+
 static int tegra_hdmi_probe(struct platform_device *pdev)
 {
         const struct of_device_id *match;
@@ -1454,8 +1673,10 @@ static int tegra_hdmi_probe(struct platform_device *pdev)
 
         hdmi->config = match->data;
         hdmi->dev = &pdev->dev;
+
         hdmi->audio_source = AUTO;
-        hdmi->audio_freq = 44100;
+        hdmi->audio_sample_rate = 48000;
+        hdmi->audio_channels = 2;
         hdmi->stereo = false;
         hdmi->dvi = false;
 
@@ -1516,6 +1737,17 @@ static int tegra_hdmi_probe(struct platform_device *pdev)
 
         hdmi->irq = err;
 
+        err = devm_request_irq(hdmi->dev, hdmi->irq, tegra_hdmi_irq, 0,
+                               dev_name(hdmi->dev), hdmi);
+        if (err < 0) {
+                dev_err(&pdev->dev, "failed to request IRQ#%u: %d\n",
+                        hdmi->irq, err);
+                return err;
+        }
+
+        platform_set_drvdata(pdev, hdmi);
+        pm_runtime_enable(&pdev->dev);
+
         INIT_LIST_HEAD(&hdmi->client.list);
         hdmi->client.ops = &hdmi_client_ops;
         hdmi->client.dev = &pdev->dev;
@@ -1527,8 +1759,6 @@ static int tegra_hdmi_probe(struct platform_device *pdev)
                 return err;
         }
 
-        platform_set_drvdata(pdev, hdmi);
-
         return 0;
 }
 
@@ -1537,6 +1767,8 @@ static int tegra_hdmi_remove(struct platform_device *pdev)
         struct tegra_hdmi *hdmi = platform_get_drvdata(pdev);
         int err;
 
+        pm_runtime_disable(&pdev->dev);
+
         err = host1x_client_unregister(&hdmi->client);
         if (err < 0) {
                 dev_err(&pdev->dev, "failed to unregister host1x client: %d\n",
@@ -1546,17 +1778,61 @@ static int tegra_hdmi_remove(struct platform_device *pdev)
 
         tegra_output_remove(&hdmi->output);
 
-        clk_disable_unprepare(hdmi->clk_parent);
+        return 0;
+}
+
+#ifdef CONFIG_PM
+static int tegra_hdmi_suspend(struct device *dev)
+{
+        struct tegra_hdmi *hdmi = dev_get_drvdata(dev);
+        int err;
+
+        err = reset_control_assert(hdmi->rst);
+        if (err < 0) {
+                dev_err(dev, "failed to assert reset: %d\n", err);
+                return err;
+        }
+
+        usleep_range(1000, 2000);
+
         clk_disable_unprepare(hdmi->clk);
 
         return 0;
 }
 
+static int tegra_hdmi_resume(struct device *dev)
+{
+        struct tegra_hdmi *hdmi = dev_get_drvdata(dev);
+        int err;
+
+        err = clk_prepare_enable(hdmi->clk);
+        if (err < 0) {
+                dev_err(dev, "failed to enable clock: %d\n", err);
+                return err;
+        }
+
+        usleep_range(1000, 2000);
+
+        err = reset_control_deassert(hdmi->rst);
+        if (err < 0) {
+                dev_err(dev, "failed to deassert reset: %d\n", err);
+                clk_disable_unprepare(hdmi->clk);
+                return err;
+        }
+
+        return 0;
+}
+#endif
+
+static const struct dev_pm_ops tegra_hdmi_pm_ops = {
+        SET_RUNTIME_PM_OPS(tegra_hdmi_suspend, tegra_hdmi_resume, NULL)
+};
+
 struct platform_driver tegra_hdmi_driver = {
         .driver = {
                 .name = "tegra-hdmi",
-                .owner = THIS_MODULE,
                 .of_match_table = tegra_hdmi_of_match,
+                .pm = &tegra_hdmi_pm_ops,
         },
         .probe = tegra_hdmi_probe,
         .remove = tegra_hdmi_remove,
diff --git a/drivers/gpu/drm/tegra/hdmi.h b/drivers/gpu/drm/tegra/hdmi.h
index a882514389cd..2339f134a09a 100644
--- a/drivers/gpu/drm/tegra/hdmi.h
+++ b/drivers/gpu/drm/tegra/hdmi.h
@@ -468,9 +468,20 @@
 #define HDMI_NV_PDISP_KEY_SKEY_INDEX                            0xa3
 
 #define HDMI_NV_PDISP_SOR_AUDIO_CNTRL0                          0xac
-#define AUDIO_CNTRL0_INJECT_NULLSMPL (1 << 29)
+#define  SOR_AUDIO_CNTRL0_SOURCE_SELECT_AUTO    (0 << 20)
+#define  SOR_AUDIO_CNTRL0_SOURCE_SELECT_SPDIF   (1 << 20)
+#define  SOR_AUDIO_CNTRL0_SOURCE_SELECT_HDAL    (2 << 20)
+#define  SOR_AUDIO_CNTRL0_INJECT_NULLSMPL       (1 << 29)
+#define HDMI_NV_PDISP_SOR_AUDIO_SPARE0                          0xae
+#define  SOR_AUDIO_SPARE0_HBR_ENABLE            (1 << 27)
+#define HDMI_NV_PDISP_SOR_AUDIO_HDA_CODEC_SCRATCH0              0xba
+#define  SOR_AUDIO_HDA_CODEC_SCRATCH0_VALID     (1 << 30)
+#define  SOR_AUDIO_HDA_CODEC_SCRATCH0_FMT_MASK  0xffff
+#define HDMI_NV_PDISP_SOR_AUDIO_HDA_CODEC_SCRATCH1              0xbb
 #define HDMI_NV_PDISP_SOR_AUDIO_HDA_ELD_BUFWR                   0xbc
 #define HDMI_NV_PDISP_SOR_AUDIO_HDA_PRESENSE                    0xbd
+#define  SOR_AUDIO_HDA_PRESENSE_VALID           (1 << 1)
+#define  SOR_AUDIO_HDA_PRESENSE_PRESENT         (1 << 0)
 
 #define HDMI_NV_PDISP_SOR_AUDIO_AVAL_0320    0xbf
 #define HDMI_NV_PDISP_SOR_AUDIO_AVAL_0441    0xc0
@@ -481,6 +492,14 @@
 #define HDMI_NV_PDISP_SOR_AUDIO_AVAL_1920    0xc5
 #define HDMI_NV_PDISP_SOR_AUDIO_AVAL_DEFAULT 0xc5
 
+#define HDMI_NV_PDISP_INT_STATUS                        0xcc
+#define  INT_SCRATCH            (1 << 3)
+#define  INT_CP_REQUEST         (1 << 2)
+#define  INT_CODEC_SCRATCH1     (1 << 1)
+#define  INT_CODEC_SCRATCH0     (1 << 0)
+#define HDMI_NV_PDISP_INT_MASK                          0xcd
+#define HDMI_NV_PDISP_INT_ENABLE                        0xce
+
 #define HDMI_NV_PDISP_SOR_IO_PEAK_CURRENT               0xd1
 #define PEAK_CURRENT_LANE0(x) (((x) & 0x7f) <<  0)
 #define PEAK_CURRENT_LANE1(x) (((x) & 0x7f) <<  8)
diff --git a/drivers/gpu/drm/tegra/output.c b/drivers/gpu/drm/tegra/output.c
index 46664b622270..595d1ec3e02e 100644
--- a/drivers/gpu/drm/tegra/output.c
+++ b/drivers/gpu/drm/tegra/output.c
@@ -36,20 +36,13 @@ int tegra_output_connector_get_modes(struct drm_connector *connector)
 
         if (edid) {
                 err = drm_add_edid_modes(connector, edid);
+                drm_edid_to_eld(connector, edid);
                 kfree(edid);
         }
 
         return err;
 }
 
-struct drm_encoder *
-tegra_output_connector_best_encoder(struct drm_connector *connector)
-{
-        struct tegra_output *output = connector_to_output(connector);
-
-        return &output->encoder;
-}
-
 enum drm_connector_status
 tegra_output_connector_detect(struct drm_connector *connector, bool force)
 {
diff --git a/drivers/gpu/drm/tegra/rgb.c b/drivers/gpu/drm/tegra/rgb.c
index e246334e0252..a131b44e2d6f 100644
--- a/drivers/gpu/drm/tegra/rgb.c
+++ b/drivers/gpu/drm/tegra/rgb.c
@@ -112,7 +112,6 @@ tegra_rgb_connector_mode_valid(struct drm_connector *connector,
 static const struct drm_connector_helper_funcs tegra_rgb_connector_helper_funcs = {
         .get_modes = tegra_output_connector_get_modes,
         .mode_valid = tegra_rgb_connector_mode_valid,
-        .best_encoder = tegra_output_connector_best_encoder,
 };
 
 static const struct drm_encoder_funcs tegra_rgb_encoder_funcs = {
diff --git a/drivers/gpu/drm/tegra/sor.c b/drivers/gpu/drm/tegra/sor.c
index 757c6e8603af..74d0540b8d4c 100644
--- a/drivers/gpu/drm/tegra/sor.c
+++ b/drivers/gpu/drm/tegra/sor.c
@@ -7,11 +7,13 @@
  */
 
 #include <linux/clk.h>
+#include <linux/clk-provider.h>
 #include <linux/debugfs.h>
 #include <linux/gpio.h>
 #include <linux/io.h>
 #include <linux/of_device.h>
 #include <linux/platform_device.h>
+#include <linux/pm_runtime.h>
 #include <linux/regulator/consumer.h>
 #include <linux/reset.h>
 
@@ -149,6 +151,8 @@ struct tegra_sor_soc {
 
         const struct tegra_sor_hdmi_settings *settings;
         unsigned int num_settings;
+
+        const u8 *xbar_cfg;
 };
 
 struct tegra_sor;
@@ -169,7 +173,9 @@ struct tegra_sor {
 
         struct reset_control *rst;
         struct clk *clk_parent;
+        struct clk *clk_brick;
         struct clk *clk_safe;
+        struct clk *clk_src;
         struct clk *clk_dp;
         struct clk *clk;
 
@@ -190,6 +196,18 @@ struct tegra_sor {
         struct regulator *hdmi_supply;
 };
 
+struct tegra_sor_state {
+        struct drm_connector_state base;
+
+        unsigned int bpc;
+};
+
+static inline struct tegra_sor_state *
+to_sor_state(struct drm_connector_state *state)
+{
+        return container_of(state, struct tegra_sor_state, base);
+}
+
 struct tegra_sor_config {
         u32 bits_per_pixel;
 
@@ -225,6 +243,118 @@ static inline void tegra_sor_writel(struct tegra_sor *sor, u32 value,
         writel(value, sor->regs + (offset << 2));
 }
 
+static int tegra_sor_set_parent_clock(struct tegra_sor *sor, struct clk *parent)
+{
+        int err;
+
+        clk_disable_unprepare(sor->clk);
+
+        err = clk_set_parent(sor->clk, parent);
+        if (err < 0)
+                return err;
+
+        err = clk_prepare_enable(sor->clk);
+        if (err < 0)
+                return err;
+
+        return 0;
+}
+
+struct tegra_clk_sor_brick {
+        struct clk_hw hw;
+        struct tegra_sor *sor;
+};
+
+static inline struct tegra_clk_sor_brick *to_brick(struct clk_hw *hw)
+{
+        return container_of(hw, struct tegra_clk_sor_brick, hw);
+}
+
+static const char * const tegra_clk_sor_brick_parents[] = {
+        "pll_d2_out0", "pll_dp"
+};
+
+static int tegra_clk_sor_brick_set_parent(struct clk_hw *hw, u8 index)
+{
+        struct tegra_clk_sor_brick *brick = to_brick(hw);
+        struct tegra_sor *sor = brick->sor;
+        u32 value;
+
+        value = tegra_sor_readl(sor, SOR_CLK_CNTRL);
+        value &= ~SOR_CLK_CNTRL_DP_CLK_SEL_MASK;
+
+        switch (index) {
+        case 0:
+                value |= SOR_CLK_CNTRL_DP_CLK_SEL_SINGLE_PCLK;
+                break;
+
+        case 1:
+                value |= SOR_CLK_CNTRL_DP_CLK_SEL_SINGLE_DPCLK;
+                break;
+        }
+
+        tegra_sor_writel(sor, value, SOR_CLK_CNTRL);
+
+        return 0;
+}
+
+static u8 tegra_clk_sor_brick_get_parent(struct clk_hw *hw)
+{
+        struct tegra_clk_sor_brick *brick = to_brick(hw);
+        struct tegra_sor *sor = brick->sor;
+        u8 parent = U8_MAX;
+        u32 value;
+
+        value = tegra_sor_readl(sor, SOR_CLK_CNTRL);
+
+        switch (value & SOR_CLK_CNTRL_DP_CLK_SEL_MASK) {
+        case SOR_CLK_CNTRL_DP_CLK_SEL_SINGLE_PCLK:
+        case SOR_CLK_CNTRL_DP_CLK_SEL_DIFF_PCLK:
+                parent = 0;
+                break;
+
+        case SOR_CLK_CNTRL_DP_CLK_SEL_SINGLE_DPCLK:
+        case SOR_CLK_CNTRL_DP_CLK_SEL_DIFF_DPCLK:
+                parent = 1;
+                break;
+        }
+
+        return parent;
+}
+
+static const struct clk_ops tegra_clk_sor_brick_ops = {
+        .set_parent = tegra_clk_sor_brick_set_parent,
+        .get_parent = tegra_clk_sor_brick_get_parent,
+};
+
+static struct clk *tegra_clk_sor_brick_register(struct tegra_sor *sor,
+                                                const char *name)
+{
+        struct tegra_clk_sor_brick *brick;
+        struct clk_init_data init;
+        struct clk *clk;
+
+        brick = devm_kzalloc(sor->dev, sizeof(*brick), GFP_KERNEL);
+        if (!brick)
+                return ERR_PTR(-ENOMEM);
+
+        brick->sor = sor;
+
+        init.name = name;
+        init.flags = 0;
+        init.parent_names = tegra_clk_sor_brick_parents;
+        init.num_parents = ARRAY_SIZE(tegra_clk_sor_brick_parents);
+        init.ops = &tegra_clk_sor_brick_ops;
+
+        brick->hw.init = &init;
+
+        clk = devm_clk_register(sor->dev, &brick->hw);
+        if (IS_ERR(clk))
+                kfree(brick);
+
+        return clk;
+}
+
 static int tegra_sor_dp_train_fast(struct tegra_sor *sor,
                                    struct drm_dp_link *link)
 {
@@ -569,10 +699,10 @@ static int tegra_sor_compute_params(struct tegra_sor *sor,
         return false;
 }
 
-static int tegra_sor_calc_config(struct tegra_sor *sor,
-                                 const struct drm_display_mode *mode,
-                                 struct tegra_sor_config *config,
-                                 struct drm_dp_link *link)
+static int tegra_sor_compute_config(struct tegra_sor *sor,
+                                    const struct drm_display_mode *mode,
+                                    struct tegra_sor_config *config,
+                                    struct drm_dp_link *link)
 {
         const u64 f = 100000, link_rate = link->rate * 1000;
         const u64 pclk = mode->clock * 1000;
@@ -661,6 +791,135 @@ static int tegra_sor_calc_config(struct tegra_sor *sor,
         return 0;
 }
 
+static void tegra_sor_apply_config(struct tegra_sor *sor,
+                                   const struct tegra_sor_config *config)
+{
+        u32 value;
+
+        value = tegra_sor_readl(sor, SOR_DP_LINKCTL0);
+        value &= ~SOR_DP_LINKCTL_TU_SIZE_MASK;
+        value |= SOR_DP_LINKCTL_TU_SIZE(config->tu_size);
+        tegra_sor_writel(sor, value, SOR_DP_LINKCTL0);
+
+        value = tegra_sor_readl(sor, SOR_DP_CONFIG0);
+        value &= ~SOR_DP_CONFIG_WATERMARK_MASK;
+        value |= SOR_DP_CONFIG_WATERMARK(config->watermark);
+
+        value &= ~SOR_DP_CONFIG_ACTIVE_SYM_COUNT_MASK;
+        value |= SOR_DP_CONFIG_ACTIVE_SYM_COUNT(config->active_count);
+
+        value &= ~SOR_DP_CONFIG_ACTIVE_SYM_FRAC_MASK;
+        value |= SOR_DP_CONFIG_ACTIVE_SYM_FRAC(config->active_frac);
+
+        if (config->active_polarity)
+                value |= SOR_DP_CONFIG_ACTIVE_SYM_POLARITY;
+        else
+                value &= ~SOR_DP_CONFIG_ACTIVE_SYM_POLARITY;
+
+        value |= SOR_DP_CONFIG_ACTIVE_SYM_ENABLE;
+        value |= SOR_DP_CONFIG_DISPARITY_NEGATIVE;
+        tegra_sor_writel(sor, value, SOR_DP_CONFIG0);
+
+        value = tegra_sor_readl(sor, SOR_DP_AUDIO_HBLANK_SYMBOLS);
+        value &= ~SOR_DP_AUDIO_HBLANK_SYMBOLS_MASK;
+        value |= config->hblank_symbols & 0xffff;
+        tegra_sor_writel(sor, value, SOR_DP_AUDIO_HBLANK_SYMBOLS);
+
+        value = tegra_sor_readl(sor, SOR_DP_AUDIO_VBLANK_SYMBOLS);
+        value &= ~SOR_DP_AUDIO_VBLANK_SYMBOLS_MASK;
+        value |= config->vblank_symbols & 0xffff;
+        tegra_sor_writel(sor, value, SOR_DP_AUDIO_VBLANK_SYMBOLS);
+}
+
+static void tegra_sor_mode_set(struct tegra_sor *sor,
+                               const struct drm_display_mode *mode,
+                               struct tegra_sor_state *state)
+{
+        struct tegra_dc *dc = to_tegra_dc(sor->output.encoder.crtc);
+        unsigned int vbe, vse, hbe, hse, vbs, hbs;
+        u32 value;
+
+        value = tegra_sor_readl(sor, SOR_STATE1);
+        value &= ~SOR_STATE_ASY_PIXELDEPTH_MASK;
+        value &= ~SOR_STATE_ASY_CRC_MODE_MASK;
+        value &= ~SOR_STATE_ASY_OWNER_MASK;
+
+        value |= SOR_STATE_ASY_CRC_MODE_COMPLETE |
+                 SOR_STATE_ASY_OWNER(dc->pipe + 1);
+
+        if (mode->flags & DRM_MODE_FLAG_PHSYNC)
+                value &= ~SOR_STATE_ASY_HSYNCPOL;
+
+        if (mode->flags & DRM_MODE_FLAG_NHSYNC)
+                value |= SOR_STATE_ASY_HSYNCPOL;
+
+        if (mode->flags & DRM_MODE_FLAG_PVSYNC)
+                value &= ~SOR_STATE_ASY_VSYNCPOL;
+
+        if (mode->flags & DRM_MODE_FLAG_NVSYNC)
+                value |= SOR_STATE_ASY_VSYNCPOL;
+
+        switch (state->bpc) {
+        case 16:
+                value |= SOR_STATE_ASY_PIXELDEPTH_BPP_48_444;
+                break;
+
+        case 12:
+                value |= SOR_STATE_ASY_PIXELDEPTH_BPP_36_444;
+                break;
+
+        case 10:
+                value |= SOR_STATE_ASY_PIXELDEPTH_BPP_30_444;
+                break;
+
+        case 8:
+                value |= SOR_STATE_ASY_PIXELDEPTH_BPP_24_444;
+                break;
+
+        case 6:
+                value |= SOR_STATE_ASY_PIXELDEPTH_BPP_18_444;
+                break;
+
+        default:
+                value |= SOR_STATE_ASY_PIXELDEPTH_BPP_24_444;
+                break;
+        }
+
+        tegra_sor_writel(sor, value, SOR_STATE1);
+
+        /*
+         * TODO: The video timing programming below doesn't seem to match the
+         * register definitions.
+         */
+
+        value = ((mode->vtotal & 0x7fff) << 16) | (mode->htotal & 0x7fff);
+        tegra_sor_writel(sor, value, SOR_HEAD_STATE1(dc->pipe));
+
+        /* sync end = sync width - 1 */
+        vse = mode->vsync_end - mode->vsync_start - 1;
+        hse = mode->hsync_end - mode->hsync_start - 1;
+
+        value = ((vse & 0x7fff) << 16) | (hse & 0x7fff);
+        tegra_sor_writel(sor, value, SOR_HEAD_STATE2(dc->pipe));
+
+        /* blank end = sync end + back porch */
+        vbe = vse + (mode->vtotal - mode->vsync_end);
+        hbe = hse + (mode->htotal - mode->hsync_end);
+
+        value = ((vbe & 0x7fff) << 16) | (hbe & 0x7fff);
+        tegra_sor_writel(sor, value, SOR_HEAD_STATE3(dc->pipe));
+
+        /* blank start = blank end + active */
+        vbs = vbe + mode->vdisplay;
+        hbs = hbe + mode->hdisplay;
+
+        value = ((vbs & 0x7fff) << 16) | (hbs & 0x7fff);
+        tegra_sor_writel(sor, value, SOR_HEAD_STATE4(dc->pipe));
+
+        /* XXX interlacing support */
+        tegra_sor_writel(sor, 0x001, SOR_HEAD_STATE5(dc->pipe));
+}
+
 static int tegra_sor_detach(struct tegra_sor *sor)
 {
         unsigned long value, timeout;
@@ -733,7 +992,8 @@ static int tegra_sor_power_down(struct tegra_sor *sor)
         if ((value & SOR_PWR_TRIGGER) != 0)
                 return -ETIMEDOUT;
 
-        err = clk_set_parent(sor->clk, sor->clk_safe);
+        /* switch to safe parent clock */
+        err = tegra_sor_set_parent_clock(sor, sor->clk_safe);
         if (err < 0)
                 dev_err(sor->dev, "failed to set safe parent clock: %d\n", err);
 
@@ -1038,6 +1298,22 @@ static void tegra_sor_debugfs_exit(struct tegra_sor *sor)
         sor->debugfs = NULL;
 }
 
+static void tegra_sor_connector_reset(struct drm_connector *connector)
+{
+        struct tegra_sor_state *state;
+
+        state = kzalloc(sizeof(*state), GFP_KERNEL);
+        if (!state)
+                return;
+
+        if (connector->state) {
+                __drm_atomic_helper_connector_destroy_state(connector->state);
+                kfree(connector->state);
+        }
+
+        __drm_atomic_helper_connector_reset(connector, &state->base);
+}
+
 static enum drm_connector_status
 tegra_sor_connector_detect(struct drm_connector *connector, bool force)
 {
@@ -1050,13 +1326,28 @@ tegra_sor_connector_detect(struct drm_connector *connector, bool force)
         return tegra_output_connector_detect(connector, force);
 }
 
+static struct drm_connector_state *
+tegra_sor_connector_duplicate_state(struct drm_connector *connector)
+{
+        struct tegra_sor_state *state = to_sor_state(connector->state);
+        struct tegra_sor_state *copy;
+
+        copy = kmemdup(state, sizeof(*state), GFP_KERNEL);
+        if (!copy)
+                return NULL;
+
+        __drm_atomic_helper_connector_duplicate_state(connector, &copy->base);
+
+        return &copy->base;
+}
+
 static const struct drm_connector_funcs tegra_sor_connector_funcs = {
         .dpms = drm_atomic_helper_connector_dpms,
-        .reset = drm_atomic_helper_connector_reset,
+        .reset = tegra_sor_connector_reset,
         .detect = tegra_sor_connector_detect,
         .fill_modes = drm_helper_probe_single_connector_modes,
         .destroy = tegra_output_connector_destroy,
-        .atomic_duplicate_state = drm_atomic_helper_connector_duplicate_state,
+        .atomic_duplicate_state = tegra_sor_connector_duplicate_state,
         .atomic_destroy_state = drm_atomic_helper_connector_destroy_state,
 };
 
@@ -1081,13 +1372,16 @@ static enum drm_mode_status
 tegra_sor_connector_mode_valid(struct drm_connector *connector,
                                struct drm_display_mode *mode)
 {
+        /* HDMI 2.0 modes are not yet supported */
+        if (mode->clock > 340000)
+                return MODE_NOCLOCK;
+
         return MODE_OK;
 }
 
 static const struct drm_connector_helper_funcs tegra_sor_connector_helper_funcs = {
         .get_modes = tegra_sor_connector_get_modes,
         .mode_valid = tegra_sor_connector_mode_valid,
-        .best_encoder = tegra_output_connector_best_encoder,
 };
 
 static const struct drm_encoder_funcs tegra_sor_encoder_funcs = {
@@ -1141,8 +1435,7 @@ static void tegra_sor_edp_disable(struct drm_encoder *encoder)
         if (output->panel)
                 drm_panel_unprepare(output->panel);
 
-        reset_control_assert(sor->rst);
-        clk_disable_unprepare(sor->clk);
+        pm_runtime_put(sor->dev);
 }
 
 #if 0
@@ -1192,19 +1485,18 @@ static void tegra_sor_edp_enable(struct drm_encoder *encoder)
         struct drm_display_mode *mode = &encoder->crtc->state->adjusted_mode;
         struct tegra_output *output = encoder_to_output(encoder);
         struct tegra_dc *dc = to_tegra_dc(encoder->crtc);
-        unsigned int vbe, vse, hbe, hse, vbs, hbs, i;
         struct tegra_sor *sor = to_sor(output);
         struct tegra_sor_config config;
+        struct tegra_sor_state *state;
         struct drm_dp_link link;
         u8 rate, lanes;
+        unsigned int i;
         int err = 0;
         u32 value;
 
-        err = clk_prepare_enable(sor->clk);
-        if (err < 0)
-                dev_err(sor->dev, "failed to enable clock: %d\n", err);
+        state = to_sor_state(output->connector.state);
 
-        reset_control_deassert(sor->rst);
+        pm_runtime_get_sync(sor->dev);
 
         if (output->panel)
                 drm_panel_prepare(output->panel);
@@ -1219,17 +1511,17 @@ static void tegra_sor_edp_enable(struct drm_encoder *encoder)
                 return;
         }
 
-        err = clk_set_parent(sor->clk, sor->clk_safe);
+        /* switch to safe parent clock */
+        err = tegra_sor_set_parent_clock(sor, sor->clk_safe);
         if (err < 0)
                 dev_err(sor->dev, "failed to set safe parent clock: %d\n", err);
 
         memset(&config, 0, sizeof(config));
-        config.bits_per_pixel = output->connector.display_info.bpc * 3;
+        config.bits_per_pixel = state->bpc * 3;
 
-        err = tegra_sor_calc_config(sor, mode, &config, &link);
+        err = tegra_sor_compute_config(sor, mode, &config, &link);
         if (err < 0)
-                dev_err(sor->dev, "failed to compute link configuration: %d\n",
-                        err);
+                dev_err(sor->dev, "failed to compute configuration: %d\n", err);
 
         value = tegra_sor_readl(sor, SOR_CLK_CNTRL);
         value &= ~SOR_CLK_CNTRL_DP_CLK_SEL_MASK;
@@ -1326,10 +1618,18 @@ static void tegra_sor_edp_enable(struct drm_encoder *encoder)
         value &= ~SOR_PLL2_PORT_POWERDOWN;
         tegra_sor_writel(sor, value, SOR_PLL2);
 
-        /* switch to DP clock */
-        err = clk_set_parent(sor->clk, sor->clk_dp);
+        /* XXX not in TRM */
+        for (value = 0, i = 0; i < 5; i++)
+                value |= SOR_XBAR_CTRL_LINK0_XSEL(i, sor->soc->xbar_cfg[i]) |
+                         SOR_XBAR_CTRL_LINK1_XSEL(i, i);
+
+        tegra_sor_writel(sor, 0x00000000, SOR_XBAR_POL);
+        tegra_sor_writel(sor, value, SOR_XBAR_CTRL);
+
+        /* switch to DP parent clock */
+        err = tegra_sor_set_parent_clock(sor, sor->clk_dp);
         if (err < 0)
-                dev_err(sor->dev, "failed to set DP parent clock: %d\n", err);
+                dev_err(sor->dev, "failed to set parent clock: %d\n", err);
 
         /* power DP lanes */
         value = tegra_sor_readl(sor, SOR_DP_PADCTL0);
@@ -1375,13 +1675,11 @@ static void tegra_sor_edp_enable(struct drm_encoder *encoder)
         value |= drm_dp_link_rate_to_bw_code(link.rate) << 2;
         tegra_sor_writel(sor, value, SOR_CLK_CNTRL);
 
-        /* set linkctl */
+        tegra_sor_apply_config(sor, &config);
+
+        /* enable link */
         value = tegra_sor_readl(sor, SOR_DP_LINKCTL0);
         value |= SOR_DP_LINKCTL_ENABLE;
-
-        value &= ~SOR_DP_LINKCTL_TU_SIZE_MASK;
-        value |= SOR_DP_LINKCTL_TU_SIZE(config.tu_size);
-
         value |= SOR_DP_LINKCTL_ENHANCED_FRAME;
         tegra_sor_writel(sor, value, SOR_DP_LINKCTL0);
 
@@ -1394,35 +1692,6 @@ static void tegra_sor_edp_enable(struct drm_encoder *encoder)
 
         tegra_sor_writel(sor, value, SOR_DP_TPG);
 
-        value = tegra_sor_readl(sor, SOR_DP_CONFIG0);
-        value &= ~SOR_DP_CONFIG_WATERMARK_MASK;
-        value |= SOR_DP_CONFIG_WATERMARK(config.watermark);
-
-        value &= ~SOR_DP_CONFIG_ACTIVE_SYM_COUNT_MASK;
-        value |= SOR_DP_CONFIG_ACTIVE_SYM_COUNT(config.active_count);
-
-        value &= ~SOR_DP_CONFIG_ACTIVE_SYM_FRAC_MASK;
-        value |= SOR_DP_CONFIG_ACTIVE_SYM_FRAC(config.active_frac);
-
-        if (config.active_polarity)
-                value |= SOR_DP_CONFIG_ACTIVE_SYM_POLARITY;
-        else
-                value &= ~SOR_DP_CONFIG_ACTIVE_SYM_POLARITY;
-
-        value |= SOR_DP_CONFIG_ACTIVE_SYM_ENABLE;
-        value |= SOR_DP_CONFIG_DISPARITY_NEGATIVE;
-        tegra_sor_writel(sor, value, SOR_DP_CONFIG0);
-
-        value = tegra_sor_readl(sor, SOR_DP_AUDIO_HBLANK_SYMBOLS);
-        value &= ~SOR_DP_AUDIO_HBLANK_SYMBOLS_MASK;
-        value |= config.hblank_symbols & 0xffff;
-        tegra_sor_writel(sor, value, SOR_DP_AUDIO_HBLANK_SYMBOLS);
-
-        value = tegra_sor_readl(sor, SOR_DP_AUDIO_VBLANK_SYMBOLS);
-        value &= ~SOR_DP_AUDIO_VBLANK_SYMBOLS_MASK;
-        value |= config.vblank_symbols & 0xffff;
-        tegra_sor_writel(sor, value, SOR_DP_AUDIO_VBLANK_SYMBOLS);
-
         /* enable pad calibration logic */
         value = tegra_sor_readl(sor, SOR_DP_PADCTL0);
         value |= SOR_DP_PADCTL_PAD_CAL_PD;
@@ -1478,75 +1747,19 @@ static void tegra_sor_edp_enable(struct drm_encoder *encoder)
         if (err < 0)
                 dev_err(sor->dev, "failed to power up SOR: %d\n", err);
 
-        /*
-         * configure panel (24bpp, vsync-, hsync-, DP-A protocol, complete
-         * raster, associate with display controller)
-         */
-        value = SOR_STATE_ASY_PROTOCOL_DP_A |
-                SOR_STATE_ASY_CRC_MODE_COMPLETE |
-                SOR_STATE_ASY_OWNER(dc->pipe + 1);
-
-        if (mode->flags & DRM_MODE_FLAG_PHSYNC)
-                value &= ~SOR_STATE_ASY_HSYNCPOL;
-
-        if (mode->flags & DRM_MODE_FLAG_NHSYNC)
-                value |= SOR_STATE_ASY_HSYNCPOL;
-
-        if (mode->flags & DRM_MODE_FLAG_PVSYNC)
-                value &= ~SOR_STATE_ASY_VSYNCPOL;
-
-        if (mode->flags & DRM_MODE_FLAG_NVSYNC)
-                value |= SOR_STATE_ASY_VSYNCPOL;
-
-        switch (config.bits_per_pixel) {
-        case 24:
-                value |= SOR_STATE_ASY_PIXELDEPTH_BPP_24_444;
-                break;
-
-        case 18:
-                value |= SOR_STATE_ASY_PIXELDEPTH_BPP_18_444;
-                break;
-
-        default:
-                BUG();
-                break;
-        }
-
-        tegra_sor_writel(sor, value, SOR_STATE1);
-
-        /*
-         * TODO: The video timing programming below doesn't seem to match the
-         * register definitions.
-         */
-
-        value = ((mode->vtotal & 0x7fff) << 16) | (mode->htotal & 0x7fff);
-        tegra_sor_writel(sor, value, SOR_HEAD_STATE1(dc->pipe));
-
-        vse = mode->vsync_end - mode->vsync_start - 1;
-        hse = mode->hsync_end - mode->hsync_start - 1;
-
-        value = ((vse & 0x7fff) << 16) | (hse & 0x7fff);
-        tegra_sor_writel(sor, value, SOR_HEAD_STATE2(dc->pipe));
-
-        vbe = vse + (mode->vsync_start - mode->vdisplay);
-        hbe = hse + (mode->hsync_start - mode->hdisplay);
-
-        value = ((vbe & 0x7fff) << 16) | (hbe & 0x7fff);
-        tegra_sor_writel(sor, value, SOR_HEAD_STATE3(dc->pipe));
-
-        vbs = vbe + mode->vdisplay;
-        hbs = hbe + mode->hdisplay;
-
-        value = ((vbs & 0x7fff) << 16) | (hbs & 0x7fff);
-        tegra_sor_writel(sor, value, SOR_HEAD_STATE4(dc->pipe));
-
-        tegra_sor_writel(sor, 0x1, SOR_HEAD_STATE5(dc->pipe));
-
         /* CSTM (LVDS, link A/B, upper) */
         value = SOR_CSTM_LVDS | SOR_CSTM_LINK_ACT_A | SOR_CSTM_LINK_ACT_B |
                 SOR_CSTM_UPPER;
         tegra_sor_writel(sor, value, SOR_CSTM);
 
+        /* use DP-A protocol */
+        value = tegra_sor_readl(sor, SOR_STATE1);
+        value &= ~SOR_STATE_ASY_PROTOCOL_MASK;
+        value |= SOR_STATE_ASY_PROTOCOL_DP_A;
+        tegra_sor_writel(sor, value, SOR_STATE1);
+
+        tegra_sor_mode_set(sor, mode, state);
+
         /* PWM setup */
         err = tegra_sor_setup_pwm(sor, 250);
         if (err < 0)
@@ -1578,11 +1791,15 @@ tegra_sor_encoder_atomic_check(struct drm_encoder *encoder,
                                struct drm_connector_state *conn_state)
 {
         struct tegra_output *output = encoder_to_output(encoder);
+        struct tegra_sor_state *state = to_sor_state(conn_state);
         struct tegra_dc *dc = to_tegra_dc(conn_state->crtc);
         unsigned long pclk = crtc_state->mode.clock * 1000;
         struct tegra_sor *sor = to_sor(output);
+        struct drm_display_info *info;
         int err;
 
+        info = &output->connector.display_info;
+
         err = tegra_dc_state_setup_clock(dc, crtc_state, sor->clk_parent,
                                          pclk, 0);
         if (err < 0) {
@@ -1590,6 +1807,18 @@ tegra_sor_encoder_atomic_check(struct drm_encoder *encoder,
                 return err;
         }
 
+        switch (info->bpc) {
+        case 8:
+        case 6:
+                state->bpc = info->bpc;
+                break;
+
+        default:
+                DRM_DEBUG_KMS("%u bits-per-color not supported\n", info->bpc);
+                state->bpc = 8;
+                break;
+        }
+
         return 0;
 }
 
@@ -1752,9 +1981,7 @@ static void tegra_sor_hdmi_disable(struct drm_encoder *encoder)
         if (err < 0)
                 dev_err(sor->dev, "failed to power off HDMI rail: %d\n", err);
 
-        reset_control_assert(sor->rst);
-        usleep_range(1000, 2000);
-        clk_disable_unprepare(sor->clk);
+        pm_runtime_put(sor->dev);
 }
 
 static void tegra_sor_hdmi_enable(struct drm_encoder *encoder)
@@ -1762,26 +1989,21 @@ static void tegra_sor_hdmi_enable(struct drm_encoder *encoder)
         struct tegra_output *output = encoder_to_output(encoder);
         unsigned int h_ref_to_sync = 1, pulse_start, max_ac;
         struct tegra_dc *dc = to_tegra_dc(encoder->crtc);
-        unsigned int vbe, vse, hbe, hse, vbs, hbs, div;
         struct tegra_sor_hdmi_settings *settings;
         struct tegra_sor *sor = to_sor(output);
+        struct tegra_sor_state *state;
         struct drm_display_mode *mode;
-        struct drm_display_info *info;
+        unsigned int div, i;
         u32 value;
         int err;
 
+        state = to_sor_state(output->connector.state);
         mode = &encoder->crtc->state->adjusted_mode;
-        info = &output->connector.display_info;
 
-        err = clk_prepare_enable(sor->clk);
-        if (err < 0)
-                dev_err(sor->dev, "failed to enable clock: %d\n", err);
+        pm_runtime_get_sync(sor->dev);
 
-        usleep_range(1000, 2000);
-
-        reset_control_deassert(sor->rst);
-
-        err = clk_set_parent(sor->clk, sor->clk_safe);
+        /* switch to safe parent clock */
+        err = tegra_sor_set_parent_clock(sor, sor->clk_safe);
         if (err < 0)
                 dev_err(sor->dev, "failed to set safe parent clock: %d\n", err);
 
@@ -1877,22 +2099,20 @@ static void tegra_sor_hdmi_enable(struct drm_encoder *encoder)
         value = SOR_REFCLK_DIV_INT(div) | SOR_REFCLK_DIV_FRAC(div);
         tegra_sor_writel(sor, value, SOR_REFCLK);
 
-        /* XXX don't hardcode */
-        value = SOR_XBAR_CTRL_LINK1_XSEL(4, 4) |
-                SOR_XBAR_CTRL_LINK1_XSEL(3, 3) |
-                SOR_XBAR_CTRL_LINK1_XSEL(2, 2) |
-                SOR_XBAR_CTRL_LINK1_XSEL(1, 1) |
-                SOR_XBAR_CTRL_LINK1_XSEL(0, 0) |
-                SOR_XBAR_CTRL_LINK0_XSEL(4, 4) |
-                SOR_XBAR_CTRL_LINK0_XSEL(3, 3) |
-                SOR_XBAR_CTRL_LINK0_XSEL(2, 0) |
-                SOR_XBAR_CTRL_LINK0_XSEL(1, 1) |
-                SOR_XBAR_CTRL_LINK0_XSEL(0, 2);
-        tegra_sor_writel(sor, value, SOR_XBAR_CTRL);
+        /* XXX not in TRM */
+        for (value = 0, i = 0; i < 5; i++)
+                value |= SOR_XBAR_CTRL_LINK0_XSEL(i, sor->soc->xbar_cfg[i]) |
+                         SOR_XBAR_CTRL_LINK1_XSEL(i, i);
 
         tegra_sor_writel(sor, 0x00000000, SOR_XBAR_POL);
+        tegra_sor_writel(sor, value, SOR_XBAR_CTRL);
 
-        err = clk_set_parent(sor->clk, sor->clk_parent);
+        /* switch to parent clock */
+        err = clk_set_parent(sor->clk_src, sor->clk_parent);
+        if (err < 0)
+                dev_err(sor->dev, "failed to set source clock: %d\n", err);
+
+        err = tegra_sor_set_parent_clock(sor, sor->clk_src);
         if (err < 0)
                 dev_err(sor->dev, "failed to set parent clock: %d\n", err);
 
@@ -2002,7 +2222,7 @@ static void tegra_sor_hdmi_enable(struct drm_encoder *encoder)
         value &= ~DITHER_CONTROL_MASK;
         value &= ~BASE_COLOR_SIZE_MASK;
 
-        switch (info->bpc) {
+        switch (state->bpc) {
         case 6:
                 value |= BASE_COLOR_SIZE_666;
                 break;
@@ -2012,7 +2232,8 @@ static void tegra_sor_hdmi_enable(struct drm_encoder *encoder)
                 break;
 
         default:
-                WARN(1, "%u bits-per-color not supported\n", info->bpc);
+                WARN(1, "%u bits-per-color not supported\n", state->bpc);
+                value |= BASE_COLOR_SIZE_888;
                 break;
         }
 
@@ -2022,83 +2243,19 @@ static void tegra_sor_hdmi_enable(struct drm_encoder *encoder)
         if (err < 0)
                 dev_err(sor->dev, "failed to power up SOR: %d\n", err);
 
-        /* configure mode */
-        value = tegra_sor_readl(sor, SOR_STATE1);
-        value &= ~SOR_STATE_ASY_PIXELDEPTH_MASK;
-        value &= ~SOR_STATE_ASY_CRC_MODE_MASK;
-        value &= ~SOR_STATE_ASY_OWNER_MASK;
-
-        value |= SOR_STATE_ASY_CRC_MODE_COMPLETE |
-                 SOR_STATE_ASY_OWNER(dc->pipe + 1);
-
-        if (mode->flags & DRM_MODE_FLAG_PHSYNC)
-                value &= ~SOR_STATE_ASY_HSYNCPOL;
-
-        if (mode->flags & DRM_MODE_FLAG_NHSYNC)
-                value |= SOR_STATE_ASY_HSYNCPOL;
-
-        if (mode->flags & DRM_MODE_FLAG_PVSYNC)
-                value &= ~SOR_STATE_ASY_VSYNCPOL;
-
-        if (mode->flags & DRM_MODE_FLAG_NVSYNC)
-                value |= SOR_STATE_ASY_VSYNCPOL;
-
-        switch (info->bpc) {
-        case 8:
-                value |= SOR_STATE_ASY_PIXELDEPTH_BPP_24_444;
-                break;
-
-        case 6:
-                value |= SOR_STATE_ASY_PIXELDEPTH_BPP_18_444;
-                break;
-
-        default:
-                BUG();
-                break;
-        }
-
-        tegra_sor_writel(sor, value, SOR_STATE1);
-
+        /* configure dynamic range of output */
         value = tegra_sor_readl(sor, SOR_HEAD_STATE0(dc->pipe));
         value &= ~SOR_HEAD_STATE_RANGECOMPRESS_MASK;
         value &= ~SOR_HEAD_STATE_DYNRANGE_MASK;
         tegra_sor_writel(sor, value, SOR_HEAD_STATE0(dc->pipe));
 
+        /* configure colorspace */
         value = tegra_sor_readl(sor, SOR_HEAD_STATE0(dc->pipe));
         value &= ~SOR_HEAD_STATE_COLORSPACE_MASK;
         value |= SOR_HEAD_STATE_COLORSPACE_RGB;
         tegra_sor_writel(sor, value, SOR_HEAD_STATE0(dc->pipe));
 
-        /*
-         * TODO: The video timing programming below doesn't seem to match the
-         * register definitions.
-         */
-
-        value = ((mode->vtotal & 0x7fff) << 16) | (mode->htotal & 0x7fff);
-        tegra_sor_writel(sor, value, SOR_HEAD_STATE1(dc->pipe));
-
-        /* sync end = sync width - 1 */
-        vse = mode->vsync_end - mode->vsync_start - 1;
-        hse = mode->hsync_end - mode->hsync_start - 1;
-
-        value = ((vse & 0x7fff) << 16) | (hse & 0x7fff);
-        tegra_sor_writel(sor, value, SOR_HEAD_STATE2(dc->pipe));
-
-        /* blank end = sync end + back porch */
-        vbe = vse + (mode->vtotal - mode->vsync_end);
-        hbe = hse + (mode->htotal - mode->hsync_end);
-
-        value = ((vbe & 0x7fff) << 16) | (hbe & 0x7fff);
-        tegra_sor_writel(sor, value, SOR_HEAD_STATE3(dc->pipe));
-
-        /* blank start = blank end + active */
-        vbs = vbe + mode->vdisplay;
-        hbs = hbe + mode->hdisplay;
-
-        value = ((vbs & 0x7fff) << 16) | (hbs & 0x7fff);
-        tegra_sor_writel(sor, value, SOR_HEAD_STATE4(dc->pipe));
-
-        tegra_sor_writel(sor, 0x1, SOR_HEAD_STATE5(dc->pipe));
+        tegra_sor_mode_set(sor, mode, state);
 
         tegra_sor_update(sor);
 
@@ -2196,10 +2353,13 @@ static int tegra_sor_init(struct host1x_client *client)
          * XXX: Remove this reset once proper hand-over from firmware to
          * kernel is possible.
          */
-        err = reset_control_assert(sor->rst);
-        if (err < 0) {
-                dev_err(sor->dev, "failed to assert SOR reset: %d\n", err);
-                return err;
+        if (sor->rst) {
+                err = reset_control_assert(sor->rst);
+                if (err < 0) {
+                        dev_err(sor->dev, "failed to assert SOR reset: %d\n",
+                                err);
+                        return err;
+                }
         }
 
         err = clk_prepare_enable(sor->clk);
@@ -2210,10 +2370,13 @@ static int tegra_sor_init(struct host1x_client *client)
 
         usleep_range(1000, 3000);
 
-        err = reset_control_deassert(sor->rst);
-        if (err < 0) {
-                dev_err(sor->dev, "failed to deassert SOR reset: %d\n", err);
-                return err;
+        if (sor->rst) {
+                err = reset_control_deassert(sor->rst);
+                if (err < 0) {
+                        dev_err(sor->dev, "failed to deassert SOR reset: %d\n",
+                                err);
+                        return err;
+                }
         }
 
         err = clk_prepare_enable(sor->clk_safe);
@@ -2324,11 +2487,16 @@ static const struct tegra_sor_ops tegra_sor_hdmi_ops = {
         .remove = tegra_sor_hdmi_remove,
 };
 
+static const u8 tegra124_sor_xbar_cfg[5] = {
+        0, 1, 2, 3, 4
+};
+
 static const struct tegra_sor_soc tegra124_sor = {
         .supports_edp = true,
         .supports_lvds = true,
         .supports_hdmi = false,
         .supports_dp = false,
+        .xbar_cfg = tegra124_sor_xbar_cfg,
 };
 
 static const struct tegra_sor_soc tegra210_sor = {
@@ -2336,6 +2504,11 @@ static const struct tegra_sor_soc tegra210_sor = {
         .supports_lvds = false,
         .supports_hdmi = false,
         .supports_dp = false,
+        .xbar_cfg = tegra124_sor_xbar_cfg,
+};
+
+static const u8 tegra210_sor_xbar_cfg[5] = {
+        2, 1, 0, 3, 4
 };
 
 static const struct tegra_sor_soc tegra210_sor1 = {
@@ -2346,6 +2519,8 @@ static const struct tegra_sor_soc tegra210_sor1 = {
 
         .num_settings = ARRAY_SIZE(tegra210_sor_hdmi_defaults),
         .settings = tegra210_sor_hdmi_defaults,
+
+        .xbar_cfg = tegra210_sor_xbar_cfg,
 };
 
 static const struct of_device_id tegra_sor_of_match[] = {
@@ -2435,11 +2610,14 @@ static int tegra_sor_probe(struct platform_device *pdev)
                 goto remove;
         }
 
-        sor->rst = devm_reset_control_get(&pdev->dev, "sor");
-        if (IS_ERR(sor->rst)) {
-                err = PTR_ERR(sor->rst);
-                dev_err(&pdev->dev, "failed to get reset control: %d\n", err);
-                goto remove;
+        if (!pdev->dev.pm_domain) {
+                sor->rst = devm_reset_control_get(&pdev->dev, "sor");
+                if (IS_ERR(sor->rst)) {
+                        err = PTR_ERR(sor->rst);
+                        dev_err(&pdev->dev, "failed to get reset control: %d\n",
+                                err);
+                        goto remove;
+                }
         }
 
         sor->clk = devm_clk_get(&pdev->dev, NULL);
@@ -2449,6 +2627,16 @@ static int tegra_sor_probe(struct platform_device *pdev)
                 goto remove;
         }
 
+        if (sor->soc->supports_hdmi || sor->soc->supports_dp) {
+                sor->clk_src = devm_clk_get(&pdev->dev, "source");
+                if (IS_ERR(sor->clk_src)) {
+                        err = PTR_ERR(sor->clk_src);
+                        dev_err(sor->dev, "failed to get source clock: %d\n",
+                                err);
+                        goto remove;
+                }
+        }
+
         sor->clk_parent = devm_clk_get(&pdev->dev, "parent");
         if (IS_ERR(sor->clk_parent)) {
                 err = PTR_ERR(sor->clk_parent);
@@ -2470,6 +2658,19 @@ static int tegra_sor_probe(struct platform_device *pdev)
                 goto remove;
         }
 
+        platform_set_drvdata(pdev, sor);
+        pm_runtime_enable(&pdev->dev);
+
+        pm_runtime_get_sync(&pdev->dev);
+        sor->clk_brick = tegra_clk_sor_brick_register(sor, "sor1_brick");
+        pm_runtime_put(&pdev->dev);
+
+        if (IS_ERR(sor->clk_brick)) {
+                err = PTR_ERR(sor->clk_brick);
+                dev_err(&pdev->dev, "failed to register SOR clock: %d\n", err);
+                goto remove;
+        }
+
         INIT_LIST_HEAD(&sor->client.list);
         sor->client.ops = &sor_client_ops;
         sor->client.dev = &pdev->dev;
@@ -2481,8 +2682,6 @@ static int tegra_sor_probe(struct platform_device *pdev)
                 goto remove;
         }
 
-        platform_set_drvdata(pdev, sor);
-
         return 0;
 
 remove:
@@ -2498,6 +2697,8 @@ static int tegra_sor_remove(struct platform_device *pdev)
         struct tegra_sor *sor = platform_get_drvdata(pdev);
         int err;
 
+        pm_runtime_disable(&pdev->dev);
+
         err = host1x_client_unregister(&sor->client);
         if (err < 0) {
                 dev_err(&pdev->dev, "failed to unregister host1x client: %d\n",
@@ -2516,10 +2717,62 @@ static int tegra_sor_remove(struct platform_device *pdev)
         return 0;
 }
 
+#ifdef CONFIG_PM
+static int tegra_sor_suspend(struct device *dev)
+{
+        struct tegra_sor *sor = dev_get_drvdata(dev);
+        int err;
+
+        if (sor->rst) {
+                err = reset_control_assert(sor->rst);
+                if (err < 0) {
+                        dev_err(dev, "failed to assert reset: %d\n", err);
+                        return err;
+                }
+        }
+
+        usleep_range(1000, 2000);
+
+        clk_disable_unprepare(sor->clk);
+
+        return 0;
+}
+
+static int tegra_sor_resume(struct device *dev)
+{
+        struct tegra_sor *sor = dev_get_drvdata(dev);
+        int err;
+
+        err = clk_prepare_enable(sor->clk);
+        if (err < 0) {
+                dev_err(dev, "failed to enable clock: %d\n", err);
+                return err;
+        }
+
+        usleep_range(1000, 2000);
+
+        if (sor->rst) {
+                err = reset_control_deassert(sor->rst);
+                if (err < 0) {
+                        dev_err(dev, "failed to deassert reset: %d\n", err);
+                        clk_disable_unprepare(sor->clk);
+                        return err;
+                }
+        }
+
+        return 0;
+}
+#endif
+
+static const struct dev_pm_ops tegra_sor_pm_ops = {
+        SET_RUNTIME_PM_OPS(tegra_sor_suspend, tegra_sor_resume, NULL)
+};
+
 struct platform_driver tegra_sor_driver = {
         .driver = {
                 .name = "tegra-sor",
                 .of_match_table = tegra_sor_of_match,
+                .pm = &tegra_sor_pm_ops,
         },
         .probe = tegra_sor_probe,
         .remove = tegra_sor_remove,
diff --git a/drivers/gpu/drm/tegra/sor.h b/drivers/gpu/drm/tegra/sor.h
index 2d31d027e3f6..865c73b48968 100644
--- a/drivers/gpu/drm/tegra/sor.h
+++ b/drivers/gpu/drm/tegra/sor.h
@@ -27,6 +27,9 @@
 #define  SOR_STATE_ASY_PIXELDEPTH_MASK          (0xf << 17)
 #define  SOR_STATE_ASY_PIXELDEPTH_BPP_18_444    (0x2 << 17)
 #define  SOR_STATE_ASY_PIXELDEPTH_BPP_24_444    (0x5 << 17)
+#define  SOR_STATE_ASY_PIXELDEPTH_BPP_30_444    (0x6 << 17)
+#define  SOR_STATE_ASY_PIXELDEPTH_BPP_36_444    (0x8 << 17)
+#define  SOR_STATE_ASY_PIXELDEPTH_BPP_48_444    (0x9 << 17)
 #define  SOR_STATE_ASY_VSYNCPOL                 (1 << 13)
 #define  SOR_STATE_ASY_HSYNCPOL                 (1 << 12)
 #define  SOR_STATE_ASY_PROTOCOL_MASK            (0xf << 8)
diff --git a/drivers/gpu/drm/tilcdc/Kconfig b/drivers/gpu/drm/tilcdc/Kconfig
index f60a1ec84fa4..28fed7e206d0 100644
--- a/drivers/gpu/drm/tilcdc/Kconfig
+++ b/drivers/gpu/drm/tilcdc/Kconfig
@@ -2,7 +2,6 @@ config DRM_TILCDC
         tristate "DRM Support for TI LCDC Display Controller"
         depends on DRM && OF && ARM
         select DRM_KMS_HELPER
-        select DRM_KMS_FB_HELPER
         select DRM_KMS_CMA_HELPER
         select DRM_GEM_CMA_HELPER
         select VIDEOMODE_HELPERS
diff --git a/drivers/gpu/drm/tilcdc/tilcdc_crtc.c b/drivers/gpu/drm/tilcdc/tilcdc_crtc.c
index 79027b1c64d3..107c8bd04f6d 100644
--- a/drivers/gpu/drm/tilcdc/tilcdc_crtc.c
+++ b/drivers/gpu/drm/tilcdc/tilcdc_crtc.c
@@ -697,7 +697,7 @@ irqreturn_t tilcdc_crtc_irq(struct drm_crtc *crtc)
 
                 spin_unlock_irqrestore(&tilcdc_crtc->irq_lock, flags);
 
-                drm_handle_vblank(dev, 0);
+                drm_crtc_handle_vblank(crtc);
 
                 if (!skip_event) {
                         struct drm_pending_vblank_event *event;
diff --git a/drivers/gpu/drm/tilcdc/tilcdc_drv.c b/drivers/gpu/drm/tilcdc/tilcdc_drv.c
index 709bc903524d..d27809372d54 100644
--- a/drivers/gpu/drm/tilcdc/tilcdc_drv.c
+++ b/drivers/gpu/drm/tilcdc/tilcdc_drv.c
@@ -541,7 +541,6 @@ static struct drm_driver tilcdc_driver = {
         .load               = tilcdc_load,
         .unload             = tilcdc_unload,
         .lastclose          = tilcdc_lastclose,
-        .set_busid          = drm_platform_set_busid,
         .irq_handler        = tilcdc_irq,
         .irq_preinstall     = tilcdc_irq_preinstall,
         .irq_postinstall    = tilcdc_irq_postinstall,
@@ -549,7 +548,7 @@ static struct drm_driver tilcdc_driver = {
         .get_vblank_counter = drm_vblank_no_hw_counter,
         .enable_vblank      = tilcdc_enable_vblank,
         .disable_vblank     = tilcdc_disable_vblank,
-        .gem_free_object    = drm_gem_cma_free_object,
+        .gem_free_object_unlocked = drm_gem_cma_free_object,
         .gem_vm_ops         = &drm_gem_cma_vm_ops,
         .dumb_create        = drm_gem_cma_dumb_create,
         .dumb_map_offset    = drm_gem_cma_dumb_map_offset,
diff --git a/drivers/gpu/drm/ttm/ttm_bo.c b/drivers/gpu/drm/ttm/ttm_bo.c
index a71cf98c655f..4054d804fe06 100644
--- a/drivers/gpu/drm/ttm/ttm_bo.c
+++ b/drivers/gpu/drm/ttm/ttm_bo.c
@@ -146,10 +146,9 @@ static void ttm_bo_release_list(struct kref *list_kref)
         BUG_ON(bo->mem.mm_node != NULL);
         BUG_ON(!list_empty(&bo->lru));
         BUG_ON(!list_empty(&bo->ddestroy));
-
-        if (bo->ttm)
-                ttm_tt_destroy(bo->ttm);
+        ttm_tt_destroy(bo->ttm);
         atomic_dec(&bo->glob->bo_count);
+        fence_put(bo->moving);
         if (bo->resv == &bo->ttm_resv)
                 reservation_object_fini(&bo->ttm_resv);
         mutex_destroy(&bo->wu_mutex);
@@ -360,7 +359,8 @@ static int ttm_bo_handle_move_mem(struct ttm_buffer_object *bo,
                 ret = bdev->driver->move(bo, evict, interruptible,
                                          no_wait_gpu, mem);
         else
-                ret = ttm_bo_move_memcpy(bo, evict, no_wait_gpu, mem);
+                ret = ttm_bo_move_memcpy(bo, evict, interruptible,
+                                         no_wait_gpu, mem);
 
         if (ret) {
                 if (bdev->driver->move_notify) {
@@ -396,8 +396,7 @@ moved:
 
 out_err:
         new_man = &bdev->man[bo->mem.mem_type];
-        if ((new_man->flags & TTM_MEMTYPE_FLAG_FIXED) && bo->ttm) {
-                ttm_tt_unbind(bo->ttm);
+        if (new_man->flags & TTM_MEMTYPE_FLAG_FIXED) {
                 ttm_tt_destroy(bo->ttm);
                 bo->ttm = NULL;
         }
@@ -418,11 +417,8 @@ static void ttm_bo_cleanup_memtype_use(struct ttm_buffer_object *bo)
         if (bo->bdev->driver->move_notify)
                 bo->bdev->driver->move_notify(bo, NULL);
 
-        if (bo->ttm) {
-                ttm_tt_unbind(bo->ttm);
-                ttm_tt_destroy(bo->ttm);
-                bo->ttm = NULL;
-        }
+        ttm_tt_destroy(bo->ttm);
+        bo->ttm = NULL;
         ttm_bo_mem_put(bo, &bo->mem);
 
         ww_mutex_unlock (&bo->resv->lock);
@@ -688,15 +684,6 @@ static int ttm_bo_evict(struct ttm_buffer_object *bo, bool interruptible,
         struct ttm_placement placement;
         int ret = 0;
 
-        ret = ttm_bo_wait(bo, interruptible, no_wait_gpu);
-
-        if (unlikely(ret != 0)) {
-                if (ret != -ERESTARTSYS) {
-                        pr_err("Failed to expire sync object before buffer eviction\n");
-                }
-                goto out;
-        }
-
         lockdep_assert_held(&bo->resv->lock.base);
 
         evict_mem = bo->mem;
@@ -720,7 +707,7 @@ static int ttm_bo_evict(struct ttm_buffer_object *bo, bool interruptible,
 
         ret = ttm_bo_handle_move_mem(bo, &evict_mem, true, interruptible,
                                      no_wait_gpu);
-        if (ret) {
+        if (unlikely(ret)) {
                 if (ret != -ERESTARTSYS)
                         pr_err("Buffer eviction failed\n");
                 ttm_bo_mem_put(bo, &evict_mem);
@@ -800,6 +787,34 @@ void ttm_bo_mem_put(struct ttm_buffer_object *bo, struct ttm_mem_reg *mem)
 EXPORT_SYMBOL(ttm_bo_mem_put);
 
 /**
+ * Add the last move fence to the BO and reserve a new shared slot.
+ */
+static int ttm_bo_add_move_fence(struct ttm_buffer_object *bo,
+                                 struct ttm_mem_type_manager *man,
+                                 struct ttm_mem_reg *mem)
+{
+        struct fence *fence;
+        int ret;
+
+        spin_lock(&man->move_lock);
+        fence = fence_get(man->move);
+        spin_unlock(&man->move_lock);
+
+        if (fence) {
+                reservation_object_add_shared_fence(bo->resv, fence);
+
+                ret = reservation_object_reserve_shared(bo->resv);
+                if (unlikely(ret))
+                        return ret;
+
+                fence_put(bo->moving);
+                bo->moving = fence;
+        }
+
+        return 0;
+}
+
+/**
  * Repeatedly evict memory from the LRU for @mem_type until we create enough
  * space, or we've evicted everything and there isn't enough space.
  */
@@ -825,10 +840,8 @@ static int ttm_bo_mem_force_space(struct ttm_buffer_object *bo,
                 if (unlikely(ret != 0))
                         return ret;
         } while (1);
-        if (mem->mm_node == NULL)
-                return -ENOMEM;
         mem->mem_type = mem_type;
-        return 0;
+        return ttm_bo_add_move_fence(bo, man, mem);
 }
 
 static uint32_t ttm_bo_select_caching(struct ttm_mem_type_manager *man,
@@ -898,6 +911,10 @@ int ttm_bo_mem_space(struct ttm_buffer_object *bo,
         bool has_erestartsys = false;
         int i, ret;
 
+        ret = reservation_object_reserve_shared(bo->resv);
+        if (unlikely(ret))
+                return ret;
+
         mem->mm_node = NULL;
         for (i = 0; i < placement->num_placement; ++i) {
                 const struct ttm_place *place = &placement->placement[i];
@@ -931,9 +948,15 @@ int ttm_bo_mem_space(struct ttm_buffer_object *bo,
                 ret = (*man->func->get_node)(man, bo, place, mem);
                 if (unlikely(ret))
                         return ret;
-                
-                if (mem->mm_node)
+
+                if (mem->mm_node) {
+                        ret = ttm_bo_add_move_fence(bo, man, mem);
+                        if (unlikely(ret)) {
+                                (*man->func->put_node)(man, mem);
+                                return ret;
+                        }
                         break;
+                }
         }
 
         if ((type_ok && (mem_type == TTM_PL_SYSTEM)) || mem->mm_node) {
@@ -1000,20 +1023,6 @@ static int ttm_bo_move_buffer(struct ttm_buffer_object *bo,
 
         lockdep_assert_held(&bo->resv->lock.base);
 
-        /*
-         * Don't wait for the BO on initial allocation. This is important when
-         * the BO has an imported reservation object.
-         */
-        if (bo->mem.mem_type != TTM_PL_SYSTEM || bo->ttm != NULL) {
-                /*
-                 * FIXME: It's possible to pipeline buffer moves.
-                 * Have the driver move function wait for idle when necessary,
-                 * instead of doing it here.
-                 */
-                ret = ttm_bo_wait(bo, interruptible, no_wait_gpu);
-                if (ret)
-                        return ret;
-        }
         mem.num_pages = bo->num_pages;
         mem.size = mem.num_pages << PAGE_SHIFT;
         mem.page_alignment = bo->mem.page_alignment;
@@ -1166,7 +1175,7 @@ int ttm_bo_init(struct ttm_bo_device *bdev,
         bo->mem.page_alignment = page_alignment;
         bo->mem.bus.io_reserved_vm = false;
         bo->mem.bus.io_reserved_count = 0;
-        bo->priv_flags = 0;
+        bo->moving = NULL;
         bo->mem.placement = (TTM_PL_FLAG_SYSTEM | TTM_PL_FLAG_CACHED);
         bo->persistent_swap_storage = persistent_swap_storage;
         bo->acc_size = acc_size;
@@ -1278,6 +1287,7 @@ static int ttm_bo_force_list_clean(struct ttm_bo_device *bdev,
 {
         struct ttm_mem_type_manager *man = &bdev->man[mem_type];
         struct ttm_bo_global *glob = bdev->glob;
+        struct fence *fence;
         int ret;
 
         /*
@@ -1298,6 +1308,23 @@ static int ttm_bo_force_list_clean(struct ttm_bo_device *bdev,
                 spin_lock(&glob->lru_lock);
         }
         spin_unlock(&glob->lru_lock);
+
+        spin_lock(&man->move_lock);
+        fence = fence_get(man->move);
+        spin_unlock(&man->move_lock);
+
+        if (fence) {
+                ret = fence_wait(fence, false);
+                fence_put(fence);
+                if (ret) {
+                        if (allow_errors) {
+                                return ret;
+                        } else {
+                                pr_err("Cleanup eviction failed\n");
+                        }
+                }
+        }
+
         return 0;
 }
 
@@ -1317,6 +1344,7 @@ int ttm_bo_clean_mm(struct ttm_bo_device *bdev, unsigned mem_type)
                        mem_type);
                 return ret;
         }
+        fence_put(man->move);
 
         man->use_type = false;
         man->has_type = false;
@@ -1362,6 +1390,7 @@ int ttm_bo_init_mm(struct ttm_bo_device *bdev, unsigned type,
         man->io_reserve_fastpath = true;
         man->use_io_reserve_lru = false;
         mutex_init(&man->io_reserve_mutex);
+        spin_lock_init(&man->move_lock);
         INIT_LIST_HEAD(&man->io_reserve_lru);
 
         ret = bdev->driver->init_mem_type(bdev, type, man);
@@ -1380,6 +1409,7 @@ int ttm_bo_init_mm(struct ttm_bo_device *bdev, unsigned type,
         man->size = p_size;
 
         INIT_LIST_HEAD(&man->lru);
+        man->move = NULL;
 
         return 0;
 }
@@ -1573,47 +1603,17 @@ EXPORT_SYMBOL(ttm_bo_unmap_virtual);
 int ttm_bo_wait(struct ttm_buffer_object *bo,
                 bool interruptible, bool no_wait)
 {
-        struct reservation_object_list *fobj;
-        struct reservation_object *resv;
-        struct fence *excl;
-        long timeout = 15 * HZ;
-        int i;
-
-        resv = bo->resv;
-        fobj = reservation_object_get_list(resv);
-        excl = reservation_object_get_excl(resv);
-        if (excl) {
-                if (!fence_is_signaled(excl)) {
-                        if (no_wait)
-                                return -EBUSY;
-
-                        timeout = fence_wait_timeout(excl,
-                                                     interruptible, timeout);
-                }
-        }
-
-        for (i = 0; fobj && timeout > 0 && i < fobj->shared_count; ++i) {
-                struct fence *fence;
-                fence = rcu_dereference_protected(fobj->shared[i],
-                                                reservation_object_held(resv));
-
-                if (!fence_is_signaled(fence)) {
-                        if (no_wait)
-                                return -EBUSY;
-
-                        timeout = fence_wait_timeout(fence,
-                                                     interruptible, timeout);
-                }
-        }
+        long timeout = no_wait ? 0 : 15 * HZ;
 
+        timeout = reservation_object_wait_timeout_rcu(bo->resv, true,
+                                                      interruptible, timeout);
         if (timeout < 0)
                 return timeout;
 
         if (timeout == 0)
                 return -EBUSY;
 
-        reservation_object_add_excl_fence(resv, NULL);
-        clear_bit(TTM_BO_PRIV_FLAG_MOVING, &bo->priv_flags);
+        reservation_object_add_excl_fence(bo->resv, NULL);
         return 0;
 }
 EXPORT_SYMBOL(ttm_bo_wait);
@@ -1683,14 +1683,9 @@ static int ttm_bo_swapout(struct ttm_mem_shrink *shrink)
         ttm_bo_list_ref_sub(bo, put_count, true);
 
         /**
-         * Wait for GPU, then move to system cached.
+         * Move to system cached
          */
 
-        ret = ttm_bo_wait(bo, false, false);
-
-        if (unlikely(ret != 0))
-                goto out;
-
         if ((bo->mem.placement & swap_placement) != swap_placement) {
                 struct ttm_mem_reg evict_mem;
 
@@ -1705,6 +1700,14 @@ static int ttm_bo_swapout(struct ttm_mem_shrink *shrink)
                         goto out;
         }
 
+        /**
+         * Make sure BO is idle.
+         */
+
+        ret = ttm_bo_wait(bo, false, false);
+        if (unlikely(ret != 0))
+                goto out;
+
         ttm_bo_unmap_virtual(bo);
 
         /**
diff --git a/drivers/gpu/drm/ttm/ttm_bo_util.c b/drivers/gpu/drm/ttm/ttm_bo_util.c
index d9831559706e..2df602a35f92 100644
--- a/drivers/gpu/drm/ttm/ttm_bo_util.c
+++ b/drivers/gpu/drm/ttm/ttm_bo_util.c
@@ -321,7 +321,8 @@ static int ttm_copy_ttm_io_page(struct ttm_tt *ttm, void *dst,
 }
 
 int ttm_bo_move_memcpy(struct ttm_buffer_object *bo,
-                       bool evict, bool no_wait_gpu,
+                       bool evict, bool interruptible,
+                       bool no_wait_gpu,
                        struct ttm_mem_reg *new_mem)
 {
         struct ttm_bo_device *bdev = bo->bdev;
@@ -337,6 +338,10 @@ int ttm_bo_move_memcpy(struct ttm_buffer_object *bo,
         unsigned long add = 0;
         int dir;
 
+        ret = ttm_bo_wait(bo, interruptible, no_wait_gpu);
+        if (ret)
+                return ret;
+
         ret = ttm_mem_reg_ioremap(bdev, old_mem, &old_iomap);
         if (ret)
                 return ret;
@@ -401,8 +406,7 @@ out2:
         *old_mem = *new_mem;
         new_mem->mm_node = NULL;
 
-        if ((man->flags & TTM_MEMTYPE_FLAG_FIXED) && (ttm != NULL)) {
-                ttm_tt_unbind(ttm);
+        if (man->flags & TTM_MEMTYPE_FLAG_FIXED) {
                 ttm_tt_destroy(ttm);
                 bo->ttm = NULL;
         }
@@ -462,6 +466,7 @@ static int ttm_buffer_object_transfer(struct ttm_buffer_object *bo,
         INIT_LIST_HEAD(&fbo->lru);
         INIT_LIST_HEAD(&fbo->swap);
         INIT_LIST_HEAD(&fbo->io_reserve_lru);
+        fbo->moving = NULL;
         drm_vma_node_reset(&fbo->vma_node);
         atomic_set(&fbo->cpu_writers, 0);
 
@@ -634,7 +639,6 @@ EXPORT_SYMBOL(ttm_bo_kunmap);
 int ttm_bo_move_accel_cleanup(struct ttm_buffer_object *bo,
                               struct fence *fence,
                               bool evict,
-                              bool no_wait_gpu,
                               struct ttm_mem_reg *new_mem)
 {
         struct ttm_bo_device *bdev = bo->bdev;
@@ -649,9 +653,7 @@ int ttm_bo_move_accel_cleanup(struct ttm_buffer_object *bo,
                 if (ret)
                         return ret;
 
-                if ((man->flags & TTM_MEMTYPE_FLAG_FIXED) &&
-                    (bo->ttm != NULL)) {
-                        ttm_tt_unbind(bo->ttm);
+                if (man->flags & TTM_MEMTYPE_FLAG_FIXED) {
                         ttm_tt_destroy(bo->ttm);
                         bo->ttm = NULL;
                 }
@@ -665,7 +667,8 @@ int ttm_bo_move_accel_cleanup(struct ttm_buffer_object *bo,
                  * operation has completed.
                  */
 
-                set_bit(TTM_BO_PRIV_FLAG_MOVING, &bo->priv_flags);
+                fence_put(bo->moving);
+                bo->moving = fence_get(fence);
 
                 ret = ttm_buffer_object_transfer(bo, &ghost_obj);
                 if (ret)
@@ -694,3 +697,95 @@ int ttm_bo_move_accel_cleanup(struct ttm_buffer_object *bo,
         return 0;
 }
 EXPORT_SYMBOL(ttm_bo_move_accel_cleanup);
+
+int ttm_bo_pipeline_move(struct ttm_buffer_object *bo,
+                         struct fence *fence, bool evict,
+                         struct ttm_mem_reg *new_mem)
+{
+        struct ttm_bo_device *bdev = bo->bdev;
+        struct ttm_mem_reg *old_mem = &bo->mem;
+
+        struct ttm_mem_type_manager *from = &bdev->man[old_mem->mem_type];
+        struct ttm_mem_type_manager *to = &bdev->man[new_mem->mem_type];
+
+        int ret;
+
+        reservation_object_add_excl_fence(bo->resv, fence);
+
+        if (!evict) {
+                struct ttm_buffer_object *ghost_obj;
+
+                /**
+                 * This should help pipeline ordinary buffer moves.
+                 *
+                 * Hang old buffer memory on a new buffer object,
+                 * and leave it to be released when the GPU
+                 * operation has completed.
+                 */
+
+                fence_put(bo->moving);
+                bo->moving = fence_get(fence);
+
+                ret = ttm_buffer_object_transfer(bo, &ghost_obj);
+                if (ret)
+                        return ret;
+
+                reservation_object_add_excl_fence(ghost_obj->resv, fence);
+
+                /**
+                 * If we're not moving to fixed memory, the TTM object
+                 * needs to stay alive. Otherwhise hang it on the ghost
+                 * bo to be unbound and destroyed.
+                 */
+
+                if (!(to->flags & TTM_MEMTYPE_FLAG_FIXED))
+                        ghost_obj->ttm = NULL;
+                else
+                        bo->ttm = NULL;
+
+                ttm_bo_unreserve(ghost_obj);
+                ttm_bo_unref(&ghost_obj);
+
+        } else if (from->flags & TTM_MEMTYPE_FLAG_FIXED) {
+
+                /**
+                 * BO doesn't have a TTM we need to bind/unbind. Just remember
+                 * this eviction and free up the allocation
+                 */
+
+                spin_lock(&from->move_lock);
+                if (!from->move || fence_is_later(fence, from->move)) {
+                        fence_put(from->move);
+                        from->move = fence_get(fence);
+                }
+                spin_unlock(&from->move_lock);
+
+                ttm_bo_free_old_node(bo);
+
+                fence_put(bo->moving);
+                bo->moving = fence_get(fence);
+
+        } else {
+                /**
+                 * Last resort, wait for the move to be completed.
+                 *
+                 * Should never happen in pratice.
+                 */
+
+                ret = ttm_bo_wait(bo, false, false);
+                if (ret)
+                        return ret;
+
+                if (to->flags & TTM_MEMTYPE_FLAG_FIXED) {
+                        ttm_tt_destroy(bo->ttm);
+                        bo->ttm = NULL;
+                }
+                ttm_bo_free_old_node(bo);
+        }
+
+        *old_mem = *new_mem;
+        new_mem->mm_node = NULL;
+
+        return 0;
+}
+EXPORT_SYMBOL(ttm_bo_pipeline_move);
diff --git a/drivers/gpu/drm/ttm/ttm_bo_vm.c b/drivers/gpu/drm/ttm/ttm_bo_vm.c
index 3216878bced3..a6ed9d5e5167 100644
--- a/drivers/gpu/drm/ttm/ttm_bo_vm.c
+++ b/drivers/gpu/drm/ttm/ttm_bo_vm.c
@@ -48,15 +48,14 @@ static int ttm_bo_vm_fault_idle(struct ttm_buffer_object *bo,
 {
         int ret = 0;
 
-        if (likely(!test_bit(TTM_BO_PRIV_FLAG_MOVING, &bo->priv_flags)))
+        if (likely(!bo->moving))
                 goto out_unlock;
 
         /*
          * Quick non-stalling check for idle.
          */
-        ret = ttm_bo_wait(bo, false, true);
-        if (likely(ret == 0))
-                goto out_unlock;
+        if (fence_is_signaled(bo->moving))
+                goto out_clear;
 
         /*
          * If possible, avoid waiting for GPU with mmap_sem
@@ -68,17 +67,23 @@ static int ttm_bo_vm_fault_idle(struct ttm_buffer_object *bo,
                         goto out_unlock;
 
                 up_read(&vma->vm_mm->mmap_sem);
-                (void) ttm_bo_wait(bo, true, false);
+                (void) fence_wait(bo->moving, true);
                 goto out_unlock;
         }
 
         /*
          * Ordinary wait.
          */
-        ret = ttm_bo_wait(bo, true, false);
-        if (unlikely(ret != 0))
+        ret = fence_wait(bo->moving, true);
+        if (unlikely(ret != 0)) {
                 ret = (ret != -ERESTARTSYS) ? VM_FAULT_SIGBUS :
                         VM_FAULT_NOPAGE;
+                goto out_unlock;
+        }
+
+out_clear:
+        fence_put(bo->moving);
+        bo->moving = NULL;
 
 out_unlock:
         return ret;
diff --git a/drivers/gpu/drm/ttm/ttm_tt.c b/drivers/gpu/drm/ttm/ttm_tt.c
index 97542c35d6ef..bc5aa573f466 100644
--- a/drivers/gpu/drm/ttm/ttm_tt.c
+++ b/drivers/gpu/drm/ttm/ttm_tt.c
@@ -166,12 +166,10 @@ EXPORT_SYMBOL(ttm_tt_set_placement_caching);
 
 void ttm_tt_destroy(struct ttm_tt *ttm)
 {
-        if (unlikely(ttm == NULL))
+        if (ttm == NULL)
                 return;
 
-        if (ttm->state == tt_bound) {
-                ttm_tt_unbind(ttm);
-        }
+        ttm_tt_unbind(ttm);
 
         if (ttm->state == tt_unbound)
                 ttm_tt_unpopulate(ttm);
diff --git a/drivers/gpu/drm/udl/Kconfig b/drivers/gpu/drm/udl/Kconfig
index 613ab0622d6e..1616ec4f4d84 100644
--- a/drivers/gpu/drm/udl/Kconfig
+++ b/drivers/gpu/drm/udl/Kconfig
@@ -4,12 +4,7 @@ config DRM_UDL
         depends on USB_SUPPORT
         depends on USB_ARCH_HAS_HCD
         select USB
-        select FB_SYS_FILLRECT
-        select FB_SYS_COPYAREA
-        select FB_SYS_IMAGEBLIT
-        select FB_DEFERRED_IO
         select DRM_KMS_HELPER
-        select DRM_KMS_FB_HELPER
         help
           This is a KMS driver for the USB displaylink video adapters.
           Say M/Y to add support for these devices via drm/kms interfaces.
diff --git a/drivers/gpu/drm/udl/udl_drv.c b/drivers/gpu/drm/udl/udl_drv.c
index c20408940cd0..17d34e0edbdd 100644
--- a/drivers/gpu/drm/udl/udl_drv.c
+++ b/drivers/gpu/drm/udl/udl_drv.c
@@ -94,7 +94,6 @@ static void udl_usb_disconnect(struct usb_interface *interface)
         struct drm_device *dev = usb_get_intfdata(interface);
 
         drm_kms_helper_poll_disable(dev);
-        drm_connector_unregister_all(dev);
         udl_fbdev_unplug(dev);
         udl_drop_usb(dev);
         drm_unplug_dev(dev);
diff --git a/drivers/gpu/drm/udl/udl_modeset.c b/drivers/gpu/drm/udl/udl_modeset.c
index b87afee44995..f92ea9579674 100644
--- a/drivers/gpu/drm/udl/udl_modeset.c
+++ b/drivers/gpu/drm/udl/udl_modeset.c
@@ -376,7 +376,7 @@ static int udl_crtc_page_flip(struct drm_crtc *crtc,
 
         spin_lock_irqsave(&dev->event_lock, flags);
         if (event)
-                drm_send_vblank_event(dev, 0, event);
+                drm_crtc_send_vblank_event(crtc, event);
         spin_unlock_irqrestore(&dev->event_lock, flags);
         crtc->primary->fb = fb;
 
diff --git a/drivers/gpu/drm/vc4/vc4_bo.c b/drivers/gpu/drm/vc4/vc4_bo.c
index e5a9d3aaf45f..59adcf8532dd 100644
--- a/drivers/gpu/drm/vc4/vc4_bo.c
+++ b/drivers/gpu/drm/vc4/vc4_bo.c
@@ -291,8 +291,6 @@ static void vc4_bo_cache_free_old(struct drm_device *dev)
 
 /* Called on the last userspace/kernel unreference of the BO.  Returns
  * it to the BO cache if possible, otherwise frees it.
- *
- * Note that this is called with the struct_mutex held.
  */
 void vc4_free_object(struct drm_gem_object *gem_bo)
 {
diff --git a/drivers/gpu/drm/vc4/vc4_crtc.c b/drivers/gpu/drm/vc4/vc4_crtc.c
index 0f18b76c7906..8fc2b731b59a 100644
--- a/drivers/gpu/drm/vc4/vc4_crtc.c
+++ b/drivers/gpu/drm/vc4/vc4_crtc.c
@@ -46,12 +46,17 @@ struct vc4_crtc {
         const struct vc4_crtc_data *data;
         void __iomem *regs;
 
+        /* Timestamp at start of vblank irq - unaffected by lock delays. */
+        ktime_t t_vblank;
+
         /* Which HVS channel we're using for our CRTC. */
         int channel;
 
         u8 lut_r[256];
         u8 lut_g[256];
         u8 lut_b[256];
+        /* Size in pixels of the COB memory allocated to this CRTC. */
+        u32 cob_size;
 
         struct drm_pending_vblank_event *event;
 };
@@ -146,6 +151,144 @@ int vc4_crtc_debugfs_regs(struct seq_file *m, void *unused)
 }
 #endif
 
+int vc4_crtc_get_scanoutpos(struct drm_device *dev, unsigned int crtc_id,
+                            unsigned int flags, int *vpos, int *hpos,
+                            ktime_t *stime, ktime_t *etime,
+                            const struct drm_display_mode *mode)
+{
+        struct vc4_dev *vc4 = to_vc4_dev(dev);
+        struct vc4_crtc *vc4_crtc = vc4->crtc[crtc_id];
+        u32 val;
+        int fifo_lines;
+        int vblank_lines;
+        int ret = 0;
+
+        /*
+         * XXX Doesn't work well in interlaced mode yet, partially due
+         * to problems in vc4 kms or drm core interlaced mode handling,
+         * so disable for now in interlaced mode.
+         */
+        if (mode->flags & DRM_MODE_FLAG_INTERLACE)
+                return ret;
+
+        /* preempt_disable_rt() should go right here in PREEMPT_RT patchset. */
+
+        /* Get optional system timestamp before query. */
+        if (stime)
+                *stime = ktime_get();
+
+        /*
+         * Read vertical scanline which is currently composed for our
+         * pixelvalve by the HVS, and also the scaler status.
+         */
+        val = HVS_READ(SCALER_DISPSTATX(vc4_crtc->channel));
+
+        /* Get optional system timestamp after query. */
+        if (etime)
+                *etime = ktime_get();
+
+        /* preempt_enable_rt() should go right here in PREEMPT_RT patchset. */
+
+        /* Vertical position of hvs composed scanline. */
+        *vpos = VC4_GET_FIELD(val, SCALER_DISPSTATX_LINE);
+
+        /* No hpos info available. */
+        if (hpos)
+                *hpos = 0;
+
+        /* This is the offset we need for translating hvs -> pv scanout pos. */
+        fifo_lines = vc4_crtc->cob_size / mode->crtc_hdisplay;
+
+        if (fifo_lines > 0)
+                ret |= DRM_SCANOUTPOS_VALID;
+
+        /* HVS more than fifo_lines into frame for compositing? */
+        if (*vpos > fifo_lines) {
+                /*
+                 * We are in active scanout and can get some meaningful results
+                 * from HVS. The actual PV scanout can not trail behind more
+                 * than fifo_lines as that is the fifo's capacity. Assume that
+                 * in active scanout the HVS and PV work in lockstep wrt. HVS
+                 * refilling the fifo and PV consuming from the fifo, ie.
+                 * whenever the PV consumes and frees up a scanline in the
+                 * fifo, the HVS will immediately refill it, therefore
+                 * incrementing vpos. Therefore we choose HVS read position -
+                 * fifo size in scanlines as a estimate of the real scanout
+                 * position of the PV.
+                 */
+                *vpos -= fifo_lines + 1;
+                if (mode->flags & DRM_MODE_FLAG_INTERLACE)
+                        *vpos /= 2;
+
+                ret |= DRM_SCANOUTPOS_ACCURATE;
+                return ret;
+        }
+
+        /*
+         * Less: This happens when we are in vblank and the HVS, after getting
+         * the VSTART restart signal from the PV, just started refilling its
+         * fifo with new lines from the top-most lines of the new framebuffers.
+         * The PV does not scan out in vblank, so does not remove lines from
+         * the fifo, so the fifo will be full quickly and the HVS has to pause.
+         * We can't get meaningful readings wrt. scanline position of the PV
+         * and need to make things up in a approximative but consistent way.
+         */
+        ret |= DRM_SCANOUTPOS_IN_VBLANK;
+        vblank_lines = mode->crtc_vtotal - mode->crtc_vdisplay;
+
+        if (flags & DRM_CALLED_FROM_VBLIRQ) {
+                /*
+                 * Assume the irq handler got called close to first
+                 * line of vblank, so PV has about a full vblank
+                 * scanlines to go, and as a base timestamp use the
+                 * one taken at entry into vblank irq handler, so it
+                 * is not affected by random delays due to lock
+                 * contention on event_lock or vblank_time lock in
+                 * the core.
+                 */
+                *vpos = -vblank_lines;
+
+                if (stime)
+                        *stime = vc4_crtc->t_vblank;
+                if (etime)
+                        *etime = vc4_crtc->t_vblank;
+
+                /*
+                 * If the HVS fifo is not yet full then we know for certain
+                 * we are at the very beginning of vblank, as the hvs just
+                 * started refilling, and the stime and etime timestamps
+                 * truly correspond to start of vblank.
+                 */
+                if ((val & SCALER_DISPSTATX_FULL) != SCALER_DISPSTATX_FULL)
+                        ret |= DRM_SCANOUTPOS_ACCURATE;
+        } else {
+                /*
+                 * No clue where we are inside vblank. Return a vpos of zero,
+                 * which will cause calling code to just return the etime
+                 * timestamp uncorrected. At least this is no worse than the
+                 * standard fallback.
+                 */
+                *vpos = 0;
+        }
+
+        return ret;
+}
+
+int vc4_crtc_get_vblank_timestamp(struct drm_device *dev, unsigned int crtc_id,
+                                  int *max_error, struct timeval *vblank_time,
+                                  unsigned flags)
+{
+        struct vc4_dev *vc4 = to_vc4_dev(dev);
+        struct vc4_crtc *vc4_crtc = vc4->crtc[crtc_id];
+        struct drm_crtc *crtc = &vc4_crtc->base;
+        struct drm_crtc_state *state = crtc->state;
+
+        /* Helper routine in DRM core does all the work: */
+        return drm_calc_vbltimestamp_from_scanoutpos(dev, crtc_id, max_error,
+                                                     vblank_time, flags,
+                                                     &state->adjusted_mode);
+}
+
 static void vc4_crtc_destroy(struct drm_crtc *crtc)
 {
         drm_crtc_cleanup(crtc);
@@ -175,20 +318,22 @@ vc4_crtc_lut_load(struct drm_crtc *crtc)
                 HVS_WRITE(SCALER_GAMDATA, vc4_crtc->lut_b[i]);
 }
 
-static void
+static int
 vc4_crtc_gamma_set(struct drm_crtc *crtc, u16 *r, u16 *g, u16 *b,
-                   uint32_t start, uint32_t size)
+                   uint32_t size)
 {
         struct vc4_crtc *vc4_crtc = to_vc4_crtc(crtc);
         u32 i;
 
-        for (i = start; i < start + size; i++) {
+        for (i = 0; i < size; i++) {
                 vc4_crtc->lut_r[i] = r[i] >> 8;
                 vc4_crtc->lut_g[i] = g[i] >> 8;
                 vc4_crtc->lut_b[i] = b[i] >> 8;
         }
 
         vc4_crtc_lut_load(crtc);
+
+        return 0;
 }
 
 static u32 vc4_get_fifo_full_level(u32 format)
@@ -395,6 +540,7 @@ static int vc4_crtc_atomic_check(struct drm_crtc *crtc,
         struct vc4_dev *vc4 = to_vc4_dev(dev);
         struct drm_plane *plane;
         unsigned long flags;
+        const struct drm_plane_state *plane_state;
         u32 dlist_count = 0;
         int ret;
 
@@ -404,18 +550,8 @@ static int vc4_crtc_atomic_check(struct drm_crtc *crtc,
         if (hweight32(state->connector_mask) > 1)
                 return -EINVAL;
 
-        drm_atomic_crtc_state_for_each_plane(plane, state) {
-                struct drm_plane_state *plane_state =
-                        state->state->plane_states[drm_plane_index(plane)];
-
-                /* plane might not have changed, in which case take
-                 * current state:
-                 */
-                if (!plane_state)
-                        plane_state = plane->state;
-
+        drm_atomic_crtc_state_for_each_plane_state(plane, plane_state, state)
                 dlist_count += vc4_plane_dlist_size(plane_state);
-        }
 
         dlist_count++; /* Account for SCALER_CTL0_END. */
 
@@ -526,6 +662,7 @@ static irqreturn_t vc4_crtc_irq_handler(int irq, void *data)
         irqreturn_t ret = IRQ_NONE;
 
         if (stat & PV_INT_VFP_START) {
+                vc4_crtc->t_vblank = ktime_get();
                 CRTC_WRITE(PV_INTSTAT, PV_INT_VFP_START);
                 drm_crtc_handle_vblank(&vc4_crtc->base);
                 vc4_crtc_handle_page_flip(vc4_crtc);
@@ -730,6 +867,22 @@ static void vc4_set_crtc_possible_masks(struct drm_device *drm,
         }
 }
 
+static void
+vc4_crtc_get_cob_allocation(struct vc4_crtc *vc4_crtc)
+{
+        struct drm_device *drm = vc4_crtc->base.dev;
+        struct vc4_dev *vc4 = to_vc4_dev(drm);
+        u32 dispbase = HVS_READ(SCALER_DISPBASEX(vc4_crtc->channel));
+        /* Top/base are supposed to be 4-pixel aligned, but the
+         * Raspberry Pi firmware fills the low bits (which are
+         * presumably ignored).
+         */
+        u32 top = VC4_GET_FIELD(dispbase, SCALER_DISPBASEX_TOP) & ~3;
+        u32 base = VC4_GET_FIELD(dispbase, SCALER_DISPBASEX_BASE) & ~3;
+
+        vc4_crtc->cob_size = top - base + 4;
+}
+
 static int vc4_crtc_bind(struct device *dev, struct device *master, void *data)
 {
         struct platform_device *pdev = to_platform_device(dev);
@@ -806,6 +959,8 @@ static int vc4_crtc_bind(struct device *dev, struct device *master, void *data)
                 crtc->cursor = cursor_plane;
         }
 
+        vc4_crtc_get_cob_allocation(vc4_crtc);
+
         CRTC_WRITE(PV_INTEN, 0);
         CRTC_WRITE(PV_INTSTAT, PV_INT_VFP_START);
         ret = devm_request_irq(dev, platform_get_irq(pdev, 0),
diff --git a/drivers/gpu/drm/vc4/vc4_dpi.c b/drivers/gpu/drm/vc4/vc4_dpi.c
index 9817dbfa4ac3..275fedbdbd9e 100644
--- a/drivers/gpu/drm/vc4/vc4_dpi.c
+++ b/drivers/gpu/drm/vc4/vc4_dpi.c
@@ -208,14 +208,6 @@ static int vc4_dpi_connector_get_modes(struct drm_connector *connector)
         return 0;
 }
 
-static struct drm_encoder *
-vc4_dpi_connector_best_encoder(struct drm_connector *connector)
-{
-        struct vc4_dpi_connector *dpi_connector =
-                to_vc4_dpi_connector(connector);
-        return dpi_connector->encoder;
-}
-
 static const struct drm_connector_funcs vc4_dpi_connector_funcs = {
         .dpms = drm_atomic_helper_connector_dpms,
         .detect = vc4_dpi_connector_detect,
@@ -228,7 +220,6 @@ static const struct drm_connector_funcs vc4_dpi_connector_funcs = {
 
 static const struct drm_connector_helper_funcs vc4_dpi_connector_helper_funcs = {
         .get_modes = vc4_dpi_connector_get_modes,
-        .best_encoder = vc4_dpi_connector_best_encoder,
 };
 
 static struct drm_connector *vc4_dpi_connector_init(struct drm_device *dev,
@@ -236,14 +227,12 @@ static struct drm_connector *vc4_dpi_connector_init(struct drm_device *dev,
 {
         struct drm_connector *connector = NULL;
         struct vc4_dpi_connector *dpi_connector;
-        int ret = 0;
 
         dpi_connector = devm_kzalloc(dev->dev, sizeof(*dpi_connector),
                                      GFP_KERNEL);
-        if (!dpi_connector) {
-                ret = -ENOMEM;
-                goto fail;
-        }
+        if (!dpi_connector)
+                return ERR_PTR(-ENOMEM);
+
         connector = &dpi_connector->base;
 
         dpi_connector->encoder = dpi->encoder;
@@ -260,12 +249,6 @@ static struct drm_connector *vc4_dpi_connector_init(struct drm_device *dev,
         drm_mode_connector_attach_encoder(connector, dpi->encoder);
 
         return connector;
-
- fail:
-        if (connector)
-                vc4_dpi_connector_destroy(connector);
-
-        return ERR_PTR(ret);
 }
 
 static const struct drm_encoder_funcs vc4_dpi_encoder_funcs = {
diff --git a/drivers/gpu/drm/vc4/vc4_drv.c b/drivers/gpu/drm/vc4/vc4_drv.c
index 250ed7e3754c..8b42d31a7f0e 100644
--- a/drivers/gpu/drm/vc4/vc4_drv.c
+++ b/drivers/gpu/drm/vc4/vc4_drv.c
@@ -14,6 +14,7 @@
 #include <linux/module.h>
 #include <linux/of_platform.h>
 #include <linux/platform_device.h>
+#include <linux/pm_runtime.h>
 #include "drm_fb_cma_helper.h"
 
 #include "uapi/drm/vc4_drm.h"
@@ -43,12 +44,54 @@ void __iomem *vc4_ioremap_regs(struct platform_device *dev, int index)
         return map;
 }
 
+static int vc4_get_param_ioctl(struct drm_device *dev, void *data,
+                               struct drm_file *file_priv)
+{
+        struct vc4_dev *vc4 = to_vc4_dev(dev);
+        struct drm_vc4_get_param *args = data;
+        int ret;
+
+        if (args->pad != 0)
+                return -EINVAL;
+
+        switch (args->param) {
+        case DRM_VC4_PARAM_V3D_IDENT0:
+                ret = pm_runtime_get_sync(&vc4->v3d->pdev->dev);
+                if (ret)
+                        return ret;
+                args->value = V3D_READ(V3D_IDENT0);
+                pm_runtime_put(&vc4->v3d->pdev->dev);
+                break;
+        case DRM_VC4_PARAM_V3D_IDENT1:
+                ret = pm_runtime_get_sync(&vc4->v3d->pdev->dev);
+                if (ret)
+                        return ret;
+                args->value = V3D_READ(V3D_IDENT1);
+                pm_runtime_put(&vc4->v3d->pdev->dev);
+                break;
+        case DRM_VC4_PARAM_V3D_IDENT2:
+                ret = pm_runtime_get_sync(&vc4->v3d->pdev->dev);
+                if (ret)
+                        return ret;
+                args->value = V3D_READ(V3D_IDENT2);
+                pm_runtime_put(&vc4->v3d->pdev->dev);
+                break;
+        case DRM_VC4_PARAM_SUPPORTS_BRANCHES:
+                args->value = true;
+                break;
+        default:
+                DRM_DEBUG("Unknown parameter %d\n", args->param);
+                return -EINVAL;
+        }
+
+        return 0;
+}
+
 static void vc4_lastclose(struct drm_device *dev)
 {
         struct vc4_dev *vc4 = to_vc4_dev(dev);
 
-        if (vc4->fbdev)
-                drm_fbdev_cma_restore_mode(vc4->fbdev);
+        drm_fbdev_cma_restore_mode(vc4->fbdev);
 }
 
 static const struct file_operations vc4_drm_fops = {
@@ -74,6 +117,7 @@ static const struct drm_ioctl_desc vc4_drm_ioctls[] = {
         DRM_IOCTL_DEF_DRV(VC4_CREATE_SHADER_BO, vc4_create_shader_bo_ioctl, DRM_RENDER_ALLOW),
         DRM_IOCTL_DEF_DRV(VC4_GET_HANG_STATE, vc4_get_hang_state_ioctl,
                           DRM_ROOT_ONLY),
+        DRM_IOCTL_DEF_DRV(VC4_GET_PARAM, vc4_get_param_ioctl, DRM_RENDER_ALLOW),
 };
 
 static struct drm_driver vc4_drm_driver = {
@@ -92,6 +136,8 @@ static struct drm_driver vc4_drm_driver = {
         .enable_vblank = vc4_enable_vblank,
         .disable_vblank = vc4_disable_vblank,
         .get_vblank_counter = drm_vblank_no_hw_counter,
+        .get_scanout_position = vc4_crtc_get_scanoutpos,
+        .get_vblank_timestamp = vc4_crtc_get_vblank_timestamp,
 
 #if defined(CONFIG_DEBUG_FS)
         .debugfs_init = vc4_debugfs_init,
@@ -99,7 +145,7 @@ static struct drm_driver vc4_drm_driver = {
 #endif
 
         .gem_create_object = vc4_create_object,
-        .gem_free_object = vc4_free_object,
+        .gem_free_object_unlocked = vc4_free_object,
         .gem_vm_ops = &drm_gem_cma_vm_ops,
 
         .prime_handle_to_fd = drm_gem_prime_handle_to_fd,
@@ -176,7 +222,6 @@ static int vc4_drm_bind(struct device *dev)
 {
         struct platform_device *pdev = to_platform_device(dev);
         struct drm_device *drm;
-        struct drm_connector *connector;
         struct vc4_dev *vc4;
         int ret = 0;
 
@@ -196,8 +241,6 @@ static int vc4_drm_bind(struct device *dev)
         vc4_bo_cache_init(drm);
 
         drm_mode_config_init(drm);
-        if (ret)
-                goto unref;
 
         vc4_gem_init(drm);
 
@@ -211,27 +254,14 @@ static int vc4_drm_bind(struct device *dev)
         if (ret < 0)
                 goto unbind_all;
 
-        /* Connector registration has to occur after DRM device
-         * registration, because it creates sysfs entries based on the
-         * DRM device.
-         */
-        list_for_each_entry(connector, &drm->mode_config.connector_list, head) {
-                ret = drm_connector_register(connector);
-                if (ret)
-                        goto unregister;
-        }
-
         vc4_kms_load(drm);
 
         return 0;
 
-unregister:
-        drm_dev_unregister(drm);
 unbind_all:
         component_unbind_all(dev, drm);
 gem_destroy:
         vc4_gem_destroy(drm);
-unref:
         drm_dev_unref(drm);
         vc4_bo_cache_destroy(drm);
         return ret;
@@ -259,8 +289,8 @@ static const struct component_master_ops vc4_drm_ops = {
 static struct platform_driver *const component_drivers[] = {
         &vc4_hdmi_driver,
         &vc4_dpi_driver,
-        &vc4_crtc_driver,
         &vc4_hvs_driver,
+        &vc4_crtc_driver,
         &vc4_v3d_driver,
 };
 
diff --git a/drivers/gpu/drm/vc4/vc4_drv.h b/drivers/gpu/drm/vc4/vc4_drv.h
index 37cac59401d7..489e3de0c050 100644
--- a/drivers/gpu/drm/vc4/vc4_drv.h
+++ b/drivers/gpu/drm/vc4/vc4_drv.h
@@ -355,6 +355,9 @@ struct vc4_validated_shader_info {
         uint32_t uniforms_src_size;
         uint32_t num_texture_samples;
         struct vc4_texture_sample_info *texture_samples;
+
+        uint32_t num_uniform_addr_offsets;
+        uint32_t *uniform_addr_offsets;
 };
 
 /**
@@ -415,6 +418,13 @@ extern struct platform_driver vc4_crtc_driver;
 int vc4_enable_vblank(struct drm_device *dev, unsigned int crtc_id);
 void vc4_disable_vblank(struct drm_device *dev, unsigned int crtc_id);
 int vc4_crtc_debugfs_regs(struct seq_file *m, void *arg);
+int vc4_crtc_get_scanoutpos(struct drm_device *dev, unsigned int crtc_id,
+                            unsigned int flags, int *vpos, int *hpos,
+                            ktime_t *stime, ktime_t *etime,
+                            const struct drm_display_mode *mode);
+int vc4_crtc_get_vblank_timestamp(struct drm_device *dev, unsigned int crtc_id,
+                                  int *max_error, struct timeval *vblank_time,
+                                  unsigned flags);
 
 /* vc4_debugfs.c */
 int vc4_debugfs_init(struct drm_minor *minor);
@@ -469,7 +479,7 @@ int vc4_kms_load(struct drm_device *dev);
 struct drm_plane *vc4_plane_init(struct drm_device *dev,
                                  enum drm_plane_type type);
 u32 vc4_plane_write_dlist(struct drm_plane *plane, u32 __iomem *dlist);
-u32 vc4_plane_dlist_size(struct drm_plane_state *state);
+u32 vc4_plane_dlist_size(const struct drm_plane_state *state);
 void vc4_plane_async_set_fb(struct drm_plane *plane,
                             struct drm_framebuffer *fb);
 
diff --git a/drivers/gpu/drm/vc4/vc4_gem.c b/drivers/gpu/drm/vc4/vc4_gem.c
index 46899d6de675..6155e8aca1c6 100644
--- a/drivers/gpu/drm/vc4/vc4_gem.c
+++ b/drivers/gpu/drm/vc4/vc4_gem.c
@@ -53,10 +53,8 @@ vc4_free_hang_state(struct drm_device *dev, struct vc4_hang_state *state)
 {
         unsigned int i;
 
-        mutex_lock(&dev->struct_mutex);
         for (i = 0; i < state->user_state.bo_count; i++)
-                drm_gem_object_unreference(state->bo[i]);
-        mutex_unlock(&dev->struct_mutex);
+                drm_gem_object_unreference_unlocked(state->bo[i]);
 
         kfree(state);
 }
@@ -687,11 +685,9 @@ vc4_complete_exec(struct drm_device *dev, struct vc4_exec_info *exec)
         struct vc4_dev *vc4 = to_vc4_dev(dev);
         unsigned i;
 
-        /* Need the struct lock for drm_gem_object_unreference(). */
-        mutex_lock(&dev->struct_mutex);
         if (exec->bo) {
                 for (i = 0; i < exec->bo_count; i++)
-                        drm_gem_object_unreference(&exec->bo[i]->base);
+                        drm_gem_object_unreference_unlocked(&exec->bo[i]->base);
                 kfree(exec->bo);
         }
 
@@ -699,9 +695,8 @@ vc4_complete_exec(struct drm_device *dev, struct vc4_exec_info *exec)
                 struct vc4_bo *bo = list_first_entry(&exec->unref_list,
                                                      struct vc4_bo, unref_head);
                 list_del(&bo->unref_head);
-                drm_gem_object_unreference(&bo->base.base);
+                drm_gem_object_unreference_unlocked(&bo->base.base);
         }
-        mutex_unlock(&dev->struct_mutex);
 
         mutex_lock(&vc4->power_lock);
         if (--vc4->power_refcount == 0)
diff --git a/drivers/gpu/drm/vc4/vc4_hdmi.c b/drivers/gpu/drm/vc4/vc4_hdmi.c
index fd2644d231ff..4452f3631cac 100644
--- a/drivers/gpu/drm/vc4/vc4_hdmi.c
+++ b/drivers/gpu/drm/vc4/vc4_hdmi.c
@@ -208,14 +208,6 @@ static int vc4_hdmi_connector_get_modes(struct drm_connector *connector)
         return ret;
 }
 
-static struct drm_encoder *
-vc4_hdmi_connector_best_encoder(struct drm_connector *connector)
-{
-        struct vc4_hdmi_connector *hdmi_connector =
-                to_vc4_hdmi_connector(connector);
-        return hdmi_connector->encoder;
-}
-
 static const struct drm_connector_funcs vc4_hdmi_connector_funcs = {
         .dpms = drm_atomic_helper_connector_dpms,
         .detect = vc4_hdmi_connector_detect,
@@ -228,7 +220,6 @@ static const struct drm_connector_funcs vc4_hdmi_connector_funcs = {
 
 static const struct drm_connector_helper_funcs vc4_hdmi_connector_helper_funcs = {
         .get_modes = vc4_hdmi_connector_get_modes,
-        .best_encoder = vc4_hdmi_connector_best_encoder,
 };
 
 static struct drm_connector *vc4_hdmi_connector_init(struct drm_device *dev,
@@ -465,12 +456,6 @@ static int vc4_hdmi_bind(struct device *dev, struct device *master, void *data)
         if (IS_ERR(hdmi->hd_regs))
                 return PTR_ERR(hdmi->hd_regs);
 
-        ddc_node = of_parse_phandle(dev->of_node, "ddc", 0);
-        if (!ddc_node) {
-                DRM_ERROR("Failed to find ddc node in device tree\n");
-                return -ENODEV;
-        }
-
         hdmi->pixel_clock = devm_clk_get(dev, "pixel");
         if (IS_ERR(hdmi->pixel_clock)) {
                 DRM_ERROR("Failed to get pixel clock\n");
@@ -482,7 +467,14 @@ static int vc4_hdmi_bind(struct device *dev, struct device *master, void *data)
                 return PTR_ERR(hdmi->hsm_clock);
         }
 
+        ddc_node = of_parse_phandle(dev->of_node, "ddc", 0);
+        if (!ddc_node) {
+                DRM_ERROR("Failed to find ddc node in device tree\n");
+                return -ENODEV;
+        }
+
         hdmi->ddc = of_find_i2c_adapter_by_node(ddc_node);
+        of_node_put(ddc_node);
         if (!hdmi->ddc) {
                 DRM_DEBUG("Failed to get ddc i2c adapter by node\n");
                 return -EPROBE_DEFER;
diff --git a/drivers/gpu/drm/vc4/vc4_kms.c b/drivers/gpu/drm/vc4/vc4_kms.c
index 861a623bc185..4ac894d993cd 100644
--- a/drivers/gpu/drm/vc4/vc4_kms.c
+++ b/drivers/gpu/drm/vc4/vc4_kms.c
@@ -26,8 +26,7 @@ static void vc4_output_poll_changed(struct drm_device *dev)
 {
         struct vc4_dev *vc4 = to_vc4_dev(dev);
 
-        if (vc4->fbdev)
-                drm_fbdev_cma_hotplug_event(vc4->fbdev);
+        drm_fbdev_cma_hotplug_event(vc4->fbdev);
 }
 
 struct vc4_commit {
@@ -111,6 +110,8 @@ static int vc4_atomic_commit(struct drm_device *dev,
         int i;
         uint64_t wait_seqno = 0;
         struct vc4_commit *c;
+        struct drm_plane *plane;
+        struct drm_plane_state *new_state;
 
         c = commit_init(state);
         if (!c)
@@ -138,13 +139,7 @@ static int vc4_atomic_commit(struct drm_device *dev,
                 return ret;
         }
 
-        for (i = 0; i < dev->mode_config.num_total_plane; i++) {
-                struct drm_plane *plane = state->planes[i];
-                struct drm_plane_state *new_state = state->plane_states[i];
-
-                if (!plane)
-                        continue;
-
+        for_each_plane_in_state(state, plane, new_state, i) {
                 if ((plane->state->fb != new_state->fb) && new_state->fb) {
                         struct drm_gem_cma_object *cma_bo =
                                 drm_fb_cma_get_gem_obj(new_state->fb, 0);
@@ -160,7 +155,7 @@ static int vc4_atomic_commit(struct drm_device *dev,
          * the software side now.
          */
 
-        drm_atomic_helper_swap_state(dev, state);
+        drm_atomic_helper_swap_state(state, true);
 
         /*
          * Everything below can be run asynchronously without the need to grab
diff --git a/drivers/gpu/drm/vc4/vc4_plane.c b/drivers/gpu/drm/vc4/vc4_plane.c
index 4037b52fde31..29e4b400e25e 100644
--- a/drivers/gpu/drm/vc4/vc4_plane.c
+++ b/drivers/gpu/drm/vc4/vc4_plane.c
@@ -94,6 +94,14 @@ static const struct hvs_format {
                 .pixel_order = HVS_PIXEL_ORDER_ABGR, .has_alpha = true,
         },
         {
+                .drm = DRM_FORMAT_ABGR8888, .hvs = HVS_PIXEL_FORMAT_RGBA8888,
+                .pixel_order = HVS_PIXEL_ORDER_ARGB, .has_alpha = true,
+        },
+        {
+                .drm = DRM_FORMAT_XBGR8888, .hvs = HVS_PIXEL_FORMAT_RGBA8888,
+                .pixel_order = HVS_PIXEL_ORDER_ARGB, .has_alpha = false,
+        },
+        {
                 .drm = DRM_FORMAT_RGB565, .hvs = HVS_PIXEL_FORMAT_RGB565,
                 .pixel_order = HVS_PIXEL_ORDER_XRGB, .has_alpha = false,
         },
@@ -690,9 +698,10 @@ u32 vc4_plane_write_dlist(struct drm_plane *plane, u32 __iomem *dlist)
         return vc4_state->dlist_count;
 }
 
-u32 vc4_plane_dlist_size(struct drm_plane_state *state)
+u32 vc4_plane_dlist_size(const struct drm_plane_state *state)
 {
-        struct vc4_plane_state *vc4_state = to_vc4_plane_state(state);
+        const struct vc4_plane_state *vc4_state =
+                container_of(state, typeof(*vc4_state), base);
 
         return vc4_state->dlist_count;
 }
diff --git a/drivers/gpu/drm/vc4/vc4_qpu_defines.h b/drivers/gpu/drm/vc4/vc4_qpu_defines.h
index d5c2f3c85ebb..f4e795a0d3f6 100644
--- a/drivers/gpu/drm/vc4/vc4_qpu_defines.h
+++ b/drivers/gpu/drm/vc4/vc4_qpu_defines.h
@@ -70,7 +70,7 @@ enum qpu_raddr {
         QPU_R_ELEM_QPU = 38,
         QPU_R_NOP,
         QPU_R_XY_PIXEL_COORD = 41,
-        QPU_R_MS_REV_FLAGS = 41,
+        QPU_R_MS_REV_FLAGS = 42,
         QPU_R_VPM = 48,
         QPU_R_VPM_LD_BUSY,
         QPU_R_VPM_LD_WAIT,
@@ -230,6 +230,15 @@ enum qpu_unpack_r4 {
 #define QPU_COND_MUL_SHIFT              46
 #define QPU_COND_MUL_MASK               QPU_MASK(48, 46)
 
+#define QPU_BRANCH_COND_SHIFT           52
+#define QPU_BRANCH_COND_MASK            QPU_MASK(55, 52)
+
+#define QPU_BRANCH_REL                  ((uint64_t)1 << 51)
+#define QPU_BRANCH_REG                  ((uint64_t)1 << 50)
+
+#define QPU_BRANCH_RADDR_A_SHIFT        45
+#define QPU_BRANCH_RADDR_A_MASK         QPU_MASK(49, 45)
+
 #define QPU_SF                          ((uint64_t)1 << 45)
 
 #define QPU_WADDR_ADD_SHIFT             38
@@ -261,4 +270,10 @@ enum qpu_unpack_r4 {
 #define QPU_OP_ADD_SHIFT                24
 #define QPU_OP_ADD_MASK                 QPU_MASK(28, 24)
 
+#define QPU_LOAD_IMM_SHIFT              0
+#define QPU_LOAD_IMM_MASK               QPU_MASK(31, 0)
+
+#define QPU_BRANCH_TARGET_SHIFT         0
+#define QPU_BRANCH_TARGET_MASK          QPU_MASK(31, 0)
+
 #endif /* VC4_QPU_DEFINES_H */
diff --git a/drivers/gpu/drm/vc4/vc4_regs.h b/drivers/gpu/drm/vc4/vc4_regs.h
index f99eece4cc97..160942a9180e 100644
--- a/drivers/gpu/drm/vc4/vc4_regs.h
+++ b/drivers/gpu/drm/vc4/vc4_regs.h
@@ -366,7 +366,6 @@
 # define SCALER_DISPBKGND_FILL                  BIT(24)
 
 #define SCALER_DISPSTAT0                        0x00000048
-#define SCALER_DISPBASE0                        0x0000004c
 # define SCALER_DISPSTATX_MODE_MASK             VC4_MASK(31, 30)
 # define SCALER_DISPSTATX_MODE_SHIFT            30
 # define SCALER_DISPSTATX_MODE_DISABLED         0
@@ -375,6 +374,24 @@
 # define SCALER_DISPSTATX_MODE_EOF              3
 # define SCALER_DISPSTATX_FULL                  BIT(29)
 # define SCALER_DISPSTATX_EMPTY                 BIT(28)
+# define SCALER_DISPSTATX_FRAME_COUNT_MASK      VC4_MASK(17, 12)
+# define SCALER_DISPSTATX_FRAME_COUNT_SHIFT     12
+# define SCALER_DISPSTATX_LINE_MASK             VC4_MASK(11, 0)
+# define SCALER_DISPSTATX_LINE_SHIFT            0
+
+#define SCALER_DISPBASE0                        0x0000004c
+/* Last pixel in the COB (display FIFO memory) allocated to this HVS
+ * channel.  Must be 4-pixel aligned (and thus 4 pixels less than the
+ * next COB base).
+ */
+# define SCALER_DISPBASEX_TOP_MASK              VC4_MASK(31, 16)
+# define SCALER_DISPBASEX_TOP_SHIFT             16
+/* First pixel in the COB (display FIFO memory) allocated to this HVS
+ * channel.  Must be 4-pixel aligned.
+ */
+# define SCALER_DISPBASEX_BASE_MASK             VC4_MASK(15, 0)
+# define SCALER_DISPBASEX_BASE_SHIFT            0
+
 #define SCALER_DISPCTRL1                        0x00000050
 #define SCALER_DISPBKGND1                       0x00000054
 #define SCALER_DISPBKGNDX(x)                    (SCALER_DISPBKGND0 +        \
@@ -385,6 +402,9 @@
                                                  (x) * (SCALER_DISPSTAT1 - \
                                                         SCALER_DISPSTAT0))
 #define SCALER_DISPBASE1                        0x0000005c
+#define SCALER_DISPBASEX(x)                     (SCALER_DISPBASE0 +        \
+                                                 (x) * (SCALER_DISPBASE1 - \
+                                                        SCALER_DISPBASE0))
 #define SCALER_DISPCTRL2                        0x00000060
 #define SCALER_DISPCTRLX(x)                     (SCALER_DISPCTRL0 +        \
                                                  (x) * (SCALER_DISPCTRL1 - \
diff --git a/drivers/gpu/drm/vc4/vc4_validate.c b/drivers/gpu/drm/vc4/vc4_validate.c
index 24c2c746e8f3..9ce1d0adf882 100644
--- a/drivers/gpu/drm/vc4/vc4_validate.c
+++ b/drivers/gpu/drm/vc4/vc4_validate.c
@@ -802,7 +802,7 @@ validate_gl_shader_rec(struct drm_device *dev,
                 uint32_t src_offset = *(uint32_t *)(pkt_u + o);
                 uint32_t *texture_handles_u;
                 void *uniform_data_u;
-                uint32_t tex;
+                uint32_t tex, uni;
 
                 *(uint32_t *)(pkt_v + o) = bo[i]->paddr + src_offset;
 
@@ -840,6 +840,17 @@ validate_gl_shader_rec(struct drm_device *dev,
                         }
                 }
 
+                /* Fill in the uniform slots that need this shader's
+                 * start-of-uniforms address (used for resetting the uniform
+                 * stream in the presence of control flow).
+                 */
+                for (uni = 0;
+                     uni < validated_shader->num_uniform_addr_offsets;
+                     uni++) {
+                        uint32_t o = validated_shader->uniform_addr_offsets[uni];
+                        ((uint32_t *)exec->uniforms_v)[o] = exec->uniforms_p;
+                }
+
                 *(uint32_t *)(pkt_v + o + 4) = exec->uniforms_p;
 
                 exec->uniforms_u += validated_shader->uniforms_src_size;
diff --git a/drivers/gpu/drm/vc4/vc4_validate_shaders.c b/drivers/gpu/drm/vc4/vc4_validate_shaders.c
index f67124b4c534..46527e989ce3 100644
--- a/drivers/gpu/drm/vc4/vc4_validate_shaders.c
+++ b/drivers/gpu/drm/vc4/vc4_validate_shaders.c
@@ -39,7 +39,17 @@
 #include "vc4_drv.h"
 #include "vc4_qpu_defines.h"
 
+#define LIVE_REG_COUNT (32 + 32 + 4)
+
 struct vc4_shader_validation_state {
+        /* Current IP being validated. */
+        uint32_t ip;
+
+        /* IP at the end of the BO, do not read shader[max_ip] */
+        uint32_t max_ip;
+
+        uint64_t *shader;
+
         struct vc4_texture_sample_info tmu_setup[2];
         int tmu_write_count[2];
 
@@ -49,8 +59,30 @@ struct vc4_shader_validation_state {
          *
          * This is used for the validation of direct address memory reads.
          */
-        uint32_t live_min_clamp_offsets[32 + 32 + 4];
-        bool live_max_clamp_regs[32 + 32 + 4];
+        uint32_t live_min_clamp_offsets[LIVE_REG_COUNT];
+        bool live_max_clamp_regs[LIVE_REG_COUNT];
+        uint32_t live_immediates[LIVE_REG_COUNT];
+
+        /* Bitfield of which IPs are used as branch targets.
+         *
+         * Used for validation that the uniform stream is updated at the right
+         * points and clearing the texturing/clamping state.
+         */
+        unsigned long *branch_targets;
+
+        /* Set when entering a basic block, and cleared when the uniform
+         * address update is found.  This is used to make sure that we don't
+         * read uniforms when the address is undefined.
+         */
+        bool needs_uniform_address_update;
+
+        /* Set when we find a backwards branch.  If the branch is backwards,
+         * the taraget is probably doing an address reset to read uniforms,
+         * and so we need to be sure that a uniforms address is present in the
+         * stream, even if the shader didn't need to read uniforms in later
+         * basic blocks.
+         */
+        bool needs_uniform_address_for_loop;
 };
 
 static uint32_t
@@ -129,11 +161,11 @@ record_texture_sample(struct vc4_validated_shader_info *validated_shader,
 }
 
 static bool
-check_tmu_write(uint64_t inst,
-                struct vc4_validated_shader_info *validated_shader,
+check_tmu_write(struct vc4_validated_shader_info *validated_shader,
                 struct vc4_shader_validation_state *validation_state,
                 bool is_mul)
 {
+        uint64_t inst = validation_state->shader[validation_state->ip];
         uint32_t waddr = (is_mul ?
                           QPU_GET_FIELD(inst, QPU_WADDR_MUL) :
                           QPU_GET_FIELD(inst, QPU_WADDR_ADD));
@@ -162,7 +194,7 @@ check_tmu_write(uint64_t inst,
                         return false;
                 }
 
-                /* We assert that the the clamped address is the first
+                /* We assert that the clamped address is the first
                  * argument, and the UBO base address is the second argument.
                  * This is arbitrary, but simpler than supporting flipping the
                  * two either way.
@@ -212,8 +244,14 @@ check_tmu_write(uint64_t inst,
         /* Since direct uses a RADDR uniform reference, it will get counted in
          * check_instruction_reads()
          */
-        if (!is_direct)
+        if (!is_direct) {
+                if (validation_state->needs_uniform_address_update) {
+                        DRM_ERROR("Texturing with undefined uniform address\n");
+                        return false;
+                }
+
                 validated_shader->uniforms_size += 4;
+        }
 
         if (submit) {
                 if (!record_texture_sample(validated_shader,
@@ -227,23 +265,138 @@ check_tmu_write(uint64_t inst,
         return true;
 }
 
+static bool require_uniform_address_uniform(struct vc4_validated_shader_info *validated_shader)
+{
+        uint32_t o = validated_shader->num_uniform_addr_offsets;
+        uint32_t num_uniforms = validated_shader->uniforms_size / 4;
+
+        validated_shader->uniform_addr_offsets =
+                krealloc(validated_shader->uniform_addr_offsets,
+                         (o + 1) *
+                         sizeof(*validated_shader->uniform_addr_offsets),
+                         GFP_KERNEL);
+        if (!validated_shader->uniform_addr_offsets)
+                return false;
+
+        validated_shader->uniform_addr_offsets[o] = num_uniforms;
+        validated_shader->num_uniform_addr_offsets++;
+
+        return true;
+}
+
 static bool
-check_reg_write(uint64_t inst,
-                struct vc4_validated_shader_info *validated_shader,
+validate_uniform_address_write(struct vc4_validated_shader_info *validated_shader,
+                               struct vc4_shader_validation_state *validation_state,
+                               bool is_mul)
+{
+        uint64_t inst = validation_state->shader[validation_state->ip];
+        u32 add_b = QPU_GET_FIELD(inst, QPU_ADD_B);
+        u32 raddr_a = QPU_GET_FIELD(inst, QPU_RADDR_A);
+        u32 raddr_b = QPU_GET_FIELD(inst, QPU_RADDR_B);
+        u32 add_lri = raddr_add_a_to_live_reg_index(inst);
+        /* We want our reset to be pointing at whatever uniform follows the
+         * uniforms base address.
+         */
+        u32 expected_offset = validated_shader->uniforms_size + 4;
+
+        /* We only support absolute uniform address changes, and we
+         * require that they be in the current basic block before any
+         * of its uniform reads.
+         *
+         * One could potentially emit more efficient QPU code, by
+         * noticing that (say) an if statement does uniform control
+         * flow for all threads and that the if reads the same number
+         * of uniforms on each side.  However, this scheme is easy to
+         * validate so it's all we allow for now.
+         */
+
+        if (QPU_GET_FIELD(inst, QPU_SIG) != QPU_SIG_NONE) {
+                DRM_ERROR("uniforms address change must be "
+                          "normal math\n");
+                return false;
+        }
+
+        if (is_mul || QPU_GET_FIELD(inst, QPU_OP_ADD) != QPU_A_ADD) {
+                DRM_ERROR("Uniform address reset must be an ADD.\n");
+                return false;
+        }
+
+        if (QPU_GET_FIELD(inst, QPU_COND_ADD) != QPU_COND_ALWAYS) {
+                DRM_ERROR("Uniform address reset must be unconditional.\n");
+                return false;
+        }
+
+        if (QPU_GET_FIELD(inst, QPU_PACK) != QPU_PACK_A_NOP &&
+            !(inst & QPU_PM)) {
+                DRM_ERROR("No packing allowed on uniforms reset\n");
+                return false;
+        }
+
+        if (add_lri == -1) {
+                DRM_ERROR("First argument of uniform address write must be "
+                          "an immediate value.\n");
+                return false;
+        }
+
+        if (validation_state->live_immediates[add_lri] != expected_offset) {
+                DRM_ERROR("Resetting uniforms with offset %db instead of %db\n",
+                          validation_state->live_immediates[add_lri],
+                          expected_offset);
+                return false;
+        }
+
+        if (!(add_b == QPU_MUX_A && raddr_a == QPU_R_UNIF) &&
+            !(add_b == QPU_MUX_B && raddr_b == QPU_R_UNIF)) {
+                DRM_ERROR("Second argument of uniform address write must be "
+                          "a uniform.\n");
+                return false;
+        }
+
+        validation_state->needs_uniform_address_update = false;
+        validation_state->needs_uniform_address_for_loop = false;
+        return require_uniform_address_uniform(validated_shader);
+}
+
+static bool
+check_reg_write(struct vc4_validated_shader_info *validated_shader,
                 struct vc4_shader_validation_state *validation_state,
                 bool is_mul)
 {
+        uint64_t inst = validation_state->shader[validation_state->ip];
         uint32_t waddr = (is_mul ?
                           QPU_GET_FIELD(inst, QPU_WADDR_MUL) :
                           QPU_GET_FIELD(inst, QPU_WADDR_ADD));
+        uint32_t sig = QPU_GET_FIELD(inst, QPU_SIG);
+        bool ws = inst & QPU_WS;
+        bool is_b = is_mul ^ ws;
+        u32 lri = waddr_to_live_reg_index(waddr, is_b);
+
+        if (lri != -1) {
+                uint32_t cond_add = QPU_GET_FIELD(inst, QPU_COND_ADD);
+                uint32_t cond_mul = QPU_GET_FIELD(inst, QPU_COND_MUL);
+
+                if (sig == QPU_SIG_LOAD_IMM &&
+                    QPU_GET_FIELD(inst, QPU_PACK) == QPU_PACK_A_NOP &&
+                    ((is_mul && cond_mul == QPU_COND_ALWAYS) ||
+                     (!is_mul && cond_add == QPU_COND_ALWAYS))) {
+                        validation_state->live_immediates[lri] =
+                                QPU_GET_FIELD(inst, QPU_LOAD_IMM);
+                } else {
+                        validation_state->live_immediates[lri] = ~0;
+                }
+        }
 
         switch (waddr) {
         case QPU_W_UNIFORMS_ADDRESS:
-                /* XXX: We'll probably need to support this for reladdr, but
-                 * it's definitely a security-related one.
-                 */
-                DRM_ERROR("uniforms address load unsupported\n");
-                return false;
+                if (is_b) {
+                        DRM_ERROR("relative uniforms address change "
+                                  "unsupported\n");
+                        return false;
+                }
+
+                return validate_uniform_address_write(validated_shader,
+                                                      validation_state,
+                                                      is_mul);
 
         case QPU_W_TLB_COLOR_MS:
         case QPU_W_TLB_COLOR_ALL:
@@ -261,7 +414,7 @@ check_reg_write(uint64_t inst,
         case QPU_W_TMU1_T:
         case QPU_W_TMU1_R:
         case QPU_W_TMU1_B:
-                return check_tmu_write(inst, validated_shader, validation_state,
+                return check_tmu_write(validated_shader, validation_state,
                                        is_mul);
 
         case QPU_W_HOST_INT:
@@ -294,10 +447,10 @@ check_reg_write(uint64_t inst,
 }
 
 static void
-track_live_clamps(uint64_t inst,
-                  struct vc4_validated_shader_info *validated_shader,
+track_live_clamps(struct vc4_validated_shader_info *validated_shader,
                   struct vc4_shader_validation_state *validation_state)
 {
+        uint64_t inst = validation_state->shader[validation_state->ip];
         uint32_t op_add = QPU_GET_FIELD(inst, QPU_OP_ADD);
         uint32_t waddr_add = QPU_GET_FIELD(inst, QPU_WADDR_ADD);
         uint32_t waddr_mul = QPU_GET_FIELD(inst, QPU_WADDR_MUL);
@@ -369,10 +522,10 @@ track_live_clamps(uint64_t inst,
 }
 
 static bool
-check_instruction_writes(uint64_t inst,
-                         struct vc4_validated_shader_info *validated_shader,
+check_instruction_writes(struct vc4_validated_shader_info *validated_shader,
                          struct vc4_shader_validation_state *validation_state)
 {
+        uint64_t inst = validation_state->shader[validation_state->ip];
         uint32_t waddr_add = QPU_GET_FIELD(inst, QPU_WADDR_ADD);
         uint32_t waddr_mul = QPU_GET_FIELD(inst, QPU_WADDR_MUL);
         bool ok;
@@ -382,20 +535,44 @@ check_instruction_writes(uint64_t inst,
                 return false;
         }
 
-        ok = (check_reg_write(inst, validated_shader, validation_state,
-                              false) &&
-              check_reg_write(inst, validated_shader, validation_state,
-                              true));
+        ok = (check_reg_write(validated_shader, validation_state, false) &&
+              check_reg_write(validated_shader, validation_state, true));
 
-        track_live_clamps(inst, validated_shader, validation_state);
+        track_live_clamps(validated_shader, validation_state);
 
         return ok;
 }
 
 static bool
-check_instruction_reads(uint64_t inst,
-                        struct vc4_validated_shader_info *validated_shader)
+check_branch(uint64_t inst,
+             struct vc4_validated_shader_info *validated_shader,
+             struct vc4_shader_validation_state *validation_state,
+             int ip)
+{
+        int32_t branch_imm = QPU_GET_FIELD(inst, QPU_BRANCH_TARGET);
+        uint32_t waddr_add = QPU_GET_FIELD(inst, QPU_WADDR_ADD);
+        uint32_t waddr_mul = QPU_GET_FIELD(inst, QPU_WADDR_MUL);
+
+        if ((int)branch_imm < 0)
+                validation_state->needs_uniform_address_for_loop = true;
+
+        /* We don't want to have to worry about validation of this, and
+         * there's no need for it.
+         */
+        if (waddr_add != QPU_W_NOP || waddr_mul != QPU_W_NOP) {
+                DRM_ERROR("branch instruction at %d wrote a register.\n",
+                          validation_state->ip);
+                return false;
+        }
+
+        return true;
+}
+
+static bool
+check_instruction_reads(struct vc4_validated_shader_info *validated_shader,
+                        struct vc4_shader_validation_state *validation_state)
 {
+        uint64_t inst = validation_state->shader[validation_state->ip];
         uint32_t raddr_a = QPU_GET_FIELD(inst, QPU_RADDR_A);
         uint32_t raddr_b = QPU_GET_FIELD(inst, QPU_RADDR_B);
         uint32_t sig = QPU_GET_FIELD(inst, QPU_SIG);
@@ -407,40 +584,204 @@ check_instruction_reads(uint64_t inst,
                  * already be OOM.
                  */
                 validated_shader->uniforms_size += 4;
+
+                if (validation_state->needs_uniform_address_update) {
+                        DRM_ERROR("Uniform read with undefined uniform "
+                                  "address\n");
+                        return false;
+                }
+        }
+
+        return true;
+}
+
+/* Make sure that all branches are absolute and point within the shader, and
+ * note their targets for later.
+ */
+static bool
+vc4_validate_branches(struct vc4_shader_validation_state *validation_state)
+{
+        uint32_t max_branch_target = 0;
+        bool found_shader_end = false;
+        int ip;
+        int shader_end_ip = 0;
+        int last_branch = -2;
+
+        for (ip = 0; ip < validation_state->max_ip; ip++) {
+                uint64_t inst = validation_state->shader[ip];
+                int32_t branch_imm = QPU_GET_FIELD(inst, QPU_BRANCH_TARGET);
+                uint32_t sig = QPU_GET_FIELD(inst, QPU_SIG);
+                uint32_t after_delay_ip = ip + 4;
+                uint32_t branch_target_ip;
+
+                if (sig == QPU_SIG_PROG_END) {
+                        shader_end_ip = ip;
+                        found_shader_end = true;
+                        continue;
+                }
+
+                if (sig != QPU_SIG_BRANCH)
+                        continue;
+
+                if (ip - last_branch < 4) {
+                        DRM_ERROR("Branch at %d during delay slots\n", ip);
+                        return false;
+                }
+                last_branch = ip;
+
+                if (inst & QPU_BRANCH_REG) {
+                        DRM_ERROR("branching from register relative "
+                                  "not supported\n");
+                        return false;
+                }
+
+                if (!(inst & QPU_BRANCH_REL)) {
+                        DRM_ERROR("relative branching required\n");
+                        return false;
+                }
+
+                /* The actual branch target is the instruction after the delay
+                 * slots, plus whatever byte offset is in the low 32 bits of
+                 * the instruction.  Make sure we're not branching beyond the
+                 * end of the shader object.
+                 */
+                if (branch_imm % sizeof(inst) != 0) {
+                        DRM_ERROR("branch target not aligned\n");
+                        return false;
+                }
+
+                branch_target_ip = after_delay_ip + (branch_imm >> 3);
+                if (branch_target_ip >= validation_state->max_ip) {
+                        DRM_ERROR("Branch at %d outside of shader (ip %d/%d)\n",
+                                  ip, branch_target_ip,
+                                  validation_state->max_ip);
+                        return false;
+                }
+                set_bit(branch_target_ip, validation_state->branch_targets);
+
+                /* Make sure that the non-branching path is also not outside
+                 * the shader.
+                 */
+                if (after_delay_ip >= validation_state->max_ip) {
+                        DRM_ERROR("Branch at %d continues past shader end "
+                                  "(%d/%d)\n",
+                                  ip, after_delay_ip, validation_state->max_ip);
+                        return false;
+                }
+                set_bit(after_delay_ip, validation_state->branch_targets);
+                max_branch_target = max(max_branch_target, after_delay_ip);
+
+                /* There are two delay slots after program end is signaled
+                 * that are still executed, then we're finished.
+                 */
+                if (found_shader_end && ip == shader_end_ip + 2)
+                        break;
+        }
+
+        if (max_branch_target > shader_end_ip) {
+                DRM_ERROR("Branch landed after QPU_SIG_PROG_END");
+                return false;
         }
 
         return true;
 }
 
+/* Resets any known state for the shader, used when we may be branched to from
+ * multiple locations in the program (or at shader start).
+ */
+static void
+reset_validation_state(struct vc4_shader_validation_state *validation_state)
+{
+        int i;
+
+        for (i = 0; i < 8; i++)
+                validation_state->tmu_setup[i / 4].p_offset[i % 4] = ~0;
+
+        for (i = 0; i < LIVE_REG_COUNT; i++) {
+                validation_state->live_min_clamp_offsets[i] = ~0;
+                validation_state->live_max_clamp_regs[i] = false;
+                validation_state->live_immediates[i] = ~0;
+        }
+}
+
+static bool
+texturing_in_progress(struct vc4_shader_validation_state *validation_state)
+{
+        return (validation_state->tmu_write_count[0] != 0 ||
+                validation_state->tmu_write_count[1] != 0);
+}
+
+static bool
+vc4_handle_branch_target(struct vc4_shader_validation_state *validation_state)
+{
+        uint32_t ip = validation_state->ip;
+
+        if (!test_bit(ip, validation_state->branch_targets))
+                return true;
+
+        if (texturing_in_progress(validation_state)) {
+                DRM_ERROR("Branch target landed during TMU setup\n");
+                return false;
+        }
+
+        /* Reset our live values tracking, since this instruction may have
+         * multiple predecessors.
+         *
+         * One could potentially do analysis to determine that, for
+         * example, all predecessors have a live max clamp in the same
+         * register, but we don't bother with that.
+         */
+        reset_validation_state(validation_state);
+
+        /* Since we've entered a basic block from potentially multiple
+         * predecessors, we need the uniforms address to be updated before any
+         * unforms are read.  We require that after any branch point, the next
+         * uniform to be loaded is a uniform address offset.  That uniform's
+         * offset will be marked by the uniform address register write
+         * validation, or a one-off the end-of-program check.
+         */
+        validation_state->needs_uniform_address_update = true;
+
+        return true;
+}
+
 struct vc4_validated_shader_info *
 vc4_validate_shader(struct drm_gem_cma_object *shader_obj)
 {
         bool found_shader_end = false;
         int shader_end_ip = 0;
-        uint32_t ip, max_ip;
-        uint64_t *shader;
-        struct vc4_validated_shader_info *validated_shader;
+        uint32_t ip;
+        struct vc4_validated_shader_info *validated_shader = NULL;
         struct vc4_shader_validation_state validation_state;
-        int i;
 
         memset(&validation_state, 0, sizeof(validation_state));
+        validation_state.shader = shader_obj->vaddr;
+        validation_state.max_ip = shader_obj->base.size / sizeof(uint64_t);
 
-        for (i = 0; i < 8; i++)
-                validation_state.tmu_setup[i / 4].p_offset[i % 4] = ~0;
-        for (i = 0; i < ARRAY_SIZE(validation_state.live_min_clamp_offsets); i++)
-                validation_state.live_min_clamp_offsets[i] = ~0;
+        reset_validation_state(&validation_state);
 
-        shader = shader_obj->vaddr;
-        max_ip = shader_obj->base.size / sizeof(uint64_t);
+        validation_state.branch_targets =
+                kcalloc(BITS_TO_LONGS(validation_state.max_ip),
+                        sizeof(unsigned long), GFP_KERNEL);
+        if (!validation_state.branch_targets)
+                goto fail;
 
         validated_shader = kcalloc(1, sizeof(*validated_shader), GFP_KERNEL);
         if (!validated_shader)
-                return NULL;
+                goto fail;
+
+        if (!vc4_validate_branches(&validation_state))
+                goto fail;
 
-        for (ip = 0; ip < max_ip; ip++) {
-                uint64_t inst = shader[ip];
+        for (ip = 0; ip < validation_state.max_ip; ip++) {
+                uint64_t inst = validation_state.shader[ip];
                 uint32_t sig = QPU_GET_FIELD(inst, QPU_SIG);
 
+                validation_state.ip = ip;
+
+                if (!vc4_handle_branch_target(&validation_state))
+                        goto fail;
+
                 switch (sig) {
                 case QPU_SIG_NONE:
                 case QPU_SIG_WAIT_FOR_SCOREBOARD:
@@ -450,13 +791,14 @@ vc4_validate_shader(struct drm_gem_cma_object *shader_obj)
                 case QPU_SIG_LOAD_TMU1:
                 case QPU_SIG_PROG_END:
                 case QPU_SIG_SMALL_IMM:
-                        if (!check_instruction_writes(inst, validated_shader,
+                        if (!check_instruction_writes(validated_shader,
                                                       &validation_state)) {
                                 DRM_ERROR("Bad write at ip %d\n", ip);
                                 goto fail;
                         }
 
-                        if (!check_instruction_reads(inst, validated_shader))
+                        if (!check_instruction_reads(validated_shader,
+                                                     &validation_state))
                                 goto fail;
 
                         if (sig == QPU_SIG_PROG_END) {
@@ -467,13 +809,18 @@ vc4_validate_shader(struct drm_gem_cma_object *shader_obj)
                         break;
 
                 case QPU_SIG_LOAD_IMM:
-                        if (!check_instruction_writes(inst, validated_shader,
+                        if (!check_instruction_writes(validated_shader,
                                                       &validation_state)) {
                                 DRM_ERROR("Bad LOAD_IMM write at ip %d\n", ip);
                                 goto fail;
                         }
                         break;
 
+                case QPU_SIG_BRANCH:
+                        if (!check_branch(inst, validated_shader,
+                                          &validation_state, ip))
+                                goto fail;
+                        break;
                 default:
                         DRM_ERROR("Unsupported QPU signal %d at "
                                   "instruction %d\n", sig, ip);
@@ -487,13 +834,28 @@ vc4_validate_shader(struct drm_gem_cma_object *shader_obj)
                         break;
         }
 
-        if (ip == max_ip) {
+        if (ip == validation_state.max_ip) {
                 DRM_ERROR("shader failed to terminate before "
                           "shader BO end at %zd\n",
                           shader_obj->base.size);
                 goto fail;
         }
 
+        /* If we did a backwards branch and we haven't emitted a uniforms
+         * reset since then, we still need the uniforms stream to have the
+         * uniforms address available so that the backwards branch can do its
+         * uniforms reset.
+         *
+         * We could potentially prove that the backwards branch doesn't
+         * contain any uses of uniforms until program exit, but that doesn't
+         * seem to be worth the trouble.
+         */
+        if (validation_state.needs_uniform_address_for_loop) {
+                if (!require_uniform_address_uniform(validated_shader))
+                        goto fail;
+                validated_shader->uniforms_size += 4;
+        }
+
         /* Again, no chance of integer overflow here because the worst case
          * scenario is 8 bytes of uniforms plus handles per 8-byte
          * instruction.
@@ -502,9 +864,12 @@ vc4_validate_shader(struct drm_gem_cma_object *shader_obj)
                 (validated_shader->uniforms_size +
                  4 * validated_shader->num_texture_samples);
 
+        kfree(validation_state.branch_targets);
+
         return validated_shader;
 
 fail:
+        kfree(validation_state.branch_targets);
         if (validated_shader) {
                 kfree(validated_shader->texture_samples);
                 kfree(validated_shader);
diff --git a/drivers/gpu/drm/vgem/Makefile b/drivers/gpu/drm/vgem/Makefile
index 3f4c7b842028..bfcdea1330e6 100644
--- a/drivers/gpu/drm/vgem/Makefile
+++ b/drivers/gpu/drm/vgem/Makefile
@@ -1,4 +1,4 @@
 ccflags-y := -Iinclude/drm
-vgem-y := vgem_drv.o
+vgem-y := vgem_drv.o vgem_fence.o
 
 obj-$(CONFIG_DRM_VGEM)  += vgem.o
diff --git a/drivers/gpu/drm/vgem/vgem_drv.c b/drivers/gpu/drm/vgem/vgem_drv.c
index 341f9be3dde6..c15bafb06665 100644
--- a/drivers/gpu/drm/vgem/vgem_drv.c
+++ b/drivers/gpu/drm/vgem/vgem_drv.c
@@ -42,81 +42,38 @@
 #define DRIVER_MAJOR    1
 #define DRIVER_MINOR    0
 
-void vgem_gem_put_pages(struct drm_vgem_gem_object *obj)
-{
-        drm_gem_put_pages(&obj->base, obj->pages, false, false);
-        obj->pages = NULL;
-}
-
 static void vgem_gem_free_object(struct drm_gem_object *obj)
 {
         struct drm_vgem_gem_object *vgem_obj = to_vgem_bo(obj);
 
-        drm_gem_free_mmap_offset(obj);
-
-        if (vgem_obj->use_dma_buf && obj->dma_buf) {
-                dma_buf_put(obj->dma_buf);
-                obj->dma_buf = NULL;
-        }
-
         drm_gem_object_release(obj);
-
-        if (vgem_obj->pages)
-                vgem_gem_put_pages(vgem_obj);
-
-        vgem_obj->pages = NULL;
-
         kfree(vgem_obj);
 }
 
-int vgem_gem_get_pages(struct drm_vgem_gem_object *obj)
-{
-        struct page **pages;
-
-        if (obj->pages || obj->use_dma_buf)
-                return 0;
-
-        pages = drm_gem_get_pages(&obj->base);
-        if (IS_ERR(pages)) {
-                return PTR_ERR(pages);
-        }
-
-        obj->pages = pages;
-
-        return 0;
-}
-
 static int vgem_gem_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
 {
         struct drm_vgem_gem_object *obj = vma->vm_private_data;
-        loff_t num_pages;
-        pgoff_t page_offset;
-        int ret;
-
         /* We don't use vmf->pgoff since that has the fake offset */
-        page_offset = ((unsigned long)vmf->virtual_address - vma->vm_start) >>
-                PAGE_SHIFT;
-
-        num_pages = DIV_ROUND_UP(obj->base.size, PAGE_SIZE);
-
-        if (page_offset > num_pages)
-                return VM_FAULT_SIGBUS;
-
-        ret = vm_insert_page(vma, (unsigned long)vmf->virtual_address,
-                             obj->pages[page_offset]);
-        switch (ret) {
-        case 0:
-                return VM_FAULT_NOPAGE;
-        case -ENOMEM:
-                return VM_FAULT_OOM;
-        case -EBUSY:
-                return VM_FAULT_RETRY;
-        case -EFAULT:
-        case -EINVAL:
-                return VM_FAULT_SIGBUS;
-        default:
-                WARN_ON(1);
-                return VM_FAULT_SIGBUS;
+        unsigned long vaddr = (unsigned long)vmf->virtual_address;
+        struct page *page;
+
+        page = shmem_read_mapping_page(file_inode(obj->base.filp)->i_mapping,
+                                       (vaddr - vma->vm_start) >> PAGE_SHIFT);
+        if (!IS_ERR(page)) {
+                vmf->page = page;
+                return 0;
+        } else switch (PTR_ERR(page)) {
+                case -ENOSPC:
+                case -ENOMEM:
+                        return VM_FAULT_OOM;
+                case -EBUSY:
+                        return VM_FAULT_RETRY;
+                case -EFAULT:
+                case -EINVAL:
+                        return VM_FAULT_SIGBUS;
+                default:
+                        WARN_ON_ONCE(PTR_ERR(page));
+                        return VM_FAULT_SIGBUS;
         }
 }
 
@@ -126,6 +83,34 @@ static const struct vm_operations_struct vgem_gem_vm_ops = {
         .close = drm_gem_vm_close,
 };
 
+static int vgem_open(struct drm_device *dev, struct drm_file *file)
+{
+        struct vgem_file *vfile;
+        int ret;
+
+        vfile = kzalloc(sizeof(*vfile), GFP_KERNEL);
+        if (!vfile)
+                return -ENOMEM;
+
+        file->driver_priv = vfile;
+
+        ret = vgem_fence_open(vfile);
+        if (ret) {
+                kfree(vfile);
+                return ret;
+        }
+
+        return 0;
+}
+
+static void vgem_preclose(struct drm_device *dev, struct drm_file *file)
+{
+        struct vgem_file *vfile = file->driver_priv;
+
+        vgem_fence_close(vfile);
+        kfree(vfile);
+}
+
 /* ioctls */
 
 static struct drm_gem_object *vgem_gem_create(struct drm_device *dev,
@@ -134,57 +119,43 @@ static struct drm_gem_object *vgem_gem_create(struct drm_device *dev,
                                               unsigned long size)
 {
         struct drm_vgem_gem_object *obj;
-        struct drm_gem_object *gem_object;
-        int err;
-
-        size = roundup(size, PAGE_SIZE);
+        int ret;
 
         obj = kzalloc(sizeof(*obj), GFP_KERNEL);
         if (!obj)
                 return ERR_PTR(-ENOMEM);
 
-        gem_object = &obj->base;
-
-        err = drm_gem_object_init(dev, gem_object, size);
-        if (err)
-                goto out;
-
-        err = vgem_gem_get_pages(obj);
-        if (err)
-                goto out;
-
-        err = drm_gem_handle_create(file, gem_object, handle);
-        if (err)
-                goto handle_out;
+        ret = drm_gem_object_init(dev, &obj->base, roundup(size, PAGE_SIZE));
+        if (ret)
+                goto err_free;
 
-        drm_gem_object_unreference_unlocked(gem_object);
+        ret = drm_gem_handle_create(file, &obj->base, handle);
+        drm_gem_object_unreference_unlocked(&obj->base);
+        if (ret)
+                goto err;
 
-        return gem_object;
+        return &obj->base;
 
-handle_out:
-        drm_gem_object_release(gem_object);
-out:
+err_free:
         kfree(obj);
-        return ERR_PTR(err);
+err:
+        return ERR_PTR(ret);
 }
 
 static int vgem_gem_dumb_create(struct drm_file *file, struct drm_device *dev,
                                 struct drm_mode_create_dumb *args)
 {
         struct drm_gem_object *gem_object;
-        uint64_t size;
-        uint64_t pitch = args->width * DIV_ROUND_UP(args->bpp, 8);
+        u64 pitch, size;
 
+        pitch = args->width * DIV_ROUND_UP(args->bpp, 8);
         size = args->height * pitch;
         if (size == 0)
                 return -EINVAL;
 
         gem_object = vgem_gem_create(dev, file, &args->handle, size);
-
-        if (IS_ERR(gem_object)) {
-                DRM_DEBUG_DRIVER("object creation failed\n");
+        if (IS_ERR(gem_object))
                 return PTR_ERR(gem_object);
-        }
 
         args->size = gem_object->size;
         args->pitch = pitch;
@@ -194,26 +165,26 @@ static int vgem_gem_dumb_create(struct drm_file *file, struct drm_device *dev,
         return 0;
 }
 
-int vgem_gem_dumb_map(struct drm_file *file, struct drm_device *dev,
-                      uint32_t handle, uint64_t *offset)
+static int vgem_gem_dumb_map(struct drm_file *file, struct drm_device *dev,
+                             uint32_t handle, uint64_t *offset)
 {
-        int ret = 0;
         struct drm_gem_object *obj;
+        int ret;
 
         obj = drm_gem_object_lookup(file, handle);
         if (!obj)
                 return -ENOENT;
 
+        if (!obj->filp) {
+                ret = -EINVAL;
+                goto unref;
+        }
+
         ret = drm_gem_create_mmap_offset(obj);
         if (ret)
                 goto unref;
 
-        BUG_ON(!obj->filp);
-
-        obj->filp->private_data = obj;
-
         *offset = drm_vma_node_offset_addr(&obj->vma_node);
-
 unref:
         drm_gem_object_unreference_unlocked(obj);
 
@@ -221,26 +192,134 @@ unref:
 }
 
 static struct drm_ioctl_desc vgem_ioctls[] = {
+        DRM_IOCTL_DEF_DRV(VGEM_FENCE_ATTACH, vgem_fence_attach_ioctl, DRM_AUTH|DRM_RENDER_ALLOW),
+        DRM_IOCTL_DEF_DRV(VGEM_FENCE_SIGNAL, vgem_fence_signal_ioctl, DRM_AUTH|DRM_RENDER_ALLOW),
 };
 
+static int vgem_mmap(struct file *filp, struct vm_area_struct *vma)
+{
+        unsigned long flags = vma->vm_flags;
+        int ret;
+
+        ret = drm_gem_mmap(filp, vma);
+        if (ret)
+                return ret;
+
+        /* Keep the WC mmaping set by drm_gem_mmap() but our pages
+         * are ordinary and not special.
+         */
+        vma->vm_flags = flags | VM_DONTEXPAND | VM_DONTDUMP;
+        return 0;
+}
+
 static const struct file_operations vgem_driver_fops = {
         .owner          = THIS_MODULE,
         .open           = drm_open,
-        .mmap           = drm_gem_mmap,
+        .mmap           = vgem_mmap,
         .poll           = drm_poll,
         .read           = drm_read,
         .unlocked_ioctl = drm_ioctl,
         .release        = drm_release,
 };
 
+static int vgem_prime_pin(struct drm_gem_object *obj)
+{
+        long n_pages = obj->size >> PAGE_SHIFT;
+        struct page **pages;
+
+        /* Flush the object from the CPU cache so that importers can rely
+         * on coherent indirect access via the exported dma-address.
+         */
+        pages = drm_gem_get_pages(obj);
+        if (IS_ERR(pages))
+                return PTR_ERR(pages);
+
+        drm_clflush_pages(pages, n_pages);
+        drm_gem_put_pages(obj, pages, true, false);
+
+        return 0;
+}
+
+static struct sg_table *vgem_prime_get_sg_table(struct drm_gem_object *obj)
+{
+        struct sg_table *st;
+        struct page **pages;
+
+        pages = drm_gem_get_pages(obj);
+        if (IS_ERR(pages))
+                return ERR_CAST(pages);
+
+        st = drm_prime_pages_to_sg(pages, obj->size >> PAGE_SHIFT);
+        drm_gem_put_pages(obj, pages, false, false);
+
+        return st;
+}
+
+static void *vgem_prime_vmap(struct drm_gem_object *obj)
+{
+        long n_pages = obj->size >> PAGE_SHIFT;
+        struct page **pages;
+        void *addr;
+
+        pages = drm_gem_get_pages(obj);
+        if (IS_ERR(pages))
+                return NULL;
+
+        addr = vmap(pages, n_pages, 0, pgprot_writecombine(PAGE_KERNEL));
+        drm_gem_put_pages(obj, pages, false, false);
+
+        return addr;
+}
+
+static void vgem_prime_vunmap(struct drm_gem_object *obj, void *vaddr)
+{
+        vunmap(vaddr);
+}
+
+static int vgem_prime_mmap(struct drm_gem_object *obj,
+                           struct vm_area_struct *vma)
+{
+        int ret;
+
+        if (obj->size < vma->vm_end - vma->vm_start)
+                return -EINVAL;
+
+        if (!obj->filp)
+                return -ENODEV;
+
+        ret = obj->filp->f_op->mmap(obj->filp, vma);
+        if (ret)
+                return ret;
+
+        fput(vma->vm_file);
+        vma->vm_file = get_file(obj->filp);
+        vma->vm_flags |= VM_DONTEXPAND | VM_DONTDUMP;
+        vma->vm_page_prot = pgprot_writecombine(vm_get_page_prot(vma->vm_flags));
+
+        return 0;
+}
+
 static struct drm_driver vgem_driver = {
-        .driver_features                = DRIVER_GEM,
-        .gem_free_object                = vgem_gem_free_object,
+        .driver_features                = DRIVER_GEM | DRIVER_PRIME,
+        .open                           = vgem_open,
+        .preclose                       = vgem_preclose,
+        .gem_free_object_unlocked       = vgem_gem_free_object,
         .gem_vm_ops                     = &vgem_gem_vm_ops,
         .ioctls                         = vgem_ioctls,
+        .num_ioctls                     = ARRAY_SIZE(vgem_ioctls),
         .fops                           = &vgem_driver_fops,
+
         .dumb_create                    = vgem_gem_dumb_create,
         .dumb_map_offset                = vgem_gem_dumb_map,
+
+        .prime_handle_to_fd = drm_gem_prime_handle_to_fd,
+        .gem_prime_pin = vgem_prime_pin,
+        .gem_prime_export = drm_gem_prime_export,
+        .gem_prime_get_sg_table = vgem_prime_get_sg_table,
+        .gem_prime_vmap = vgem_prime_vmap,
+        .gem_prime_vunmap = vgem_prime_vunmap,
+        .gem_prime_mmap = vgem_prime_mmap,
+
         .name   = DRIVER_NAME,
         .desc   = DRIVER_DESC,
         .date   = DRIVER_DATE,
@@ -248,7 +327,7 @@ static struct drm_driver vgem_driver = {
         .minor  = DRIVER_MINOR,
 };
 
-struct drm_device *vgem_device;
+static struct drm_device *vgem_device;
 
 static int __init vgem_init(void)
 {
@@ -260,10 +339,7 @@ static int __init vgem_init(void)
                 goto out;
         }
 
-        drm_dev_set_unique(vgem_device, "vgem");
-
         ret  = drm_dev_register(vgem_device, 0);
-
         if (ret)
                 goto out_unref;
 
@@ -285,5 +361,6 @@ module_init(vgem_init);
 module_exit(vgem_exit);
 
 MODULE_AUTHOR("Red Hat, Inc.");
+MODULE_AUTHOR("Intel Corporation");
 MODULE_DESCRIPTION(DRIVER_DESC);
 MODULE_LICENSE("GPL and additional rights");
diff --git a/drivers/gpu/drm/vgem/vgem_drv.h b/drivers/gpu/drm/vgem/vgem_drv.h
index e9f92f7ee275..1f8798ad329c 100644
--- a/drivers/gpu/drm/vgem/vgem_drv.h
+++ b/drivers/gpu/drm/vgem/vgem_drv.h
@@ -32,15 +32,25 @@
 #include <drm/drmP.h>
 #include <drm/drm_gem.h>
 
+#include <uapi/drm/vgem_drm.h>
+
+struct vgem_file {
+        struct idr fence_idr;
+        struct mutex fence_mutex;
+};
+
 #define to_vgem_bo(x) container_of(x, struct drm_vgem_gem_object, base)
 struct drm_vgem_gem_object {
         struct drm_gem_object base;
-        struct page **pages;
-        bool use_dma_buf;
 };
 
-/* vgem_drv.c */
-extern void vgem_gem_put_pages(struct drm_vgem_gem_object *obj);
-extern int vgem_gem_get_pages(struct drm_vgem_gem_object *obj);
+int vgem_fence_open(struct vgem_file *file);
+int vgem_fence_attach_ioctl(struct drm_device *dev,
+                            void *data,
+                            struct drm_file *file);
+int vgem_fence_signal_ioctl(struct drm_device *dev,
+                            void *data,
+                            struct drm_file *file);
+void vgem_fence_close(struct vgem_file *file);
 
 #endif
diff --git a/drivers/gpu/drm/vgem/vgem_fence.c b/drivers/gpu/drm/vgem/vgem_fence.c
new file mode 100644
index 000000000000..5c57c1ffa1f9
--- /dev/null
+++ b/drivers/gpu/drm/vgem/vgem_fence.c
@@ -0,0 +1,283 @@
+/*
+ * Copyright 2016 Intel Corporation
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software")
+ * to deal in the software without restriction, including without limitation
+ * on the rights to use, copy, modify, merge, publish, distribute, sub
+ * license, and/or sell copies of the Software, and to permit persons to whom
+ * them Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTIBILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES, OR OTHER LIABILITY, WHETHER
+ * IN AN ACTION OF CONTRACT, TORT, OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+#include <linux/dma-buf.h>
+#include <linux/reservation.h>
+
+#include "vgem_drv.h"
+
+#define VGEM_FENCE_TIMEOUT (10*HZ)
+
+struct vgem_fence {
+        struct fence base;
+        struct spinlock lock;
+        struct timer_list timer;
+};
+
+static const char *vgem_fence_get_driver_name(struct fence *fence)
+{
+        return "vgem";
+}
+
+static const char *vgem_fence_get_timeline_name(struct fence *fence)
+{
+        return "unbound";
+}
+
+static bool vgem_fence_signaled(struct fence *fence)
+{
+        return false;
+}
+
+static bool vgem_fence_enable_signaling(struct fence *fence)
+{
+        return true;
+}
+
+static void vgem_fence_release(struct fence *base)
+{
+        struct vgem_fence *fence = container_of(base, typeof(*fence), base);
+
+        del_timer_sync(&fence->timer);
+        fence_free(&fence->base);
+}
+
+static void vgem_fence_value_str(struct fence *fence, char *str, int size)
+{
+        snprintf(str, size, "%u", fence->seqno);
+}
+
+static void vgem_fence_timeline_value_str(struct fence *fence, char *str,
+                                          int size)
+{
+        snprintf(str, size, "%u", fence_is_signaled(fence) ? fence->seqno : 0);
+}
+
+static const struct fence_ops vgem_fence_ops = {
+        .get_driver_name = vgem_fence_get_driver_name,
+        .get_timeline_name = vgem_fence_get_timeline_name,
+        .enable_signaling = vgem_fence_enable_signaling,
+        .signaled = vgem_fence_signaled,
+        .wait = fence_default_wait,
+        .release = vgem_fence_release,
+
+        .fence_value_str = vgem_fence_value_str,
+        .timeline_value_str = vgem_fence_timeline_value_str,
+};
+
+static void vgem_fence_timeout(unsigned long data)
+{
+        struct vgem_fence *fence = (struct vgem_fence *)data;
+
+        fence_signal(&fence->base);
+}
+
+static struct fence *vgem_fence_create(struct vgem_file *vfile,
+                                       unsigned int flags)
+{
+        struct vgem_fence *fence;
+
+        fence = kzalloc(sizeof(*fence), GFP_KERNEL);
+        if (!fence)
+                return NULL;
+
+        spin_lock_init(&fence->lock);
+        fence_init(&fence->base, &vgem_fence_ops, &fence->lock,
+                   fence_context_alloc(1), 1);
+
+        setup_timer(&fence->timer, vgem_fence_timeout, (unsigned long)fence);
+
+        /* We force the fence to expire within 10s to prevent driver hangs */
+        mod_timer(&fence->timer, jiffies + VGEM_FENCE_TIMEOUT);
+
+        return &fence->base;
+}
+
+static int attach_dmabuf(struct drm_device *dev,
+                         struct drm_gem_object *obj)
+{
+        struct dma_buf *dmabuf;
+
+        if (obj->dma_buf)
+                return 0;
+
+        dmabuf = dev->driver->gem_prime_export(dev, obj, 0);
+        if (IS_ERR(dmabuf))
+                return PTR_ERR(dmabuf);
+
+        obj->dma_buf = dmabuf;
+        drm_gem_object_reference(obj);
+        return 0;
+}
+
+/*
+ * vgem_fence_attach_ioctl (DRM_IOCTL_VGEM_FENCE_ATTACH):
+ *
+ * Create and attach a fence to the vGEM handle. This fence is then exposed
+ * via the dma-buf reservation object and visible to consumers of the exported
+ * dma-buf. If the flags contain VGEM_FENCE_WRITE, the fence indicates the
+ * vGEM buffer is being written to by the client and is exposed as an exclusive
+ * fence, otherwise the fence indicates the client is current reading from the
+ * buffer and all future writes should wait for the client to signal its
+ * completion. Note that if a conflicting fence is already on the dma-buf (i.e.
+ * an exclusive fence when adding a read, or any fence when adding a write),
+ * -EBUSY is reported. Serialisation between operations should be handled
+ * by waiting upon the dma-buf.
+ *
+ * This returns the handle for the new fence that must be signaled within 10
+ * seconds (or otherwise it will automatically expire). See
+ * vgem_fence_signal_ioctl (DRM_IOCTL_VGEM_FENCE_SIGNAL).
+ *
+ * If the vGEM handle does not exist, vgem_fence_attach_ioctl returns -ENOENT.
+ */
+int vgem_fence_attach_ioctl(struct drm_device *dev,
+                            void *data,
+                            struct drm_file *file)
+{
+        struct drm_vgem_fence_attach *arg = data;
+        struct vgem_file *vfile = file->driver_priv;
+        struct reservation_object *resv;
+        struct drm_gem_object *obj;
+        struct fence *fence;
+        int ret;
+
+        if (arg->flags & ~VGEM_FENCE_WRITE)
+                return -EINVAL;
+
+        if (arg->pad)
+                return -EINVAL;
+
+        obj = drm_gem_object_lookup(file, arg->handle);
+        if (!obj)
+                return -ENOENT;
+
+        ret = attach_dmabuf(dev, obj);
+        if (ret)
+                goto err;
+
+        fence = vgem_fence_create(vfile, arg->flags);
+        if (!fence) {
+                ret = -ENOMEM;
+                goto err;
+        }
+
+        /* Check for a conflicting fence */
+        resv = obj->dma_buf->resv;
+        if (!reservation_object_test_signaled_rcu(resv,
+                                                  arg->flags & VGEM_FENCE_WRITE)) {
+                ret = -EBUSY;
+                goto err_fence;
+        }
+
+        /* Expose the fence via the dma-buf */
+        ret = 0;
+        mutex_lock(&resv->lock.base);
+        if (arg->flags & VGEM_FENCE_WRITE)
+                reservation_object_add_excl_fence(resv, fence);
+        else if ((ret = reservation_object_reserve_shared(resv)) == 0)
+                reservation_object_add_shared_fence(resv, fence);
+        mutex_unlock(&resv->lock.base);
+
+        /* Record the fence in our idr for later signaling */
+        if (ret == 0) {
+                mutex_lock(&vfile->fence_mutex);
+                ret = idr_alloc(&vfile->fence_idr, fence, 1, 0, GFP_KERNEL);
+                mutex_unlock(&vfile->fence_mutex);
+                if (ret > 0) {
+                        arg->out_fence = ret;
+                        ret = 0;
+                }
+        }
+err_fence:
+        if (ret) {
+                fence_signal(fence);
+                fence_put(fence);
+        }
+err:
+        drm_gem_object_unreference_unlocked(obj);
+        return ret;
+}
+
+/*
+ * vgem_fence_signal_ioctl (DRM_IOCTL_VGEM_FENCE_SIGNAL):
+ *
+ * Signal and consume a fence ealier attached to a vGEM handle using
+ * vgem_fence_attach_ioctl (DRM_IOCTL_VGEM_FENCE_ATTACH).
+ *
+ * All fences must be signaled within 10s of attachment or otherwise they
+ * will automatically expire (and a vgem_fence_signal_ioctl returns -ETIMEDOUT).
+ *
+ * Signaling a fence indicates to all consumers of the dma-buf that the
+ * client has completed the operation associated with the fence, and that the
+ * buffer is then ready for consumption.
+ *
+ * If the fence does not exist (or has already been signaled by the client),
+ * vgem_fence_signal_ioctl returns -ENOENT.
+ */
+int vgem_fence_signal_ioctl(struct drm_device *dev,
+                            void *data,
+                            struct drm_file *file)
+{
+        struct vgem_file *vfile = file->driver_priv;
+        struct drm_vgem_fence_signal *arg = data;
+        struct fence *fence;
+        int ret = 0;
+
+        if (arg->flags)
+                return -EINVAL;
+
+        mutex_lock(&vfile->fence_mutex);
+        fence = idr_replace(&vfile->fence_idr, NULL, arg->fence);
+        mutex_unlock(&vfile->fence_mutex);
+        if (!fence)
+                return -ENOENT;
+        if (IS_ERR(fence))
+                return PTR_ERR(fence);
+
+        if (fence_is_signaled(fence))
+                ret = -ETIMEDOUT;
+
+        fence_signal(fence);
+        fence_put(fence);
+        return ret;
+}
+
+int vgem_fence_open(struct vgem_file *vfile)
+{
+        mutex_init(&vfile->fence_mutex);
+        idr_init(&vfile->fence_idr);
+
+        return 0;
+}
+
+static int __vgem_fence_idr_fini(int id, void *p, void *data)
+{
+        fence_signal(p);
+        fence_put(p);
+        return 0;
+}
+
+void vgem_fence_close(struct vgem_file *vfile)
+{
+        idr_for_each(&vfile->fence_idr, __vgem_fence_idr_fini, vfile);
+        idr_destroy(&vfile->fence_idr);
+}
diff --git a/drivers/gpu/drm/via/via_mm.c b/drivers/gpu/drm/via/via_mm.c
index 4f20742e7788..a04ef1c992d9 100644
--- a/drivers/gpu/drm/via/via_mm.c
+++ b/drivers/gpu/drm/via/via_mm.c
@@ -208,7 +208,7 @@ void via_reclaim_buffers_locked(struct drm_device *dev,
         struct via_file_private *file_priv = file->driver_priv;
         struct via_memblock *entry, *next;
 
-        if (!(file->minor->master && file->master->lock.hw_lock))
+        if (!(dev->master && file->master->lock.hw_lock))
                 return;
 
         drm_legacy_idlelock_take(&file->master->lock);
diff --git a/drivers/gpu/drm/virtio/Kconfig b/drivers/gpu/drm/virtio/Kconfig
index 9983eadb81b6..e1afc3d3f8d9 100644
--- a/drivers/gpu/drm/virtio/Kconfig
+++ b/drivers/gpu/drm/virtio/Kconfig
@@ -1,11 +1,7 @@
 config DRM_VIRTIO_GPU
         tristate "Virtio GPU driver"
         depends on DRM && VIRTIO
-        select FB_SYS_FILLRECT
-        select FB_SYS_COPYAREA
-        select FB_SYS_IMAGEBLIT
         select DRM_KMS_HELPER
-        select DRM_KMS_FB_HELPER
         select DRM_TTM
         help
            This is the virtual GPU driver for virtio.  It can be used with
diff --git a/drivers/gpu/drm/virtio/virtgpu_display.c b/drivers/gpu/drm/virtio/virtgpu_display.c
index d4305da88f44..4e192aa2d021 100644
--- a/drivers/gpu/drm/virtio/virtgpu_display.c
+++ b/drivers/gpu/drm/virtio/virtgpu_display.c
@@ -29,8 +29,8 @@
 #include <drm/drm_crtc_helper.h>
 #include <drm/drm_atomic_helper.h>
 
-#define XRES_MIN   320
-#define YRES_MIN   200
+#define XRES_MIN    32
+#define YRES_MIN    32
 
 #define XRES_DEF  1024
 #define YRES_DEF   768
@@ -38,138 +38,11 @@
 #define XRES_MAX  8192
 #define YRES_MAX  8192
 
-static void
-virtio_gpu_hide_cursor(struct virtio_gpu_device *vgdev,
-                       struct virtio_gpu_output *output)
-{
-        output->cursor.hdr.type = cpu_to_le32(VIRTIO_GPU_CMD_UPDATE_CURSOR);
-        output->cursor.resource_id = 0;
-        virtio_gpu_cursor_ping(vgdev, output);
-}
-
-static int virtio_gpu_crtc_cursor_set(struct drm_crtc *crtc,
-                                      struct drm_file *file_priv,
-                                      uint32_t handle,
-                                      uint32_t width,
-                                      uint32_t height,
-                                      int32_t hot_x, int32_t hot_y)
-{
-        struct virtio_gpu_device *vgdev = crtc->dev->dev_private;
-        struct virtio_gpu_output *output =
-                container_of(crtc, struct virtio_gpu_output, crtc);
-        struct drm_gem_object *gobj = NULL;
-        struct virtio_gpu_object *qobj = NULL;
-        struct virtio_gpu_fence *fence = NULL;
-        int ret = 0;
-
-        if (handle == 0) {
-                virtio_gpu_hide_cursor(vgdev, output);
-                return 0;
-        }
-
-        /* lookup the cursor */
-        gobj = drm_gem_object_lookup(file_priv, handle);
-        if (gobj == NULL)
-                return -ENOENT;
-
-        qobj = gem_to_virtio_gpu_obj(gobj);
-
-        if (!qobj->hw_res_handle) {
-                ret = -EINVAL;
-                goto out;
-        }
-
-        virtio_gpu_cmd_transfer_to_host_2d(vgdev, qobj->hw_res_handle, 0,
-                                           cpu_to_le32(64),
-                                           cpu_to_le32(64),
-                                           0, 0, &fence);
-        ret = virtio_gpu_object_reserve(qobj, false);
-        if (!ret) {
-                reservation_object_add_excl_fence(qobj->tbo.resv,
-                                                  &fence->f);
-                fence_put(&fence->f);
-                virtio_gpu_object_unreserve(qobj);
-                virtio_gpu_object_wait(qobj, false);
-        }
-
-        output->cursor.hdr.type = cpu_to_le32(VIRTIO_GPU_CMD_UPDATE_CURSOR);
-        output->cursor.resource_id = cpu_to_le32(qobj->hw_res_handle);
-        output->cursor.hot_x = cpu_to_le32(hot_x);
-        output->cursor.hot_y = cpu_to_le32(hot_y);
-        virtio_gpu_cursor_ping(vgdev, output);
-        ret = 0;
-
-out:
-        drm_gem_object_unreference_unlocked(gobj);
-        return ret;
-}
-
-static int virtio_gpu_crtc_cursor_move(struct drm_crtc *crtc,
-                                    int x, int y)
-{
-        struct virtio_gpu_device *vgdev = crtc->dev->dev_private;
-        struct virtio_gpu_output *output =
-                container_of(crtc, struct virtio_gpu_output, crtc);
-
-        output->cursor.hdr.type = cpu_to_le32(VIRTIO_GPU_CMD_MOVE_CURSOR);
-        output->cursor.pos.x = cpu_to_le32(x);
-        output->cursor.pos.y = cpu_to_le32(y);
-        virtio_gpu_cursor_ping(vgdev, output);
-        return 0;
-}
-
-static int virtio_gpu_page_flip(struct drm_crtc *crtc,
-                                struct drm_framebuffer *fb,
-                                struct drm_pending_vblank_event *event,
-                                uint32_t flags)
-{
-        struct virtio_gpu_device *vgdev = crtc->dev->dev_private;
-        struct virtio_gpu_output *output =
-                container_of(crtc, struct virtio_gpu_output, crtc);
-        struct drm_plane *plane = crtc->primary;
-        struct virtio_gpu_framebuffer *vgfb;
-        struct virtio_gpu_object *bo;
-        unsigned long irqflags;
-        uint32_t handle;
-
-        plane->fb = fb;
-        vgfb = to_virtio_gpu_framebuffer(plane->fb);
-        bo = gem_to_virtio_gpu_obj(vgfb->obj);
-        handle = bo->hw_res_handle;
-
-        DRM_DEBUG("handle 0x%x%s, crtc %dx%d\n", handle,
-                  bo->dumb ? ", dumb" : "",
-                  crtc->mode.hdisplay, crtc->mode.vdisplay);
-        if (bo->dumb) {
-                virtio_gpu_cmd_transfer_to_host_2d
-                        (vgdev, handle, 0,
-                         cpu_to_le32(crtc->mode.hdisplay),
-                         cpu_to_le32(crtc->mode.vdisplay),
-                         0, 0, NULL);
-        }
-        virtio_gpu_cmd_set_scanout(vgdev, output->index, handle,
-                                   crtc->mode.hdisplay,
-                                   crtc->mode.vdisplay, 0, 0);
-        virtio_gpu_cmd_resource_flush(vgdev, handle, 0, 0,
-                                      crtc->mode.hdisplay,
-                                      crtc->mode.vdisplay);
-
-        if (event) {
-                spin_lock_irqsave(&crtc->dev->event_lock, irqflags);
-                drm_send_vblank_event(crtc->dev, -1, event);
-                spin_unlock_irqrestore(&crtc->dev->event_lock, irqflags);
-        }
-
-        return 0;
-}
-
 static const struct drm_crtc_funcs virtio_gpu_crtc_funcs = {
-        .cursor_set2            = virtio_gpu_crtc_cursor_set,
-        .cursor_move            = virtio_gpu_crtc_cursor_move,
         .set_config             = drm_atomic_helper_set_config,
         .destroy                = drm_crtc_cleanup,
 
-        .page_flip              = virtio_gpu_page_flip,
+        .page_flip              = drm_atomic_helper_page_flip,
         .reset                  = drm_atomic_helper_crtc_reset,
         .atomic_duplicate_state = drm_atomic_helper_crtc_duplicate_state,
         .atomic_destroy_state   = drm_atomic_helper_crtc_destroy_state,
@@ -180,8 +53,7 @@ static void virtio_gpu_user_framebuffer_destroy(struct drm_framebuffer *fb)
         struct virtio_gpu_framebuffer *virtio_gpu_fb
                 = to_virtio_gpu_framebuffer(fb);
 
-        if (virtio_gpu_fb->obj)
-                drm_gem_object_unreference_unlocked(virtio_gpu_fb->obj);
+        drm_gem_object_unreference_unlocked(virtio_gpu_fb->obj);
         drm_framebuffer_cleanup(fb);
         kfree(virtio_gpu_fb);
 }
@@ -267,6 +139,7 @@ static void virtio_gpu_crtc_atomic_flush(struct drm_crtc *crtc,
         spin_lock_irqsave(&crtc->dev->event_lock, flags);
         if (crtc->state->event)
                 drm_crtc_send_vblank_event(crtc, crtc->state->event);
+        crtc->state->event = NULL;
         spin_unlock_irqrestore(&crtc->dev->event_lock, flags);
 }
 
@@ -341,15 +214,6 @@ static int virtio_gpu_conn_mode_valid(struct drm_connector *connector,
         return MODE_BAD;
 }
 
-static struct drm_encoder*
-virtio_gpu_best_encoder(struct drm_connector *connector)
-{
-        struct virtio_gpu_output *virtio_gpu_output =
-                drm_connector_to_virtio_gpu_output(connector);
-
-        return &virtio_gpu_output->enc;
-}
-
 static const struct drm_encoder_helper_funcs virtio_gpu_enc_helper_funcs = {
         .mode_set   = virtio_gpu_enc_mode_set,
         .enable     = virtio_gpu_enc_enable,
@@ -359,7 +223,6 @@ static const struct drm_encoder_helper_funcs virtio_gpu_enc_helper_funcs = {
 static const struct drm_connector_helper_funcs virtio_gpu_conn_helper_funcs = {
         .get_modes    = virtio_gpu_conn_get_modes,
         .mode_valid   = virtio_gpu_conn_mode_valid,
-        .best_encoder = virtio_gpu_best_encoder,
 };
 
 static enum drm_connector_status virtio_gpu_conn_detect(
@@ -406,7 +269,7 @@ static int vgdev_output_init(struct virtio_gpu_device *vgdev, int index)
         struct drm_connector *connector = &output->conn;
         struct drm_encoder *encoder = &output->enc;
         struct drm_crtc *crtc = &output->crtc;
-        struct drm_plane *plane;
+        struct drm_plane *primary, *cursor;
 
         output->index = index;
         if (index == 0) {
@@ -415,13 +278,17 @@ static int vgdev_output_init(struct virtio_gpu_device *vgdev, int index)
                 output->info.r.height = cpu_to_le32(YRES_DEF);
         }
 
-        plane = virtio_gpu_plane_init(vgdev, index);
-        if (IS_ERR(plane))
-                return PTR_ERR(plane);
-        drm_crtc_init_with_planes(dev, crtc, plane, NULL,
+        primary = virtio_gpu_plane_init(vgdev, DRM_PLANE_TYPE_PRIMARY, index);
+        if (IS_ERR(primary))
+                return PTR_ERR(primary);
+        cursor = virtio_gpu_plane_init(vgdev, DRM_PLANE_TYPE_CURSOR, index);
+        if (IS_ERR(cursor))
+                return PTR_ERR(cursor);
+        drm_crtc_init_with_planes(dev, crtc, primary, cursor,
                                   &virtio_gpu_crtc_funcs, NULL);
         drm_crtc_helper_add(crtc, &virtio_gpu_crtc_helper_funcs);
-        plane->crtc = crtc;
+        primary->crtc = crtc;
+        cursor->crtc = crtc;
 
         drm_connector_init(dev, connector, &virtio_gpu_connector_funcs,
                            DRM_MODE_CONNECTOR_VIRTUAL);
@@ -458,14 +325,31 @@ virtio_gpu_user_framebuffer_create(struct drm_device *dev,
         ret = virtio_gpu_framebuffer_init(dev, virtio_gpu_fb, mode_cmd, obj);
         if (ret) {
                 kfree(virtio_gpu_fb);
-                if (obj)
-                        drm_gem_object_unreference_unlocked(obj);
+                drm_gem_object_unreference_unlocked(obj);
                 return NULL;
         }
 
         return &virtio_gpu_fb->base;
 }
 
+static void vgdev_atomic_commit_tail(struct drm_atomic_state *state)
+{
+        struct drm_device *dev = state->dev;
+
+        drm_atomic_helper_commit_modeset_disables(dev, state);
+        drm_atomic_helper_commit_modeset_enables(dev, state);
+        drm_atomic_helper_commit_planes(dev, state, true);
+
+        drm_atomic_helper_commit_hw_done(state);
+
+        drm_atomic_helper_wait_for_vblanks(dev, state);
+        drm_atomic_helper_cleanup_planes(dev, state);
+}
+
+static struct drm_mode_config_helper_funcs virtio_mode_config_helpers = {
+        .atomic_commit_tail = vgdev_atomic_commit_tail,
+};
+
 static const struct drm_mode_config_funcs virtio_gpu_mode_funcs = {
         .fb_create = virtio_gpu_user_framebuffer_create,
         .atomic_check = drm_atomic_helper_check,
@@ -477,7 +361,8 @@ int virtio_gpu_modeset_init(struct virtio_gpu_device *vgdev)
         int i;
 
         drm_mode_config_init(vgdev->ddev);
-        vgdev->ddev->mode_config.funcs = (void *)&virtio_gpu_mode_funcs;
+        vgdev->ddev->mode_config.funcs = &virtio_gpu_mode_funcs;
+        vgdev->ddev->mode_config.helper_private = &virtio_mode_config_helpers;
 
         /* modes will be validated against the framebuffer size */
         vgdev->ddev->mode_config.min_width = XRES_MIN;
diff --git a/drivers/gpu/drm/virtio/virtgpu_drm_bus.c b/drivers/gpu/drm/virtio/virtgpu_drm_bus.c
index 88a39165edd5..7f0e93f87a55 100644
--- a/drivers/gpu/drm/virtio/virtgpu_drm_bus.c
+++ b/drivers/gpu/drm/virtio/virtgpu_drm_bus.c
@@ -27,16 +27,6 @@
 
 #include "virtgpu_drv.h"
 
-int drm_virtio_set_busid(struct drm_device *dev, struct drm_master *master)
-{
-        struct pci_dev *pdev = dev->pdev;
-
-        if (pdev) {
-                return drm_pci_set_busid(dev, master);
-        }
-        return 0;
-}
-
 static void virtio_pci_kick_out_firmware_fb(struct pci_dev *pci_dev)
 {
         struct apertures_struct *ap;
diff --git a/drivers/gpu/drm/virtio/virtgpu_drv.c b/drivers/gpu/drm/virtio/virtgpu_drv.c
index 3cc7afa77a35..c13f70cfc461 100644
--- a/drivers/gpu/drm/virtio/virtgpu_drv.c
+++ b/drivers/gpu/drm/virtio/virtgpu_drv.c
@@ -117,7 +117,6 @@ static const struct file_operations virtio_gpu_driver_fops = {
 
 static struct drm_driver driver = {
         .driver_features = DRIVER_MODESET | DRIVER_GEM | DRIVER_PRIME | DRIVER_RENDER | DRIVER_ATOMIC,
-        .set_busid = drm_virtio_set_busid,
         .load = virtio_gpu_driver_load,
         .unload = virtio_gpu_driver_unload,
         .open = virtio_gpu_driver_open,
@@ -143,7 +142,7 @@ static struct drm_driver driver = {
         .gem_prime_vunmap = virtgpu_gem_prime_vunmap,
         .gem_prime_mmap = virtgpu_gem_prime_mmap,
 
-        .gem_free_object = virtio_gpu_gem_free_object,
+        .gem_free_object_unlocked = virtio_gpu_gem_free_object,
         .gem_open_object = virtio_gpu_gem_object_open,
         .gem_close_object = virtio_gpu_gem_object_close,
         .fops = &virtio_gpu_driver_fops,
diff --git a/drivers/gpu/drm/virtio/virtgpu_drv.h b/drivers/gpu/drm/virtio/virtgpu_drv.h
index 0a54f43f846a..b18ef3111f0c 100644
--- a/drivers/gpu/drm/virtio/virtgpu_drv.h
+++ b/drivers/gpu/drm/virtio/virtgpu_drv.h
@@ -33,6 +33,7 @@
 
 #include <drm/drmP.h>
 #include <drm/drm_gem.h>
+#include <drm/drm_atomic.h>
 #include <drm/drm_crtc_helper.h>
 #include <ttm/ttm_bo_api.h>
 #include <ttm/ttm_bo_driver.h>
@@ -48,7 +49,6 @@
 #define DRIVER_PATCHLEVEL 1
 
 /* virtgpu_drm_bus.c */
-int drm_virtio_set_busid(struct drm_device *dev, struct drm_master *master);
 int drm_virtio_init(struct drm_driver *driver, struct virtio_device *vdev);
 
 struct virtio_gpu_object {
@@ -335,6 +335,7 @@ void virtio_gpu_modeset_fini(struct virtio_gpu_device *vgdev);
 
 /* virtio_gpu_plane.c */
 struct drm_plane *virtio_gpu_plane_init(struct virtio_gpu_device *vgdev,
+                                        enum drm_plane_type type,
                                         int index);
 
 /* virtio_gpu_ttm.c */
diff --git a/drivers/gpu/drm/virtio/virtgpu_plane.c b/drivers/gpu/drm/virtio/virtgpu_plane.c
index 70b44a2345ab..925ca25209df 100644
--- a/drivers/gpu/drm/virtio/virtgpu_plane.c
+++ b/drivers/gpu/drm/virtio/virtgpu_plane.c
@@ -38,6 +38,10 @@ static const uint32_t virtio_gpu_formats[] = {
         DRM_FORMAT_ABGR8888,
 };
 
+static const uint32_t virtio_gpu_cursor_formats[] = {
+        DRM_FORMAT_ARGB8888,
+};
+
 static void virtio_gpu_plane_destroy(struct drm_plane *plane)
 {
         kfree(plane);
@@ -58,16 +62,22 @@ static int virtio_gpu_plane_atomic_check(struct drm_plane *plane,
         return 0;
 }
 
-static void virtio_gpu_plane_atomic_update(struct drm_plane *plane,
-                                           struct drm_plane_state *old_state)
+static void virtio_gpu_primary_plane_update(struct drm_plane *plane,
+                                            struct drm_plane_state *old_state)
 {
         struct drm_device *dev = plane->dev;
         struct virtio_gpu_device *vgdev = dev->dev_private;
-        struct virtio_gpu_output *output = drm_crtc_to_virtio_gpu_output(plane->crtc);
+        struct virtio_gpu_output *output = NULL;
         struct virtio_gpu_framebuffer *vgfb;
         struct virtio_gpu_object *bo;
         uint32_t handle;
 
+        if (plane->state->crtc)
+                output = drm_crtc_to_virtio_gpu_output(plane->state->crtc);
+        if (old_state->crtc)
+                output = drm_crtc_to_virtio_gpu_output(old_state->crtc);
+        WARN_ON(!output);
+
         if (plane->state->fb) {
                 vgfb = to_virtio_gpu_framebuffer(plane->state->fb);
                 bo = gem_to_virtio_gpu_obj(vgfb->obj);
@@ -75,55 +85,149 @@ static void virtio_gpu_plane_atomic_update(struct drm_plane *plane,
                 if (bo->dumb) {
                         virtio_gpu_cmd_transfer_to_host_2d
                                 (vgdev, handle, 0,
-                                 cpu_to_le32(plane->state->crtc_w),
-                                 cpu_to_le32(plane->state->crtc_h),
-                                 plane->state->crtc_x, plane->state->crtc_y, NULL);
+                                 cpu_to_le32(plane->state->src_w >> 16),
+                                 cpu_to_le32(plane->state->src_h >> 16),
+                                 plane->state->src_x >> 16,
+                                 plane->state->src_y >> 16, NULL);
                 }
         } else {
                 handle = 0;
         }
 
-        DRM_DEBUG("handle 0x%x, crtc %dx%d+%d+%d\n", handle,
+        DRM_DEBUG("handle 0x%x, crtc %dx%d+%d+%d, src %dx%d+%d+%d\n", handle,
                   plane->state->crtc_w, plane->state->crtc_h,
-                  plane->state->crtc_x, plane->state->crtc_y);
+                  plane->state->crtc_x, plane->state->crtc_y,
+                  plane->state->src_w >> 16,
+                  plane->state->src_h >> 16,
+                  plane->state->src_x >> 16,
+                  plane->state->src_y >> 16);
         virtio_gpu_cmd_set_scanout(vgdev, output->index, handle,
-                                   plane->state->crtc_w,
-                                   plane->state->crtc_h,
-                                   plane->state->crtc_x,
-                                   plane->state->crtc_y);
+                                   plane->state->src_w >> 16,
+                                   plane->state->src_h >> 16,
+                                   plane->state->src_x >> 16,
+                                   plane->state->src_y >> 16);
         virtio_gpu_cmd_resource_flush(vgdev, handle,
-                                      plane->state->crtc_x,
-                                      plane->state->crtc_y,
-                                      plane->state->crtc_w,
-                                      plane->state->crtc_h);
+                                      plane->state->src_x >> 16,
+                                      plane->state->src_y >> 16,
+                                      plane->state->src_w >> 16,
+                                      plane->state->src_h >> 16);
 }
 
+static void virtio_gpu_cursor_plane_update(struct drm_plane *plane,
+                                           struct drm_plane_state *old_state)
+{
+        struct drm_device *dev = plane->dev;
+        struct virtio_gpu_device *vgdev = dev->dev_private;
+        struct virtio_gpu_output *output = NULL;
+        struct virtio_gpu_framebuffer *vgfb;
+        struct virtio_gpu_fence *fence = NULL;
+        struct virtio_gpu_object *bo = NULL;
+        uint32_t handle;
+        int ret = 0;
 
-static const struct drm_plane_helper_funcs virtio_gpu_plane_helper_funcs = {
+        if (plane->state->crtc)
+                output = drm_crtc_to_virtio_gpu_output(plane->state->crtc);
+        if (old_state->crtc)
+                output = drm_crtc_to_virtio_gpu_output(old_state->crtc);
+        WARN_ON(!output);
+
+        if (plane->state->fb) {
+                vgfb = to_virtio_gpu_framebuffer(plane->state->fb);
+                bo = gem_to_virtio_gpu_obj(vgfb->obj);
+                handle = bo->hw_res_handle;
+        } else {
+                handle = 0;
+        }
+
+        if (bo && bo->dumb && (plane->state->fb != old_state->fb)) {
+                /* new cursor -- update & wait */
+                virtio_gpu_cmd_transfer_to_host_2d
+                        (vgdev, handle, 0,
+                         cpu_to_le32(plane->state->crtc_w),
+                         cpu_to_le32(plane->state->crtc_h),
+                         0, 0, &fence);
+                ret = virtio_gpu_object_reserve(bo, false);
+                if (!ret) {
+                        reservation_object_add_excl_fence(bo->tbo.resv,
+                                                          &fence->f);
+                        fence_put(&fence->f);
+                        fence = NULL;
+                        virtio_gpu_object_unreserve(bo);
+                        virtio_gpu_object_wait(bo, false);
+                }
+        }
+
+        if (plane->state->fb != old_state->fb) {
+                DRM_DEBUG("update, handle %d, pos +%d+%d, hot %d,%d\n", handle,
+                          plane->state->crtc_x,
+                          plane->state->crtc_y,
+                          plane->state->fb ? plane->state->fb->hot_x : 0,
+                          plane->state->fb ? plane->state->fb->hot_y : 0);
+                output->cursor.hdr.type =
+                        cpu_to_le32(VIRTIO_GPU_CMD_UPDATE_CURSOR);
+                output->cursor.resource_id = cpu_to_le32(handle);
+                if (plane->state->fb) {
+                        output->cursor.hot_x =
+                                cpu_to_le32(plane->state->fb->hot_x);
+                        output->cursor.hot_y =
+                                cpu_to_le32(plane->state->fb->hot_y);
+                } else {
+                        output->cursor.hot_x = cpu_to_le32(0);
+                        output->cursor.hot_y = cpu_to_le32(0);
+                }
+        } else {
+                DRM_DEBUG("move +%d+%d\n",
+                          plane->state->crtc_x,
+                          plane->state->crtc_y);
+                output->cursor.hdr.type =
+                        cpu_to_le32(VIRTIO_GPU_CMD_MOVE_CURSOR);
+        }
+        output->cursor.pos.x = cpu_to_le32(plane->state->crtc_x);
+        output->cursor.pos.y = cpu_to_le32(plane->state->crtc_y);
+        virtio_gpu_cursor_ping(vgdev, output);
+}
+
+static const struct drm_plane_helper_funcs virtio_gpu_primary_helper_funcs = {
+        .atomic_check           = virtio_gpu_plane_atomic_check,
+        .atomic_update          = virtio_gpu_primary_plane_update,
+};
+
+static const struct drm_plane_helper_funcs virtio_gpu_cursor_helper_funcs = {
         .atomic_check           = virtio_gpu_plane_atomic_check,
-        .atomic_update          = virtio_gpu_plane_atomic_update,
+        .atomic_update          = virtio_gpu_cursor_plane_update,
 };
 
 struct drm_plane *virtio_gpu_plane_init(struct virtio_gpu_device *vgdev,
+                                        enum drm_plane_type type,
                                         int index)
 {
         struct drm_device *dev = vgdev->ddev;
+        const struct drm_plane_helper_funcs *funcs;
         struct drm_plane *plane;
-        int ret;
+        const uint32_t *formats;
+        int ret, nformats;
 
         plane = kzalloc(sizeof(*plane), GFP_KERNEL);
         if (!plane)
                 return ERR_PTR(-ENOMEM);
 
+        if (type == DRM_PLANE_TYPE_CURSOR) {
+                formats = virtio_gpu_cursor_formats;
+                nformats = ARRAY_SIZE(virtio_gpu_cursor_formats);
+                funcs = &virtio_gpu_cursor_helper_funcs;
+        } else {
+                formats = virtio_gpu_formats;
+                nformats = ARRAY_SIZE(virtio_gpu_formats);
+                funcs = &virtio_gpu_primary_helper_funcs;
+        }
         ret = drm_universal_plane_init(dev, plane, 1 << index,
                                        &virtio_gpu_plane_funcs,
-                                       virtio_gpu_formats,
-                                       ARRAY_SIZE(virtio_gpu_formats),
-                                       DRM_PLANE_TYPE_PRIMARY, NULL);
+                                       formats, nformats,
+                                       type, NULL);
         if (ret)
                 goto err_plane_init;
 
-        drm_plane_helper_add(plane, &virtio_gpu_plane_helper_funcs);
+        drm_plane_helper_add(plane, funcs);
         return plane;
 
 err_plane_init:
diff --git a/drivers/gpu/drm/virtio/virtgpu_ttm.c b/drivers/gpu/drm/virtio/virtgpu_ttm.c
index a0580815629f..80482ac5f95d 100644
--- a/drivers/gpu/drm/virtio/virtgpu_ttm.c
+++ b/drivers/gpu/drm/virtio/virtgpu_ttm.c
@@ -375,6 +375,12 @@ static int virtio_gpu_bo_move(struct ttm_buffer_object *bo,
                               bool no_wait_gpu,
                               struct ttm_mem_reg *new_mem)
 {
+        int ret;
+
+        ret = ttm_bo_wait(bo, interruptible, no_wait_gpu);
+        if (ret)
+                return ret;
+
         virtio_gpu_move_null(bo, new_mem);
         return 0;
 }
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_cmdbuf.c b/drivers/gpu/drm/vmwgfx/vmwgfx_cmdbuf.c
index 67cebb23c940..aa04fb0159a7 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_cmdbuf.c
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_cmdbuf.c
@@ -293,13 +293,10 @@ static int vmw_cmdbuf_header_submit(struct vmw_cmdbuf_header *header)
         struct vmw_cmdbuf_man *man = header->man;
         u32 val;
 
-        if (sizeof(header->handle) > 4)
-                val = (header->handle >> 32);
-        else
-                val = 0;
+        val = upper_32_bits(header->handle);
         vmw_write(man->dev_priv, SVGA_REG_COMMAND_HIGH, val);
 
-        val = (header->handle & 0xFFFFFFFFULL);
+        val = lower_32_bits(header->handle);
         val |= header->cb_context & SVGA_CB_CONTEXT_MASK;
         vmw_write(man->dev_priv, SVGA_REG_COMMAND_LOW, val);
 
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_drv.c b/drivers/gpu/drm/vmwgfx/vmwgfx_drv.c
index 8d528fcf6e96..e8ae3dc476d1 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_drv.c
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_drv.c
@@ -1053,15 +1053,14 @@ static struct vmw_master *vmw_master_check(struct drm_device *dev,
         struct vmw_fpriv *vmw_fp = vmw_fpriv(file_priv);
         struct vmw_master *vmaster;
 
-        if (file_priv->minor->type != DRM_MINOR_LEGACY ||
-            !(flags & DRM_AUTH))
+        if (!drm_is_primary_client(file_priv) || !(flags & DRM_AUTH))
                 return NULL;
 
         ret = mutex_lock_interruptible(&dev->master_mutex);
         if (unlikely(ret != 0))
                 return ERR_PTR(-ERESTARTSYS);
 
-        if (file_priv->is_master) {
+        if (drm_is_current_master(file_priv)) {
                 mutex_unlock(&dev->master_mutex);
                 return NULL;
         }
@@ -1240,8 +1239,7 @@ static int vmw_master_set(struct drm_device *dev,
 }
 
 static void vmw_master_drop(struct drm_device *dev,
-                            struct drm_file *file_priv,
-                            bool from_release)
+                            struct drm_file *file_priv)
 {
         struct vmw_private *dev_priv = vmw_priv(dev);
         struct vmw_fpriv *vmw_fp = vmw_fpriv(file_priv);
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_drv.h b/drivers/gpu/drm/vmwgfx/vmwgfx_drv.h
index 89fb19443a3f..74304b03f9d4 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_drv.h
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_drv.h
@@ -32,6 +32,7 @@
 #include <drm/drmP.h>
 #include <drm/vmwgfx_drm.h>
 #include <drm/drm_hashtab.h>
+#include <drm/drm_auth.h>
 #include <linux/suspend.h>
 #include <drm/ttm/ttm_bo_driver.h>
 #include <drm/ttm/ttm_object.h>
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_execbuf.c b/drivers/gpu/drm/vmwgfx/vmwgfx_execbuf.c
index 1a1a87cbf109..dc5beff2b4aa 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_execbuf.c
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_execbuf.c
@@ -3625,9 +3625,7 @@ static int vmw_resize_cmd_bounce(struct vmw_sw_context *sw_context,
                                    (sw_context->cmd_bounce_size >> 1));
         }
 
-        if (sw_context->cmd_bounce != NULL)
-                vfree(sw_context->cmd_bounce);
-
+        vfree(sw_context->cmd_bounce);
         sw_context->cmd_bounce = vmalloc(sw_context->cmd_bounce_size);
 
         if (sw_context->cmd_bounce == NULL) {
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_fence.c b/drivers/gpu/drm/vmwgfx/vmwgfx_fence.c
index e959df6ede83..26ac8e80a478 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_fence.c
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_fence.c
@@ -46,7 +46,7 @@ struct vmw_fence_manager {
         bool goal_irq_on; /* Protected by @goal_irq_mutex */
         bool seqno_valid; /* Protected by @lock, and may not be set to true
                              without the @goal_irq_mutex held. */
-        unsigned ctx;
+        u64 ctx;
 };
 
 struct vmw_user_fence {
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_kms.c b/drivers/gpu/drm/vmwgfx/vmwgfx_kms.c
index e29da45a2847..bf28ccc150df 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_kms.c
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_kms.c
@@ -1404,9 +1404,9 @@ static int vmw_du_update_layout(struct vmw_private *dev_priv, unsigned num,
         return 0;
 }
 
-void vmw_du_crtc_gamma_set(struct drm_crtc *crtc,
-                           u16 *r, u16 *g, u16 *b,
-                           uint32_t start, uint32_t size)
+int vmw_du_crtc_gamma_set(struct drm_crtc *crtc,
+                          u16 *r, u16 *g, u16 *b,
+                          uint32_t size)
 {
         struct vmw_private *dev_priv = vmw_priv(crtc->dev);
         int i;
@@ -1418,6 +1418,8 @@ void vmw_du_crtc_gamma_set(struct drm_crtc *crtc,
                 vmw_write(dev_priv, SVGA_PALETTE_BASE + i * 3 + 1, g[i] >> 8);
                 vmw_write(dev_priv, SVGA_PALETTE_BASE + i * 3 + 2, b[i] >> 8);
         }
+
+        return 0;
 }
 
 int vmw_du_connector_dpms(struct drm_connector *connector, int mode)
diff --git a/drivers/gpu/drm/vmwgfx/vmwgfx_kms.h b/drivers/gpu/drm/vmwgfx/vmwgfx_kms.h
index 57203212c501..ff4803c107bc 100644
--- a/drivers/gpu/drm/vmwgfx/vmwgfx_kms.h
+++ b/drivers/gpu/drm/vmwgfx/vmwgfx_kms.h
@@ -195,9 +195,9 @@ struct vmw_display_unit {
 void vmw_du_cleanup(struct vmw_display_unit *du);
 void vmw_du_crtc_save(struct drm_crtc *crtc);
 void vmw_du_crtc_restore(struct drm_crtc *crtc);
-void vmw_du_crtc_gamma_set(struct drm_crtc *crtc,
+int vmw_du_crtc_gamma_set(struct drm_crtc *crtc,
                            u16 *r, u16 *g, u16 *b,
-                           uint32_t start, uint32_t size);
+                           uint32_t size);
 int vmw_du_crtc_cursor_set2(struct drm_crtc *crtc, struct drm_file *file_priv,
                             uint32_t handle, uint32_t width, uint32_t height,
                             int32_t hot_x, int32_t hot_y);
diff --git a/drivers/gpu/host1x/cdma.c b/drivers/gpu/host1x/cdma.c
index a18db4d5347c..c5d82a8a2ec9 100644
--- a/drivers/gpu/host1x/cdma.c
+++ b/drivers/gpu/host1x/cdma.c
@@ -96,12 +96,12 @@ fail:
  */
 static void host1x_pushbuffer_push(struct push_buffer *pb, u32 op1, u32 op2)
 {
-        u32 pos = pb->pos;
-        u32 *p = (u32 *)((void *)pb->mapped + pos);
-        WARN_ON(pos == pb->fence);
+        u32 *p = (u32 *)((void *)pb->mapped + pb->pos);
+
+        WARN_ON(pb->pos == pb->fence);
         *(p++) = op1;
         *(p++) = op2;
-        pb->pos = (pos + 8) & (pb->size_bytes - 1);
+        pb->pos = (pb->pos + 8) & (pb->size_bytes - 1);
 }
 
 /*
@@ -134,14 +134,19 @@ unsigned int host1x_cdma_wait_locked(struct host1x_cdma *cdma,
                                      enum cdma_event event)
 {
         for (;;) {
+                struct push_buffer *pb = &cdma->push_buffer;
                 unsigned int space;
 
-                if (event == CDMA_EVENT_SYNC_QUEUE_EMPTY)
+                switch (event) {
+                case CDMA_EVENT_SYNC_QUEUE_EMPTY:
                         space = list_empty(&cdma->sync_queue) ? 1 : 0;
-                else if (event == CDMA_EVENT_PUSH_BUFFER_SPACE) {
-                        struct push_buffer *pb = &cdma->push_buffer;
+                        break;
+
+                case CDMA_EVENT_PUSH_BUFFER_SPACE:
                         space = host1x_pushbuffer_space(pb);
-                } else {
+                        break;
+
+                default:
                         WARN_ON(1);
                         return -EINVAL;
                 }
@@ -159,12 +164,14 @@ unsigned int host1x_cdma_wait_locked(struct host1x_cdma *cdma,
                         mutex_lock(&cdma->lock);
                         continue;
                 }
+
                 cdma->event = event;
 
                 mutex_unlock(&cdma->lock);
                 down(&cdma->sem);
                 mutex_lock(&cdma->lock);
         }
+
         return 0;
 }
 
@@ -234,6 +241,7 @@ static void update_cdma_locked(struct host1x_cdma *cdma)
                         /* Start timer on next pending syncpt */
                         if (job->timeout)
                                 cdma_start_timer_locked(cdma, job);
+
                         break;
                 }
 
@@ -247,7 +255,9 @@ static void update_cdma_locked(struct host1x_cdma *cdma)
                 /* Pop push buffer slots */
                 if (job->num_slots) {
                         struct push_buffer *pb = &cdma->push_buffer;
+
                         host1x_pushbuffer_pop(pb, job->num_slots);
+
                         if (cdma->event == CDMA_EVENT_PUSH_BUFFER_SPACE)
                                 signal = true;
                 }
@@ -269,11 +279,9 @@ static void update_cdma_locked(struct host1x_cdma *cdma)
 void host1x_cdma_update_sync_queue(struct host1x_cdma *cdma,
                                    struct device *dev)
 {
-        u32 restart_addr;
-        u32 syncpt_incrs;
-        struct host1x_job *job = NULL;
-        u32 syncpt_val;
         struct host1x *host1x = cdma_to_host1x(cdma);
+        u32 restart_addr, syncpt_incrs, syncpt_val;
+        struct host1x_job *job = NULL;
 
         syncpt_val = host1x_syncpt_load(cdma->timeout.syncpt);
 
@@ -342,9 +350,11 @@ void host1x_cdma_update_sync_queue(struct host1x_cdma *cdma,
                 syncpt_val += syncpt_incrs;
         }
 
-        /* The following sumbits from the same client may be dependent on the
+        /*
+         * The following sumbits from the same client may be dependent on the
          * failed submit and therefore they may fail. Force a small timeout
-         * to make the queue cleanup faster */
+         * to make the queue cleanup faster.
+         */
 
         list_for_each_entry_from(job, &cdma->sync_queue, list)
                 if (job->client == cdma->timeout.client)
@@ -375,6 +385,7 @@ int host1x_cdma_init(struct host1x_cdma *cdma)
         err = host1x_pushbuffer_init(&cdma->push_buffer);
         if (err)
                 return err;
+
         return 0;
 }
 
@@ -410,6 +421,7 @@ int host1x_cdma_begin(struct host1x_cdma *cdma, struct host1x_job *job)
                 /* init state on first submit with timeout value */
                 if (!cdma->timeout.initialized) {
                         int err;
+
                         err = host1x_hw_cdma_timeout_init(host1x, cdma,
                                                           job->syncpt_id);
                         if (err) {
@@ -418,6 +430,7 @@ int host1x_cdma_begin(struct host1x_cdma *cdma, struct host1x_job *job)
                         }
                 }
         }
+
         if (!cdma->running)
                 host1x_hw_cdma_start(host1x, cdma);
 
@@ -448,6 +461,7 @@ void host1x_cdma_push(struct host1x_cdma *cdma, u32 op1, u32 op2)
                 slots_free = host1x_cdma_wait_locked(cdma,
                                                 CDMA_EVENT_PUSH_BUFFER_SPACE);
         }
+
         cdma->slots_free = slots_free - 1;
         cdma->slots_used++;
         host1x_pushbuffer_push(pb, op1, op2);
diff --git a/drivers/gpu/host1x/channel.c b/drivers/gpu/host1x/channel.c
index b4ae3affb987..8f437d924c10 100644
--- a/drivers/gpu/host1x/channel.c
+++ b/drivers/gpu/host1x/channel.c
@@ -83,9 +83,10 @@ EXPORT_SYMBOL(host1x_channel_put);
 struct host1x_channel *host1x_channel_request(struct device *dev)
 {
         struct host1x *host = dev_get_drvdata(dev->parent);
-        int max_channels = host->info->nb_channels;
+        unsigned int max_channels = host->info->nb_channels;
         struct host1x_channel *channel = NULL;
-        int index, err;
+        unsigned long index;
+        int err;
 
         mutex_lock(&host->chlist_mutex);
 
diff --git a/drivers/gpu/host1x/debug.c b/drivers/gpu/host1x/debug.c
index ee3d12b51c50..d9330fcc62ad 100644
--- a/drivers/gpu/host1x/debug.c
+++ b/drivers/gpu/host1x/debug.c
@@ -39,6 +39,7 @@ void host1x_debug_output(struct output *o, const char *fmt, ...)
         va_start(args, fmt);
         len = vsnprintf(o->buf, sizeof(o->buf), fmt, args);
         va_end(args);
+
         o->fn(o->ctx, o->buf, len);
 }
 
@@ -48,13 +49,17 @@ static int show_channels(struct host1x_channel *ch, void *data, bool show_fifo)
         struct output *o = data;
 
         mutex_lock(&ch->reflock);
+
         if (ch->refcount) {
                 mutex_lock(&ch->cdma.lock);
+
                 if (show_fifo)
                         host1x_hw_show_channel_fifo(m, ch, o);
+
                 host1x_hw_show_channel_cdma(m, ch, o);
                 mutex_unlock(&ch->cdma.lock);
         }
+
         mutex_unlock(&ch->reflock);
 
         return 0;
@@ -62,22 +67,27 @@ static int show_channels(struct host1x_channel *ch, void *data, bool show_fifo)
 
 static void show_syncpts(struct host1x *m, struct output *o)
 {
-        int i;
+        unsigned int i;
+
         host1x_debug_output(o, "---- syncpts ----\n");
+
         for (i = 0; i < host1x_syncpt_nb_pts(m); i++) {
                 u32 max = host1x_syncpt_read_max(m->syncpt + i);
                 u32 min = host1x_syncpt_load(m->syncpt + i);
+
                 if (!min && !max)
                         continue;
-                host1x_debug_output(o, "id %d (%s) min %d max %d\n",
+
+                host1x_debug_output(o, "id %u (%s) min %d max %d\n",
                                     i, m->syncpt[i].name, min, max);
         }
 
         for (i = 0; i < host1x_syncpt_nb_bases(m); i++) {
                 u32 base_val;
+
                 base_val = host1x_syncpt_load_wait_base(m->syncpt + i);
                 if (base_val)
-                        host1x_debug_output(o, "waitbase id %d val %d\n", i,
+                        host1x_debug_output(o, "waitbase id %u val %d\n", i,
                                             base_val);
         }
 
@@ -114,7 +124,9 @@ static int host1x_debug_show_all(struct seq_file *s, void *unused)
                 .fn = write_to_seqfile,
                 .ctx = s
         };
+
         show_all(s->private, &o);
+
         return 0;
 }
 
@@ -124,7 +136,9 @@ static int host1x_debug_show(struct seq_file *s, void *unused)
                 .fn = write_to_seqfile,
                 .ctx = s
         };
+
         show_all_no_fifo(s->private, &o);
+
         return 0;
 }
 
@@ -134,10 +148,10 @@ static int host1x_debug_open_all(struct inode *inode, struct file *file)
 }
 
 static const struct file_operations host1x_debug_all_fops = {
-        .open           = host1x_debug_open_all,
-        .read           = seq_read,
-        .llseek         = seq_lseek,
-        .release        = single_release,
+        .open = host1x_debug_open_all,
+        .read = seq_read,
+        .llseek = seq_lseek,
+        .release = single_release,
 };
 
 static int host1x_debug_open(struct inode *inode, struct file *file)
@@ -146,10 +160,10 @@ static int host1x_debug_open(struct inode *inode, struct file *file)
 }
 
 static const struct file_operations host1x_debug_fops = {
-        .open           = host1x_debug_open,
-        .read           = seq_read,
-        .llseek         = seq_lseek,
-        .release        = single_release,
+        .open = host1x_debug_open,
+        .read = seq_read,
+        .llseek = seq_lseek,
+        .release = single_release,
 };
 
 static void host1x_debugfs_init(struct host1x *host1x)
@@ -201,6 +215,7 @@ void host1x_debug_dump(struct host1x *host1x)
         struct output o = {
                 .fn = write_to_printk
         };
+
         show_all(host1x, &o);
 }
 
@@ -209,5 +224,6 @@ void host1x_debug_dump_syncpts(struct host1x *host1x)
         struct output o = {
                 .fn = write_to_printk
         };
+
         show_syncpts(host1x, &o);
 }
diff --git a/drivers/gpu/host1x/dev.c b/drivers/gpu/host1x/dev.c
index ff348690df94..a62317af76ad 100644
--- a/drivers/gpu/host1x/dev.c
+++ b/drivers/gpu/host1x/dev.c
@@ -63,13 +63,13 @@ u32 host1x_ch_readl(struct host1x_channel *ch, u32 r)
 }
 
 static const struct host1x_info host1x01_info = {
-        .nb_channels    = 8,
-        .nb_pts         = 32,
-        .nb_mlocks      = 16,
-        .nb_bases       = 8,
-        .init           = host1x01_init,
-        .sync_offset    = 0x3000,
-        .dma_mask       = DMA_BIT_MASK(32),
+        .nb_channels = 8,
+        .nb_pts = 32,
+        .nb_mlocks = 16,
+        .nb_bases = 8,
+        .init = host1x01_init,
+        .sync_offset = 0x3000,
+        .dma_mask = DMA_BIT_MASK(32),
 };
 
 static const struct host1x_info host1x02_info = {
@@ -102,7 +102,7 @@ static const struct host1x_info host1x05_info = {
         .dma_mask = DMA_BIT_MASK(34),
 };
 
-static struct of_device_id host1x_of_match[] = {
+static const struct of_device_id host1x_of_match[] = {
         { .compatible = "nvidia,tegra210-host1x", .data = &host1x05_info, },
         { .compatible = "nvidia,tegra124-host1x", .data = &host1x04_info, },
         { .compatible = "nvidia,tegra114-host1x", .data = &host1x02_info, },
diff --git a/drivers/gpu/host1x/dev.h b/drivers/gpu/host1x/dev.h
index dace124994bb..5220510f39da 100644
--- a/drivers/gpu/host1x/dev.h
+++ b/drivers/gpu/host1x/dev.h
@@ -45,7 +45,7 @@ struct host1x_cdma_ops {
         void (*start)(struct host1x_cdma *cdma);
         void (*stop)(struct host1x_cdma *cdma);
         void (*flush)(struct  host1x_cdma *cdma);
-        int (*timeout_init)(struct host1x_cdma *cdma, u32 syncpt_id);
+        int (*timeout_init)(struct host1x_cdma *cdma, unsigned int syncpt);
         void (*timeout_destroy)(struct host1x_cdma *cdma);
         void (*freeze)(struct host1x_cdma *cdma);
         void (*resume)(struct host1x_cdma *cdma, u32 getptr);
@@ -82,21 +82,21 @@ struct host1x_intr_ops {
         int (*init_host_sync)(struct host1x *host, u32 cpm,
                 void (*syncpt_thresh_work)(struct work_struct *work));
         void (*set_syncpt_threshold)(
-                struct host1x *host, u32 id, u32 thresh);
-        void (*enable_syncpt_intr)(struct host1x *host, u32 id);
-        void (*disable_syncpt_intr)(struct host1x *host, u32 id);
+                struct host1x *host, unsigned int id, u32 thresh);
+        void (*enable_syncpt_intr)(struct host1x *host, unsigned int id);
+        void (*disable_syncpt_intr)(struct host1x *host, unsigned int id);
         void (*disable_all_syncpt_intrs)(struct host1x *host);
         int (*free_syncpt_irq)(struct host1x *host);
 };
 
 struct host1x_info {
-        int     nb_channels;            /* host1x: num channels supported */
-        int     nb_pts;                 /* host1x: num syncpoints supported */
-        int     nb_bases;               /* host1x: num syncpoints supported */
-        int     nb_mlocks;              /* host1x: number of mlocks */
-        int     (*init)(struct host1x *); /* initialize per SoC ops */
-        int     sync_offset;
-        u64     dma_mask;               /* mask of addressable memory */
+        unsigned int nb_channels; /* host1x: number of channels supported */
+        unsigned int nb_pts; /* host1x: number of syncpoints supported */
+        unsigned int nb_bases; /* host1x: number of syncpoint bases supported */
+        unsigned int nb_mlocks; /* host1x: number of mlocks supported */
+        int (*init)(struct host1x *host1x); /* initialize per SoC ops */
+        unsigned int sync_offset; /* offset of syncpoint registers */
+        u64 dma_mask; /* mask of addressable memory */
 };
 
 struct host1x {
@@ -109,7 +109,6 @@ struct host1x {
         struct clk *clk;
 
         struct mutex intr_mutex;
-        struct workqueue_struct *intr_wq;
         int intr_syncpt_irq;
 
         const struct host1x_syncpt_ops *syncpt_op;
@@ -183,19 +182,20 @@ static inline int host1x_hw_intr_init_host_sync(struct host1x *host, u32 cpm,
 }
 
 static inline void host1x_hw_intr_set_syncpt_threshold(struct host1x *host,
-                                                       u32 id, u32 thresh)
+                                                       unsigned int id,
+                                                       u32 thresh)
 {
         host->intr_op->set_syncpt_threshold(host, id, thresh);
 }
 
 static inline void host1x_hw_intr_enable_syncpt_intr(struct host1x *host,
-                                                     u32 id)
+                                                     unsigned int id)
 {
         host->intr_op->enable_syncpt_intr(host, id);
 }
 
 static inline void host1x_hw_intr_disable_syncpt_intr(struct host1x *host,
-                                                      u32 id)
+                                                      unsigned int id)
 {
         host->intr_op->disable_syncpt_intr(host, id);
 }
@@ -212,9 +212,9 @@ static inline int host1x_hw_intr_free_syncpt_irq(struct host1x *host)
 
 static inline int host1x_hw_channel_init(struct host1x *host,
                                          struct host1x_channel *channel,
-                                         int chid)
+                                         unsigned int id)
 {
-        return host->channel_op->init(channel, host, chid);
+        return host->channel_op->init(channel, host, id);
 }
 
 static inline int host1x_hw_channel_submit(struct host1x *host,
@@ -243,9 +243,9 @@ static inline void host1x_hw_cdma_flush(struct host1x *host,
 
 static inline int host1x_hw_cdma_timeout_init(struct host1x *host,
                                               struct host1x_cdma *cdma,
-                                              u32 syncpt_id)
+                                              unsigned int syncpt)
 {
-        return host->cdma_op->timeout_init(cdma, syncpt_id);
+        return host->cdma_op->timeout_init(cdma, syncpt);
 }
 
 static inline void host1x_hw_cdma_timeout_destroy(struct host1x *host,
diff --git a/drivers/gpu/host1x/hw/cdma_hw.c b/drivers/gpu/host1x/hw/cdma_hw.c
index 305ea8f3382d..659c1bbfeeba 100644
--- a/drivers/gpu/host1x/hw/cdma_hw.c
+++ b/drivers/gpu/host1x/hw/cdma_hw.c
@@ -41,7 +41,7 @@ static void cdma_timeout_cpu_incr(struct host1x_cdma *cdma, u32 getptr,
 {
         struct host1x *host1x = cdma_to_host1x(cdma);
         struct push_buffer *pb = &cdma->push_buffer;
-        u32 i;
+        unsigned int i;
 
         for (i = 0; i < syncpt_incrs; i++)
                 host1x_syncpt_incr(cdma->timeout.syncpt);
@@ -58,6 +58,7 @@ static void cdma_timeout_cpu_incr(struct host1x_cdma *cdma, u32 getptr,
                         &pb->phys, getptr);
                 getptr = (getptr + 8) & (pb->size_bytes - 1);
         }
+
         wmb();
 }
 
@@ -162,12 +163,14 @@ static void cdma_stop(struct host1x_cdma *cdma)
         struct host1x_channel *ch = cdma_to_channel(cdma);
 
         mutex_lock(&cdma->lock);
+
         if (cdma->running) {
                 host1x_cdma_wait_locked(cdma, CDMA_EVENT_SYNC_QUEUE_EMPTY);
                 host1x_ch_writel(ch, HOST1X_CHANNEL_DMACTRL_DMASTOP,
                                  HOST1X_CHANNEL_DMACTRL);
                 cdma->running = false;
         }
+
         mutex_unlock(&cdma->lock);
 }
 
@@ -213,11 +216,11 @@ static void cdma_resume(struct host1x_cdma *cdma, u32 getptr)
         u32 cmdproc_stop;
 
         dev_dbg(host1x->dev,
-                "resuming channel (id %d, DMAGET restart = 0x%x)\n",
+                "resuming channel (id %u, DMAGET restart = 0x%x)\n",
                 ch->id, getptr);
 
         cmdproc_stop = host1x_sync_readl(host1x, HOST1X_SYNC_CMDPROC_STOP);
-        cmdproc_stop &= ~(BIT(ch->id));
+        cmdproc_stop &= ~BIT(ch->id);
         host1x_sync_writel(host1x, cmdproc_stop, HOST1X_SYNC_CMDPROC_STOP);
 
         cdma->torndown = false;
@@ -231,14 +234,11 @@ static void cdma_resume(struct host1x_cdma *cdma, u32 getptr)
  */
 static void cdma_timeout_handler(struct work_struct *work)
 {
+        u32 prev_cmdproc, cmdproc_stop, syncpt_val;
         struct host1x_cdma *cdma;
         struct host1x *host1x;
         struct host1x_channel *ch;
 
-        u32 syncpt_val;
-
-        u32 prev_cmdproc, cmdproc_stop;
-
         cdma = container_of(to_delayed_work(work), struct host1x_cdma,
                             timeout.wq);
         host1x = cdma_to_host1x(cdma);
@@ -277,9 +277,9 @@ static void cdma_timeout_handler(struct work_struct *work)
                 return;
         }
 
-        dev_warn(host1x->dev, "%s: timeout: %d (%s), HW thresh %d, done %d\n",
-                __func__, cdma->timeout.syncpt->id, cdma->timeout.syncpt->name,
-                syncpt_val, cdma->timeout.syncpt_val);
+        dev_warn(host1x->dev, "%s: timeout: %u (%s), HW thresh %d, done %d\n",
+                 __func__, cdma->timeout.syncpt->id, cdma->timeout.syncpt->name,
+                 syncpt_val, cdma->timeout.syncpt_val);
 
         /* stop HW, resetting channel/module */
         host1x_hw_cdma_freeze(host1x, cdma);
@@ -291,7 +291,7 @@ static void cdma_timeout_handler(struct work_struct *work)
 /*
  * Init timeout resources
  */
-static int cdma_timeout_init(struct host1x_cdma *cdma, u32 syncpt_id)
+static int cdma_timeout_init(struct host1x_cdma *cdma, unsigned int syncpt)
 {
         INIT_DELAYED_WORK(&cdma->timeout.wq, cdma_timeout_handler);
         cdma->timeout.initialized = true;
@@ -306,6 +306,7 @@ static void cdma_timeout_destroy(struct host1x_cdma *cdma)
 {
         if (cdma->timeout.initialized)
                 cancel_delayed_work(&cdma->timeout.wq);
+
         cdma->timeout.initialized = false;
 }
 
diff --git a/drivers/gpu/host1x/hw/channel_hw.c b/drivers/gpu/host1x/hw/channel_hw.c
index 946c332c3906..5e8df78b7acd 100644
--- a/drivers/gpu/host1x/hw/channel_hw.c
+++ b/drivers/gpu/host1x/hw/channel_hw.c
@@ -46,6 +46,7 @@ static void trace_write_gather(struct host1x_cdma *cdma, struct host1x_bo *bo,
                  */
                 for (i = 0; i < words; i += TRACE_MAX_LENGTH) {
                         u32 num_words = min(words - i, TRACE_MAX_LENGTH);
+
                         offset += i * sizeof(u32);
 
                         trace_host1x_cdma_push_gather(dev_name(dev), bo,
@@ -66,6 +67,7 @@ static void submit_gathers(struct host1x_job *job)
                 struct host1x_job_gather *g = &job->gathers[i];
                 u32 op1 = host1x_opcode_gather(g->words);
                 u32 op2 = g->base + g->offset;
+
                 trace_write_gather(cdma, g->bo, g->offset, op1 & 0xffff);
                 host1x_cdma_push(cdma, op1, op2);
         }
@@ -75,7 +77,8 @@ static inline void synchronize_syncpt_base(struct host1x_job *job)
 {
         struct host1x *host = dev_get_drvdata(job->channel->dev->parent);
         struct host1x_syncpt *sp = host->syncpt + job->syncpt_id;
-        u32 id, value;
+        unsigned int id;
+        u32 value;
 
         value = host1x_syncpt_read_max(sp);
         id = sp->base->id;
diff --git a/drivers/gpu/host1x/hw/debug_hw.c b/drivers/gpu/host1x/hw/debug_hw.c
index cc3f1825c735..7a4a3286e4a7 100644
--- a/drivers/gpu/host1x/hw/debug_hw.c
+++ b/drivers/gpu/host1x/hw/debug_hw.c
@@ -40,8 +40,7 @@ enum {
 
 static unsigned int show_channel_command(struct output *o, u32 val)
 {
-        unsigned mask;
-        unsigned subop;
+        unsigned int mask, subop;
 
         switch (val >> 28) {
         case HOST1X_OPCODE_SETCLASS:
@@ -51,12 +50,11 @@ static unsigned int show_channel_command(struct output *o, u32 val)
                                             val >> 6 & 0x3ff,
                                             val >> 16 & 0xfff, mask);
                         return hweight8(mask);
-                } else {
-                        host1x_debug_output(o, "SETCL(class=%03x)\n",
-                                            val >> 6 & 0x3ff);
-                        return 0;
                 }
 
+                host1x_debug_output(o, "SETCL(class=%03x)\n", val >> 6 & 0x3ff);
+                return 0;
+
         case HOST1X_OPCODE_INCR:
                 host1x_debug_output(o, "INCR(offset=%03x, [",
                                     val >> 16 & 0xfff);
@@ -143,7 +141,8 @@ static void show_channel_gathers(struct output *o, struct host1x_cdma *cdma)
         struct host1x_job *job;
 
         list_for_each_entry(job, &cdma->sync_queue, list) {
-                int i;
+                unsigned int i;
+
                 host1x_debug_output(o, "\n%p: JOB, syncpt_id=%d, syncpt_val=%d, first_get=%08x, timeout=%d num_slots=%d, num_handles=%d\n",
                                     job, job->syncpt_id, job->syncpt_end,
                                     job->first_get, job->timeout,
@@ -190,7 +189,7 @@ static void host1x_debug_show_channel_cdma(struct host1x *host,
         cbread = host1x_sync_readl(host, HOST1X_SYNC_CBREAD(ch->id));
         cbstat = host1x_sync_readl(host, HOST1X_SYNC_CBSTAT(ch->id));
 
-        host1x_debug_output(o, "%d-%s: ", ch->id, dev_name(ch->dev));
+        host1x_debug_output(o, "%u-%s: ", ch->id, dev_name(ch->dev));
 
         if (HOST1X_CHANNEL_DMACTRL_DMASTOP_V(dmactrl) ||
             !ch->cdma.push_buffer.mapped) {
@@ -200,14 +199,13 @@ static void host1x_debug_show_channel_cdma(struct host1x *host,
 
         if (HOST1X_SYNC_CBSTAT_CBCLASS_V(cbstat) == HOST1X_CLASS_HOST1X &&
             HOST1X_SYNC_CBSTAT_CBOFFSET_V(cbstat) ==
-            HOST1X_UCLASS_WAIT_SYNCPT)
+                        HOST1X_UCLASS_WAIT_SYNCPT)
                 host1x_debug_output(o, "waiting on syncpt %d val %d\n",
                                     cbread >> 24, cbread & 0xffffff);
         else if (HOST1X_SYNC_CBSTAT_CBCLASS_V(cbstat) ==
-           HOST1X_CLASS_HOST1X &&
-           HOST1X_SYNC_CBSTAT_CBOFFSET_V(cbstat) ==
-           HOST1X_UCLASS_WAIT_SYNCPT_BASE) {
-
+                                HOST1X_CLASS_HOST1X &&
+                 HOST1X_SYNC_CBSTAT_CBOFFSET_V(cbstat) ==
+                                HOST1X_UCLASS_WAIT_SYNCPT_BASE) {
                 base = (cbread >> 16) & 0xff;
                 baseval =
                         host1x_sync_readl(host, HOST1X_SYNC_SYNCPT_BASE(base));
@@ -236,7 +234,7 @@ static void host1x_debug_show_channel_fifo(struct host1x *host,
         u32 val, rd_ptr, wr_ptr, start, end;
         unsigned int data_count = 0;
 
-        host1x_debug_output(o, "%d: fifo:\n", ch->id);
+        host1x_debug_output(o, "%u: fifo:\n", ch->id);
 
         val = host1x_ch_readl(ch, HOST1X_CHANNEL_FIFOSTAT);
         host1x_debug_output(o, "FIFOSTAT %08x\n", val);
@@ -290,20 +288,22 @@ static void host1x_debug_show_channel_fifo(struct host1x *host,
 
 static void host1x_debug_show_mlocks(struct host1x *host, struct output *o)
 {
-        int i;
+        unsigned int i;
 
         host1x_debug_output(o, "---- mlocks ----\n");
+
         for (i = 0; i < host1x_syncpt_nb_mlocks(host); i++) {
                 u32 owner =
                         host1x_sync_readl(host, HOST1X_SYNC_MLOCK_OWNER(i));
                 if (HOST1X_SYNC_MLOCK_OWNER_CH_OWNS_V(owner))
-                        host1x_debug_output(o, "%d: locked by channel %d\n",
+                        host1x_debug_output(o, "%u: locked by channel %u\n",
                                 i, HOST1X_SYNC_MLOCK_OWNER_CHID_V(owner));
                 else if (HOST1X_SYNC_MLOCK_OWNER_CPU_OWNS_V(owner))
-                        host1x_debug_output(o, "%d: locked by cpu\n", i);
+                        host1x_debug_output(o, "%u: locked by cpu\n", i);
                 else
-                        host1x_debug_output(o, "%d: unlocked\n", i);
+                        host1x_debug_output(o, "%u: unlocked\n", i);
         }
+
         host1x_debug_output(o, "\n");
 }
 
diff --git a/drivers/gpu/host1x/hw/intr_hw.c b/drivers/gpu/host1x/hw/intr_hw.c
index e1e31e9e67cd..dacb8009a605 100644
--- a/drivers/gpu/host1x/hw/intr_hw.c
+++ b/drivers/gpu/host1x/hw/intr_hw.c
@@ -38,14 +38,14 @@ static void host1x_intr_syncpt_handle(struct host1x_syncpt *syncpt)
         host1x_sync_writel(host, BIT_MASK(id),
                 HOST1X_SYNC_SYNCPT_THRESH_CPU0_INT_STATUS(BIT_WORD(id)));
 
-        queue_work(host->intr_wq, &syncpt->intr.work);
+        schedule_work(&syncpt->intr.work);
 }
 
 static irqreturn_t syncpt_thresh_isr(int irq, void *dev_id)
 {
         struct host1x *host = dev_id;
         unsigned long reg;
-        int i, id;
+        unsigned int i, id;
 
         for (i = 0; i < DIV_ROUND_UP(host->info->nb_pts, 32); i++) {
                 reg = host1x_sync_readl(host,
@@ -62,7 +62,7 @@ static irqreturn_t syncpt_thresh_isr(int irq, void *dev_id)
 
 static void _host1x_intr_disable_all_syncpt_intrs(struct host1x *host)
 {
-        u32 i;
+        unsigned int i;
 
         for (i = 0; i < DIV_ROUND_UP(host->info->nb_pts, 32); ++i) {
                 host1x_sync_writel(host, 0xffffffffu,
@@ -72,10 +72,12 @@ static void _host1x_intr_disable_all_syncpt_intrs(struct host1x *host)
         }
 }
 
-static int _host1x_intr_init_host_sync(struct host1x *host, u32 cpm,
-        void (*syncpt_thresh_work)(struct work_struct *))
+static int
+_host1x_intr_init_host_sync(struct host1x *host, u32 cpm,
+                            void (*syncpt_thresh_work)(struct work_struct *))
 {
-        int i, err;
+        unsigned int i;
+        int err;
 
         host1x_hw_intr_disable_all_syncpt_intrs(host);
 
@@ -106,18 +108,21 @@ static int _host1x_intr_init_host_sync(struct host1x *host, u32 cpm,
 }
 
 static void _host1x_intr_set_syncpt_threshold(struct host1x *host,
-        u32 id, u32 thresh)
+                                              unsigned int id,
+                                              u32 thresh)
 {
         host1x_sync_writel(host, thresh, HOST1X_SYNC_SYNCPT_INT_THRESH(id));
 }
 
-static void _host1x_intr_enable_syncpt_intr(struct host1x *host, u32 id)
+static void _host1x_intr_enable_syncpt_intr(struct host1x *host,
+                                            unsigned int id)
 {
         host1x_sync_writel(host, BIT_MASK(id),
                 HOST1X_SYNC_SYNCPT_THRESH_INT_ENABLE_CPU0(BIT_WORD(id)));
 }
 
-static void _host1x_intr_disable_syncpt_intr(struct host1x *host, u32 id)
+static void _host1x_intr_disable_syncpt_intr(struct host1x *host,
+                                             unsigned int id)
 {
         host1x_sync_writel(host, BIT_MASK(id),
                 HOST1X_SYNC_SYNCPT_THRESH_INT_DISABLE(BIT_WORD(id)));
@@ -127,8 +132,13 @@ static void _host1x_intr_disable_syncpt_intr(struct host1x *host, u32 id)
 
 static int _host1x_free_syncpt_irq(struct host1x *host)
 {
+        unsigned int i;
+
         devm_free_irq(host->dev, host->intr_syncpt_irq, host);
-        flush_workqueue(host->intr_wq);
+
+        for (i = 0; i < host->info->nb_pts; i++)
+                cancel_work_sync(&host->syncpt[i].intr.work);
+
         return 0;
 }
 
diff --git a/drivers/gpu/host1x/hw/syncpt_hw.c b/drivers/gpu/host1x/hw/syncpt_hw.c
index 56e85395ac24..c93f74fcce72 100644
--- a/drivers/gpu/host1x/hw/syncpt_hw.c
+++ b/drivers/gpu/host1x/hw/syncpt_hw.c
@@ -26,8 +26,9 @@
  */
 static void syncpt_restore(struct host1x_syncpt *sp)
 {
+        u32 min = host1x_syncpt_read_min(sp);
         struct host1x *host = sp->host;
-        int min = host1x_syncpt_read_min(sp);
+
         host1x_sync_writel(host, min, HOST1X_SYNC_SYNCPT(sp->id));
 }
 
@@ -37,6 +38,7 @@ static void syncpt_restore(struct host1x_syncpt *sp)
 static void syncpt_restore_wait_base(struct host1x_syncpt *sp)
 {
         struct host1x *host = sp->host;
+
         host1x_sync_writel(host, sp->base_val,
                            HOST1X_SYNC_SYNCPT_BASE(sp->id));
 }
@@ -47,6 +49,7 @@ static void syncpt_restore_wait_base(struct host1x_syncpt *sp)
 static void syncpt_read_wait_base(struct host1x_syncpt *sp)
 {
         struct host1x *host = sp->host;
+
         sp->base_val =
                 host1x_sync_readl(host, HOST1X_SYNC_SYNCPT_BASE(sp->id));
 }
@@ -85,6 +88,7 @@ static int syncpt_cpu_incr(struct host1x_syncpt *sp)
         if (!host1x_syncpt_client_managed(sp) &&
             host1x_syncpt_idle(sp))
                 return -EINVAL;
+
         host1x_sync_writel(host, BIT_MASK(sp->id),
                            HOST1X_SYNC_SYNCPT_CPU_INCR(reg_offset));
         wmb();
@@ -95,10 +99,10 @@ static int syncpt_cpu_incr(struct host1x_syncpt *sp)
 /* remove a wait pointed to by patch_addr */
 static int syncpt_patch_wait(struct host1x_syncpt *sp, void *patch_addr)
 {
-        u32 override = host1x_class_host_wait_syncpt(
-                HOST1X_SYNCPT_RESERVED, 0);
+        u32 override = host1x_class_host_wait_syncpt(HOST1X_SYNCPT_RESERVED, 0);
 
         *((u32 *)patch_addr) = override;
+
         return 0;
 }
 
diff --git a/drivers/gpu/host1x/intr.c b/drivers/gpu/host1x/intr.c
index 2491bf82e30c..8b4fad0ab35d 100644
--- a/drivers/gpu/host1x/intr.c
+++ b/drivers/gpu/host1x/intr.c
@@ -122,18 +122,20 @@ static void action_submit_complete(struct host1x_waitlist *waiter)
 static void action_wakeup(struct host1x_waitlist *waiter)
 {
         wait_queue_head_t *wq = waiter->data;
+
         wake_up(wq);
 }
 
 static void action_wakeup_interruptible(struct host1x_waitlist *waiter)
 {
         wait_queue_head_t *wq = waiter->data;
+
         wake_up_interruptible(wq);
 }
 
 typedef void (*action_handler)(struct host1x_waitlist *waiter);
 
-static action_handler action_handlers[HOST1X_INTR_ACTION_COUNT] = {
+static const action_handler action_handlers[HOST1X_INTR_ACTION_COUNT] = {
         action_submit_complete,
         action_wakeup,
         action_wakeup_interruptible,
@@ -209,7 +211,7 @@ static void syncpt_thresh_work(struct work_struct *work)
                                 host1x_syncpt_load(host->syncpt + id));
 }
 
-int host1x_intr_add_action(struct host1x *host, u32 id, u32 thresh,
+int host1x_intr_add_action(struct host1x *host, unsigned int id, u32 thresh,
                            enum host1x_intr_action action, void *data,
                            struct host1x_waitlist *waiter, void **ref)
 {
@@ -254,7 +256,7 @@ int host1x_intr_add_action(struct host1x *host, u32 id, u32 thresh,
         return 0;
 }
 
-void host1x_intr_put_ref(struct host1x *host, u32 id, void *ref)
+void host1x_intr_put_ref(struct host1x *host, unsigned int id, void *ref)
 {
         struct host1x_waitlist *waiter = ref;
         struct host1x_syncpt *syncpt;
@@ -277,9 +279,6 @@ int host1x_intr_init(struct host1x *host, unsigned int irq_sync)
 
         mutex_init(&host->intr_mutex);
         host->intr_syncpt_irq = irq_sync;
-        host->intr_wq = create_workqueue("host_syncpt");
-        if (!host->intr_wq)
-                return -ENOMEM;
 
         for (id = 0; id < nb_pts; ++id) {
                 struct host1x_syncpt *syncpt = host->syncpt + id;
@@ -288,7 +287,7 @@ int host1x_intr_init(struct host1x *host, unsigned int irq_sync)
                 INIT_LIST_HEAD(&syncpt->intr.wait_head);
                 snprintf(syncpt->intr.thresh_irq_name,
                          sizeof(syncpt->intr.thresh_irq_name),
-                         "host1x_sp_%02d", id);
+                         "host1x_sp_%02u", id);
         }
 
         host1x_intr_start(host);
@@ -299,7 +298,6 @@ int host1x_intr_init(struct host1x *host, unsigned int irq_sync)
 void host1x_intr_deinit(struct host1x *host)
 {
         host1x_intr_stop(host);
-        destroy_workqueue(host->intr_wq);
 }
 
 void host1x_intr_start(struct host1x *host)
@@ -342,7 +340,7 @@ void host1x_intr_stop(struct host1x *host)
                 if (!list_empty(&syncpt[id].intr.wait_head)) {
                         /* output diagnostics */
                         mutex_unlock(&host->intr_mutex);
-                        pr_warn("%s cannot stop syncpt intr id=%d\n",
+                        pr_warn("%s cannot stop syncpt intr id=%u\n",
                                 __func__, id);
                         return;
                 }
diff --git a/drivers/gpu/host1x/intr.h b/drivers/gpu/host1x/intr.h
index 2b8adf016a05..1370c2bb75b8 100644
--- a/drivers/gpu/host1x/intr.h
+++ b/drivers/gpu/host1x/intr.h
@@ -75,7 +75,7 @@ struct host1x_waitlist {
  *
  * This is a non-blocking api.
  */
-int host1x_intr_add_action(struct host1x *host, u32 id, u32 thresh,
+int host1x_intr_add_action(struct host1x *host, unsigned int id, u32 thresh,
         enum host1x_intr_action action, void *data,
         struct host1x_waitlist *waiter, void **ref);
 
@@ -84,7 +84,7 @@ int host1x_intr_add_action(struct host1x *host, u32 id, u32 thresh,
  * You must call this if you passed non-NULL as ref.
  * @ref the ref returned from host1x_intr_add_action()
  */
-void host1x_intr_put_ref(struct host1x *host, u32 id, void *ref);
+void host1x_intr_put_ref(struct host1x *host, unsigned int id, void *ref);
 
 /* Initialize host1x sync point interrupt */
 int host1x_intr_init(struct host1x *host, unsigned int irq_sync);
diff --git a/drivers/gpu/host1x/job.c b/drivers/gpu/host1x/job.c
index b4515d544039..a91b7c4a6110 100644
--- a/drivers/gpu/host1x/job.c
+++ b/drivers/gpu/host1x/job.c
@@ -161,7 +161,7 @@ static int do_waitchks(struct host1x_job *job, struct host1x *host,
 
                 if (host1x_syncpt_is_expired(sp, wait->thresh)) {
                         dev_dbg(host->dev,
-                                "drop WAIT id %d (%s) thresh 0x%x, min 0x%x\n",
+                                "drop WAIT id %u (%s) thresh 0x%x, min 0x%x\n",
                                 wait->syncpt_id, sp->name, wait->thresh,
                                 host1x_syncpt_read_min(sp));
 
@@ -464,6 +464,7 @@ static inline int copy_gathers(struct host1x_job *job, struct device *dev)
 
         for (i = 0; i < job->num_gathers; i++) {
                 struct host1x_job_gather *g = &job->gathers[i];
+
                 size += g->words * sizeof(u32);
         }
 
@@ -514,6 +515,7 @@ int host1x_job_pin(struct host1x_job *job, struct device *dev)
         bitmap_zero(waitchk_mask, host1x_syncpt_nb_pts(host));
         for (i = 0; i < job->num_waitchk; i++) {
                 u32 syncpt_id = job->waitchk[i].syncpt_id;
+
                 if (syncpt_id < host1x_syncpt_nb_pts(host))
                         set_bit(syncpt_id, waitchk_mask);
         }
@@ -571,14 +573,16 @@ void host1x_job_unpin(struct host1x_job *job)
 
         for (i = 0; i < job->num_unpins; i++) {
                 struct host1x_job_unpin_data *unpin = &job->unpins[i];
+
                 host1x_bo_unpin(unpin->bo, unpin->sgt);
                 host1x_bo_put(unpin->bo);
         }
+
         job->num_unpins = 0;
 
         if (job->gather_copy_size)
                 dma_free_wc(job->channel->dev, job->gather_copy_size,
-                            job->gather_copy_mapped, job->gather_copy);
+                            job->gather_copy_mapped, job->gather_copy);
 }
 EXPORT_SYMBOL(host1x_job_unpin);
 
diff --git a/drivers/gpu/host1x/syncpt.c b/drivers/gpu/host1x/syncpt.c
index 6b7fdc1e2ed0..95589328ad52 100644
--- a/drivers/gpu/host1x/syncpt.c
+++ b/drivers/gpu/host1x/syncpt.c
@@ -73,7 +73,7 @@ static struct host1x_syncpt *host1x_syncpt_alloc(struct host1x *host,
                         return NULL;
         }
 
-        name = kasprintf(GFP_KERNEL, "%02d-%s", sp->id,
+        name = kasprintf(GFP_KERNEL, "%02u-%s", sp->id,
                         dev ? dev_name(dev) : NULL);
         if (!name)
                 return NULL;
@@ -110,12 +110,14 @@ EXPORT_SYMBOL(host1x_syncpt_incr_max);
 void host1x_syncpt_restore(struct host1x *host)
 {
         struct host1x_syncpt *sp_base = host->syncpt;
-        u32 i;
+        unsigned int i;
 
         for (i = 0; i < host1x_syncpt_nb_pts(host); i++)
                 host1x_hw_syncpt_restore(host, sp_base + i);
+
         for (i = 0; i < host1x_syncpt_nb_bases(host); i++)
                 host1x_hw_syncpt_restore_wait_base(host, sp_base + i);
+
         wmb();
 }
 
@@ -126,7 +128,7 @@ void host1x_syncpt_restore(struct host1x *host)
 void host1x_syncpt_save(struct host1x *host)
 {
         struct host1x_syncpt *sp_base = host->syncpt;
-        u32 i;
+        unsigned int i;
 
         for (i = 0; i < host1x_syncpt_nb_pts(host); i++) {
                 if (host1x_syncpt_client_managed(sp_base + i))
@@ -146,6 +148,7 @@ void host1x_syncpt_save(struct host1x *host)
 u32 host1x_syncpt_load(struct host1x_syncpt *sp)
 {
         u32 val;
+
         val = host1x_hw_syncpt_load(sp->host, sp);
         trace_host1x_syncpt_load_min(sp->id, val);
 
@@ -157,10 +160,9 @@ u32 host1x_syncpt_load(struct host1x_syncpt *sp)
  */
 u32 host1x_syncpt_load_wait_base(struct host1x_syncpt *sp)
 {
-        u32 val;
         host1x_hw_syncpt_load_wait_base(sp->host, sp);
-        val = sp->base_val;
-        return val;
+
+        return sp->base_val;
 }
 
 /*
@@ -179,6 +181,7 @@ EXPORT_SYMBOL(host1x_syncpt_incr);
 static bool syncpt_load_min_is_expired(struct host1x_syncpt *sp, u32 thresh)
 {
         host1x_hw_syncpt_load(sp->host, sp);
+
         return host1x_syncpt_is_expired(sp, thresh);
 }
 
@@ -186,7 +189,7 @@ static bool syncpt_load_min_is_expired(struct host1x_syncpt *sp, u32 thresh)
  * Main entrypoint for syncpoint value waits.
  */
 int host1x_syncpt_wait(struct host1x_syncpt *sp, u32 thresh, long timeout,
-                        u32 *value)
+                       u32 *value)
 {
         DECLARE_WAIT_QUEUE_HEAD_ONSTACK(wq);
         void *ref;
@@ -201,6 +204,7 @@ int host1x_syncpt_wait(struct host1x_syncpt *sp, u32 thresh, long timeout,
         if (host1x_syncpt_is_expired(sp, thresh)) {
                 if (value)
                         *value = host1x_syncpt_load(sp);
+
                 return 0;
         }
 
@@ -209,6 +213,7 @@ int host1x_syncpt_wait(struct host1x_syncpt *sp, u32 thresh, long timeout,
         if (host1x_syncpt_is_expired(sp, thresh)) {
                 if (value)
                         *value = val;
+
                 goto done;
         }
 
@@ -239,32 +244,42 @@ int host1x_syncpt_wait(struct host1x_syncpt *sp, u32 thresh, long timeout,
         /* wait for the syncpoint, or timeout, or signal */
         while (timeout) {
                 long check = min_t(long, SYNCPT_CHECK_PERIOD, timeout);
-                int remain = wait_event_interruptible_timeout(wq,
+                int remain;
+
+                remain = wait_event_interruptible_timeout(wq,
                                 syncpt_load_min_is_expired(sp, thresh),
                                 check);
                 if (remain > 0 || host1x_syncpt_is_expired(sp, thresh)) {
                         if (value)
                                 *value = host1x_syncpt_load(sp);
+
                         err = 0;
+
                         break;
                 }
+
                 if (remain < 0) {
                         err = remain;
                         break;
                 }
+
                 timeout -= check;
+
                 if (timeout && check_count <= MAX_STUCK_CHECK_COUNT) {
                         dev_warn(sp->host->dev,
-                                "%s: syncpoint id %d (%s) stuck waiting %d, timeout=%ld\n",
+                                "%s: syncpoint id %u (%s) stuck waiting %d, timeout=%ld\n",
                                  current->comm, sp->id, sp->name,
                                  thresh, timeout);
 
                         host1x_debug_dump_syncpts(sp->host);
+
                         if (check_count == MAX_STUCK_CHECK_COUNT)
                                 host1x_debug_dump(sp->host);
+
                         check_count++;
                 }
         }
+
         host1x_intr_put_ref(sp->host, sp->id, ref);
 
 done:
@@ -279,7 +294,9 @@ bool host1x_syncpt_is_expired(struct host1x_syncpt *sp, u32 thresh)
 {
         u32 current_val;
         u32 future_val;
+
         smp_rmb();
+
         current_val = (u32)atomic_read(&sp->min_val);
         future_val = (u32)atomic_read(&sp->max_val);
 
@@ -341,14 +358,14 @@ int host1x_syncpt_init(struct host1x *host)
 {
         struct host1x_syncpt_base *bases;
         struct host1x_syncpt *syncpt;
-        int i;
+        unsigned int i;
 
-        syncpt = devm_kzalloc(host->dev, sizeof(*syncpt) * host->info->nb_pts,
+        syncpt = devm_kcalloc(host->dev, host->info->nb_pts, sizeof(*syncpt),
                               GFP_KERNEL);
         if (!syncpt)
                 return -ENOMEM;
 
-        bases = devm_kzalloc(host->dev, sizeof(*bases) * host->info->nb_bases,
+        bases = devm_kcalloc(host->dev, host->info->nb_bases, sizeof(*bases),
                              GFP_KERNEL);
         if (!bases)
                 return -ENOMEM;
@@ -378,6 +395,7 @@ struct host1x_syncpt *host1x_syncpt_request(struct device *dev,
                                             unsigned long flags)
 {
         struct host1x *host = dev_get_drvdata(dev->parent);
+
         return host1x_syncpt_alloc(host, dev, flags);
 }
 EXPORT_SYMBOL(host1x_syncpt_request);
@@ -398,8 +416,9 @@ EXPORT_SYMBOL(host1x_syncpt_free);
 
 void host1x_syncpt_deinit(struct host1x *host)
 {
-        int i;
         struct host1x_syncpt *sp = host->syncpt;
+        unsigned int i;
+
         for (i = 0; i < host->info->nb_pts; i++, sp++)
                 kfree(sp->name);
 }
@@ -407,10 +426,11 @@ void host1x_syncpt_deinit(struct host1x *host)
 /*
  * Read max. It indicates how many operations there are in queue, either in
  * channel or in a software thread.
- * */
+ */
 u32 host1x_syncpt_read_max(struct host1x_syncpt *sp)
 {
         smp_rmb();
+
         return (u32)atomic_read(&sp->max_val);
 }
 EXPORT_SYMBOL(host1x_syncpt_read_max);
@@ -421,6 +441,7 @@ EXPORT_SYMBOL(host1x_syncpt_read_max);
 u32 host1x_syncpt_read_min(struct host1x_syncpt *sp)
 {
         smp_rmb();
+
         return (u32)atomic_read(&sp->min_val);
 }
 EXPORT_SYMBOL(host1x_syncpt_read_min);
@@ -431,25 +452,26 @@ u32 host1x_syncpt_read(struct host1x_syncpt *sp)
 }
 EXPORT_SYMBOL(host1x_syncpt_read);
 
-int host1x_syncpt_nb_pts(struct host1x *host)
+unsigned int host1x_syncpt_nb_pts(struct host1x *host)
 {
         return host->info->nb_pts;
 }
 
-int host1x_syncpt_nb_bases(struct host1x *host)
+unsigned int host1x_syncpt_nb_bases(struct host1x *host)
 {
         return host->info->nb_bases;
 }
 
-int host1x_syncpt_nb_mlocks(struct host1x *host)
+unsigned int host1x_syncpt_nb_mlocks(struct host1x *host)
 {
         return host->info->nb_mlocks;
 }
 
-struct host1x_syncpt *host1x_syncpt_get(struct host1x *host, u32 id)
+struct host1x_syncpt *host1x_syncpt_get(struct host1x *host, unsigned int id)
 {
         if (host->info->nb_pts < id)
                 return NULL;
+
         return host->syncpt + id;
 }
 EXPORT_SYMBOL(host1x_syncpt_get);
diff --git a/drivers/gpu/host1x/syncpt.h b/drivers/gpu/host1x/syncpt.h
index 9056465ecd3f..f719205105ac 100644
--- a/drivers/gpu/host1x/syncpt.h
+++ b/drivers/gpu/host1x/syncpt.h
@@ -37,7 +37,7 @@ struct host1x_syncpt_base {
 };
 
 struct host1x_syncpt {
-        int id;
+        unsigned int id;
         atomic_t min_val;
         atomic_t max_val;
         u32 base_val;
@@ -58,13 +58,13 @@ int host1x_syncpt_init(struct host1x *host);
 void host1x_syncpt_deinit(struct host1x *host);
 
 /* Return number of sync point supported. */
-int host1x_syncpt_nb_pts(struct host1x *host);
+unsigned int host1x_syncpt_nb_pts(struct host1x *host);
 
 /* Return number of wait bases supported. */
-int host1x_syncpt_nb_bases(struct host1x *host);
+unsigned int host1x_syncpt_nb_bases(struct host1x *host);
 
 /* Return number of mlocks supported. */
-int host1x_syncpt_nb_mlocks(struct host1x *host);
+unsigned int host1x_syncpt_nb_mlocks(struct host1x *host);
 
 /*
  * Check sync point sanity. If max is larger than min, there have too many
diff --git a/drivers/gpu/ipu-v3/ipu-dc.c b/drivers/gpu/ipu-v3/ipu-dc.c
index 2f29780e7c68..659475c1e44a 100644
--- a/drivers/gpu/ipu-v3/ipu-dc.c
+++ b/drivers/gpu/ipu-v3/ipu-dc.c
@@ -150,6 +150,9 @@ static void dc_write_tmpl(struct ipu_dc *dc, int word, u32 opcode, u32 operand,
 static int ipu_bus_format_to_map(u32 fmt)
 {
         switch (fmt) {
+        default:
+                WARN_ON(1);
+                /* fall-through */
         case MEDIA_BUS_FMT_RGB888_1X24:
                 return IPU_DC_MAP_RGB24;
         case MEDIA_BUS_FMT_RGB565_1X16:
@@ -162,8 +165,6 @@ static int ipu_bus_format_to_map(u32 fmt)
                 return IPU_DC_MAP_LVDS666;
         case MEDIA_BUS_FMT_BGR888_1X24:
                 return IPU_DC_MAP_BGR24;
-        default:
-                return -EINVAL;
         }
 }
 
@@ -178,10 +179,6 @@ int ipu_dc_init_sync(struct ipu_dc *dc, struct ipu_di *di, bool interlaced,
         dc->di = ipu_di_get_num(di);
 
         map = ipu_bus_format_to_map(bus_format);
-        if (map < 0) {
-                dev_dbg(priv->dev, "IPU_DISP: No MAP\n");
-                return map;
-        }
 
         /*
          * In interlaced mode we need more counters to create the asymmetric
diff --git a/drivers/gpu/ipu-v3/ipu-di.c b/drivers/gpu/ipu-v3/ipu-di.c
index 359268e3a166..a8d87ddd8a17 100644
--- a/drivers/gpu/ipu-v3/ipu-di.c
+++ b/drivers/gpu/ipu-v3/ipu-di.c
@@ -572,9 +572,6 @@ int ipu_di_init_sync_panel(struct ipu_di *di, struct ipu_di_signal_cfg *sig)
         dev_dbg(di->ipu->dev, "disp %d: panel size = %d x %d\n",
                 di->id, sig->mode.hactive, sig->mode.vactive);
 
-        if ((sig->mode.vsync_len == 0) || (sig->mode.hsync_len == 0))
-                return -EINVAL;
-
         dev_dbg(di->ipu->dev, "Clocks: IPU %luHz DI %luHz Needed %luHz\n",
                 clk_get_rate(di->clk_ipu),
                 clk_get_rate(di->clk_di),
diff --git a/drivers/gpu/ipu-v3/ipu-dmfc.c b/drivers/gpu/ipu-v3/ipu-dmfc.c
index 837b1ec22800..42705bb5aaa3 100644
--- a/drivers/gpu/ipu-v3/ipu-dmfc.c
+++ b/drivers/gpu/ipu-v3/ipu-dmfc.c
@@ -45,17 +45,6 @@
 #define DMFC_DP_CHAN_6B_24              16
 #define DMFC_DP_CHAN_6F_29              24
 
-#define DMFC_FIFO_SIZE_64               (3 << 3)
-#define DMFC_FIFO_SIZE_128              (2 << 3)
-#define DMFC_FIFO_SIZE_256              (1 << 3)
-#define DMFC_FIFO_SIZE_512              (0 << 3)
-
-#define DMFC_SEGMENT(x)                 ((x & 0x7) << 0)
-#define DMFC_BURSTSIZE_128              (0 << 6)
-#define DMFC_BURSTSIZE_64               (1 << 6)
-#define DMFC_BURSTSIZE_32               (2 << 6)
-#define DMFC_BURSTSIZE_16               (3 << 6)
-
 struct dmfc_channel_data {
         int             ipu_channel;
         unsigned long   channel_reg;
@@ -104,9 +93,6 @@ struct ipu_dmfc_priv;
 
 struct dmfc_channel {
         unsigned                        slots;
-        unsigned                        slotmask;
-        unsigned                        segment;
-        int                             burstsize;
         struct ipu_soc                  *ipu;
         struct ipu_dmfc_priv            *priv;
         const struct dmfc_channel_data  *data;
@@ -117,7 +103,6 @@ struct ipu_dmfc_priv {
         struct device *dev;
         struct dmfc_channel channels[DMFC_NUM_CHANNELS];
         struct mutex mutex;
-        unsigned long bandwidth_per_slot;
         void __iomem *base;
         int use_count;
 };
@@ -172,184 +157,6 @@ void ipu_dmfc_disable_channel(struct dmfc_channel *dmfc)
 }
 EXPORT_SYMBOL_GPL(ipu_dmfc_disable_channel);
 
-static int ipu_dmfc_setup_channel(struct dmfc_channel *dmfc, int slots,
-                int segment, int burstsize)
-{
-        struct ipu_dmfc_priv *priv = dmfc->priv;
-        u32 val, field;
-
-        dev_dbg(priv->dev,
-                        "dmfc: using %d slots starting from segment %d for IPU channel %d\n",
-                        slots, segment, dmfc->data->ipu_channel);
-
-        switch (slots) {
-        case 1:
-                field = DMFC_FIFO_SIZE_64;
-                break;
-        case 2:
-                field = DMFC_FIFO_SIZE_128;
-                break;
-        case 4:
-                field = DMFC_FIFO_SIZE_256;
-                break;
-        case 8:
-                field = DMFC_FIFO_SIZE_512;
-                break;
-        default:
-                return -EINVAL;
-        }
-
-        switch (burstsize) {
-        case 16:
-                field |= DMFC_BURSTSIZE_16;
-                break;
-        case 32:
-                field |= DMFC_BURSTSIZE_32;
-                break;
-        case 64:
-                field |= DMFC_BURSTSIZE_64;
-                break;
-        case 128:
-                field |= DMFC_BURSTSIZE_128;
-                break;
-        }
-
-        field |= DMFC_SEGMENT(segment);
-
-        val = readl(priv->base + dmfc->data->channel_reg);
-
-        val &= ~(0xff << dmfc->data->shift);
-        val |= field << dmfc->data->shift;
-
-        writel(val, priv->base + dmfc->data->channel_reg);
-
-        dmfc->slots = slots;
-        dmfc->segment = segment;
-        dmfc->burstsize = burstsize;
-        dmfc->slotmask = ((1 << slots) - 1) << segment;
-
-        return 0;
-}
-
-static int dmfc_bandwidth_to_slots(struct ipu_dmfc_priv *priv,
-                unsigned long bandwidth)
-{
-        int slots = 1;
-
-        while (slots * priv->bandwidth_per_slot < bandwidth)
-                slots *= 2;
-
-        return slots;
-}
-
-static int dmfc_find_slots(struct ipu_dmfc_priv *priv, int slots)
-{
-        unsigned slotmask_need, slotmask_used = 0;
-        int i, segment = 0;
-
-        slotmask_need = (1 << slots) - 1;
-
-        for (i = 0; i < DMFC_NUM_CHANNELS; i++)
-                slotmask_used |= priv->channels[i].slotmask;
-
-        while (slotmask_need <= 0xff) {
-                if (!(slotmask_used & slotmask_need))
-                        return segment;
-
-                slotmask_need <<= 1;
-                segment++;
-        }
-
-        return -EBUSY;
-}
-
-void ipu_dmfc_free_bandwidth(struct dmfc_channel *dmfc)
-{
-        struct ipu_dmfc_priv *priv = dmfc->priv;
-        int i;
-
-        dev_dbg(priv->dev, "dmfc: freeing %d slots starting from segment %d\n",
-                        dmfc->slots, dmfc->segment);
-
-        mutex_lock(&priv->mutex);
-
-        if (!dmfc->slots)
-                goto out;
-
-        dmfc->slotmask = 0;
-        dmfc->slots = 0;
-        dmfc->segment = 0;
-
-        for (i = 0; i < DMFC_NUM_CHANNELS; i++)
-                priv->channels[i].slotmask = 0;
-
-        for (i = 0; i < DMFC_NUM_CHANNELS; i++) {
-                if (priv->channels[i].slots > 0) {
-                        priv->channels[i].segment =
-                                dmfc_find_slots(priv, priv->channels[i].slots);
-                        priv->channels[i].slotmask =
-                                ((1 << priv->channels[i].slots) - 1) <<
-                                priv->channels[i].segment;
-                }
-        }
-
-        for (i = 0; i < DMFC_NUM_CHANNELS; i++) {
-                if (priv->channels[i].slots > 0)
-                        ipu_dmfc_setup_channel(&priv->channels[i],
-                                        priv->channels[i].slots,
-                                        priv->channels[i].segment,
-                                        priv->channels[i].burstsize);
-        }
-out:
-        mutex_unlock(&priv->mutex);
-}
-EXPORT_SYMBOL_GPL(ipu_dmfc_free_bandwidth);
-
-int ipu_dmfc_alloc_bandwidth(struct dmfc_channel *dmfc,
-                unsigned long bandwidth_pixel_per_second, int burstsize)
-{
-        struct ipu_dmfc_priv *priv = dmfc->priv;
-        int slots = dmfc_bandwidth_to_slots(priv, bandwidth_pixel_per_second);
-        int segment = -1, ret = 0;
-
-        dev_dbg(priv->dev, "dmfc: trying to allocate %ldMpixel/s for IPU channel %d\n",
-                        bandwidth_pixel_per_second / 1000000,
-                        dmfc->data->ipu_channel);
-
-        ipu_dmfc_free_bandwidth(dmfc);
-
-        mutex_lock(&priv->mutex);
-
-        if (slots > 8) {
-                ret = -EBUSY;
-                goto out;
-        }
-
-        /* For the MEM_BG channel, first try to allocate twice the slots */
-        if (dmfc->data->ipu_channel == IPUV3_CHANNEL_MEM_BG_SYNC)
-                segment = dmfc_find_slots(priv, slots * 2);
-        else if (slots < 2)
-                /* Always allocate at least 128*4 bytes (2 slots) */
-                slots = 2;
-
-        if (segment >= 0)
-                slots *= 2;
-        else
-                segment = dmfc_find_slots(priv, slots);
-        if (segment < 0) {
-                ret = -EBUSY;
-                goto out;
-        }
-
-        ipu_dmfc_setup_channel(dmfc, slots, segment, burstsize);
-
-out:
-        mutex_unlock(&priv->mutex);
-
-        return ret;
-}
-EXPORT_SYMBOL_GPL(ipu_dmfc_alloc_bandwidth);
-
 void ipu_dmfc_config_wait4eot(struct dmfc_channel *dmfc, int width)
 {
         struct ipu_dmfc_priv *priv = dmfc->priv;
@@ -384,7 +191,6 @@ EXPORT_SYMBOL_GPL(ipu_dmfc_get);
 
 void ipu_dmfc_put(struct dmfc_channel *dmfc)
 {
-        ipu_dmfc_free_bandwidth(dmfc);
 }
 EXPORT_SYMBOL_GPL(ipu_dmfc_put);
 
@@ -412,20 +218,15 @@ int ipu_dmfc_init(struct ipu_soc *ipu, struct device *dev, unsigned long base,
                 priv->channels[i].priv = priv;
                 priv->channels[i].ipu = ipu;
                 priv->channels[i].data = &dmfcdata[i];
-        }
-
-        writel(0x0, priv->base + DMFC_WR_CHAN);
-        writel(0x0, priv->base + DMFC_DP_CHAN);
 
-        /*
-         * We have a total bandwidth of clkrate * 4pixel divided
-         * into 8 slots.
-         */
-        priv->bandwidth_per_slot = clk_get_rate(ipu_clk) * 4 / 8;
-
-        dev_dbg(dev, "dmfc: 8 slots with %ldMpixel/s bandwidth each\n",
-                        priv->bandwidth_per_slot / 1000000);
+                if (dmfcdata[i].ipu_channel == IPUV3_CHANNEL_MEM_BG_SYNC ||
+                    dmfcdata[i].ipu_channel == IPUV3_CHANNEL_MEM_FG_SYNC ||
+                    dmfcdata[i].ipu_channel == IPUV3_CHANNEL_MEM_DC_SYNC)
+                        priv->channels[i].slots = 2;
+        }
 
+        writel(0x00000050, priv->base + DMFC_WR_CHAN);
+        writel(0x00005654, priv->base + DMFC_DP_CHAN);
         writel(0x202020f6, priv->base + DMFC_WR_CHAN_DEF);
         writel(0x2020f6f6, priv->base + DMFC_DP_CHAN_DEF);
         writel(0x00000003, priv->base + DMFC_GENERAL1);
diff --git a/drivers/gpu/vga/vga_switcheroo.c b/drivers/gpu/vga/vga_switcheroo.c
index cbd7c986d926..5f962bfcb43c 100644
--- a/drivers/gpu/vga/vga_switcheroo.c
+++ b/drivers/gpu/vga/vga_switcheroo.c
@@ -30,6 +30,7 @@
 
 #define pr_fmt(fmt) "vga_switcheroo: " fmt
 
+#include <linux/apple-gmux.h>
 #include <linux/console.h>
 #include <linux/debugfs.h>
 #include <linux/fb.h>
@@ -51,9 +52,9 @@
  *
  * * muxed: Dual GPUs with a multiplexer chip to switch outputs between GPUs.
  * * muxless: Dual GPUs but only one of them is connected to outputs.
- *      The other one is merely used to offload rendering, its results
- *      are copied over PCIe into the framebuffer. On Linux this is
- *      supported with DRI PRIME.
+ *   The other one is merely used to offload rendering, its results
+ *   are copied over PCIe into the framebuffer. On Linux this is
+ *   supported with DRI PRIME.
  *
  * Hybrid graphics started to appear in the late Naughties and were initially
  * all muxed. Newer laptops moved to a muxless architecture for cost reasons.
@@ -308,7 +309,8 @@ static int register_client(struct pci_dev *pdev,
  *
  * Register vga client (GPU). Enable vga_switcheroo if another GPU and a
  * handler have already registered. The power state of the client is assumed
- * to be ON.
+ * to be ON. Beforehand, vga_switcheroo_client_probe_defer() shall be called
+ * to ensure that all prerequisites are met.
  *
  * Return: 0 on success, -ENOMEM on memory allocation error.
  */
@@ -329,7 +331,8 @@ EXPORT_SYMBOL(vga_switcheroo_register_client);
  * @id: client identifier
  *
  * Register audio client (audio device on a GPU). The power state of the
- * client is assumed to be ON.
+ * client is assumed to be ON. Beforehand, vga_switcheroo_client_probe_defer()
+ * shall be called to ensure that all prerequisites are met.
  *
  * Return: 0 on success, -ENOMEM on memory allocation error.
  */
@@ -376,6 +379,33 @@ find_active_client(struct list_head *head)
 }
 
 /**
+ * vga_switcheroo_client_probe_defer() - whether to defer probing a given client
+ * @pdev: client pci device
+ *
+ * Determine whether any prerequisites are not fulfilled to probe a given
+ * client. Drivers shall invoke this early on in their ->probe callback
+ * and return %-EPROBE_DEFER if it evaluates to %true. Thou shalt not
+ * register the client ere thou hast called this.
+ *
+ * Return: %true if probing should be deferred, otherwise %false.
+ */
+bool vga_switcheroo_client_probe_defer(struct pci_dev *pdev)
+{
+        if ((pdev->class >> 16) == PCI_BASE_CLASS_DISPLAY) {
+                /*
+                 * apple-gmux is needed on pre-retina MacBook Pro
+                 * to probe the panel if pdev is the inactive GPU.
+                 */
+                if (apple_gmux_present() && pdev != vga_default_device() &&
+                    !vgasr_priv.handler_flags)
+                        return true;
+        }
+
+        return false;
+}
+EXPORT_SYMBOL(vga_switcheroo_client_probe_defer);
+
+/**
  * vga_switcheroo_get_client_state() - obtain power state of a given client
  * @pdev: client pci device
  *
@@ -530,21 +560,21 @@ EXPORT_SYMBOL(vga_switcheroo_unlock_ddc);
  * * OFF: Power off the device not in use.
  * * ON: Power on the device not in use.
  * * IGD: Switch to the integrated graphics device.
- *      Power on the integrated GPU if necessary, power off the discrete GPU.
- *      Prerequisite is that no user space processes (e.g. Xorg, alsactl)
- *      have opened device files of the GPUs or the audio client. If the
- *      switch fails, the user may invoke lsof(8) or fuser(1) on /dev/dri/
- *      and /dev/snd/controlC1 to identify processes blocking the switch.
+ *   Power on the integrated GPU if necessary, power off the discrete GPU.
+ *   Prerequisite is that no user space processes (e.g. Xorg, alsactl)
+ *   have opened device files of the GPUs or the audio client. If the
+ *   switch fails, the user may invoke lsof(8) or fuser(1) on /dev/dri/
+ *   and /dev/snd/controlC1 to identify processes blocking the switch.
  * * DIS: Switch to the discrete graphics device.
  * * DIGD: Delayed switch to the integrated graphics device.
- *      This will perform the switch once the last user space process has
- *      closed the device files of the GPUs and the audio client.
+ *   This will perform the switch once the last user space process has
+ *   closed the device files of the GPUs and the audio client.
  * * DDIS: Delayed switch to the discrete graphics device.
  * * MIGD: Mux-only switch to the integrated graphics device.
- *      Does not remap console or change the power state of either gpu.
- *      If the integrated GPU is currently off, the screen will turn black.
- *      If it is on, the screen will show whatever happens to be in VRAM.
- *      Either way, the user has to blindly enter the command to switch back.
+ *   Does not remap console or change the power state of either gpu.
+ *   If the integrated GPU is currently off, the screen will turn black.
+ *   If it is on, the screen will show whatever happens to be in VRAM.
+ *   Either way, the user has to blindly enter the command to switch back.
  * * MDIS: Mux-only switch to the discrete graphics device.
  *
  * For GPUs whose power state is controlled by the driver's runtime pm,
diff --git a/drivers/media/platform/omap/omap_vout.c b/drivers/media/platform/omap/omap_vout.c
index 4afc999c0780..6b01e126fe73 100644
--- a/drivers/media/platform/omap/omap_vout.c
+++ b/drivers/media/platform/omap/omap_vout.c
@@ -45,7 +45,7 @@
 #include <media/v4l2-ioctl.h>
 
 #include <video/omapvrfb.h>
-#include <video/omapdss.h>
+#include <video/omapfb_dss.h>
 
 #include "omap_voutlib.h"
 #include "omap_voutdef.h"
diff --git a/drivers/media/platform/omap/omap_voutdef.h b/drivers/media/platform/omap/omap_voutdef.h
index 49de1475e473..80c79fabdf95 100644
--- a/drivers/media/platform/omap/omap_voutdef.h
+++ b/drivers/media/platform/omap/omap_voutdef.h
@@ -12,7 +12,7 @@
 #define OMAP_VOUTDEF_H
 
 #include <media/v4l2-ctrls.h>
-#include <video/omapdss.h>
+#include <video/omapfb_dss.h>
 #include <video/omapvrfb.h>
 
 #define YUYV_BPP        2
diff --git a/drivers/media/platform/omap/omap_voutlib.c b/drivers/media/platform/omap/omap_voutlib.c
index 80b0d88f125c..58a25fdf0cce 100644
--- a/drivers/media/platform/omap/omap_voutlib.c
+++ b/drivers/media/platform/omap/omap_voutlib.c
@@ -26,7 +26,7 @@
 
 #include <linux/dma-mapping.h>
 
-#include <video/omapdss.h>
+#include <video/omapfb_dss.h>
 
 #include "omap_voutlib.h"
 
diff --git a/drivers/net/ethernet/mellanox/mlx4/pd.c b/drivers/net/ethernet/mellanox/mlx4/pd.c
index b3cc3ab63799..6fc156a3918d 100644
--- a/drivers/net/ethernet/mellanox/mlx4/pd.c
+++ b/drivers/net/ethernet/mellanox/mlx4/pd.c
@@ -205,7 +205,9 @@ int mlx4_bf_alloc(struct mlx4_dev *dev, struct mlx4_bf *bf, int node)
                         goto free_uar;
                 }
 
-                uar->bf_map = io_mapping_map_wc(priv->bf_mapping, uar->index << PAGE_SHIFT);
+                uar->bf_map = io_mapping_map_wc(priv->bf_mapping,
+                                                uar->index << PAGE_SHIFT,
+                                                PAGE_SIZE);
                 if (!uar->bf_map) {
                         err = -ENOMEM;
                         goto unamp_uar;
diff --git a/drivers/platform/x86/apple-gmux.c b/drivers/platform/x86/apple-gmux.c
index 4034d2d4c507..a66be137324c 100644
--- a/drivers/platform/x86/apple-gmux.c
+++ b/drivers/platform/x86/apple-gmux.c
@@ -31,19 +31,21 @@
 /**
  * DOC: Overview
  *
- * :1:  http://www.latticesemi.com/en/Products/FPGAandCPLD/LatticeXP2.aspx
- * :2:  http://www.renesas.com/products/mpumcu/h8s/h8s2100/h8s2113/index.jsp
- *
  * gmux is a microcontroller built into the MacBook Pro to support dual GPUs:
- * A {1}[Lattice XP2] on pre-retinas, a {2}[Renesas R4F2113] on retinas.
+ * A `Lattice XP2`_ on pre-retinas, a `Renesas R4F2113`_ on retinas.
  *
  * (The MacPro6,1 2013 also has a gmux, however it is unclear why since it has
  * dual GPUs but no built-in display.)
  *
  * gmux is connected to the LPC bus of the southbridge. Its I/O ports are
  * accessed differently depending on the microcontroller: Driver functions
- * to access a pre-retina gmux are infixed `_pio_`, those for a retina gmux
- * are infixed `_index_`.
+ * to access a pre-retina gmux are infixed ``_pio_``, those for a retina gmux
+ * are infixed ``_index_``.
+ *
+ * .. _Lattice XP2:
+ *     http://www.latticesemi.com/en/Products/FPGAandCPLD/LatticeXP2.aspx
+ * .. _Renesas R4F2113:
+ *     http://www.renesas.com/products/mpumcu/h8s/h8s2100/h8s2113/index.jsp
  */
 
 struct apple_gmux_data {
@@ -272,15 +274,15 @@ static bool gmux_is_indexed(struct apple_gmux_data *gmux_data)
 /**
  * DOC: Backlight control
  *
- * :3:  http://www.ti.com/lit/ds/symlink/lp8543.pdf
- * :4:  http://www.ti.com/lit/ds/symlink/lp8545.pdf
- *
  * On single GPU MacBooks, the PWM signal for the backlight is generated by
  * the GPU. On dual GPU MacBook Pros by contrast, either GPU may be suspended
  * to conserve energy. Hence the PWM signal needs to be generated by a separate
  * backlight driver which is controlled by gmux. The earliest generation
- * MBP5 2008/09 uses a {3}[TI LP8543] backlight driver. All newer models
- * use a {4}[TI LP8545].
+ * MBP5 2008/09 uses a `TI LP8543`_ backlight driver. All newer models
+ * use a `TI LP8545`_.
+ *
+ * .. _TI LP8543: http://www.ti.com/lit/ds/symlink/lp8543.pdf
+ * .. _TI LP8545: http://www.ti.com/lit/ds/symlink/lp8545.pdf
  */
 
 static int gmux_get_brightness(struct backlight_device *bd)
@@ -312,28 +314,20 @@ static const struct backlight_ops gmux_bl_ops = {
 /**
  * DOC: Graphics mux
  *
- * :5:  http://pimg-fpiw.uspto.gov/fdd/07/870/086/0.pdf
- * :6:  http://www.nxp.com/documents/data_sheet/CBTL06141.pdf
- * :7:  http://www.ti.com/lit/ds/symlink/hd3ss212.pdf
- * :8:  https://www.pericom.com/assets/Datasheets/PI3VDP12412.pdf
- * :9:  http://www.ti.com/lit/ds/symlink/sn74lv4066a.pdf
- * :10: http://pdf.datasheetarchive.com/indexerfiles/Datasheets-SW16/DSASW00308511.pdf
- * :11: http://www.ti.com/lit/ds/symlink/ts3ds10224.pdf
- *
  * On pre-retinas, the LVDS outputs of both GPUs feed into gmux which muxes
  * either of them to the panel. One of the tricks gmux has up its sleeve is
  * to lengthen the blanking interval of its output during a switch to
  * synchronize it with the GPU switched to. This allows for a flicker-free
- * switch that is imperceptible by the user ({5}[US 8,687,007 B2]).
+ * switch that is imperceptible by the user (`US 8,687,007 B2`_).
  *
  * On retinas, muxing is no longer done by gmux itself, but by a separate
  * chip which is controlled by gmux. The chip is triple sourced, it is
- * either an {6}[NXP CBTL06142], {7}[TI HD3SS212] or {8}[Pericom PI3VDP12412].
+ * either an `NXP CBTL06142`_, `TI HD3SS212`_ or `Pericom PI3VDP12412`_.
  * The panel is driven with eDP instead of LVDS since the pixel clock
  * required for retina resolution exceeds LVDS' limits.
  *
  * Pre-retinas are able to switch the panel's DDC pins separately.
- * This is handled by a {9}[TI SN74LV4066A] which is controlled by gmux.
+ * This is handled by a `TI SN74LV4066A`_ which is controlled by gmux.
  * The inactive GPU can thus probe the panel's EDID without switching over
  * the entire panel. Retinas lack this functionality as the chips used for
  * eDP muxing are incapable of switching the AUX channel separately (see
@@ -344,15 +338,15 @@ static const struct backlight_ops gmux_bl_ops = {
  *
  * The external DP port is only fully switchable on the first two unibody
  * MacBook Pro generations, MBP5 2008/09 and MBP6 2010. This is done by an
- * {6}[NXP CBTL06141] which is controlled by gmux. It's the predecessor of the
+ * `NXP CBTL06141`_ which is controlled by gmux. It's the predecessor of the
  * eDP mux on retinas, the difference being support for 2.7 versus 5.4 Gbit/s.
  *
  * The following MacBook Pro generations replaced the external DP port with a
  * combined DP/Thunderbolt port and lost the ability to switch it between GPUs,
  * connecting it either to the discrete GPU or the Thunderbolt controller.
  * Oddly enough, while the full port is no longer switchable, AUX and HPD
- * are still switchable by way of an {10}[NXP CBTL03062] (on pre-retinas
- * MBP8 2011 and MBP9 2012) or two {11}[TI TS3DS10224] (on retinas) under the
+ * are still switchable by way of an `NXP CBTL03062`_ (on pre-retinas
+ * MBP8 2011 and MBP9 2012) or two `TI TS3DS10224`_ (on retinas) under the
  * control of gmux. Since the integrated GPU is missing the main link,
  * external displays appear to it as phantoms which fail to link-train.
  *
@@ -365,10 +359,19 @@ static const struct backlight_ops gmux_bl_ops = {
  * of this feature.
  *
  * gmux' initial switch state on bootup is user configurable via the EFI
- * variable `gpu-power-prefs-fa4ce28d-b62f-4c99-9cc3-6815686e30f9` (5th byte,
+ * variable ``gpu-power-prefs-fa4ce28d-b62f-4c99-9cc3-6815686e30f9`` (5th byte,
  * 1 = IGD, 0 = DIS). Based on this setting, the EFI firmware tells gmux to
  * switch the panel and the external DP connector and allocates a framebuffer
  * for the selected GPU.
+ *
+ * .. _US 8,687,007 B2: http://pimg-fpiw.uspto.gov/fdd/07/870/086/0.pdf
+ * .. _NXP CBTL06141:   http://www.nxp.com/documents/data_sheet/CBTL06141.pdf
+ * .. _NXP CBTL06142:   http://www.nxp.com/documents/data_sheet/CBTL06141.pdf
+ * .. _TI HD3SS212:     http://www.ti.com/lit/ds/symlink/hd3ss212.pdf
+ * .. _Pericom PI3VDP12412: https://www.pericom.com/assets/Datasheets/PI3VDP12412.pdf
+ * .. _TI SN74LV4066A:  http://www.ti.com/lit/ds/symlink/sn74lv4066a.pdf
+ * .. _NXP CBTL03062:   http://pdf.datasheetarchive.com/indexerfiles/Datasheets-SW16/DSASW00308511.pdf
+ * .. _TI TS3DS10224:   http://www.ti.com/lit/ds/symlink/ts3ds10224.pdf
  */
 
 static void gmux_read_switch_state(struct apple_gmux_data *gmux_data)
diff --git a/drivers/staging/android/sync_debug.h b/drivers/staging/android/sync_debug.h
index 425ebc5c32aa..fab66396d421 100644
--- a/drivers/staging/android/sync_debug.h
+++ b/drivers/staging/android/sync_debug.h
@@ -34,7 +34,8 @@ struct sync_timeline {
         char                    name[32];
 
         /* protected by child_list_lock */
-        int                     context, value;
+        u64                     context;
+        int                     value;
 
         struct list_head        child_list_head;
         spinlock_t              child_list_lock;
diff --git a/drivers/video/fbdev/omap2/omapfb/displays/connector-analog-tv.c b/drivers/video/fbdev/omap2/omapfb/displays/connector-analog-tv.c
index 8511c648a15c..9d78411a3bf7 100644
--- a/drivers/video/fbdev/omap2/omapfb/displays/connector-analog-tv.c
+++ b/drivers/video/fbdev/omap2/omapfb/displays/connector-analog-tv.c
@@ -14,7 +14,7 @@
 #include <linux/platform_device.h>
 #include <linux/of.h>
 
-#include <video/omapdss.h>
+#include <video/omapfb_dss.h>
 #include <video/omap-panel-data.h>
 
 struct panel_drv_data {
@@ -25,7 +25,6 @@ struct panel_drv_data {
 
         struct omap_video_timings timings;
 
-        enum omap_dss_venc_type connector_type;
         bool invert_polarity;
 };
 
@@ -45,10 +44,6 @@ static const struct omap_video_timings tvc_pal_timings = {
 
 static const struct of_device_id tvc_of_match[];
 
-struct tvc_of_data {
-        enum omap_dss_venc_type connector_type;
-};
-
 #define to_panel_data(x) container_of(x, struct panel_drv_data, dssdev)
 
 static int tvc_connect(struct omap_dss_device *dssdev)
@@ -99,7 +94,7 @@ static int tvc_enable(struct omap_dss_device *dssdev)
         in->ops.atv->set_timings(in, &ddata->timings);
 
         if (!ddata->dev->of_node) {
-                in->ops.atv->set_type(in, ddata->connector_type);
+                in->ops.atv->set_type(in, OMAP_DSS_VENC_TYPE_COMPOSITE);
 
                 in->ops.atv->invert_vid_out_polarity(in,
                         ddata->invert_polarity);
@@ -207,7 +202,6 @@ static int tvc_probe_pdata(struct platform_device *pdev)
 
         ddata->in = in;
 
-        ddata->connector_type = pdata->connector_type;
         ddata->invert_polarity = pdata->invert_polarity;
 
         dssdev = &ddata->dssdev;
diff --git a/drivers/video/fbdev/omap2/omapfb/displays/connector-dvi.c b/drivers/video/fbdev/omap2/omapfb/displays/connector-dvi.c
index d811e6dcaef7..06e1db34541e 100644
--- a/drivers/video/fbdev/omap2/omapfb/displays/connector-dvi.c
+++ b/drivers/video/fbdev/omap2/omapfb/displays/connector-dvi.c
@@ -16,8 +16,7 @@
 
 #include <drm/drm_edid.h>
 
-#include <video/omapdss.h>
-#include <video/omap-panel-data.h>
+#include <video/omapfb_dss.h>
 
 static const struct omap_video_timings dvic_default_timings = {
         .x_res          = 640,
@@ -236,46 +235,6 @@ static struct omap_dss_driver dvic_driver = {
         .detect         = dvic_detect,
 };
 
-static int dvic_probe_pdata(struct platform_device *pdev)
-{
-        struct panel_drv_data *ddata = platform_get_drvdata(pdev);
-        struct connector_dvi_platform_data *pdata;
-        struct omap_dss_device *in, *dssdev;
-        int i2c_bus_num;
-
-        pdata = dev_get_platdata(&pdev->dev);
-        i2c_bus_num = pdata->i2c_bus_num;
-
-        if (i2c_bus_num != -1) {
-                struct i2c_adapter *adapter;
-
-                adapter = i2c_get_adapter(i2c_bus_num);
-                if (!adapter) {
-                        dev_err(&pdev->dev,
-                                        "Failed to get I2C adapter, bus %d\n",
-                                        i2c_bus_num);
-                        return -EPROBE_DEFER;
-                }
-
-                ddata->i2c_adapter = adapter;
-        }
-
-        in = omap_dss_find_output(pdata->source);
-        if (in == NULL) {
-                i2c_put_adapter(ddata->i2c_adapter);
-
-                dev_err(&pdev->dev, "Failed to find video source\n");
-                return -EPROBE_DEFER;
-        }
-
-        ddata->in = in;
-
-        dssdev = &ddata->dssdev;
-        dssdev->name = pdata->name;
-
-        return 0;
-}
-
 static int dvic_probe_of(struct platform_device *pdev)
 {
         struct panel_drv_data *ddata = platform_get_drvdata(pdev);
@@ -313,23 +272,18 @@ static int dvic_probe(struct platform_device *pdev)
         struct omap_dss_device *dssdev;
         int r;
 
+        if (!pdev->dev.of_node)
+                return -ENODEV;
+
         ddata = devm_kzalloc(&pdev->dev, sizeof(*ddata), GFP_KERNEL);
         if (!ddata)
                 return -ENOMEM;
 
         platform_set_drvdata(pdev, ddata);
 
-        if (dev_get_platdata(&pdev->dev)) {
-                r = dvic_probe_pdata(pdev);
-                if (r)
-                        return r;
-        } else if (pdev->dev.of_node) {
-                r = dvic_probe_of(pdev);
-                if (r)
-                        return r;
-        } else {
-                return -ENODEV;
-        }
+        r = dvic_probe_of(pdev);
+        if (r)
+                return r;
 
         ddata->timings = dvic_default_timings;
 
diff --git a/drivers/video/fbdev/omap2/omapfb/displays/connector-hdmi.c b/drivers/video/fbdev/omap2/omapfb/displays/connector-hdmi.c
index 6ee4129bc0c0..58d5803ede67 100644
--- a/drivers/video/fbdev/omap2/omapfb/displays/connector-hdmi.c
+++ b/drivers/video/fbdev/omap2/omapfb/displays/connector-hdmi.c
@@ -17,8 +17,7 @@
 
 #include <drm/drm_edid.h>
 
-#include <video/omapdss.h>
-#include <video/omap-panel-data.h>
+#include <video/omapfb_dss.h>
 
 static const struct omap_video_timings hdmic_default_timings = {
         .x_res          = 640,
@@ -206,30 +205,6 @@ static struct omap_dss_driver hdmic_driver = {
         .set_hdmi_infoframe     = hdmic_set_infoframe,
 };
 
-static int hdmic_probe_pdata(struct platform_device *pdev)
-{
-        struct panel_drv_data *ddata = platform_get_drvdata(pdev);
-        struct connector_hdmi_platform_data *pdata;
-        struct omap_dss_device *in, *dssdev;
-
-        pdata = dev_get_platdata(&pdev->dev);
-
-        ddata->hpd_gpio = -ENODEV;
-
-        in = omap_dss_find_output(pdata->source);
-        if (in == NULL) {
-                dev_err(&pdev->dev, "Failed to find video source\n");
-                return -EPROBE_DEFER;
-        }
-
-        ddata->in = in;
-
-        dssdev = &ddata->dssdev;
-        dssdev->name = pdata->name;
-
-        return 0;
-}
-
 static int hdmic_probe_of(struct platform_device *pdev)
 {
         struct panel_drv_data *ddata = platform_get_drvdata(pdev);
@@ -261,6 +236,9 @@ static int hdmic_probe(struct platform_device *pdev)
         struct omap_dss_device *dssdev;
         int r;
 
+        if (!pdev->dev.of_node)
+                return -ENODEV;
+
         ddata = devm_kzalloc(&pdev->dev, sizeof(*ddata), GFP_KERNEL);
         if (!ddata)
                 return -ENOMEM;
@@ -268,17 +246,9 @@ static int hdmic_probe(struct platform_device *pdev)
         platform_set_drvdata(pdev, ddata);
         ddata->dev = &pdev->dev;
 
-        if (dev_get_platdata(&pdev->dev)) {
-                r = hdmic_probe_pdata(pdev);
-                if (r)
-                        return r;
-        } else if (pdev->dev.of_node) {
-                r = hdmic_probe_of(pdev);
-                if (r)
-                        return r;
-        } else {
-                return -ENODEV;
-        }
+        r = hdmic_probe_of(pdev);
+        if (r)
+                return r;
 
         if (gpio_is_valid(ddata->hpd_gpio)) {
                 r = devm_gpio_request_one(&pdev->dev, ddata->hpd_gpio,
diff --git a/drivers/video/fbdev/omap2/omapfb/displays/encoder-opa362.c b/drivers/video/fbdev/omap2/omapfb/displays/encoder-opa362.c
index 8c246c213e06..a9a67167cc3d 100644
--- a/drivers/video/fbdev/omap2/omapfb/displays/encoder-opa362.c
+++ b/drivers/video/fbdev/omap2/omapfb/displays/encoder-opa362.c
@@ -20,7 +20,7 @@
 #include <linux/slab.h>
 #include <linux/of_gpio.h>
 
-#include <video/omapdss.h>
+#include <video/omapfb_dss.h>
 
 struct panel_drv_data {
         struct omap_dss_device dssdev;
diff --git a/drivers/video/fbdev/omap2/omapfb/displays/encoder-tfp410.c b/drivers/video/fbdev/omap2/omapfb/displays/encoder-tfp410.c
index d9048b3df495..8c0953d069b7 100644
--- a/drivers/video/fbdev/omap2/omapfb/displays/encoder-tfp410.c
+++ b/drivers/video/fbdev/omap2/omapfb/displays/encoder-tfp410.c
@@ -15,8 +15,7 @@
 #include <linux/slab.h>
 #include <linux/of_gpio.h>
 
-#include <video/omapdss.h>
-#include <video/omap-panel-data.h>
+#include <video/omapfb_dss.h>
 
 struct panel_drv_data {
         struct omap_dss_device dssdev;
@@ -166,32 +165,6 @@ static const struct omapdss_dvi_ops tfp410_dvi_ops = {
         .get_timings    = tfp410_get_timings,
 };
 
-static int tfp410_probe_pdata(struct platform_device *pdev)
-{
-        struct panel_drv_data *ddata = platform_get_drvdata(pdev);
-        struct encoder_tfp410_platform_data *pdata;
-        struct omap_dss_device *dssdev, *in;
-
-        pdata = dev_get_platdata(&pdev->dev);
-
-        ddata->pd_gpio = pdata->power_down_gpio;
-
-        ddata->data_lines = pdata->data_lines;
-
-        in = omap_dss_find_output(pdata->source);
-        if (in == NULL) {
-                dev_err(&pdev->dev, "Failed to find video source\n");
-                return -ENODEV;
-        }
-
-        ddata->in = in;
-
-        dssdev = &ddata->dssdev;
-        dssdev->name = pdata->name;
-
-        return 0;
-}
-
 static int tfp410_probe_of(struct platform_device *pdev)
 {
         struct panel_drv_data *ddata = platform_get_drvdata(pdev);
@@ -225,23 +198,18 @@ static int tfp410_probe(struct platform_device *pdev)
         struct omap_dss_device *dssdev;
         int r;
 
+        if (!pdev->dev.of_node)
+                return -ENODEV;
+
         ddata = devm_kzalloc(&pdev->dev, sizeof(*ddata), GFP_KERNEL);
         if (!ddata)
                 return -ENOMEM;
 
         platform_set_drvdata(pdev, ddata);
 
-        if (dev_get_platdata(&pdev->dev)) {
-                r = tfp410_probe_pdata(pdev);
-                if (r)
-                        return r;
-        } else if (pdev->dev.of_node) {
-                r = tfp410_probe_of(pdev);
-                if (r)
-                        return r;
-        } else {
-                return -ENODEV;
-        }
+        r = tfp410_probe_of(pdev);
+        if (r)
+                return r;
 
         if (gpio_is_valid(ddata->pd_gpio)) {
                 r = devm_gpio_request_one(&pdev->dev, ddata->pd_gpio,
diff --git a/drivers/video/fbdev/omap2/omapfb/displays/encoder-tpd12s015.c b/drivers/video/fbdev/omap2/omapfb/displays/encoder-tpd12s015.c
index 677e2545fcbe..80dc47347e21 100644
--- a/drivers/video/fbdev/omap2/omapfb/displays/encoder-tpd12s015.c
+++ b/drivers/video/fbdev/omap2/omapfb/displays/encoder-tpd12s015.c
@@ -16,8 +16,7 @@
 #include <linux/platform_device.h>
 #include <linux/gpio/consumer.h>
 
-#include <video/omapdss.h>
-#include <video/omap-panel-data.h>
+#include <video/omapfb_dss.h>
 
 struct panel_drv_data {
         struct omap_dss_device dssdev;
diff --git a/drivers/video/fbdev/omap2/omapfb/displays/panel-dpi.c b/drivers/video/fbdev/omap2/omapfb/displays/panel-dpi.c
index e780fd4f8b46..ace3d818afe5 100644
--- a/drivers/video/fbdev/omap2/omapfb/displays/panel-dpi.c
+++ b/drivers/video/fbdev/omap2/omapfb/displays/panel-dpi.c
@@ -16,7 +16,7 @@
 #include <linux/of.h>
 #include <linux/of_gpio.h>
 
-#include <video/omapdss.h>
+#include <video/omapfb_dss.h>
 #include <video/omap-panel-data.h>
 #include <video/of_display_timing.h>
 
diff --git a/drivers/video/fbdev/omap2/omapfb/displays/panel-dsi-cm.c b/drivers/video/fbdev/omap2/omapfb/displays/panel-dsi-cm.c
index 3414c2609320..b58012b82b6f 100644
--- a/drivers/video/fbdev/omap2/omapfb/displays/panel-dsi-cm.c
+++ b/drivers/video/fbdev/omap2/omapfb/displays/panel-dsi-cm.c
@@ -25,8 +25,7 @@
 #include <linux/of_device.h>
 #include <linux/of_gpio.h>
 
-#include <video/omapdss.h>
-#include <video/omap-panel-data.h>
+#include <video/omapfb_dss.h>
 #include <video/mipi_display.h>
 
 /* DSI Virtual channel. Hardcoded for now. */
@@ -1127,40 +1126,6 @@ static struct omap_dss_driver dsicm_ops = {
         .memory_read    = dsicm_memory_read,
 };
 
-static int dsicm_probe_pdata(struct platform_device *pdev)
-{
-        const struct panel_dsicm_platform_data *pdata;
-        struct panel_drv_data *ddata = platform_get_drvdata(pdev);
-        struct omap_dss_device *dssdev, *in;
-
-        pdata = dev_get_platdata(&pdev->dev);
-
-        in = omap_dss_find_output(pdata->source);
-        if (in == NULL) {
-                dev_err(&pdev->dev, "failed to find video source\n");
-                return -EPROBE_DEFER;
-        }
-        ddata->in = in;
-
-        ddata->reset_gpio = pdata->reset_gpio;
-
-        if (pdata->use_ext_te)
-                ddata->ext_te_gpio = pdata->ext_te_gpio;
-        else
-                ddata->ext_te_gpio = -1;
-
-        ddata->ulps_timeout = pdata->ulps_timeout;
-
-        ddata->use_dsi_backlight = pdata->use_dsi_backlight;
-
-        ddata->pin_config = pdata->pin_config;
-
-        dssdev = &ddata->dssdev;
-        dssdev->name = pdata->name;
-
-        return 0;
-}
-
 static int dsicm_probe_of(struct platform_device *pdev)
 {
         struct device_node *node = pdev->dev.of_node;
@@ -1207,6 +1172,9 @@ static int dsicm_probe(struct platform_device *pdev)
 
         dev_dbg(dev, "probe\n");
 
+        if (!pdev->dev.of_node)
+                return -ENODEV;
+
         ddata = devm_kzalloc(dev, sizeof(*ddata), GFP_KERNEL);
         if (!ddata)
                 return -ENOMEM;
@@ -1214,17 +1182,9 @@ static int dsicm_probe(struct platform_device *pdev)
         platform_set_drvdata(pdev, ddata);
         ddata->pdev = pdev;
 
-        if (dev_get_platdata(dev)) {
-                r = dsicm_probe_pdata(pdev);
-                if (r)
-                        return r;
-        } else if (pdev->dev.of_node) {
-                r = dsicm_probe_of(pdev);
-                if (r)
-                        return r;
-        } else {
-                return -ENODEV;
-        }
+        r = dsicm_probe_of(pdev);
+        if (r)
+                return r;
 
         ddata->timings.x_res = 864;
         ddata->timings.y_res = 480;
diff --git a/drivers/video/fbdev/omap2/omapfb/displays/panel-lgphilips-lb035q02.c b/drivers/video/fbdev/omap2/omapfb/displays/panel-lgphilips-lb035q02.c
index 18eb60e9c9ec..f14691ce8d02 100644
--- a/drivers/video/fbdev/omap2/omapfb/displays/panel-lgphilips-lb035q02.c
+++ b/drivers/video/fbdev/omap2/omapfb/displays/panel-lgphilips-lb035q02.c
@@ -16,8 +16,7 @@
 #include <linux/mutex.h>
 #include <linux/gpio.h>
 
-#include <video/omapdss.h>
-#include <video/omap-panel-data.h>
+#include <video/omapfb_dss.h>
 
 static struct omap_video_timings lb035q02_timings = {
         .x_res = 320,
@@ -240,44 +239,6 @@ static struct omap_dss_driver lb035q02_ops = {
         .get_resolution = omapdss_default_get_resolution,
 };
 
-static int lb035q02_probe_pdata(struct spi_device *spi)
-{
-        const struct panel_lb035q02_platform_data *pdata;
-        struct panel_drv_data *ddata = dev_get_drvdata(&spi->dev);
-        struct omap_dss_device *dssdev, *in;
-        int r;
-
-        pdata = dev_get_platdata(&spi->dev);
-
-        in = omap_dss_find_output(pdata->source);
-        if (in == NULL) {
-                dev_err(&spi->dev, "failed to find video source '%s'\n",
-                                pdata->source);
-                return -EPROBE_DEFER;
-        }
-
-        ddata->in = in;
-
-        ddata->data_lines = pdata->data_lines;
-
-        dssdev = &ddata->dssdev;
-        dssdev->name = pdata->name;
-
-        r = devm_gpio_request_one(&spi->dev, pdata->enable_gpio,
-                                        GPIOF_OUT_INIT_LOW, "panel enable");
-        if (r)
-                goto err_gpio;
-
-        ddata->enable_gpio = gpio_to_desc(pdata->enable_gpio);
-
-        ddata->backlight_gpio = pdata->backlight_gpio;
-
-        return 0;
-err_gpio:
-        omap_dss_put_device(ddata->in);
-        return r;
-}
-
 static int lb035q02_probe_of(struct spi_device *spi)
 {
         struct device_node *node = spi->dev.of_node;
@@ -312,6 +273,9 @@ static int lb035q02_panel_spi_probe(struct spi_device *spi)
         struct omap_dss_device *dssdev;
         int r;
 
+        if (!spi->dev.of_node)
+                return -ENODEV;
+
         ddata = devm_kzalloc(&spi->dev, sizeof(*ddata), GFP_KERNEL);
         if (ddata == NULL)
                 return -ENOMEM;
@@ -320,17 +284,9 @@ static int lb035q02_panel_spi_probe(struct spi_device *spi)
 
         ddata->spi = spi;
 
-        if (dev_get_platdata(&spi->dev)) {
-                r = lb035q02_probe_pdata(spi);
-                if (r)
-                        return r;
-        } else if (spi->dev.of_node) {
-                r = lb035q02_probe_of(spi);
-                if (r)
-                        return r;
-        } else {
-                return -ENODEV;
-        }
+        r = lb035q02_probe_of(spi);
+        if (r)
+                return r;
 
         if (gpio_is_valid(ddata->backlight_gpio)) {
                 r = devm_gpio_request_one(&spi->dev, ddata->backlight_gpio,
diff --git a/drivers/video/fbdev/omap2/omapfb/displays/panel-nec-nl8048hl11.c b/drivers/video/fbdev/omap2/omapfb/displays/panel-nec-nl8048hl11.c
index 8a928c9a2fc9..a2cbadd3eca3 100644
--- a/drivers/video/fbdev/omap2/omapfb/displays/panel-nec-nl8048hl11.c
+++ b/drivers/video/fbdev/omap2/omapfb/displays/panel-nec-nl8048hl11.c
@@ -18,8 +18,7 @@
 #include <linux/gpio.h>
 #include <linux/of_gpio.h>
 
-#include <video/omapdss.h>
-#include <video/omap-panel-data.h>
+#include <video/omapfb_dss.h>
 
 struct panel_drv_data {
         struct omap_dss_device  dssdev;
@@ -233,33 +232,6 @@ static struct omap_dss_driver nec_8048_ops = {
 };
 
 
-static int nec_8048_probe_pdata(struct spi_device *spi)
-{
-        const struct panel_nec_nl8048hl11_platform_data *pdata;
-        struct panel_drv_data *ddata = dev_get_drvdata(&spi->dev);
-        struct omap_dss_device *dssdev, *in;
-
-        pdata = dev_get_platdata(&spi->dev);
-
-        ddata->qvga_gpio = pdata->qvga_gpio;
-        ddata->res_gpio = pdata->res_gpio;
-
-        in = omap_dss_find_output(pdata->source);
-        if (in == NULL) {
-                dev_err(&spi->dev, "failed to find video source '%s'\n",
-                                pdata->source);
-                return -EPROBE_DEFER;
-        }
-        ddata->in = in;
-
-        ddata->data_lines = pdata->data_lines;
-
-        dssdev = &ddata->dssdev;
-        dssdev->name = pdata->name;
-
-        return 0;
-}
-
 static int nec_8048_probe_of(struct spi_device *spi)
 {
         struct device_node *node = spi->dev.of_node;
@@ -296,6 +268,9 @@ static int nec_8048_probe(struct spi_device *spi)
 
         dev_dbg(&spi->dev, "%s\n", __func__);
 
+        if (!spi->dev.of_node)
+                return -ENODEV;
+
         spi->mode = SPI_MODE_0;
         spi->bits_per_word = 32;
 
@@ -315,17 +290,9 @@ static int nec_8048_probe(struct spi_device *spi)
 
         ddata->spi = spi;
 
-        if (dev_get_platdata(&spi->dev)) {
-                r = nec_8048_probe_pdata(spi);
-                if (r)
-                        return r;
-        } else if (spi->dev.of_node) {
-                r = nec_8048_probe_of(spi);
-                if (r)
-                        return r;
-        } else {
-                return -ENODEV;
-        }
+        r = nec_8048_probe_of(spi);
+        if (r)
+                return r;
 
         if (gpio_is_valid(ddata->qvga_gpio)) {
                 r = devm_gpio_request_one(&spi->dev, ddata->qvga_gpio,
diff --git a/drivers/video/fbdev/omap2/omapfb/displays/panel-sharp-ls037v7dw01.c b/drivers/video/fbdev/omap2/omapfb/displays/panel-sharp-ls037v7dw01.c
index 1954ec913ce5..a8be18a87fa0 100644
--- a/drivers/video/fbdev/omap2/omapfb/displays/panel-sharp-ls037v7dw01.c
+++ b/drivers/video/fbdev/omap2/omapfb/displays/panel-sharp-ls037v7dw01.c
@@ -17,8 +17,7 @@
 #include <linux/platform_device.h>
 #include <linux/slab.h>
 #include <linux/regulator/consumer.h>
-#include <video/omapdss.h>
-#include <video/omap-panel-data.h>
+#include <video/omapfb_dss.h>
 
 struct panel_drv_data {
         struct omap_dss_device dssdev;
@@ -197,69 +196,6 @@ static struct omap_dss_driver sharp_ls_ops = {
         .get_resolution = omapdss_default_get_resolution,
 };
 
-static int sharp_ls_get_gpio(struct device *dev, int gpio, unsigned long flags,
-                  char *desc, struct gpio_desc **gpiod)
-{
-        int r;
-
-        r = devm_gpio_request_one(dev, gpio, flags, desc);
-        if (r) {
-                *gpiod = NULL;
-                return r == -ENOENT ? 0 : r;
-        }
-
-        *gpiod = gpio_to_desc(gpio);
-
-        return 0;
-}
-
-static int sharp_ls_probe_pdata(struct platform_device *pdev)
-{
-        const struct panel_sharp_ls037v7dw01_platform_data *pdata;
-        struct panel_drv_data *ddata = platform_get_drvdata(pdev);
-        struct omap_dss_device *dssdev, *in;
-        int r;
-
-        pdata = dev_get_platdata(&pdev->dev);
-
-        in = omap_dss_find_output(pdata->source);
-        if (in == NULL) {
-                dev_err(&pdev->dev, "failed to find video source '%s'\n",
-                                pdata->source);
-                return -EPROBE_DEFER;
-        }
-
-        ddata->in = in;
-
-        ddata->data_lines = pdata->data_lines;
-
-        dssdev = &ddata->dssdev;
-        dssdev->name = pdata->name;
-
-        r = sharp_ls_get_gpio(&pdev->dev, pdata->mo_gpio, GPIOF_OUT_INIT_LOW,
-                "lcd MO", &ddata->mo_gpio);
-        if (r)
-                return r;
-        r = sharp_ls_get_gpio(&pdev->dev, pdata->lr_gpio, GPIOF_OUT_INIT_HIGH,
-                "lcd LR", &ddata->lr_gpio);
-        if (r)
-                return r;
-        r = sharp_ls_get_gpio(&pdev->dev, pdata->ud_gpio, GPIOF_OUT_INIT_HIGH,
-                "lcd UD", &ddata->ud_gpio);
-        if (r)
-                return r;
-        r = sharp_ls_get_gpio(&pdev->dev, pdata->resb_gpio, GPIOF_OUT_INIT_LOW,
-                "lcd RESB", &ddata->resb_gpio);
-        if (r)
-                return r;
-        r = sharp_ls_get_gpio(&pdev->dev, pdata->ini_gpio, GPIOF_OUT_INIT_LOW,
-                "lcd INI", &ddata->ini_gpio);
-        if (r)
-                return r;
-
-        return 0;
-}
-
 static  int sharp_ls_get_gpio_of(struct device *dev, int index, int val,
         const char *desc, struct gpio_desc **gpiod)
 {
@@ -330,23 +266,18 @@ static int sharp_ls_probe(struct platform_device *pdev)
         struct omap_dss_device *dssdev;
         int r;
 
+        if (!pdev->dev.of_node)
+                return -ENODEV;
+
         ddata = devm_kzalloc(&pdev->dev, sizeof(*ddata), GFP_KERNEL);
         if (ddata == NULL)
                 return -ENOMEM;
 
         platform_set_drvdata(pdev, ddata);
 
-        if (dev_get_platdata(&pdev->dev)) {
-                r = sharp_ls_probe_pdata(pdev);
-                if (r)
-                        return r;
-        } else if (pdev->dev.of_node) {
-                r = sharp_ls_probe_of(pdev);
-                if (r)
-                        return r;
-        } else {
-                return -ENODEV;
-        }
+        r = sharp_ls_probe_of(pdev);
+        if (r)
+                return r;
 
         ddata->videomode = sharp_ls_timings;
 
diff --git a/drivers/video/fbdev/omap2/omapfb/displays/panel-sony-acx565akm.c b/drivers/video/fbdev/omap2/omapfb/displays/panel-sony-acx565akm.c
index 31efcca801bd..468560a6daae 100644
--- a/drivers/video/fbdev/omap2/omapfb/displays/panel-sony-acx565akm.c
+++ b/drivers/video/fbdev/omap2/omapfb/displays/panel-sony-acx565akm.c
@@ -33,7 +33,7 @@
 #include <linux/of.h>
 #include <linux/of_gpio.h>
 
-#include <video/omapdss.h>
+#include <video/omapfb_dss.h>
 #include <video/omap-panel-data.h>
 
 #define MIPID_CMD_READ_DISP_ID          0x04
diff --git a/drivers/video/fbdev/omap2/omapfb/displays/panel-tpo-td028ttec1.c b/drivers/video/fbdev/omap2/omapfb/displays/panel-tpo-td028ttec1.c
index 4d657f3ab679..b529a8c2b652 100644
--- a/drivers/video/fbdev/omap2/omapfb/displays/panel-tpo-td028ttec1.c
+++ b/drivers/video/fbdev/omap2/omapfb/displays/panel-tpo-td028ttec1.c
@@ -28,8 +28,7 @@
 #include <linux/delay.h>
 #include <linux/spi/spi.h>
 #include <linux/gpio.h>
-#include <video/omapdss.h>
-#include <video/omap-panel-data.h>
+#include <video/omapfb_dss.h>
 
 struct panel_drv_data {
         struct omap_dss_device dssdev;
@@ -365,31 +364,6 @@ static struct omap_dss_driver td028ttec1_ops = {
         .check_timings  = td028ttec1_panel_check_timings,
 };
 
-static int td028ttec1_panel_probe_pdata(struct spi_device *spi)
-{
-        const struct panel_tpo_td028ttec1_platform_data *pdata;
-        struct panel_drv_data *ddata = dev_get_drvdata(&spi->dev);
-        struct omap_dss_device *dssdev, *in;
-
-        pdata = dev_get_platdata(&spi->dev);
-
-        in = omap_dss_find_output(pdata->source);
-        if (in == NULL) {
-                dev_err(&spi->dev, "failed to find video source '%s'\n",
-                                pdata->source);
-                return -EPROBE_DEFER;
-        }
-
-        ddata->in = in;
-
-        ddata->data_lines = pdata->data_lines;
-
-        dssdev = &ddata->dssdev;
-        dssdev->name = pdata->name;
-
-        return 0;
-}
-
 static int td028ttec1_probe_of(struct spi_device *spi)
 {
         struct device_node *node = spi->dev.of_node;
@@ -415,6 +389,9 @@ static int td028ttec1_panel_probe(struct spi_device *spi)
 
         dev_dbg(&spi->dev, "%s\n", __func__);
 
+        if (!spi->dev.of_node)
+                return -ENODEV;
+
         spi->bits_per_word = 9;
         spi->mode = SPI_MODE_3;
 
@@ -432,17 +409,9 @@ static int td028ttec1_panel_probe(struct spi_device *spi)
 
         ddata->spi_dev = spi;
 
-        if (dev_get_platdata(&spi->dev)) {
-                r = td028ttec1_panel_probe_pdata(spi);
-                if (r)
-                        return r;
-        } else if (spi->dev.of_node) {
-                r = td028ttec1_probe_of(spi);
-                if (r)
-                        return r;
-        } else {
-                return -ENODEV;
-        }
+        r = td028ttec1_probe_of(spi);
+        if (r)
+                return r;
 
         ddata->videomode = td028ttec1_panel_timings;
 
diff --git a/drivers/video/fbdev/omap2/omapfb/displays/panel-tpo-td043mtea1.c b/drivers/video/fbdev/omap2/omapfb/displays/panel-tpo-td043mtea1.c
index 68e3b68a2920..51e628b85f4a 100644
--- a/drivers/video/fbdev/omap2/omapfb/displays/panel-tpo-td043mtea1.c
+++ b/drivers/video/fbdev/omap2/omapfb/displays/panel-tpo-td043mtea1.c
@@ -19,8 +19,7 @@
 #include <linux/slab.h>
 #include <linux/of_gpio.h>
 
-#include <video/omapdss.h>
-#include <video/omap-panel-data.h>
+#include <video/omapfb_dss.h>
 
 #define TPO_R02_MODE(x)         ((x) & 7)
 #define TPO_R02_MODE_800x480    7
@@ -465,32 +464,6 @@ static struct omap_dss_driver tpo_td043_ops = {
 };
 
 
-static int tpo_td043_probe_pdata(struct spi_device *spi)
-{
-        const struct panel_tpo_td043mtea1_platform_data *pdata;
-        struct panel_drv_data *ddata = dev_get_drvdata(&spi->dev);
-        struct omap_dss_device *dssdev, *in;
-
-        pdata = dev_get_platdata(&spi->dev);
-
-        ddata->nreset_gpio = pdata->nreset_gpio;
-
-        in = omap_dss_find_output(pdata->source);
-        if (in == NULL) {
-                dev_err(&spi->dev, "failed to find video source '%s'\n",
-                                pdata->source);
-                return -EPROBE_DEFER;
-        }
-        ddata->in = in;
-
-        ddata->data_lines = pdata->data_lines;
-
-        dssdev = &ddata->dssdev;
-        dssdev->name = pdata->name;
-
-        return 0;
-}
-
 static int tpo_td043_probe_of(struct spi_device *spi)
 {
         struct device_node *node = spi->dev.of_node;
@@ -524,6 +497,9 @@ static int tpo_td043_probe(struct spi_device *spi)
 
         dev_dbg(&spi->dev, "%s\n", __func__);
 
+        if (!spi->dev.of_node)
+                return -ENODEV;
+
         spi->bits_per_word = 16;
         spi->mode = SPI_MODE_0;
 
@@ -541,17 +517,9 @@ static int tpo_td043_probe(struct spi_device *spi)
 
         ddata->spi = spi;
 
-        if (dev_get_platdata(&spi->dev)) {
-                r = tpo_td043_probe_pdata(spi);
-                if (r)
-                        return r;
-        } else if (spi->dev.of_node) {
-                r = tpo_td043_probe_of(spi);
-                if (r)
-                        return r;
-        } else {
-                return -ENODEV;
-        }
+        r = tpo_td043_probe_of(spi);
+        if (r)
+                return r;
 
         ddata->mode = TPO_R02_MODE_800x480;
         memcpy(ddata->gamma, tpo_td043_def_gamma, sizeof(ddata->gamma));
diff --git a/drivers/video/fbdev/omap2/omapfb/dss/apply.c b/drivers/video/fbdev/omap2/omapfb/dss/apply.c
index 663ccc3bf4e5..2481f4871f66 100644
--- a/drivers/video/fbdev/omap2/omapfb/dss/apply.c
+++ b/drivers/video/fbdev/omap2/omapfb/dss/apply.c
@@ -23,7 +23,7 @@
 #include <linux/spinlock.h>
 #include <linux/jiffies.h>
 
-#include <video/omapdss.h>
+#include <video/omapfb_dss.h>
 
 #include "dss.h"
 #include "dss_features.h"
diff --git a/drivers/video/fbdev/omap2/omapfb/dss/core.c b/drivers/video/fbdev/omap2/omapfb/dss/core.c
index 5a87179b7312..29de4827589d 100644
--- a/drivers/video/fbdev/omap2/omapfb/dss/core.c
+++ b/drivers/video/fbdev/omap2/omapfb/dss/core.c
@@ -35,7 +35,7 @@
 #include <linux/suspend.h>
 #include <linux/slab.h>
 
-#include <video/omapdss.h>
+#include <video/omapfb_dss.h>
 
 #include "dss.h"
 #include "dss_features.h"
@@ -208,8 +208,6 @@ static int __init omap_dss_probe(struct platform_device *pdev)
                 core.default_display_name = def_disp_name;
         else if (pdata->default_display_name)
                 core.default_display_name = pdata->default_display_name;
-        else if (pdata->default_device)
-                core.default_display_name = pdata->default_device->name;
 
         register_pm_notifier(&omap_dss_pm_notif_block);
 
diff --git a/drivers/video/fbdev/omap2/omapfb/dss/dispc-compat.c b/drivers/video/fbdev/omap2/omapfb/dss/dispc-compat.c
index 6607db37a5e4..3691bde4ce0a 100644
--- a/drivers/video/fbdev/omap2/omapfb/dss/dispc-compat.c
+++ b/drivers/video/fbdev/omap2/omapfb/dss/dispc-compat.c
@@ -26,7 +26,7 @@
 #include <linux/interrupt.h>
 #include <linux/seq_file.h>
 
-#include <video/omapdss.h>
+#include <video/omapfb_dss.h>
 
 #include "dss.h"
 #include "dss_features.h"
diff --git a/drivers/video/fbdev/omap2/omapfb/dss/dispc.c b/drivers/video/fbdev/omap2/omapfb/dss/dispc.c
index 5491e304f4fe..7a75dfda9845 100644
--- a/drivers/video/fbdev/omap2/omapfb/dss/dispc.c
+++ b/drivers/video/fbdev/omap2/omapfb/dss/dispc.c
@@ -41,7 +41,7 @@
 #include <linux/of.h>
 #include <linux/component.h>
 
-#include <video/omapdss.h>
+#include <video/omapfb_dss.h>
 
 #include "dss.h"
 #include "dss_features.h"
diff --git a/drivers/video/fbdev/omap2/omapfb/dss/dispc_coefs.c b/drivers/video/fbdev/omap2/omapfb/dss/dispc_coefs.c
index 038c15b04215..59c9a5c47ca9 100644
--- a/drivers/video/fbdev/omap2/omapfb/dss/dispc_coefs.c
+++ b/drivers/video/fbdev/omap2/omapfb/dss/dispc_coefs.c
@@ -18,7 +18,7 @@
  */
 
 #include <linux/kernel.h>
-#include <video/omapdss.h>
+#include <video/omapfb_dss.h>
 
 #include "dispc.h"
 
diff --git a/drivers/video/fbdev/omap2/omapfb/dss/display-sysfs.c b/drivers/video/fbdev/omap2/omapfb/dss/display-sysfs.c
index 75b5286029ee..b3fdbfd0b82d 100644
--- a/drivers/video/fbdev/omap2/omapfb/dss/display-sysfs.c
+++ b/drivers/video/fbdev/omap2/omapfb/dss/display-sysfs.c
@@ -25,7 +25,7 @@
 #include <linux/platform_device.h>
 #include <linux/sysfs.h>
 
-#include <video/omapdss.h>
+#include <video/omapfb_dss.h>
 #include "dss.h"
 
 static ssize_t display_name_show(struct omap_dss_device *dssdev, char *buf)
diff --git a/drivers/video/fbdev/omap2/omapfb/dss/display.c b/drivers/video/fbdev/omap2/omapfb/dss/display.c
index ef5b9027985d..dd5468695c43 100644
--- a/drivers/video/fbdev/omap2/omapfb/dss/display.c
+++ b/drivers/video/fbdev/omap2/omapfb/dss/display.c
@@ -28,7 +28,7 @@
 #include <linux/platform_device.h>
 #include <linux/of.h>
 
-#include <video/omapdss.h>
+#include <video/omapfb_dss.h>
 #include "dss.h"
 #include "dss_features.h"
 
diff --git a/drivers/video/fbdev/omap2/omapfb/dss/dpi.c b/drivers/video/fbdev/omap2/omapfb/dss/dpi.c
index 7953e6a52346..da09806b940c 100644
--- a/drivers/video/fbdev/omap2/omapfb/dss/dpi.c
+++ b/drivers/video/fbdev/omap2/omapfb/dss/dpi.c
@@ -34,7 +34,7 @@
 #include <linux/clk.h>
 #include <linux/component.h>
 
-#include <video/omapdss.h>
+#include <video/omapfb_dss.h>
 
 #include "dss.h"
 #include "dss_features.h"
diff --git a/drivers/video/fbdev/omap2/omapfb/dss/dsi.c b/drivers/video/fbdev/omap2/omapfb/dss/dsi.c
index d63e59807707..9e4800a4e3d1 100644
--- a/drivers/video/fbdev/omap2/omapfb/dss/dsi.c
+++ b/drivers/video/fbdev/omap2/omapfb/dss/dsi.c
@@ -42,7 +42,7 @@
 #include <linux/of_platform.h>
 #include <linux/component.h>
 
-#include <video/omapdss.h>
+#include <video/omapfb_dss.h>
 #include <video/mipi_display.h>
 
 #include "dss.h"
diff --git a/drivers/video/fbdev/omap2/omapfb/dss/dss-of.c b/drivers/video/fbdev/omap2/omapfb/dss/dss-of.c
index bf407b6ba15c..d356a252ab4a 100644
--- a/drivers/video/fbdev/omap2/omapfb/dss/dss-of.c
+++ b/drivers/video/fbdev/omap2/omapfb/dss/dss-of.c
@@ -18,7 +18,7 @@
 #include <linux/of.h>
 #include <linux/seq_file.h>
 
-#include <video/omapdss.h>
+#include <video/omapfb_dss.h>
 
 #include "dss.h"
 
diff --git a/drivers/video/fbdev/omap2/omapfb/dss/dss.c b/drivers/video/fbdev/omap2/omapfb/dss/dss.c
index 0078c4d1fc31..47d7f69ad9ad 100644
--- a/drivers/video/fbdev/omap2/omapfb/dss/dss.c
+++ b/drivers/video/fbdev/omap2/omapfb/dss/dss.c
@@ -41,7 +41,7 @@
 #include <linux/suspend.h>
 #include <linux/component.h>
 
-#include <video/omapdss.h>
+#include <video/omapfb_dss.h>
 
 #include "dss.h"
 #include "dss_features.h"
diff --git a/drivers/video/fbdev/omap2/omapfb/dss/dss.h b/drivers/video/fbdev/omap2/omapfb/dss/dss.h
index 0184a8461df1..a3cc0ca8f9d2 100644
--- a/drivers/video/fbdev/omap2/omapfb/dss/dss.h
+++ b/drivers/video/fbdev/omap2/omapfb/dss/dss.h
@@ -73,6 +73,17 @@
 #define FLD_MOD(orig, val, start, end) \
         (((orig) & ~FLD_MASK(start, end)) | FLD_VAL(val, start, end))
 
+enum omap_dss_clk_source {
+        OMAP_DSS_CLK_SRC_FCK = 0,               /* OMAP2/3: DSS1_ALWON_FCLK
+                                                 * OMAP4: DSS_FCLK */
+        OMAP_DSS_CLK_SRC_DSI_PLL_HSDIV_DISPC,   /* OMAP3: DSI1_PLL_FCLK
+                                                 * OMAP4: PLL1_CLK1 */
+        OMAP_DSS_CLK_SRC_DSI_PLL_HSDIV_DSI,     /* OMAP3: DSI2_PLL_FCLK
+                                                 * OMAP4: PLL1_CLK2 */
+        OMAP_DSS_CLK_SRC_DSI2_PLL_HSDIV_DISPC,  /* OMAP4: PLL2_CLK1 */
+        OMAP_DSS_CLK_SRC_DSI2_PLL_HSDIV_DSI,    /* OMAP4: PLL2_CLK2 */
+};
+
 enum dss_io_pad_mode {
         DSS_IO_PAD_MODE_RESET,
         DSS_IO_PAD_MODE_RFBI,
diff --git a/drivers/video/fbdev/omap2/omapfb/dss/dss_features.c b/drivers/video/fbdev/omap2/omapfb/dss/dss_features.c
index c886a2927f73..8fc843b56b26 100644
--- a/drivers/video/fbdev/omap2/omapfb/dss/dss_features.c
+++ b/drivers/video/fbdev/omap2/omapfb/dss/dss_features.c
@@ -23,7 +23,7 @@
 #include <linux/err.h>
 #include <linux/slab.h>
 
-#include <video/omapdss.h>
+#include <video/omapfb_dss.h>
 
 #include "dss.h"
 #include "dss_features.h"
diff --git a/drivers/video/fbdev/omap2/omapfb/dss/hdmi.h b/drivers/video/fbdev/omap2/omapfb/dss/hdmi.h
index 53616b02b613..f6de87e078b0 100644
--- a/drivers/video/fbdev/omap2/omapfb/dss/hdmi.h
+++ b/drivers/video/fbdev/omap2/omapfb/dss/hdmi.h
@@ -23,7 +23,8 @@
 #include <linux/io.h>
 #include <linux/platform_device.h>
 #include <linux/hdmi.h>
-#include <video/omapdss.h>
+#include <video/omapfb_dss.h>
+#include <sound/omap-hdmi-audio.h>
 
 #include "dss.h"
 
diff --git a/drivers/video/fbdev/omap2/omapfb/dss/hdmi4.c b/drivers/video/fbdev/omap2/omapfb/dss/hdmi4.c
index 2e71aec838b1..926a6f20dbb2 100644
--- a/drivers/video/fbdev/omap2/omapfb/dss/hdmi4.c
+++ b/drivers/video/fbdev/omap2/omapfb/dss/hdmi4.c
@@ -33,7 +33,7 @@
 #include <linux/gpio.h>
 #include <linux/regulator/consumer.h>
 #include <linux/component.h>
-#include <video/omapdss.h>
+#include <video/omapfb_dss.h>
 #include <sound/omap-hdmi-audio.h>
 
 #include "hdmi4_core.h"
diff --git a/drivers/video/fbdev/omap2/omapfb/dss/hdmi5.c b/drivers/video/fbdev/omap2/omapfb/dss/hdmi5.c
index aade6d99662a..0ee829a165c3 100644
--- a/drivers/video/fbdev/omap2/omapfb/dss/hdmi5.c
+++ b/drivers/video/fbdev/omap2/omapfb/dss/hdmi5.c
@@ -38,7 +38,7 @@
 #include <linux/gpio.h>
 #include <linux/regulator/consumer.h>
 #include <linux/component.h>
-#include <video/omapdss.h>
+#include <video/omapfb_dss.h>
 #include <sound/omap-hdmi-audio.h>
 
 #include "hdmi5_core.h"
diff --git a/drivers/video/fbdev/omap2/omapfb/dss/hdmi_common.c b/drivers/video/fbdev/omap2/omapfb/dss/hdmi_common.c
index 1b8fcc6c4ba1..189a5ad125a3 100644
--- a/drivers/video/fbdev/omap2/omapfb/dss/hdmi_common.c
+++ b/drivers/video/fbdev/omap2/omapfb/dss/hdmi_common.c
@@ -4,7 +4,7 @@
 #include <linux/kernel.h>
 #include <linux/err.h>
 #include <linux/of.h>
-#include <video/omapdss.h>
+#include <video/omapfb_dss.h>
 
 #include "hdmi.h"
 
diff --git a/drivers/video/fbdev/omap2/omapfb/dss/hdmi_phy.c b/drivers/video/fbdev/omap2/omapfb/dss/hdmi_phy.c
index 1f5d19c119ce..9a13c35fd6d8 100644
--- a/drivers/video/fbdev/omap2/omapfb/dss/hdmi_phy.c
+++ b/drivers/video/fbdev/omap2/omapfb/dss/hdmi_phy.c
@@ -13,7 +13,7 @@
 #include <linux/io.h>
 #include <linux/platform_device.h>
 #include <linux/slab.h>
-#include <video/omapdss.h>
+#include <video/omapfb_dss.h>
 
 #include "dss.h"
 #include "hdmi.h"
diff --git a/drivers/video/fbdev/omap2/omapfb/dss/hdmi_pll.c b/drivers/video/fbdev/omap2/omapfb/dss/hdmi_pll.c
index 06e23a7c432c..eac3665aba6c 100644
--- a/drivers/video/fbdev/omap2/omapfb/dss/hdmi_pll.c
+++ b/drivers/video/fbdev/omap2/omapfb/dss/hdmi_pll.c
@@ -17,7 +17,7 @@
 #include <linux/platform_device.h>
 #include <linux/clk.h>
 
-#include <video/omapdss.h>
+#include <video/omapfb_dss.h>
 
 #include "dss.h"
 #include "hdmi.h"
diff --git a/drivers/video/fbdev/omap2/omapfb/dss/hdmi_wp.c b/drivers/video/fbdev/omap2/omapfb/dss/hdmi_wp.c
index 7c544bc56fb5..705373e4cf38 100644
--- a/drivers/video/fbdev/omap2/omapfb/dss/hdmi_wp.c
+++ b/drivers/video/fbdev/omap2/omapfb/dss/hdmi_wp.c
@@ -14,7 +14,7 @@
 #include <linux/err.h>
 #include <linux/io.h>
 #include <linux/platform_device.h>
-#include <video/omapdss.h>
+#include <video/omapfb_dss.h>
 
 #include "dss.h"
 #include "hdmi.h"
diff --git a/drivers/video/fbdev/omap2/omapfb/dss/manager-sysfs.c b/drivers/video/fbdev/omap2/omapfb/dss/manager-sysfs.c
index a7414fb12830..9e2a67fdf4d2 100644
--- a/drivers/video/fbdev/omap2/omapfb/dss/manager-sysfs.c
+++ b/drivers/video/fbdev/omap2/omapfb/dss/manager-sysfs.c
@@ -26,7 +26,7 @@
 #include <linux/platform_device.h>
 #include <linux/jiffies.h>
 
-#include <video/omapdss.h>
+#include <video/omapfb_dss.h>
 
 #include "dss.h"
 #include "dss_features.h"
diff --git a/drivers/video/fbdev/omap2/omapfb/dss/manager.c b/drivers/video/fbdev/omap2/omapfb/dss/manager.c
index 08a67f4f6a20..69f86d2cc274 100644
--- a/drivers/video/fbdev/omap2/omapfb/dss/manager.c
+++ b/drivers/video/fbdev/omap2/omapfb/dss/manager.c
@@ -28,7 +28,7 @@
 #include <linux/platform_device.h>
 #include <linux/jiffies.h>
 
-#include <video/omapdss.h>
+#include <video/omapfb_dss.h>
 
 #include "dss.h"
 #include "dss_features.h"
@@ -69,7 +69,6 @@ int dss_init_overlay_managers(void)
                         break;
                 }
 
-                mgr->caps = 0;
                 mgr->supported_displays =
                         dss_feat_get_supported_displays(mgr->id);
                 mgr->supported_outputs =
diff --git a/drivers/video/fbdev/omap2/omapfb/dss/output.c b/drivers/video/fbdev/omap2/omapfb/dss/output.c
index 16072159bd24..bed9a978269d 100644
--- a/drivers/video/fbdev/omap2/omapfb/dss/output.c
+++ b/drivers/video/fbdev/omap2/omapfb/dss/output.c
@@ -21,7 +21,7 @@
 #include <linux/slab.h>
 #include <linux/of.h>
 
-#include <video/omapdss.h>
+#include <video/omapfb_dss.h>
 
 #include "dss.h"
 
diff --git a/drivers/video/fbdev/omap2/omapfb/dss/overlay-sysfs.c b/drivers/video/fbdev/omap2/omapfb/dss/overlay-sysfs.c
index 4cc5ddebfb34..f1f6c0aea752 100644
--- a/drivers/video/fbdev/omap2/omapfb/dss/overlay-sysfs.c
+++ b/drivers/video/fbdev/omap2/omapfb/dss/overlay-sysfs.c
@@ -26,7 +26,7 @@
 #include <linux/kobject.h>
 #include <linux/platform_device.h>
 
-#include <video/omapdss.h>
+#include <video/omapfb_dss.h>
 
 #include "dss.h"
 #include "dss_features.h"
diff --git a/drivers/video/fbdev/omap2/omapfb/dss/overlay.c b/drivers/video/fbdev/omap2/omapfb/dss/overlay.c
index 2f7cee985cdd..d6c5d75d2ef8 100644
--- a/drivers/video/fbdev/omap2/omapfb/dss/overlay.c
+++ b/drivers/video/fbdev/omap2/omapfb/dss/overlay.c
@@ -30,7 +30,7 @@
 #include <linux/delay.h>
 #include <linux/slab.h>
 
-#include <video/omapdss.h>
+#include <video/omapfb_dss.h>
 
 #include "dss.h"
 #include "dss_features.h"
diff --git a/drivers/video/fbdev/omap2/omapfb/dss/pll.c b/drivers/video/fbdev/omap2/omapfb/dss/pll.c
index f974ddcd3b6e..0564c5606cd0 100644
--- a/drivers/video/fbdev/omap2/omapfb/dss/pll.c
+++ b/drivers/video/fbdev/omap2/omapfb/dss/pll.c
@@ -22,7 +22,7 @@
 #include <linux/regulator/consumer.h>
 #include <linux/sched.h>
 
-#include <video/omapdss.h>
+#include <video/omapfb_dss.h>
 
 #include "dss.h"
 
diff --git a/drivers/video/fbdev/omap2/omapfb/dss/rfbi.c b/drivers/video/fbdev/omap2/omapfb/dss/rfbi.c
index aea6a1d0fb20..562b0c4ae0c6 100644
--- a/drivers/video/fbdev/omap2/omapfb/dss/rfbi.c
+++ b/drivers/video/fbdev/omap2/omapfb/dss/rfbi.c
@@ -38,7 +38,7 @@
 #include <linux/pm_runtime.h>
 #include <linux/component.h>
 
-#include <video/omapdss.h>
+#include <video/omapfb_dss.h>
 #include "dss.h"
 
 struct rfbi_reg { u16 idx; };
diff --git a/drivers/video/fbdev/omap2/omapfb/dss/sdi.c b/drivers/video/fbdev/omap2/omapfb/dss/sdi.c
index d747cc6b59e1..c4be732a4714 100644
--- a/drivers/video/fbdev/omap2/omapfb/dss/sdi.c
+++ b/drivers/video/fbdev/omap2/omapfb/dss/sdi.c
@@ -29,7 +29,7 @@
 #include <linux/of.h>
 #include <linux/component.h>
 
-#include <video/omapdss.h>
+#include <video/omapfb_dss.h>
 #include "dss.h"
 
 static struct {
diff --git a/drivers/video/fbdev/omap2/omapfb/dss/venc.c b/drivers/video/fbdev/omap2/omapfb/dss/venc.c
index 26e0ee30adf8..392464da12e4 100644
--- a/drivers/video/fbdev/omap2/omapfb/dss/venc.c
+++ b/drivers/video/fbdev/omap2/omapfb/dss/venc.c
@@ -37,7 +37,7 @@
 #include <linux/of.h>
 #include <linux/component.h>
 
-#include <video/omapdss.h>
+#include <video/omapfb_dss.h>
 
 #include "dss.h"
 #include "dss_features.h"
diff --git a/drivers/video/fbdev/omap2/omapfb/dss/video-pll.c b/drivers/video/fbdev/omap2/omapfb/dss/video-pll.c
index b1ec59e42940..a890540f2037 100644
--- a/drivers/video/fbdev/omap2/omapfb/dss/video-pll.c
+++ b/drivers/video/fbdev/omap2/omapfb/dss/video-pll.c
@@ -17,7 +17,7 @@
 #include <linux/platform_device.h>
 #include <linux/sched.h>
 
-#include <video/omapdss.h>
+#include <video/omapfb_dss.h>
 
 #include "dss.h"
 #include "dss_features.h"
diff --git a/drivers/video/fbdev/omap2/omapfb/omapfb-ioctl.c b/drivers/video/fbdev/omap2/omapfb/omapfb-ioctl.c
index 9ddfdd63b84c..ef69273074ba 100644
--- a/drivers/video/fbdev/omap2/omapfb/omapfb-ioctl.c
+++ b/drivers/video/fbdev/omap2/omapfb/omapfb-ioctl.c
@@ -30,7 +30,7 @@
 #include <linux/export.h>
 #include <linux/sizes.h>
 
-#include <video/omapdss.h>
+#include <video/omapfb_dss.h>
 #include <video/omapvrfb.h>
 
 #include "omapfb.h"
diff --git a/drivers/video/fbdev/omap2/omapfb/omapfb-main.c b/drivers/video/fbdev/omap2/omapfb/omapfb-main.c
index d3af01c94a58..2fb90cb6803f 100644
--- a/drivers/video/fbdev/omap2/omapfb/omapfb-main.c
+++ b/drivers/video/fbdev/omap2/omapfb/omapfb-main.c
@@ -30,7 +30,7 @@
 #include <linux/platform_device.h>
 #include <linux/omapfb.h>
 
-#include <video/omapdss.h>
+#include <video/omapfb_dss.h>
 #include <video/omapvrfb.h>
 
 #include "omapfb.h"
diff --git a/drivers/video/fbdev/omap2/omapfb/omapfb-sysfs.c b/drivers/video/fbdev/omap2/omapfb/omapfb-sysfs.c
index 18fa9e1d0033..8087a009c54f 100644
--- a/drivers/video/fbdev/omap2/omapfb/omapfb-sysfs.c
+++ b/drivers/video/fbdev/omap2/omapfb/omapfb-sysfs.c
@@ -29,7 +29,7 @@
 #include <linux/mm.h>
 #include <linux/omapfb.h>
 
-#include <video/omapdss.h>
+#include <video/omapfb_dss.h>
 #include <video/omapvrfb.h>
 
 #include "omapfb.h"
diff --git a/drivers/video/fbdev/omap2/omapfb/omapfb.h b/drivers/video/fbdev/omap2/omapfb/omapfb.h
index 623cd872a367..bcb9ff4a607d 100644
--- a/drivers/video/fbdev/omap2/omapfb/omapfb.h
+++ b/drivers/video/fbdev/omap2/omapfb/omapfb.h
@@ -31,7 +31,7 @@
 #include <linux/dma-attrs.h>
 #include <linux/dma-mapping.h>
 
-#include <video/omapdss.h>
+#include <video/omapfb_dss.h>
 
 #ifdef DEBUG
 extern bool omapfb_debug;