Merge tag 'drm-intel-next-2017-11-17-1' of git://anongit.freedesktop.org/drm/drm-intel into drm-next

More change sets for 4.16:

- Many improvements for selftests and other igt tests (Chris)
- Forcewake with PUNIT->PMIC bus fixes and robustness (Hans)
- Define an engine class for uABI (Tvrtko)
- Context switch fixes and improvements (Chris)
- GT powersavings and power gating simplification and fixes (Chris)
- Other general driver clean-ups (Chris, Lucas, Ville)
- Removing old, useless and/or bad workarounds (Chris, Oscar, Radhakrishna)
- IPS, pipe config, etc in preparation for another Fast Boot attempt (Maarten)
- OA perf fixes and support to Coffee Lake and Cannonlake (Lionel)
- Fixes around GPU fault registers (Michel)
- GEM Proxy (Tina)
- Refactor of Geminilake and Cannonlake plane color handling (James)
- Generalize transcoder loop (Mika Kahola)
- New HW Workaround for Cannonlake and Geminilake (Rodrigo)
- Resume GuC before using GEM (Chris)
- Stolen Memory handling improvements (Ville)
- Initialize entry in PPAT for older compilers (Chris)
- Other fixes and robustness improvements on execbuf (Chris)
- Improve logs of GEM_BUG_ON (Mika Kuoppala)
- Rework with massive rename of GuC functions and files (Sagar)
- Don't sanitize frame start delay if pipe is off (Ville)
- Cannonlake clock fixes (Rodrigo)
- Cannonlake HDMI 2.0 support (Rodrigo)
- Add a GuC doorbells selftest (Michel)
- Add might_sleep() check to our wait_for() (Chris)

Many GVT changes for 4.16:

- CSB HWSP update support (Weinan)
- GVT debug helpers, dyndbg and debugfs (Chuanxiao, Shuo)
- full virtualized opregion (Xiaolin)
- VM health check for sane fallback (Fred)
- workload submission code refactor for future enabling (Zhi)
- Updated repo URL in MAINTAINERS (Zhenyu)
- other many misc fixes

* tag 'drm-intel-next-2017-11-17-1' of git://anongit.freedesktop.org/drm/drm-intel: (260 commits)
  drm/i915: Update DRIVER_DATE to 20171117
  drm/i915: Add a policy note for removing workarounds
  drm/i915/selftests: Report ENOMEM clearly for an allocation failure
  Revert "drm/i915: Display WA #1133 WaFbcSkipSegments:cnl, glk"
  drm/i915: Calculate g4x intermediate watermarks correctly
  drm/i915: Calculate vlv/chv intermediate watermarks correctly, v3.
  drm/i915: Pass crtc_state to ips toggle functions, v2
  drm/i915: Pass idle crtc_state to intel_dp_sink_crc
  drm/i915: Enable FIFO underrun reporting after initial fastset, v4.
  drm/i915: Mark the userptr invalidate workqueue as WQ_MEM_RECLAIM
  drm/i915: Add might_sleep() check to wait_for()
  drm/i915/selftests: Add a GuC doorbells selftest
  drm/i915/cnl: Extend HDMI 2.0 support to CNL.
  drm/i915/cnl: Simplify dco_fraction calculation.
  drm/i915/cnl: Don't blindly replace qdiv.
  drm/i915/cnl: Fix wrpll math for higher freqs.
  drm/i915/cnl: Fix, simplify and unify wrpll variable sizes.
  drm/i915/cnl: Remove useless conversion.
  drm/i915/cnl: Remove spurious central_freq.
  drm/i915/selftests: exercise_ggtt may have nothing to do
  ...

129 files changed, 6526 insertions, 3154 deletions

diff --git a/Documentation/gpu/i915.rst b/Documentation/gpu/i915.rst
index 2e7ee0313c1c..21577eabaf78 100644
--- a/Documentation/gpu/i915.rst
+++ b/Documentation/gpu/i915.rst
@@ -350,10 +350,10 @@ GuC-specific firmware loader
 GuC-based command submission
 ----------------------------
 
-.. kernel-doc:: drivers/gpu/drm/i915/i915_guc_submission.c
+.. kernel-doc:: drivers/gpu/drm/i915/intel_guc_submission.c
    :doc: GuC-based command submission
 
-.. kernel-doc:: drivers/gpu/drm/i915/i915_guc_submission.c
+.. kernel-doc:: drivers/gpu/drm/i915/intel_guc_submission.c
    :internal:
 
 GuC Firmware Layout
diff --git a/MAINTAINERS b/MAINTAINERS
index d7eedb15be85..069ba63190b2 100644
--- a/MAINTAINERS
+++ b/MAINTAINERS
@@ -7030,7 +7030,7 @@ M:	Zhi Wang <zhi.a.wang@intel.com>
 L:      intel-gvt-dev@lists.freedesktop.org
 L:      intel-gfx@lists.freedesktop.org
 W:      https://01.org/igvt-g
-T:      git https://github.com/01org/gvt-linux.git
+T:      git https://github.com/intel/gvt-linux.git
 S:      Supported
 F:      drivers/gpu/drm/i915/gvt/
 
diff --git a/arch/x86/include/asm/iosf_mbi.h b/arch/x86/include/asm/iosf_mbi.h
index 7d87437bd030..3de0489deade 100644
--- a/arch/x86/include/asm/iosf_mbi.h
+++ b/arch/x86/include/asm/iosf_mbi.h
@@ -147,6 +147,18 @@ int iosf_mbi_register_pmic_bus_access_notifier(struct notifier_block *nb);
 int iosf_mbi_unregister_pmic_bus_access_notifier(struct notifier_block *nb);
 
 /**
+ * iosf_mbi_unregister_pmic_bus_access_notifier_unlocked - Unregister PMIC bus
+ *                                                         notifier, unlocked
+ *
+ * Like iosf_mbi_unregister_pmic_bus_access_notifier(), but for use when the
+ * caller has already called iosf_mbi_punit_acquire() itself.
+ *
+ * @nb: notifier_block to unregister
+ */
+int iosf_mbi_unregister_pmic_bus_access_notifier_unlocked(
+        struct notifier_block *nb);
+
+/**
  * iosf_mbi_call_pmic_bus_access_notifier_chain - Call PMIC bus notifier chain
  *
  * @val: action to pass into listener's notifier_call function
@@ -154,6 +166,11 @@ int iosf_mbi_unregister_pmic_bus_access_notifier(struct notifier_block *nb);
  */
 int iosf_mbi_call_pmic_bus_access_notifier_chain(unsigned long val, void *v);
 
+/**
+ * iosf_mbi_assert_punit_acquired - Assert that the P-Unit has been acquired.
+ */
+void iosf_mbi_assert_punit_acquired(void);
+
 #else /* CONFIG_IOSF_MBI is not enabled */
 static inline
 bool iosf_mbi_available(void)
@@ -197,12 +214,20 @@ int iosf_mbi_unregister_pmic_bus_access_notifier(struct notifier_block *nb)
         return 0;
 }
 
+static inline int
+iosf_mbi_unregister_pmic_bus_access_notifier_unlocked(struct notifier_block *nb)
+{
+        return 0;
+}
+
 static inline
 int iosf_mbi_call_pmic_bus_access_notifier_chain(unsigned long val, void *v)
 {
         return 0;
 }
 
+static inline void iosf_mbi_assert_punit_acquired(void) {}
+
 #endif /* CONFIG_IOSF_MBI */
 
 #endif /* IOSF_MBI_SYMS_H */
diff --git a/arch/x86/platform/intel/iosf_mbi.c b/arch/x86/platform/intel/iosf_mbi.c
index a952ac199741..6f37a2137a79 100644
--- a/arch/x86/platform/intel/iosf_mbi.c
+++ b/arch/x86/platform/intel/iosf_mbi.c
@@ -218,14 +218,23 @@ int iosf_mbi_register_pmic_bus_access_notifier(struct notifier_block *nb)
 }
 EXPORT_SYMBOL(iosf_mbi_register_pmic_bus_access_notifier);
 
+int iosf_mbi_unregister_pmic_bus_access_notifier_unlocked(
+        struct notifier_block *nb)
+{
+        iosf_mbi_assert_punit_acquired();
+
+        return blocking_notifier_chain_unregister(
+                                &iosf_mbi_pmic_bus_access_notifier, nb);
+}
+EXPORT_SYMBOL(iosf_mbi_unregister_pmic_bus_access_notifier_unlocked);
+
 int iosf_mbi_unregister_pmic_bus_access_notifier(struct notifier_block *nb)
 {
         int ret;
 
         /* Wait for the bus to go inactive before unregistering */
         mutex_lock(&iosf_mbi_punit_mutex);
-        ret = blocking_notifier_chain_unregister(
-                                &iosf_mbi_pmic_bus_access_notifier, nb);
+        ret = iosf_mbi_unregister_pmic_bus_access_notifier_unlocked(nb);
         mutex_unlock(&iosf_mbi_punit_mutex);
 
         return ret;
@@ -239,6 +248,12 @@ int iosf_mbi_call_pmic_bus_access_notifier_chain(unsigned long val, void *v)
 }
 EXPORT_SYMBOL(iosf_mbi_call_pmic_bus_access_notifier_chain);
 
+void iosf_mbi_assert_punit_acquired(void)
+{
+        WARN_ON(!mutex_is_locked(&iosf_mbi_punit_mutex));
+}
+EXPORT_SYMBOL(iosf_mbi_assert_punit_acquired);
+
 #ifdef CONFIG_IOSF_MBI_DEBUG
 static u32      dbg_mdr;
 static u32      dbg_mcr;
diff --git a/drivers/gpu/drm/i915/Kconfig.debug b/drivers/gpu/drm/i915/Kconfig.debug
index aed7d207ea84..9e53edbc713b 100644
--- a/drivers/gpu/drm/i915/Kconfig.debug
+++ b/drivers/gpu/drm/i915/Kconfig.debug
@@ -28,6 +28,7 @@ config DRM_I915_DEBUG
         select SW_SYNC # signaling validation framework (igt/syncobj*)
         select DRM_I915_SW_FENCE_DEBUG_OBJECTS
         select DRM_I915_SELFTEST
+        select DRM_I915_TRACE_GEM
         default n
         help
           Choose this option to turn on extra driver debugging that may affect
@@ -49,6 +50,19 @@ config DRM_I915_DEBUG_GEM
 
           If in doubt, say "N".
 
+config DRM_I915_TRACE_GEM
+        bool "Insert extra ftrace output from the GEM internals"
+        select TRACING
+        default n
+        help
+          Enable additional and verbose debugging output that will spam
+          ordinary tests, but may be vital for post-mortem debugging when
+          used with /proc/sys/kernel/ftrace_dump_on_oops
+
+          Recommended for driver developers only.
+
+          If in doubt, say "N".
+
 config DRM_I915_SW_FENCE_DEBUG_OBJECTS
         bool "Enable additional driver debugging for fence objects"
         depends on DRM_I915
@@ -90,6 +104,20 @@ config DRM_I915_SELFTEST
 
           If in doubt, say "N".
 
+config DRM_I915_SELFTEST_BROKEN
+        bool "Enable broken and dangerous selftests"
+        depends on DRM_I915_SELFTEST
+        depends on BROKEN
+        default n
+        help
+          This option enables the execution of selftests that are "dangerous"
+          and may trigger unintended HW side-effects as they break strict
+          rules given in the HW specification. For science.
+
+          Recommended for masochistic driver developers only.
+
+          If in doubt, say "N".
+
 config DRM_I915_LOW_LEVEL_TRACEPOINTS
         bool "Enable low level request tracing events"
         depends on DRM_I915
diff --git a/drivers/gpu/drm/i915/Makefile b/drivers/gpu/drm/i915/Makefile
index 2acf3b3c5f9d..49b9535e40d1 100644
--- a/drivers/gpu/drm/i915/Makefile
+++ b/drivers/gpu/drm/i915/Makefile
@@ -3,7 +3,26 @@
 # Makefile for the drm device driver.  This driver provides support for the
 # Direct Rendering Infrastructure (DRI) in XFree86 4.1.0 and higher.
 
-subdir-ccflags-$(CONFIG_DRM_I915_WERROR) := -Werror
+# Add a set of useful warning flags and enable -Werror for CI to prevent
+# trivial mistakes from creeping in. We have to do this piecemeal as we reject
+# any patch that isn't warning clean, so turning on -Wall -Wextra (or W=1) we
+# need to filter out dubious warnings.  Still it is our interest
+# to keep running locally with W=1 C=1 until we are completely clean.
+#
+# Note the danger in using -Wall -Wextra is that when CI updates gcc we
+# will most likely get a sudden build breakage... Hopefully we will fix
+# new warnings before CI updates!
+subdir-ccflags-y := -Wall -Wextra
+subdir-ccflags-y += $(call cc-disable-warning, unused-parameter)
+subdir-ccflags-y += $(call cc-disable-warning, type-limits)
+subdir-ccflags-y += $(call cc-disable-warning, missing-field-initializers)
+subdir-ccflags-y += $(call cc-disable-warning, implicit-fallthrough)
+subdir-ccflags-$(CONFIG_DRM_I915_WERROR) += -Werror
+
+# Fine grained warnings disable
+CFLAGS_i915_pci.o = $(call cc-disable-warning, override-init)
+CFLAGS_intel_fbdev.o = $(call cc-disable-warning, override-init)
+
 subdir-ccflags-y += \
         $(call as-instr,movntdqa (%eax)$(comma)%xmm0,-DCONFIG_AS_MOVNTDQA)
 
@@ -64,10 +83,10 @@ i915-y += intel_uc.o \
           intel_uc_fw.o \
           intel_guc.o \
           intel_guc_ct.o \
-          intel_guc_log.o \
           intel_guc_fw.o \
-          intel_huc.o \
-          i915_guc_submission.o
+          intel_guc_log.o \
+          intel_guc_submission.o \
+          intel_huc.o
 
 # autogenerated null render state
 i915-y += intel_renderstate_gen6.o \
@@ -144,7 +163,9 @@ i915-y += i915_perf.o \
           i915_oa_kblgt2.o \
           i915_oa_kblgt3.o \
           i915_oa_glk.o \
-          i915_oa_cflgt2.o
+          i915_oa_cflgt2.o \
+          i915_oa_cflgt3.o \
+          i915_oa_cnl.o
 
 ifeq ($(CONFIG_DRM_I915_GVT),y)
 i915-y += intel_gvt.o
diff --git a/drivers/gpu/drm/i915/gvt/Makefile b/drivers/gpu/drm/i915/gvt/Makefile
index 2641ba510a61..18e1c172e792 100644
--- a/drivers/gpu/drm/i915/gvt/Makefile
+++ b/drivers/gpu/drm/i915/gvt/Makefile
@@ -2,7 +2,7 @@
 GVT_DIR := gvt
 GVT_SOURCE := gvt.o aperture_gm.o handlers.o vgpu.o trace_points.o firmware.o \
         interrupt.o gtt.o cfg_space.o opregion.o mmio.o display.o edid.o \
-        execlist.o scheduler.o sched_policy.o render.o cmd_parser.o
+        execlist.o scheduler.o sched_policy.o render.o cmd_parser.o debugfs.o
 
 ccflags-y                               += -I$(src) -I$(src)/$(GVT_DIR)
 i915-y                                  += $(addprefix $(GVT_DIR)/, $(GVT_SOURCE))
diff --git a/drivers/gpu/drm/i915/gvt/cfg_space.c b/drivers/gpu/drm/i915/gvt/cfg_space.c
index ab19545d59a1..4ce2e6bd0680 100644
--- a/drivers/gpu/drm/i915/gvt/cfg_space.c
+++ b/drivers/gpu/drm/i915/gvt/cfg_space.c
@@ -208,6 +208,20 @@ static int emulate_pci_command_write(struct intel_vgpu *vgpu,
         return 0;
 }
 
+static int emulate_pci_rom_bar_write(struct intel_vgpu *vgpu,
+        unsigned int offset, void *p_data, unsigned int bytes)
+{
+        u32 *pval = (u32 *)(vgpu_cfg_space(vgpu) + offset);
+        u32 new = *(u32 *)(p_data);
+
+        if ((new & PCI_ROM_ADDRESS_MASK) == PCI_ROM_ADDRESS_MASK)
+                /* We don't have rom, return size of 0. */
+                *pval = 0;
+        else
+                vgpu_pci_cfg_mem_write(vgpu, offset, p_data, bytes);
+        return 0;
+}
+
 static int emulate_pci_bar_write(struct intel_vgpu *vgpu, unsigned int offset,
         void *p_data, unsigned int bytes)
 {
@@ -300,6 +314,11 @@ int intel_vgpu_emulate_cfg_write(struct intel_vgpu *vgpu, unsigned int offset,
         }
 
         switch (rounddown(offset, 4)) {
+        case PCI_ROM_ADDRESS:
+                if (WARN_ON(!IS_ALIGNED(offset, 4)))
+                        return -EINVAL;
+                return emulate_pci_rom_bar_write(vgpu, offset, p_data, bytes);
+
         case PCI_BASE_ADDRESS_0 ... PCI_BASE_ADDRESS_5:
                 if (WARN_ON(!IS_ALIGNED(offset, 4)))
                         return -EINVAL;
@@ -375,6 +394,8 @@ void intel_vgpu_init_cfg_space(struct intel_vgpu *vgpu,
                                 pci_resource_len(gvt->dev_priv->drm.pdev, 0);
         vgpu->cfg_space.bar[INTEL_GVT_PCI_BAR_APERTURE].size =
                                 pci_resource_len(gvt->dev_priv->drm.pdev, 2);
+
+        memset(vgpu_cfg_space(vgpu) + PCI_ROM_ADDRESS, 0, 4);
 }
 
 /**
diff --git a/drivers/gpu/drm/i915/gvt/cmd_parser.c b/drivers/gpu/drm/i915/gvt/cmd_parser.c
index 85d4c57870fb..18c45734c7a2 100644
--- a/drivers/gpu/drm/i915/gvt/cmd_parser.c
+++ b/drivers/gpu/drm/i915/gvt/cmd_parser.c
@@ -709,18 +709,13 @@ static void parser_exec_state_dump(struct parser_exec_state *s)
 
         print_opcode(cmd_val(s, 0), s->ring_id);
 
-        /* print the whole page to trace */
-        pr_err("    ip_va=%p: %08x %08x %08x %08x\n",
-                        s->ip_va, cmd_val(s, 0), cmd_val(s, 1),
-                        cmd_val(s, 2), cmd_val(s, 3));
-
         s->ip_va = (u32 *)((((u64)s->ip_va) >> 12) << 12);
 
         while (cnt < 1024) {
-                pr_err("ip_va=%p: ", s->ip_va);
+                gvt_dbg_cmd("ip_va=%p: ", s->ip_va);
                 for (i = 0; i < 8; i++)
-                        pr_err("%08x ", cmd_val(s, i));
-                pr_err("\n");
+                        gvt_dbg_cmd("%08x ", cmd_val(s, i));
+                gvt_dbg_cmd("\n");
 
                 s->ip_va += 8 * sizeof(u32);
                 cnt += 8;
@@ -825,7 +820,7 @@ static int force_nonpriv_reg_handler(struct parser_exec_state *s,
         if (!intel_gvt_in_force_nonpriv_whitelist(gvt, data)) {
                 gvt_err("Unexpected forcenonpriv 0x%x LRI write, value=0x%x\n",
                         offset, data);
-                return -EINVAL;
+                return -EPERM;
         }
         return 0;
 }
@@ -839,7 +834,7 @@ static int cmd_reg_handler(struct parser_exec_state *s,
         if (offset + 4 > gvt->device_info.mmio_size) {
                 gvt_vgpu_err("%s access to (%x) outside of MMIO range\n",
                                 cmd, offset);
-                return -EINVAL;
+                return -EFAULT;
         }
 
         if (!intel_gvt_mmio_is_cmd_access(gvt, offset)) {
@@ -854,8 +849,8 @@ static int cmd_reg_handler(struct parser_exec_state *s,
         }
 
         if (is_force_nonpriv_mmio(offset) &&
-            force_nonpriv_reg_handler(s, offset, index))
-                return -EINVAL;
+                force_nonpriv_reg_handler(s, offset, index))
+                return -EPERM;
 
         if (offset == i915_mmio_reg_offset(DERRMR) ||
                 offset == i915_mmio_reg_offset(FORCEWAKE_MT)) {
@@ -894,11 +889,14 @@ static int cmd_handler_lri(struct parser_exec_state *s)
                                         i915_mmio_reg_offset(DERRMR))
                                 ret |= 0;
                         else
-                                ret |= (cmd_reg_inhibit(s, i)) ? -EINVAL : 0;
+                                ret |= (cmd_reg_inhibit(s, i)) ?
+                                        -EBADRQC : 0;
                 }
                 if (ret)
                         break;
                 ret |= cmd_reg_handler(s, cmd_reg(s, i), i, "lri");
+                if (ret)
+                        break;
         }
         return ret;
 }
@@ -912,11 +910,15 @@ static int cmd_handler_lrr(struct parser_exec_state *s)
                 if (IS_BROADWELL(s->vgpu->gvt->dev_priv))
                         ret |= ((cmd_reg_inhibit(s, i) ||
                                         (cmd_reg_inhibit(s, i + 1)))) ?
-                                -EINVAL : 0;
+                                -EBADRQC : 0;
                 if (ret)
                         break;
                 ret |= cmd_reg_handler(s, cmd_reg(s, i), i, "lrr-src");
+                if (ret)
+                        break;
                 ret |= cmd_reg_handler(s, cmd_reg(s, i + 1), i, "lrr-dst");
+                if (ret)
+                        break;
         }
         return ret;
 }
@@ -934,15 +936,19 @@ static int cmd_handler_lrm(struct parser_exec_state *s)
 
         for (i = 1; i < cmd_len;) {
                 if (IS_BROADWELL(gvt->dev_priv))
-                        ret |= (cmd_reg_inhibit(s, i)) ? -EINVAL : 0;
+                        ret |= (cmd_reg_inhibit(s, i)) ? -EBADRQC : 0;
                 if (ret)
                         break;
                 ret |= cmd_reg_handler(s, cmd_reg(s, i), i, "lrm");
+                if (ret)
+                        break;
                 if (cmd_val(s, 0) & (1 << 22)) {
                         gma = cmd_gma(s, i + 1);
                         if (gmadr_bytes == 8)
                                 gma |= (cmd_gma_hi(s, i + 2)) << 32;
                         ret |= cmd_address_audit(s, gma, sizeof(u32), false);
+                        if (ret)
+                                break;
                 }
                 i += gmadr_dw_number(s) + 1;
         }
@@ -958,11 +964,15 @@ static int cmd_handler_srm(struct parser_exec_state *s)
 
         for (i = 1; i < cmd_len;) {
                 ret |= cmd_reg_handler(s, cmd_reg(s, i), i, "srm");
+                if (ret)
+                        break;
                 if (cmd_val(s, 0) & (1 << 22)) {
                         gma = cmd_gma(s, i + 1);
                         if (gmadr_bytes == 8)
                                 gma |= (cmd_gma_hi(s, i + 2)) << 32;
                         ret |= cmd_address_audit(s, gma, sizeof(u32), false);
+                        if (ret)
+                                break;
                 }
                 i += gmadr_dw_number(s) + 1;
         }
@@ -1116,7 +1126,7 @@ static int gen8_decode_mi_display_flip(struct parser_exec_state *s,
 
         v = (dword0 & GENMASK(21, 19)) >> 19;
         if (WARN_ON(v >= ARRAY_SIZE(gen8_plane_code)))
-                return -EINVAL;
+                return -EBADRQC;
 
         info->pipe = gen8_plane_code[v].pipe;
         info->plane = gen8_plane_code[v].plane;
@@ -1136,7 +1146,7 @@ static int gen8_decode_mi_display_flip(struct parser_exec_state *s,
                 info->surf_reg = SPRSURF(info->pipe);
         } else {
                 WARN_ON(1);
-                return -EINVAL;
+                return -EBADRQC;
         }
         return 0;
 }
@@ -1185,7 +1195,7 @@ static int skl_decode_mi_display_flip(struct parser_exec_state *s,
 
         default:
                 gvt_vgpu_err("unknown plane code %d\n", plane);
-                return -EINVAL;
+                return -EBADRQC;
         }
 
         info->stride_val = (dword1 & GENMASK(15, 6)) >> 6;
@@ -1348,10 +1358,13 @@ static unsigned long get_gma_bb_from_cmd(struct parser_exec_state *s, int index)
 {
         unsigned long addr;
         unsigned long gma_high, gma_low;
-        int gmadr_bytes = s->vgpu->gvt->device_info.gmadr_bytes_in_cmd;
+        struct intel_vgpu *vgpu = s->vgpu;
+        int gmadr_bytes = vgpu->gvt->device_info.gmadr_bytes_in_cmd;
 
-        if (WARN_ON(gmadr_bytes != 4 && gmadr_bytes != 8))
+        if (WARN_ON(gmadr_bytes != 4 && gmadr_bytes != 8)) {
+                gvt_vgpu_err("invalid gma bytes %d\n", gmadr_bytes);
                 return INTEL_GVT_INVALID_ADDR;
+        }
 
         gma_low = cmd_val(s, index) & BATCH_BUFFER_ADDR_MASK;
         if (gmadr_bytes == 4) {
@@ -1374,16 +1387,16 @@ static inline int cmd_address_audit(struct parser_exec_state *s,
         if (op_size > max_surface_size) {
                 gvt_vgpu_err("command address audit fail name %s\n",
                         s->info->name);
-                return -EINVAL;
+                return -EFAULT;
         }
 
         if (index_mode) {
-                if (guest_gma >= GTT_PAGE_SIZE / sizeof(u64)) {
-                        ret = -EINVAL;
+                if (guest_gma >= I915_GTT_PAGE_SIZE / sizeof(u64)) {
+                        ret = -EFAULT;
                         goto err;
                 }
         } else if (!intel_gvt_ggtt_validate_range(vgpu, guest_gma, op_size)) {
-                ret = -EINVAL;
+                ret = -EFAULT;
                 goto err;
         }
 
@@ -1439,7 +1452,7 @@ static inline int unexpected_cmd(struct parser_exec_state *s)
 
         gvt_vgpu_err("Unexpected %s in command buffer!\n", s->info->name);
 
-        return -EINVAL;
+        return -EBADRQC;
 }
 
 static int cmd_handler_mi_semaphore_wait(struct parser_exec_state *s)
@@ -1545,10 +1558,10 @@ static int copy_gma_to_hva(struct intel_vgpu *vgpu, struct intel_vgpu_mm *mm,
                         return -EFAULT;
                 }
 
-                offset = gma & (GTT_PAGE_SIZE - 1);
+                offset = gma & (I915_GTT_PAGE_SIZE - 1);
 
-                copy_len = (end_gma - gma) >= (GTT_PAGE_SIZE - offset) ?
-                        GTT_PAGE_SIZE - offset : end_gma - gma;
+                copy_len = (end_gma - gma) >= (I915_GTT_PAGE_SIZE - offset) ?
+                        I915_GTT_PAGE_SIZE - offset : end_gma - gma;
 
                 intel_gvt_hypervisor_read_gpa(vgpu, gpa, va + len, copy_len);
 
@@ -1576,110 +1589,113 @@ static int batch_buffer_needs_scan(struct parser_exec_state *s)
         return 1;
 }
 
-static int find_bb_size(struct parser_exec_state *s)
+static int find_bb_size(struct parser_exec_state *s, unsigned long *bb_size)
 {
         unsigned long gma = 0;
         struct cmd_info *info;
-        int bb_size = 0;
         uint32_t cmd_len = 0;
-        bool met_bb_end = false;
+        bool bb_end = false;
         struct intel_vgpu *vgpu = s->vgpu;
         u32 cmd;
 
+        *bb_size = 0;
+
         /* get the start gm address of the batch buffer */
         gma = get_gma_bb_from_cmd(s, 1);
-        cmd = cmd_val(s, 0);
+        if (gma == INTEL_GVT_INVALID_ADDR)
+                return -EFAULT;
 
+        cmd = cmd_val(s, 0);
         info = get_cmd_info(s->vgpu->gvt, cmd, s->ring_id);
         if (info == NULL) {
                 gvt_vgpu_err("unknown cmd 0x%x, opcode=0x%x\n",
                                 cmd, get_opcode(cmd, s->ring_id));
-                return -EINVAL;
+                return -EBADRQC;
         }
         do {
-                copy_gma_to_hva(s->vgpu, s->vgpu->gtt.ggtt_mm,
-                                gma, gma + 4, &cmd);
+                if (copy_gma_to_hva(s->vgpu, s->vgpu->gtt.ggtt_mm,
+                                gma, gma + 4, &cmd) < 0)
+                        return -EFAULT;
                 info = get_cmd_info(s->vgpu->gvt, cmd, s->ring_id);
                 if (info == NULL) {
                         gvt_vgpu_err("unknown cmd 0x%x, opcode=0x%x\n",
                                 cmd, get_opcode(cmd, s->ring_id));
-                        return -EINVAL;
+                        return -EBADRQC;
                 }
 
                 if (info->opcode == OP_MI_BATCH_BUFFER_END) {
-                        met_bb_end = true;
+                        bb_end = true;
                 } else if (info->opcode == OP_MI_BATCH_BUFFER_START) {
-                        if (BATCH_BUFFER_2ND_LEVEL_BIT(cmd) == 0) {
+                        if (BATCH_BUFFER_2ND_LEVEL_BIT(cmd) == 0)
                                 /* chained batch buffer */
-                                met_bb_end = true;
-                        }
+                                bb_end = true;
                 }
                 cmd_len = get_cmd_length(info, cmd) << 2;
-                bb_size += cmd_len;
+                *bb_size += cmd_len;
                 gma += cmd_len;
+        } while (!bb_end);
 
-        } while (!met_bb_end);
-
-        return bb_size;
+        return 0;
 }
 
 static int perform_bb_shadow(struct parser_exec_state *s)
 {
-        struct intel_shadow_bb_entry *entry_obj;
         struct intel_vgpu *vgpu = s->vgpu;
+        struct intel_vgpu_shadow_bb *bb;
         unsigned long gma = 0;
-        int bb_size;
-        void *dst = NULL;
+        unsigned long bb_size;
         int ret = 0;
 
         /* get the start gm address of the batch buffer */
         gma = get_gma_bb_from_cmd(s, 1);
+        if (gma == INTEL_GVT_INVALID_ADDR)
+                return -EFAULT;
 
-        /* get the size of the batch buffer */
-        bb_size = find_bb_size(s);
-        if (bb_size < 0)
-                return -EINVAL;
+        ret = find_bb_size(s, &bb_size);
+        if (ret)
+                return ret;
 
-        /* allocate shadow batch buffer */
-        entry_obj = kmalloc(sizeof(*entry_obj), GFP_KERNEL);
-        if (entry_obj == NULL)
+        bb = kzalloc(sizeof(*bb), GFP_KERNEL);
+        if (!bb)
                 return -ENOMEM;
 
-        entry_obj->obj =
-                i915_gem_object_create(s->vgpu->gvt->dev_priv,
-                                       roundup(bb_size, PAGE_SIZE));
-        if (IS_ERR(entry_obj->obj)) {
-                ret = PTR_ERR(entry_obj->obj);
-                goto free_entry;
+        bb->obj = i915_gem_object_create(s->vgpu->gvt->dev_priv,
+                                         roundup(bb_size, PAGE_SIZE));
+        if (IS_ERR(bb->obj)) {
+                ret = PTR_ERR(bb->obj);
+                goto err_free_bb;
         }
-        entry_obj->len = bb_size;
-        INIT_LIST_HEAD(&entry_obj->list);
 
-        dst = i915_gem_object_pin_map(entry_obj->obj, I915_MAP_WB);
-        if (IS_ERR(dst)) {
-                ret = PTR_ERR(dst);
-                goto put_obj;
-        }
+        ret = i915_gem_obj_prepare_shmem_write(bb->obj, &bb->clflush);
+        if (ret)
+                goto err_free_obj;
 
-        ret = i915_gem_object_set_to_cpu_domain(entry_obj->obj, false);
-        if (ret) {
-                gvt_vgpu_err("failed to set shadow batch to CPU\n");
-                goto unmap_src;
+        bb->va = i915_gem_object_pin_map(bb->obj, I915_MAP_WB);
+        if (IS_ERR(bb->va)) {
+                ret = PTR_ERR(bb->va);
+                goto err_finish_shmem_access;
         }
 
-        entry_obj->va = dst;
-        entry_obj->bb_start_cmd_va = s->ip_va;
+        if (bb->clflush & CLFLUSH_BEFORE) {
+                drm_clflush_virt_range(bb->va, bb->obj->base.size);
+                bb->clflush &= ~CLFLUSH_BEFORE;
+        }
 
-        /* copy batch buffer to shadow batch buffer*/
         ret = copy_gma_to_hva(s->vgpu, s->vgpu->gtt.ggtt_mm,
                               gma, gma + bb_size,
-                              dst);
+                              bb->va);
         if (ret < 0) {
                 gvt_vgpu_err("fail to copy guest ring buffer\n");
-                goto unmap_src;
+                ret = -EFAULT;
+                goto err_unmap;
         }
 
-        list_add(&entry_obj->list, &s->workload->shadow_bb);
+        INIT_LIST_HEAD(&bb->list);
+        list_add(&bb->list, &s->workload->shadow_bb);
+
+        bb->accessing = true;
+        bb->bb_start_cmd_va = s->ip_va;
+
         /*
          * ip_va saves the virtual address of the shadow batch buffer, while
          * ip_gma saves the graphics address of the original batch buffer.
@@ -1688,17 +1704,17 @@ static int perform_bb_shadow(struct parser_exec_state *s)
          * buffer's gma in pair. After all, we don't want to pin the shadow
          * buffer here (too early).
          */
-        s->ip_va = dst;
+        s->ip_va = bb->va;
         s->ip_gma = gma;
-
         return 0;
-
-unmap_src:
-        i915_gem_object_unpin_map(entry_obj->obj);
-put_obj:
-        i915_gem_object_put(entry_obj->obj);
-free_entry:
-        kfree(entry_obj);
+err_unmap:
+        i915_gem_object_unpin_map(bb->obj);
+err_finish_shmem_access:
+        i915_gem_obj_finish_shmem_access(bb->obj);
+err_free_obj:
+        i915_gem_object_put(bb->obj);
+err_free_bb:
+        kfree(bb);
         return ret;
 }
 
@@ -1710,13 +1726,13 @@ static int cmd_handler_mi_batch_buffer_start(struct parser_exec_state *s)
 
         if (s->buf_type == BATCH_BUFFER_2ND_LEVEL) {
                 gvt_vgpu_err("Found MI_BATCH_BUFFER_START in 2nd level BB\n");
-                return -EINVAL;
+                return -EFAULT;
         }
 
         second_level = BATCH_BUFFER_2ND_LEVEL_BIT(cmd_val(s, 0)) == 1;
         if (second_level && (s->buf_type != BATCH_BUFFER_INSTRUCTION)) {
                 gvt_vgpu_err("Jumping to 2nd level BB from RB is not allowed\n");
-                return -EINVAL;
+                return -EFAULT;
         }
 
         s->saved_buf_addr_type = s->buf_addr_type;
@@ -1740,7 +1756,6 @@ static int cmd_handler_mi_batch_buffer_start(struct parser_exec_state *s)
                 if (ret < 0)
                         return ret;
         }
-
         return ret;
 }
 
@@ -2430,7 +2445,7 @@ static int cmd_parser_exec(struct parser_exec_state *s)
         if (info == NULL) {
                 gvt_vgpu_err("unknown cmd 0x%x, opcode=0x%x\n",
                                 cmd, get_opcode(cmd, s->ring_id));
-                return -EINVAL;
+                return -EBADRQC;
         }
 
         s->info = info;
@@ -2465,6 +2480,10 @@ static inline bool gma_out_of_range(unsigned long gma,
                 return (gma > gma_tail) && (gma < gma_head);
 }
 
+/* Keep the consistent return type, e.g EBADRQC for unknown
+ * cmd, EFAULT for invalid address, EPERM for nonpriv. later
+ * works as the input of VM healthy status.
+ */
 static int command_scan(struct parser_exec_state *s,
                 unsigned long rb_head, unsigned long rb_tail,
                 unsigned long rb_start, unsigned long rb_len)
@@ -2487,7 +2506,7 @@ static int command_scan(struct parser_exec_state *s,
                                         s->ip_gma, rb_start,
                                         gma_bottom);
                                 parser_exec_state_dump(s);
-                                return -EINVAL;
+                                return -EFAULT;
                         }
                         if (gma_out_of_range(s->ip_gma, gma_head, gma_tail)) {
                                 gvt_vgpu_err("ip_gma %lx out of range."
@@ -2516,7 +2535,7 @@ static int scan_workload(struct intel_vgpu_workload *workload)
         int ret = 0;
 
         /* ring base is page aligned */
-        if (WARN_ON(!IS_ALIGNED(workload->rb_start, GTT_PAGE_SIZE)))
+        if (WARN_ON(!IS_ALIGNED(workload->rb_start, I915_GTT_PAGE_SIZE)))
                 return -EINVAL;
 
         gma_head = workload->rb_start + workload->rb_head;
@@ -2565,7 +2584,8 @@ static int scan_wa_ctx(struct intel_shadow_wa_ctx *wa_ctx)
                                 wa_ctx);
 
         /* ring base is page aligned */
-        if (WARN_ON(!IS_ALIGNED(wa_ctx->indirect_ctx.guest_gma, GTT_PAGE_SIZE)))
+        if (WARN_ON(!IS_ALIGNED(wa_ctx->indirect_ctx.guest_gma,
+                                        I915_GTT_PAGE_SIZE)))
                 return -EINVAL;
 
         ring_tail = wa_ctx->indirect_ctx.size + 3 * sizeof(uint32_t);
@@ -2604,6 +2624,7 @@ out:
 static int shadow_workload_ring_buffer(struct intel_vgpu_workload *workload)
 {
         struct intel_vgpu *vgpu = workload->vgpu;
+        struct intel_vgpu_submission *s = &vgpu->submission;
         unsigned long gma_head, gma_tail, gma_top, guest_rb_size;
         void *shadow_ring_buffer_va;
         int ring_id = workload->ring_id;
@@ -2619,19 +2640,21 @@ static int shadow_workload_ring_buffer(struct intel_vgpu_workload *workload)
         gma_tail = workload->rb_start + workload->rb_tail;
         gma_top = workload->rb_start + guest_rb_size;
 
-        if (workload->rb_len > vgpu->reserve_ring_buffer_size[ring_id]) {
-                void *va = vgpu->reserve_ring_buffer_va[ring_id];
+        if (workload->rb_len > s->ring_scan_buffer_size[ring_id]) {
+                void *p;
+
                 /* realloc the new ring buffer if needed */
-                vgpu->reserve_ring_buffer_va[ring_id] =
-                        krealloc(va, workload->rb_len, GFP_KERNEL);
-                if (!vgpu->reserve_ring_buffer_va[ring_id]) {
-                        gvt_vgpu_err("fail to alloc reserve ring buffer\n");
+                p = krealloc(s->ring_scan_buffer[ring_id], workload->rb_len,
+                                GFP_KERNEL);
+                if (!p) {
+                        gvt_vgpu_err("fail to re-alloc ring scan buffer\n");
                         return -ENOMEM;
                 }
-                vgpu->reserve_ring_buffer_size[ring_id] = workload->rb_len;
+                s->ring_scan_buffer[ring_id] = p;
+                s->ring_scan_buffer_size[ring_id] = workload->rb_len;
         }
 
-        shadow_ring_buffer_va = vgpu->reserve_ring_buffer_va[ring_id];
+        shadow_ring_buffer_va = s->ring_scan_buffer[ring_id];
 
         /* get shadow ring buffer va */
         workload->shadow_ring_buffer_va = shadow_ring_buffer_va;
diff --git a/drivers/gpu/drm/i915/gvt/debug.h b/drivers/gpu/drm/i915/gvt/debug.h
index b0cff4dc2684..c6027125c1ec 100644
--- a/drivers/gpu/drm/i915/gvt/debug.h
+++ b/drivers/gpu/drm/i915/gvt/debug.h
@@ -25,41 +25,41 @@
 #define __GVT_DEBUG_H__
 
 #define gvt_err(fmt, args...) \
-        DRM_ERROR("gvt: "fmt, ##args)
+        pr_err("gvt: "fmt, ##args)
 
 #define gvt_vgpu_err(fmt, args...)                                      \
 do {                                                                    \
         if (IS_ERR_OR_NULL(vgpu))                                       \
-                DRM_DEBUG_DRIVER("gvt: "fmt, ##args);                   \
+                pr_err("gvt: "fmt, ##args);                     \
         else                                                            \
-                DRM_DEBUG_DRIVER("gvt: vgpu %d: "fmt, vgpu->id, ##args);\
+                pr_err("gvt: vgpu %d: "fmt, vgpu->id, ##args);\
 } while (0)
 
 #define gvt_dbg_core(fmt, args...) \
-        DRM_DEBUG_DRIVER("gvt: core: "fmt, ##args)
+        pr_debug("gvt: core: "fmt, ##args)
 
 #define gvt_dbg_irq(fmt, args...) \
-        DRM_DEBUG_DRIVER("gvt: irq: "fmt, ##args)
+        pr_debug("gvt: irq: "fmt, ##args)
 
 #define gvt_dbg_mm(fmt, args...) \
-        DRM_DEBUG_DRIVER("gvt: mm: "fmt, ##args)
+        pr_debug("gvt: mm: "fmt, ##args)
 
 #define gvt_dbg_mmio(fmt, args...) \
-        DRM_DEBUG_DRIVER("gvt: mmio: "fmt, ##args)
+        pr_debug("gvt: mmio: "fmt, ##args)
 
 #define gvt_dbg_dpy(fmt, args...) \
-        DRM_DEBUG_DRIVER("gvt: dpy: "fmt, ##args)
+        pr_debug("gvt: dpy: "fmt, ##args)
 
 #define gvt_dbg_el(fmt, args...) \
-        DRM_DEBUG_DRIVER("gvt: el: "fmt, ##args)
+        pr_debug("gvt: el: "fmt, ##args)
 
 #define gvt_dbg_sched(fmt, args...) \
-        DRM_DEBUG_DRIVER("gvt: sched: "fmt, ##args)
+        pr_debug("gvt: sched: "fmt, ##args)
 
 #define gvt_dbg_render(fmt, args...) \
-        DRM_DEBUG_DRIVER("gvt: render: "fmt, ##args)
+        pr_debug("gvt: render: "fmt, ##args)
 
 #define gvt_dbg_cmd(fmt, args...) \
-        DRM_DEBUG_DRIVER("gvt: cmd: "fmt, ##args)
+        pr_debug("gvt: cmd: "fmt, ##args)
 
 #endif
diff --git a/drivers/gpu/drm/i915/gvt/debugfs.c b/drivers/gpu/drm/i915/gvt/debugfs.c
new file mode 100644
index 000000000000..32a66dfdf112
--- /dev/null
+++ b/drivers/gpu/drm/i915/gvt/debugfs.c
@@ -0,0 +1,212 @@
+/*
+ * Copyright(c) 2011-2017 Intel Corporation. All rights reserved.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+#include <linux/debugfs.h>
+#include <linux/list_sort.h>
+#include "i915_drv.h"
+#include "gvt.h"
+
+struct mmio_diff_param {
+        struct intel_vgpu *vgpu;
+        int total;
+        int diff;
+        struct list_head diff_mmio_list;
+};
+
+struct diff_mmio {
+        struct list_head node;
+        u32 offset;
+        u32 preg;
+        u32 vreg;
+};
+
+/* Compare two diff_mmio items. */
+static int mmio_offset_compare(void *priv,
+        struct list_head *a, struct list_head *b)
+{
+        struct diff_mmio *ma;
+        struct diff_mmio *mb;
+
+        ma = container_of(a, struct diff_mmio, node);
+        mb = container_of(b, struct diff_mmio, node);
+        if (ma->offset < mb->offset)
+                return -1;
+        else if (ma->offset > mb->offset)
+                return 1;
+        return 0;
+}
+
+static inline int mmio_diff_handler(struct intel_gvt *gvt,
+                                    u32 offset, void *data)
+{
+        struct drm_i915_private *dev_priv = gvt->dev_priv;
+        struct mmio_diff_param *param = data;
+        struct diff_mmio *node;
+        u32 preg, vreg;
+
+        preg = I915_READ_NOTRACE(_MMIO(offset));
+        vreg = vgpu_vreg(param->vgpu, offset);
+
+        if (preg != vreg) {
+                node = kmalloc(sizeof(*node), GFP_KERNEL);
+                if (!node)
+                        return -ENOMEM;
+
+                node->offset = offset;
+                node->preg = preg;
+                node->vreg = vreg;
+                list_add(&node->node, &param->diff_mmio_list);
+                param->diff++;
+        }
+        param->total++;
+        return 0;
+}
+
+/* Show the all the different values of tracked mmio. */
+static int vgpu_mmio_diff_show(struct seq_file *s, void *unused)
+{
+        struct intel_vgpu *vgpu = s->private;
+        struct intel_gvt *gvt = vgpu->gvt;
+        struct mmio_diff_param param = {
+                .vgpu = vgpu,
+                .total = 0,
+                .diff = 0,
+        };
+        struct diff_mmio *node, *next;
+
+        INIT_LIST_HEAD(&param.diff_mmio_list);
+
+        mutex_lock(&gvt->lock);
+        spin_lock_bh(&gvt->scheduler.mmio_context_lock);
+
+        mmio_hw_access_pre(gvt->dev_priv);
+        /* Recognize all the diff mmios to list. */
+        intel_gvt_for_each_tracked_mmio(gvt, mmio_diff_handler, &param);
+        mmio_hw_access_post(gvt->dev_priv);
+
+        spin_unlock_bh(&gvt->scheduler.mmio_context_lock);
+        mutex_unlock(&gvt->lock);
+
+        /* In an ascending order by mmio offset. */
+        list_sort(NULL, &param.diff_mmio_list, mmio_offset_compare);
+
+        seq_printf(s, "%-8s %-8s %-8s %-8s\n", "Offset", "HW", "vGPU", "Diff");
+        list_for_each_entry_safe(node, next, &param.diff_mmio_list, node) {
+                u32 diff = node->preg ^ node->vreg;
+
+                seq_printf(s, "%08x %08x %08x %*pbl\n",
+                           node->offset, node->preg, node->vreg,
+                           32, &diff);
+                list_del(&node->node);
+                kfree(node);
+        }
+        seq_printf(s, "Total: %d, Diff: %d\n", param.total, param.diff);
+        return 0;
+}
+
+static int vgpu_mmio_diff_open(struct inode *inode, struct file *file)
+{
+        return single_open(file, vgpu_mmio_diff_show, inode->i_private);
+}
+
+static const struct file_operations vgpu_mmio_diff_fops = {
+        .open           = vgpu_mmio_diff_open,
+        .read           = seq_read,
+        .llseek         = seq_lseek,
+        .release        = single_release,
+};
+
+/**
+ * intel_gvt_debugfs_add_vgpu - register debugfs entries for a vGPU
+ * @vgpu: a vGPU
+ *
+ * Returns:
+ * Zero on success, negative error code if failed.
+ */
+int intel_gvt_debugfs_add_vgpu(struct intel_vgpu *vgpu)
+{
+        struct dentry *ent;
+        char name[10] = "";
+
+        sprintf(name, "vgpu%d", vgpu->id);
+        vgpu->debugfs = debugfs_create_dir(name, vgpu->gvt->debugfs_root);
+        if (!vgpu->debugfs)
+                return -ENOMEM;
+
+        ent = debugfs_create_bool("active", 0444, vgpu->debugfs,
+                                  &vgpu->active);
+        if (!ent)
+                return -ENOMEM;
+
+        ent = debugfs_create_file("mmio_diff", 0444, vgpu->debugfs,
+                                  vgpu, &vgpu_mmio_diff_fops);
+        if (!ent)
+                return -ENOMEM;
+
+        return 0;
+}
+
+/**
+ * intel_gvt_debugfs_remove_vgpu - remove debugfs entries of a vGPU
+ * @vgpu: a vGPU
+ */
+void intel_gvt_debugfs_remove_vgpu(struct intel_vgpu *vgpu)
+{
+        debugfs_remove_recursive(vgpu->debugfs);
+        vgpu->debugfs = NULL;
+}
+
+/**
+ * intel_gvt_debugfs_init - register gvt debugfs root entry
+ * @gvt: GVT device
+ *
+ * Returns:
+ * zero on success, negative if failed.
+ */
+int intel_gvt_debugfs_init(struct intel_gvt *gvt)
+{
+        struct drm_minor *minor = gvt->dev_priv->drm.primary;
+        struct dentry *ent;
+
+        gvt->debugfs_root = debugfs_create_dir("gvt", minor->debugfs_root);
+        if (!gvt->debugfs_root) {
+                gvt_err("Cannot create debugfs dir\n");
+                return -ENOMEM;
+        }
+
+        ent = debugfs_create_ulong("num_tracked_mmio", 0444, gvt->debugfs_root,
+                                   &gvt->mmio.num_tracked_mmio);
+        if (!ent)
+                return -ENOMEM;
+
+        return 0;
+}
+
+/**
+ * intel_gvt_debugfs_clean - remove debugfs entries
+ * @gvt: GVT device
+ */
+void intel_gvt_debugfs_clean(struct intel_gvt *gvt)
+{
+        debugfs_remove_recursive(gvt->debugfs_root);
+        gvt->debugfs_root = NULL;
+}
diff --git a/drivers/gpu/drm/i915/gvt/execlist.c b/drivers/gpu/drm/i915/gvt/execlist.c
index 940cdaaa3f24..c9fa0fb488d3 100644
--- a/drivers/gpu/drm/i915/gvt/execlist.c
+++ b/drivers/gpu/drm/i915/gvt/execlist.c
@@ -46,8 +46,6 @@
 #define same_context(a, b) (((a)->context_id == (b)->context_id) && \
                 ((a)->lrca == (b)->lrca))
 
-static void clean_workloads(struct intel_vgpu *vgpu, unsigned long engine_mask);
-
 static int context_switch_events[] = {
         [RCS] = RCS_AS_CONTEXT_SWITCH,
         [BCS] = BCS_AS_CONTEXT_SWITCH,
@@ -135,6 +133,8 @@ static void emulate_csb_update(struct intel_vgpu_execlist *execlist,
         struct execlist_context_status_pointer_format ctx_status_ptr;
         u32 write_pointer;
         u32 ctx_status_ptr_reg, ctx_status_buf_reg, offset;
+        unsigned long hwsp_gpa;
+        struct drm_i915_private *dev_priv = vgpu->gvt->dev_priv;
 
         ctx_status_ptr_reg = execlist_ring_mmio(vgpu->gvt, ring_id,
                         _EL_OFFSET_STATUS_PTR);
@@ -160,6 +160,20 @@ static void emulate_csb_update(struct intel_vgpu_execlist *execlist,
         ctx_status_ptr.write_ptr = write_pointer;
         vgpu_vreg(vgpu, ctx_status_ptr_reg) = ctx_status_ptr.dw;
 
+        /* Update the CSB and CSB write pointer in HWSP */
+        hwsp_gpa = intel_vgpu_gma_to_gpa(vgpu->gtt.ggtt_mm,
+                                         vgpu->hws_pga[ring_id]);
+        if (hwsp_gpa != INTEL_GVT_INVALID_ADDR) {
+                intel_gvt_hypervisor_write_gpa(vgpu,
+                        hwsp_gpa + I915_HWS_CSB_BUF0_INDEX * 4 +
+                        write_pointer * 8,
+                        status, 8);
+                intel_gvt_hypervisor_write_gpa(vgpu,
+                        hwsp_gpa +
+                        intel_hws_csb_write_index(dev_priv) * 4,
+                        &write_pointer, 4);
+        }
+
         gvt_dbg_el("vgpu%d: w pointer %u reg %x csb l %x csb h %x\n",
                 vgpu->id, write_pointer, offset, status->ldw, status->udw);
 
@@ -358,218 +372,47 @@ static int emulate_execlist_schedule_in(struct intel_vgpu_execlist *execlist,
         return 0;
 }
 
-static void free_workload(struct intel_vgpu_workload *workload)
-{
-        intel_vgpu_unpin_mm(workload->shadow_mm);
-        intel_gvt_mm_unreference(workload->shadow_mm);
-        kmem_cache_free(workload->vgpu->workloads, workload);
-}
-
 #define get_desc_from_elsp_dwords(ed, i) \
         ((struct execlist_ctx_descriptor_format *)&((ed)->data[i * 2]))
 
-static int prepare_shadow_batch_buffer(struct intel_vgpu_workload *workload)
-{
-        const int gmadr_bytes = workload->vgpu->gvt->device_info.gmadr_bytes_in_cmd;
-        struct intel_shadow_bb_entry *entry_obj;
-
-        /* pin the gem object to ggtt */
-        list_for_each_entry(entry_obj, &workload->shadow_bb, list) {
-                struct i915_vma *vma;
-
-                vma = i915_gem_object_ggtt_pin(entry_obj->obj, NULL, 0, 4, 0);
-                if (IS_ERR(vma)) {
-                        return PTR_ERR(vma);
-                }
-
-                /* FIXME: we are not tracking our pinned VMA leaving it
-                 * up to the core to fix up the stray pin_count upon
-                 * free.
-                 */
-
-                /* update the relocate gma with shadow batch buffer*/
-                entry_obj->bb_start_cmd_va[1] = i915_ggtt_offset(vma);
-                if (gmadr_bytes == 8)
-                        entry_obj->bb_start_cmd_va[2] = 0;
-        }
-        return 0;
-}
-
-static int update_wa_ctx_2_shadow_ctx(struct intel_shadow_wa_ctx *wa_ctx)
-{
-        struct intel_vgpu_workload *workload = container_of(wa_ctx,
-                                        struct intel_vgpu_workload,
-                                        wa_ctx);
-        int ring_id = workload->ring_id;
-        struct i915_gem_context *shadow_ctx = workload->vgpu->shadow_ctx;
-        struct drm_i915_gem_object *ctx_obj =
-                shadow_ctx->engine[ring_id].state->obj;
-        struct execlist_ring_context *shadow_ring_context;
-        struct page *page;
-
-        page = i915_gem_object_get_page(ctx_obj, LRC_STATE_PN);
-        shadow_ring_context = kmap_atomic(page);
-
-        shadow_ring_context->bb_per_ctx_ptr.val =
-                (shadow_ring_context->bb_per_ctx_ptr.val &
-                (~PER_CTX_ADDR_MASK)) | wa_ctx->per_ctx.shadow_gma;
-        shadow_ring_context->rcs_indirect_ctx.val =
-                (shadow_ring_context->rcs_indirect_ctx.val &
-                (~INDIRECT_CTX_ADDR_MASK)) | wa_ctx->indirect_ctx.shadow_gma;
-
-        kunmap_atomic(shadow_ring_context);
-        return 0;
-}
-
-static int prepare_shadow_wa_ctx(struct intel_shadow_wa_ctx *wa_ctx)
-{
-        struct i915_vma *vma;
-        unsigned char *per_ctx_va =
-                (unsigned char *)wa_ctx->indirect_ctx.shadow_va +
-                wa_ctx->indirect_ctx.size;
-
-        if (wa_ctx->indirect_ctx.size == 0)
-                return 0;
-
-        vma = i915_gem_object_ggtt_pin(wa_ctx->indirect_ctx.obj, NULL,
-                                       0, CACHELINE_BYTES, 0);
-        if (IS_ERR(vma)) {
-                return PTR_ERR(vma);
-        }
-
-        /* FIXME: we are not tracking our pinned VMA leaving it
-         * up to the core to fix up the stray pin_count upon
-         * free.
-         */
-
-        wa_ctx->indirect_ctx.shadow_gma = i915_ggtt_offset(vma);
-
-        wa_ctx->per_ctx.shadow_gma = *((unsigned int *)per_ctx_va + 1);
-        memset(per_ctx_va, 0, CACHELINE_BYTES);
-
-        update_wa_ctx_2_shadow_ctx(wa_ctx);
-        return 0;
-}
-
-static void release_shadow_batch_buffer(struct intel_vgpu_workload *workload)
-{
-        /* release all the shadow batch buffer */
-        if (!list_empty(&workload->shadow_bb)) {
-                struct intel_shadow_bb_entry *entry_obj =
-                        list_first_entry(&workload->shadow_bb,
-                                         struct intel_shadow_bb_entry,
-                                         list);
-                struct intel_shadow_bb_entry *temp;
-
-                list_for_each_entry_safe(entry_obj, temp, &workload->shadow_bb,
-                                         list) {
-                        i915_gem_object_unpin_map(entry_obj->obj);
-                        i915_gem_object_put(entry_obj->obj);
-                        list_del(&entry_obj->list);
-                        kfree(entry_obj);
-                }
-        }
-}
-
 static int prepare_execlist_workload(struct intel_vgpu_workload *workload)
 {
         struct intel_vgpu *vgpu = workload->vgpu;
+        struct intel_vgpu_submission *s = &vgpu->submission;
         struct execlist_ctx_descriptor_format ctx[2];
         int ring_id = workload->ring_id;
         int ret;
 
-        ret = intel_vgpu_pin_mm(workload->shadow_mm);
-        if (ret) {
-                gvt_vgpu_err("fail to vgpu pin mm\n");
-                goto out;
-        }
-
-        ret = intel_vgpu_sync_oos_pages(workload->vgpu);
-        if (ret) {
-                gvt_vgpu_err("fail to vgpu sync oos pages\n");
-                goto err_unpin_mm;
-        }
-
-        ret = intel_vgpu_flush_post_shadow(workload->vgpu);
-        if (ret) {
-                gvt_vgpu_err("fail to flush post shadow\n");
-                goto err_unpin_mm;
-        }
-
-        ret = intel_gvt_generate_request(workload);
-        if (ret) {
-                gvt_vgpu_err("fail to generate request\n");
-                goto err_unpin_mm;
-        }
-
-        ret = prepare_shadow_batch_buffer(workload);
-        if (ret) {
-                gvt_vgpu_err("fail to prepare_shadow_batch_buffer\n");
-                goto err_unpin_mm;
-        }
-
-        ret = prepare_shadow_wa_ctx(&workload->wa_ctx);
-        if (ret) {
-                gvt_vgpu_err("fail to prepare_shadow_wa_ctx\n");
-                goto err_shadow_batch;
-        }
-
         if (!workload->emulate_schedule_in)
                 return 0;
 
-        ctx[0] = *get_desc_from_elsp_dwords(&workload->elsp_dwords, 1);
-        ctx[1] = *get_desc_from_elsp_dwords(&workload->elsp_dwords, 0);
+        ctx[0] = *get_desc_from_elsp_dwords(&workload->elsp_dwords, 0);
+        ctx[1] = *get_desc_from_elsp_dwords(&workload->elsp_dwords, 1);
 
-        ret = emulate_execlist_schedule_in(&vgpu->execlist[ring_id], ctx);
-        if (!ret)
-                goto out;
-        else
+        ret = emulate_execlist_schedule_in(&s->execlist[ring_id], ctx);
+        if (ret) {
                 gvt_vgpu_err("fail to emulate execlist schedule in\n");
-
-        release_shadow_wa_ctx(&workload->wa_ctx);
-err_shadow_batch:
-        release_shadow_batch_buffer(workload);
-err_unpin_mm:
-        intel_vgpu_unpin_mm(workload->shadow_mm);
-out:
-        return ret;
+                return ret;
+        }
+        return 0;
 }
 
 static int complete_execlist_workload(struct intel_vgpu_workload *workload)
 {
         struct intel_vgpu *vgpu = workload->vgpu;
         int ring_id = workload->ring_id;
-        struct intel_vgpu_execlist *execlist = &vgpu->execlist[ring_id];
+        struct intel_vgpu_submission *s = &vgpu->submission;
+        struct intel_vgpu_execlist *execlist = &s->execlist[ring_id];
         struct intel_vgpu_workload *next_workload;
         struct list_head *next = workload_q_head(vgpu, ring_id)->next;
         bool lite_restore = false;
-        int ret;
+        int ret = 0;
 
         gvt_dbg_el("complete workload %p status %d\n", workload,
                         workload->status);
 
-        if (!workload->status) {
-                release_shadow_batch_buffer(workload);
-                release_shadow_wa_ctx(&workload->wa_ctx);
-        }
-
-        if (workload->status || (vgpu->resetting_eng & ENGINE_MASK(ring_id))) {
-                /* if workload->status is not successful means HW GPU
-                 * has occurred GPU hang or something wrong with i915/GVT,
-                 * and GVT won't inject context switch interrupt to guest.
-                 * So this error is a vGPU hang actually to the guest.
-                 * According to this we should emunlate a vGPU hang. If
-                 * there are pending workloads which are already submitted
-                 * from guest, we should clean them up like HW GPU does.
-                 *
-                 * if it is in middle of engine resetting, the pending
-                 * workloads won't be submitted to HW GPU and will be
-                 * cleaned up during the resetting process later, so doing
-                 * the workload clean up here doesn't have any impact.
-                 **/
-                clean_workloads(vgpu, ENGINE_MASK(ring_id));
+        if (workload->status || (vgpu->resetting_eng & ENGINE_MASK(ring_id)))
                 goto out;
-        }
 
         if (!list_empty(workload_q_head(vgpu, ring_id))) {
                 struct execlist_ctx_descriptor_format *this_desc, *next_desc;
@@ -584,213 +427,60 @@ static int complete_execlist_workload(struct intel_vgpu_workload *workload)
 
         if (lite_restore) {
                 gvt_dbg_el("next context == current - no schedule-out\n");
-                free_workload(workload);
-                return 0;
+                goto out;
         }
 
         ret = emulate_execlist_ctx_schedule_out(execlist, &workload->ctx_desc);
-        if (ret)
-                goto err;
 out:
-        free_workload(workload);
-        return 0;
-err:
-        free_workload(workload);
+        intel_vgpu_unpin_mm(workload->shadow_mm);
+        intel_vgpu_destroy_workload(workload);
         return ret;
 }
 
-#define RING_CTX_OFF(x) \
-        offsetof(struct execlist_ring_context, x)
-
-static void read_guest_pdps(struct intel_vgpu *vgpu,
-                u64 ring_context_gpa, u32 pdp[8])
-{
-        u64 gpa;
-        int i;
-
-        gpa = ring_context_gpa + RING_CTX_OFF(pdp3_UDW.val);
-
-        for (i = 0; i < 8; i++)
-                intel_gvt_hypervisor_read_gpa(vgpu,
-                                gpa + i * 8, &pdp[7 - i], 4);
-}
-
-static int prepare_mm(struct intel_vgpu_workload *workload)
-{
-        struct execlist_ctx_descriptor_format *desc = &workload->ctx_desc;
-        struct intel_vgpu_mm *mm;
-        struct intel_vgpu *vgpu = workload->vgpu;
-        int page_table_level;
-        u32 pdp[8];
-
-        if (desc->addressing_mode == 1) { /* legacy 32-bit */
-                page_table_level = 3;
-        } else if (desc->addressing_mode == 3) { /* legacy 64 bit */
-                page_table_level = 4;
-        } else {
-                gvt_vgpu_err("Advanced Context mode(SVM) is not supported!\n");
-                return -EINVAL;
-        }
-
-        read_guest_pdps(workload->vgpu, workload->ring_context_gpa, pdp);
-
-        mm = intel_vgpu_find_ppgtt_mm(workload->vgpu, page_table_level, pdp);
-        if (mm) {
-                intel_gvt_mm_reference(mm);
-        } else {
-
-                mm = intel_vgpu_create_mm(workload->vgpu, INTEL_GVT_MM_PPGTT,
-                                pdp, page_table_level, 0);
-                if (IS_ERR(mm)) {
-                        gvt_vgpu_err("fail to create mm object.\n");
-                        return PTR_ERR(mm);
-                }
-        }
-        workload->shadow_mm = mm;
-        return 0;
-}
-
-#define get_last_workload(q) \
-        (list_empty(q) ? NULL : container_of(q->prev, \
-        struct intel_vgpu_workload, list))
-
 static int submit_context(struct intel_vgpu *vgpu, int ring_id,
                 struct execlist_ctx_descriptor_format *desc,
                 bool emulate_schedule_in)
 {
-        struct list_head *q = workload_q_head(vgpu, ring_id);
-        struct intel_vgpu_workload *last_workload = get_last_workload(q);
+        struct intel_vgpu_submission *s = &vgpu->submission;
         struct intel_vgpu_workload *workload = NULL;
-        struct drm_i915_private *dev_priv = vgpu->gvt->dev_priv;
-        u64 ring_context_gpa;
-        u32 head, tail, start, ctl, ctx_ctl, per_ctx, indirect_ctx;
-        int ret;
-
-        ring_context_gpa = intel_vgpu_gma_to_gpa(vgpu->gtt.ggtt_mm,
-                        (u32)((desc->lrca + 1) << GTT_PAGE_SHIFT));
-        if (ring_context_gpa == INTEL_GVT_INVALID_ADDR) {
-                gvt_vgpu_err("invalid guest context LRCA: %x\n", desc->lrca);
-                return -EINVAL;
-        }
-
-        intel_gvt_hypervisor_read_gpa(vgpu, ring_context_gpa +
-                        RING_CTX_OFF(ring_header.val), &head, 4);
 
-        intel_gvt_hypervisor_read_gpa(vgpu, ring_context_gpa +
-                        RING_CTX_OFF(ring_tail.val), &tail, 4);
+        workload = intel_vgpu_create_workload(vgpu, ring_id, desc);
+        if (IS_ERR(workload))
+                return PTR_ERR(workload);
 
-        head &= RB_HEAD_OFF_MASK;
-        tail &= RB_TAIL_OFF_MASK;
-
-        if (last_workload && same_context(&last_workload->ctx_desc, desc)) {
-                gvt_dbg_el("ring id %d cur workload == last\n", ring_id);
-                gvt_dbg_el("ctx head %x real head %lx\n", head,
-                                last_workload->rb_tail);
-                /*
-                 * cannot use guest context head pointer here,
-                 * as it might not be updated at this time
-                 */
-                head = last_workload->rb_tail;
-        }
-
-        gvt_dbg_el("ring id %d begin a new workload\n", ring_id);
-
-        workload = kmem_cache_zalloc(vgpu->workloads, GFP_KERNEL);
-        if (!workload)
-                return -ENOMEM;
-
-        /* record some ring buffer register values for scan and shadow */
-        intel_gvt_hypervisor_read_gpa(vgpu, ring_context_gpa +
-                        RING_CTX_OFF(rb_start.val), &start, 4);
-        intel_gvt_hypervisor_read_gpa(vgpu, ring_context_gpa +
-                        RING_CTX_OFF(rb_ctrl.val), &ctl, 4);
-        intel_gvt_hypervisor_read_gpa(vgpu, ring_context_gpa +
-                        RING_CTX_OFF(ctx_ctrl.val), &ctx_ctl, 4);
-
-        INIT_LIST_HEAD(&workload->list);
-        INIT_LIST_HEAD(&workload->shadow_bb);
-
-        init_waitqueue_head(&workload->shadow_ctx_status_wq);
-        atomic_set(&workload->shadow_ctx_active, 0);
-
-        workload->vgpu = vgpu;
-        workload->ring_id = ring_id;
-        workload->ctx_desc = *desc;
-        workload->ring_context_gpa = ring_context_gpa;
-        workload->rb_head = head;
-        workload->rb_tail = tail;
-        workload->rb_start = start;
-        workload->rb_ctl = ctl;
         workload->prepare = prepare_execlist_workload;
         workload->complete = complete_execlist_workload;
-        workload->status = -EINPROGRESS;
         workload->emulate_schedule_in = emulate_schedule_in;
-        workload->shadowed = false;
-
-        if (ring_id == RCS) {
-                intel_gvt_hypervisor_read_gpa(vgpu, ring_context_gpa +
-                        RING_CTX_OFF(bb_per_ctx_ptr.val), &per_ctx, 4);
-                intel_gvt_hypervisor_read_gpa(vgpu, ring_context_gpa +
-                        RING_CTX_OFF(rcs_indirect_ctx.val), &indirect_ctx, 4);
-
-                workload->wa_ctx.indirect_ctx.guest_gma =
-                        indirect_ctx & INDIRECT_CTX_ADDR_MASK;
-                workload->wa_ctx.indirect_ctx.size =
-                        (indirect_ctx & INDIRECT_CTX_SIZE_MASK) *
-                        CACHELINE_BYTES;
-                workload->wa_ctx.per_ctx.guest_gma =
-                        per_ctx & PER_CTX_ADDR_MASK;
-                workload->wa_ctx.per_ctx.valid = per_ctx & 1;
-        }
 
         if (emulate_schedule_in)
-                workload->elsp_dwords = vgpu->execlist[ring_id].elsp_dwords;
-
-        gvt_dbg_el("workload %p ring id %d head %x tail %x start %x ctl %x\n",
-                        workload, ring_id, head, tail, start, ctl);
+                workload->elsp_dwords = s->execlist[ring_id].elsp_dwords;
 
         gvt_dbg_el("workload %p emulate schedule_in %d\n", workload,
                         emulate_schedule_in);
 
-        ret = prepare_mm(workload);
-        if (ret) {
-                kmem_cache_free(vgpu->workloads, workload);
-                return ret;
-        }
-
-        /* Only scan and shadow the first workload in the queue
-         * as there is only one pre-allocated buf-obj for shadow.
-         */
-        if (list_empty(workload_q_head(vgpu, ring_id))) {
-                intel_runtime_pm_get(dev_priv);
-                mutex_lock(&dev_priv->drm.struct_mutex);
-                intel_gvt_scan_and_shadow_workload(workload);
-                mutex_unlock(&dev_priv->drm.struct_mutex);
-                intel_runtime_pm_put(dev_priv);
-        }
-
         queue_workload(workload);
         return 0;
 }
 
 int intel_vgpu_submit_execlist(struct intel_vgpu *vgpu, int ring_id)
 {
-        struct intel_vgpu_execlist *execlist = &vgpu->execlist[ring_id];
-        struct execlist_ctx_descriptor_format desc[2];
+        struct intel_vgpu_submission *s = &vgpu->submission;
+        struct intel_vgpu_execlist *execlist = &s->execlist[ring_id];
+        struct execlist_ctx_descriptor_format *desc[2];
         int i, ret;
 
-        desc[0] = *get_desc_from_elsp_dwords(&execlist->elsp_dwords, 1);
-        desc[1] = *get_desc_from_elsp_dwords(&execlist->elsp_dwords, 0);
+        desc[0] = get_desc_from_elsp_dwords(&execlist->elsp_dwords, 0);
+        desc[1] = get_desc_from_elsp_dwords(&execlist->elsp_dwords, 1);
 
-        if (!desc[0].valid) {
+        if (!desc[0]->valid) {
                 gvt_vgpu_err("invalid elsp submission, desc0 is invalid\n");
                 goto inv_desc;
         }
 
         for (i = 0; i < ARRAY_SIZE(desc); i++) {
-                if (!desc[i].valid)
+                if (!desc[i]->valid)
                         continue;
-                if (!desc[i].privilege_access) {
+                if (!desc[i]->privilege_access) {
                         gvt_vgpu_err("unexpected GGTT elsp submission\n");
                         goto inv_desc;
                 }
@@ -798,9 +488,9 @@ int intel_vgpu_submit_execlist(struct intel_vgpu *vgpu, int ring_id)
 
         /* submit workload */
         for (i = 0; i < ARRAY_SIZE(desc); i++) {
-                if (!desc[i].valid)
+                if (!desc[i]->valid)
                         continue;
-                ret = submit_context(vgpu, ring_id, &desc[i], i == 0);
+                ret = submit_context(vgpu, ring_id, desc[i], i == 0);
                 if (ret) {
                         gvt_vgpu_err("failed to submit desc %d\n", i);
                         return ret;
@@ -811,13 +501,14 @@ int intel_vgpu_submit_execlist(struct intel_vgpu *vgpu, int ring_id)
 
 inv_desc:
         gvt_vgpu_err("descriptors content: desc0 %08x %08x desc1 %08x %08x\n",
-                     desc[0].udw, desc[0].ldw, desc[1].udw, desc[1].ldw);
+                     desc[0]->udw, desc[0]->ldw, desc[1]->udw, desc[1]->ldw);
         return -EINVAL;
 }
 
 static void init_vgpu_execlist(struct intel_vgpu *vgpu, int ring_id)
 {
-        struct intel_vgpu_execlist *execlist = &vgpu->execlist[ring_id];
+        struct intel_vgpu_submission *s = &vgpu->submission;
+        struct intel_vgpu_execlist *execlist = &s->execlist[ring_id];
         struct execlist_context_status_pointer_format ctx_status_ptr;
         u32 ctx_status_ptr_reg;
 
@@ -837,91 +528,40 @@ static void init_vgpu_execlist(struct intel_vgpu *vgpu, int ring_id)
         vgpu_vreg(vgpu, ctx_status_ptr_reg) = ctx_status_ptr.dw;
 }
 
-static void clean_workloads(struct intel_vgpu *vgpu, unsigned long engine_mask)
-{
-        struct drm_i915_private *dev_priv = vgpu->gvt->dev_priv;
-        struct intel_engine_cs *engine;
-        struct intel_vgpu_workload *pos, *n;
-        unsigned int tmp;
-
-        /* free the unsubmited workloads in the queues. */
-        for_each_engine_masked(engine, dev_priv, engine_mask, tmp) {
-                list_for_each_entry_safe(pos, n,
-                        &vgpu->workload_q_head[engine->id], list) {
-                        list_del_init(&pos->list);
-                        free_workload(pos);
-                }
-
-                clear_bit(engine->id, vgpu->shadow_ctx_desc_updated);
-        }
-}
-
-void intel_vgpu_clean_execlist(struct intel_vgpu *vgpu)
+void clean_execlist(struct intel_vgpu *vgpu)
 {
         enum intel_engine_id i;
         struct intel_engine_cs *engine;
 
-        clean_workloads(vgpu, ALL_ENGINES);
-        kmem_cache_destroy(vgpu->workloads);
-
         for_each_engine(engine, vgpu->gvt->dev_priv, i) {
-                kfree(vgpu->reserve_ring_buffer_va[i]);
-                vgpu->reserve_ring_buffer_va[i] = NULL;
-                vgpu->reserve_ring_buffer_size[i] = 0;
-        }
+                struct intel_vgpu_submission *s = &vgpu->submission;
 
-}
-
-#define RESERVE_RING_BUFFER_SIZE                ((1 * PAGE_SIZE)/8)
-int intel_vgpu_init_execlist(struct intel_vgpu *vgpu)
-{
-        enum intel_engine_id i;
-        struct intel_engine_cs *engine;
-
-        /* each ring has a virtual execlist engine */
-        for_each_engine(engine, vgpu->gvt->dev_priv, i) {
-                init_vgpu_execlist(vgpu, i);
-                INIT_LIST_HEAD(&vgpu->workload_q_head[i]);
-        }
-
-        vgpu->workloads = kmem_cache_create("gvt-g_vgpu_workload",
-                        sizeof(struct intel_vgpu_workload), 0,
-                        SLAB_HWCACHE_ALIGN,
-                        NULL);
-
-        if (!vgpu->workloads)
-                return -ENOMEM;
-
-        /* each ring has a shadow ring buffer until vgpu destroyed */
-        for_each_engine(engine, vgpu->gvt->dev_priv, i) {
-                vgpu->reserve_ring_buffer_va[i] =
-                        kmalloc(RESERVE_RING_BUFFER_SIZE, GFP_KERNEL);
-                if (!vgpu->reserve_ring_buffer_va[i]) {
-                        gvt_vgpu_err("fail to alloc reserve ring buffer\n");
-                        goto out;
-                }
-                vgpu->reserve_ring_buffer_size[i] = RESERVE_RING_BUFFER_SIZE;
+                kfree(s->ring_scan_buffer[i]);
+                s->ring_scan_buffer[i] = NULL;
+                s->ring_scan_buffer_size[i] = 0;
         }
-        return 0;
-out:
-        for_each_engine(engine, vgpu->gvt->dev_priv, i) {
-                if (vgpu->reserve_ring_buffer_size[i]) {
-                        kfree(vgpu->reserve_ring_buffer_va[i]);
-                        vgpu->reserve_ring_buffer_va[i] = NULL;
-                        vgpu->reserve_ring_buffer_size[i] = 0;
-                }
-        }
-        return -ENOMEM;
 }
 
-void intel_vgpu_reset_execlist(struct intel_vgpu *vgpu,
+void reset_execlist(struct intel_vgpu *vgpu,
                 unsigned long engine_mask)
 {
         struct drm_i915_private *dev_priv = vgpu->gvt->dev_priv;
         struct intel_engine_cs *engine;
         unsigned int tmp;
 
-        clean_workloads(vgpu, engine_mask);
         for_each_engine_masked(engine, dev_priv, engine_mask, tmp)
                 init_vgpu_execlist(vgpu, engine->id);
 }
+
+int init_execlist(struct intel_vgpu *vgpu)
+{
+        reset_execlist(vgpu, ALL_ENGINES);
+        return 0;
+}
+
+const struct intel_vgpu_submission_ops intel_vgpu_execlist_submission_ops = {
+        .name = "execlist",
+        .init = init_execlist,
+        .reset = reset_execlist,
+        .clean = clean_execlist,
+};
diff --git a/drivers/gpu/drm/i915/gvt/execlist.h b/drivers/gpu/drm/i915/gvt/execlist.h
index 7eced40a1e30..427e40e64d41 100644
--- a/drivers/gpu/drm/i915/gvt/execlist.h
+++ b/drivers/gpu/drm/i915/gvt/execlist.h
@@ -37,10 +37,6 @@
 
 struct execlist_ctx_descriptor_format {
         union {
-                u32 udw;
-                u32 context_id;
-        };
-        union {
                 u32 ldw;
                 struct {
                         u32 valid                  : 1;
@@ -54,6 +50,10 @@ struct execlist_ctx_descriptor_format {
                         u32 lrca                   : 20;
                 };
         };
+        union {
+                u32 udw;
+                u32 context_id;
+        };
 };
 
 struct execlist_status_format {
diff --git a/drivers/gpu/drm/i915/gvt/firmware.c b/drivers/gpu/drm/i915/gvt/firmware.c
index a26c1705430e..a73e1d418c22 100644
--- a/drivers/gpu/drm/i915/gvt/firmware.c
+++ b/drivers/gpu/drm/i915/gvt/firmware.c
@@ -66,20 +66,23 @@ static struct bin_attribute firmware_attr = {
         .mmap = NULL,
 };
 
-static int expose_firmware_sysfs(struct intel_gvt *gvt)
+static int mmio_snapshot_handler(struct intel_gvt *gvt, u32 offset, void *data)
 {
         struct drm_i915_private *dev_priv = gvt->dev_priv;
+
+        *(u32 *)(data + offset) = I915_READ_NOTRACE(_MMIO(offset));
+        return 0;
+}
+
+static int expose_firmware_sysfs(struct intel_gvt *gvt)
+{
         struct intel_gvt_device_info *info = &gvt->device_info;
         struct pci_dev *pdev = gvt->dev_priv->drm.pdev;
-        struct intel_gvt_mmio_info *e;
-        struct gvt_mmio_block *block = gvt->mmio.mmio_block;
-        int num = gvt->mmio.num_mmio_block;
         struct gvt_firmware_header *h;
         void *firmware;
         void *p;
         unsigned long size, crc32_start;
-        int i, j;
-        int ret;
+        int i, ret;
 
         size = sizeof(*h) + info->mmio_size + info->cfg_space_size;
         firmware = vzalloc(size);
@@ -104,15 +107,8 @@ static int expose_firmware_sysfs(struct intel_gvt *gvt)
 
         p = firmware + h->mmio_offset;
 
-        hash_for_each(gvt->mmio.mmio_info_table, i, e, node)
-                *(u32 *)(p + e->offset) = I915_READ_NOTRACE(_MMIO(e->offset));
-
-        for (i = 0; i < num; i++, block++) {
-                for (j = 0; j < block->size; j += 4)
-                        *(u32 *)(p + INTEL_GVT_MMIO_OFFSET(block->offset) + j) =
-                                I915_READ_NOTRACE(_MMIO(INTEL_GVT_MMIO_OFFSET(
-                                                        block->offset) + j));
-        }
+        /* Take a snapshot of hw mmio registers. */
+        intel_gvt_for_each_tracked_mmio(gvt, mmio_snapshot_handler, p);
 
         memcpy(gvt->firmware.mmio, p, info->mmio_size);
 
diff --git a/drivers/gpu/drm/i915/gvt/gtt.c b/drivers/gpu/drm/i915/gvt/gtt.c
index 8e331142badb..71a0f2b87b3a 100644
--- a/drivers/gpu/drm/i915/gvt/gtt.c
+++ b/drivers/gpu/drm/i915/gvt/gtt.c
@@ -94,12 +94,12 @@ int intel_gvt_ggtt_index_g2h(struct intel_vgpu *vgpu, unsigned long g_index,
         u64 h_addr;
         int ret;
 
-        ret = intel_gvt_ggtt_gmadr_g2h(vgpu, g_index << GTT_PAGE_SHIFT,
+        ret = intel_gvt_ggtt_gmadr_g2h(vgpu, g_index << I915_GTT_PAGE_SHIFT,
                                        &h_addr);
         if (ret)
                 return ret;
 
-        *h_index = h_addr >> GTT_PAGE_SHIFT;
+        *h_index = h_addr >> I915_GTT_PAGE_SHIFT;
         return 0;
 }
 
@@ -109,12 +109,12 @@ int intel_gvt_ggtt_h2g_index(struct intel_vgpu *vgpu, unsigned long h_index,
         u64 g_addr;
         int ret;
 
-        ret = intel_gvt_ggtt_gmadr_h2g(vgpu, h_index << GTT_PAGE_SHIFT,
+        ret = intel_gvt_ggtt_gmadr_h2g(vgpu, h_index << I915_GTT_PAGE_SHIFT,
                                        &g_addr);
         if (ret)
                 return ret;
 
-        *g_index = g_addr >> GTT_PAGE_SHIFT;
+        *g_index = g_addr >> I915_GTT_PAGE_SHIFT;
         return 0;
 }
 
@@ -156,13 +156,15 @@ int intel_gvt_ggtt_h2g_index(struct intel_vgpu *vgpu, unsigned long h_index,
 
 struct gtt_type_table_entry {
         int entry_type;
+        int pt_type;
         int next_pt_type;
         int pse_entry_type;
 };
 
-#define GTT_TYPE_TABLE_ENTRY(type, e_type, npt_type, pse_type) \
+#define GTT_TYPE_TABLE_ENTRY(type, e_type, cpt_type, npt_type, pse_type) \
         [type] = { \
                 .entry_type = e_type, \
+                .pt_type = cpt_type, \
                 .next_pt_type = npt_type, \
                 .pse_entry_type = pse_type, \
         }
@@ -170,55 +172,68 @@ struct gtt_type_table_entry {
 static struct gtt_type_table_entry gtt_type_table[] = {
         GTT_TYPE_TABLE_ENTRY(GTT_TYPE_PPGTT_ROOT_L4_ENTRY,
                         GTT_TYPE_PPGTT_ROOT_L4_ENTRY,
+                        GTT_TYPE_INVALID,
                         GTT_TYPE_PPGTT_PML4_PT,
                         GTT_TYPE_INVALID),
         GTT_TYPE_TABLE_ENTRY(GTT_TYPE_PPGTT_PML4_PT,
                         GTT_TYPE_PPGTT_PML4_ENTRY,
+                        GTT_TYPE_PPGTT_PML4_PT,
                         GTT_TYPE_PPGTT_PDP_PT,
                         GTT_TYPE_INVALID),
         GTT_TYPE_TABLE_ENTRY(GTT_TYPE_PPGTT_PML4_ENTRY,
                         GTT_TYPE_PPGTT_PML4_ENTRY,
+                        GTT_TYPE_PPGTT_PML4_PT,
                         GTT_TYPE_PPGTT_PDP_PT,
                         GTT_TYPE_INVALID),
         GTT_TYPE_TABLE_ENTRY(GTT_TYPE_PPGTT_PDP_PT,
                         GTT_TYPE_PPGTT_PDP_ENTRY,
+                        GTT_TYPE_PPGTT_PDP_PT,
                         GTT_TYPE_PPGTT_PDE_PT,
                         GTT_TYPE_PPGTT_PTE_1G_ENTRY),
         GTT_TYPE_TABLE_ENTRY(GTT_TYPE_PPGTT_ROOT_L3_ENTRY,
                         GTT_TYPE_PPGTT_ROOT_L3_ENTRY,
+                        GTT_TYPE_INVALID,
                         GTT_TYPE_PPGTT_PDE_PT,
                         GTT_TYPE_PPGTT_PTE_1G_ENTRY),
         GTT_TYPE_TABLE_ENTRY(GTT_TYPE_PPGTT_PDP_ENTRY,
                         GTT_TYPE_PPGTT_PDP_ENTRY,
+                        GTT_TYPE_PPGTT_PDP_PT,
                         GTT_TYPE_PPGTT_PDE_PT,
                         GTT_TYPE_PPGTT_PTE_1G_ENTRY),
         GTT_TYPE_TABLE_ENTRY(GTT_TYPE_PPGTT_PDE_PT,
                         GTT_TYPE_PPGTT_PDE_ENTRY,
+                        GTT_TYPE_PPGTT_PDE_PT,
                         GTT_TYPE_PPGTT_PTE_PT,
                         GTT_TYPE_PPGTT_PTE_2M_ENTRY),
         GTT_TYPE_TABLE_ENTRY(GTT_TYPE_PPGTT_PDE_ENTRY,
                         GTT_TYPE_PPGTT_PDE_ENTRY,
+                        GTT_TYPE_PPGTT_PDE_PT,
                         GTT_TYPE_PPGTT_PTE_PT,
                         GTT_TYPE_PPGTT_PTE_2M_ENTRY),
         GTT_TYPE_TABLE_ENTRY(GTT_TYPE_PPGTT_PTE_PT,
                         GTT_TYPE_PPGTT_PTE_4K_ENTRY,
+                        GTT_TYPE_PPGTT_PTE_PT,
                         GTT_TYPE_INVALID,
                         GTT_TYPE_INVALID),
         GTT_TYPE_TABLE_ENTRY(GTT_TYPE_PPGTT_PTE_4K_ENTRY,
                         GTT_TYPE_PPGTT_PTE_4K_ENTRY,
+                        GTT_TYPE_PPGTT_PTE_PT,
                         GTT_TYPE_INVALID,
                         GTT_TYPE_INVALID),
         GTT_TYPE_TABLE_ENTRY(GTT_TYPE_PPGTT_PTE_2M_ENTRY,
                         GTT_TYPE_PPGTT_PDE_ENTRY,
+                        GTT_TYPE_PPGTT_PDE_PT,
                         GTT_TYPE_INVALID,
                         GTT_TYPE_PPGTT_PTE_2M_ENTRY),
         GTT_TYPE_TABLE_ENTRY(GTT_TYPE_PPGTT_PTE_1G_ENTRY,
                         GTT_TYPE_PPGTT_PDP_ENTRY,
+                        GTT_TYPE_PPGTT_PDP_PT,
                         GTT_TYPE_INVALID,
                         GTT_TYPE_PPGTT_PTE_1G_ENTRY),
         GTT_TYPE_TABLE_ENTRY(GTT_TYPE_GGTT_PTE,
                         GTT_TYPE_GGTT_PTE,
                         GTT_TYPE_INVALID,
+                        GTT_TYPE_INVALID,
                         GTT_TYPE_INVALID),
 };
 
@@ -227,6 +242,11 @@ static inline int get_next_pt_type(int type)
         return gtt_type_table[type].next_pt_type;
 }
 
+static inline int get_pt_type(int type)
+{
+        return gtt_type_table[type].pt_type;
+}
+
 static inline int get_entry_type(int type)
 {
         return gtt_type_table[type].entry_type;
@@ -351,7 +371,7 @@ static bool gen8_gtt_test_pse(struct intel_gvt_gtt_entry *e)
                 return false;
 
         e->type = get_entry_type(e->type);
-        if (!(e->val64 & (1 << 7)))
+        if (!(e->val64 & BIT(7)))
                 return false;
 
         e->type = get_pse_type(e->type);
@@ -369,12 +389,17 @@ static bool gen8_gtt_test_present(struct intel_gvt_gtt_entry *e)
                         || e->type == GTT_TYPE_PPGTT_ROOT_L4_ENTRY)
                 return (e->val64 != 0);
         else
-                return (e->val64 & (1 << 0));
+                return (e->val64 & BIT(0));
 }
 
 static void gtt_entry_clear_present(struct intel_gvt_gtt_entry *e)
 {
-        e->val64 &= ~(1 << 0);
+        e->val64 &= ~BIT(0);
+}
+
+static void gtt_entry_set_present(struct intel_gvt_gtt_entry *e)
+{
+        e->val64 |= BIT(0);
 }
 
 /*
@@ -382,7 +407,7 @@ static void gtt_entry_clear_present(struct intel_gvt_gtt_entry *e)
  */
 static unsigned long gma_to_ggtt_pte_index(unsigned long gma)
 {
-        unsigned long x = (gma >> GTT_PAGE_SHIFT);
+        unsigned long x = (gma >> I915_GTT_PAGE_SHIFT);
 
         trace_gma_index(__func__, gma, x);
         return x;
@@ -406,6 +431,7 @@ static struct intel_gvt_gtt_pte_ops gen8_gtt_pte_ops = {
         .get_entry = gtt_get_entry64,
         .set_entry = gtt_set_entry64,
         .clear_present = gtt_entry_clear_present,
+        .set_present = gtt_entry_set_present,
         .test_present = gen8_gtt_test_present,
         .test_pse = gen8_gtt_test_pse,
         .get_pfn = gen8_gtt_get_pfn,
@@ -494,7 +520,7 @@ static inline int ppgtt_spt_get_entry(
                 return -EINVAL;
 
         ret = ops->get_entry(page_table, e, index, guest,
-                        spt->guest_page.gfn << GTT_PAGE_SHIFT,
+                        spt->guest_page.track.gfn << I915_GTT_PAGE_SHIFT,
                         spt->vgpu);
         if (ret)
                 return ret;
@@ -516,7 +542,7 @@ static inline int ppgtt_spt_set_entry(
                 return -EINVAL;
 
         return ops->set_entry(page_table, e, index, guest,
-                        spt->guest_page.gfn << GTT_PAGE_SHIFT,
+                        spt->guest_page.track.gfn << I915_GTT_PAGE_SHIFT,
                         spt->vgpu);
 }
 
@@ -537,88 +563,103 @@ static inline int ppgtt_spt_set_entry(
                 spt->shadow_page.type, e, index, false)
 
 /**
- * intel_vgpu_init_guest_page - init a guest page data structure
+ * intel_vgpu_init_page_track - init a page track data structure
  * @vgpu: a vGPU
- * @p: a guest page data structure
+ * @t: a page track data structure
  * @gfn: guest memory page frame number
- * @handler: function will be called when target guest memory page has
+ * @handler: the function will be called when target guest memory page has
  * been modified.
  *
- * This function is called when user wants to track a guest memory page.
+ * This function is called when a user wants to prepare a page track data
+ * structure to track a guest memory page.
  *
  * Returns:
  * Zero on success, negative error code if failed.
  */
-int intel_vgpu_init_guest_page(struct intel_vgpu *vgpu,
-                struct intel_vgpu_guest_page *p,
+int intel_vgpu_init_page_track(struct intel_vgpu *vgpu,
+                struct intel_vgpu_page_track *t,
                 unsigned long gfn,
                 int (*handler)(void *, u64, void *, int),
                 void *data)
 {
-        INIT_HLIST_NODE(&p->node);
+        INIT_HLIST_NODE(&t->node);
 
-        p->writeprotection = false;
-        p->gfn = gfn;
-        p->handler = handler;
-        p->data = data;
-        p->oos_page = NULL;
-        p->write_cnt = 0;
+        t->tracked = false;
+        t->gfn = gfn;
+        t->handler = handler;
+        t->data = data;
 
-        hash_add(vgpu->gtt.guest_page_hash_table, &p->node, p->gfn);
+        hash_add(vgpu->gtt.tracked_guest_page_hash_table, &t->node, t->gfn);
         return 0;
 }
 
-static int detach_oos_page(struct intel_vgpu *vgpu,
-                struct intel_vgpu_oos_page *oos_page);
-
 /**
- * intel_vgpu_clean_guest_page - release the resource owned by guest page data
- * structure
+ * intel_vgpu_clean_page_track - release a page track data structure
  * @vgpu: a vGPU
- * @p: a tracked guest page
+ * @t: a page track data structure
  *
- * This function is called when user tries to stop tracking a guest memory
- * page.
+ * This function is called before a user frees a page track data structure.
  */
-void intel_vgpu_clean_guest_page(struct intel_vgpu *vgpu,
-                struct intel_vgpu_guest_page *p)
+void intel_vgpu_clean_page_track(struct intel_vgpu *vgpu,
+                struct intel_vgpu_page_track *t)
 {
-        if (!hlist_unhashed(&p->node))
-                hash_del(&p->node);
-
-        if (p->oos_page)
-                detach_oos_page(vgpu, p->oos_page);
+        if (!hlist_unhashed(&t->node))
+                hash_del(&t->node);
 
-        if (p->writeprotection)
-                intel_gvt_hypervisor_unset_wp_page(vgpu, p);
+        if (t->tracked)
+                intel_gvt_hypervisor_disable_page_track(vgpu, t);
 }
 
 /**
- * intel_vgpu_find_guest_page - find a guest page data structure by GFN.
+ * intel_vgpu_find_tracked_page - find a tracked guest page
  * @vgpu: a vGPU
  * @gfn: guest memory page frame number
  *
- * This function is called when emulation logic wants to know if a trapped GFN
- * is a tracked guest page.
+ * This function is called when the emulation layer wants to figure out if a
+ * trapped GFN is a tracked guest page.
  *
  * Returns:
- * Pointer to guest page data structure, NULL if failed.
+ * Pointer to page track data structure, NULL if not found.
  */
-struct intel_vgpu_guest_page *intel_vgpu_find_guest_page(
+struct intel_vgpu_page_track *intel_vgpu_find_tracked_page(
                 struct intel_vgpu *vgpu, unsigned long gfn)
 {
-        struct intel_vgpu_guest_page *p;
+        struct intel_vgpu_page_track *t;
 
-        hash_for_each_possible(vgpu->gtt.guest_page_hash_table,
-                p, node, gfn) {
-                if (p->gfn == gfn)
-                        return p;
+        hash_for_each_possible(vgpu->gtt.tracked_guest_page_hash_table,
+                        t, node, gfn) {
+                if (t->gfn == gfn)
+                        return t;
         }
         return NULL;
 }
 
+static int init_guest_page(struct intel_vgpu *vgpu,
+                struct intel_vgpu_guest_page *p,
+                unsigned long gfn,
+                int (*handler)(void *, u64, void *, int),
+                void *data)
+{
+        p->oos_page = NULL;
+        p->write_cnt = 0;
+
+        return intel_vgpu_init_page_track(vgpu, &p->track, gfn, handler, data);
+}
+
+static int detach_oos_page(struct intel_vgpu *vgpu,
+                struct intel_vgpu_oos_page *oos_page);
+
+static void clean_guest_page(struct intel_vgpu *vgpu,
+                struct intel_vgpu_guest_page *p)
+{
+        if (p->oos_page)
+                detach_oos_page(vgpu, p->oos_page);
+
+        intel_vgpu_clean_page_track(vgpu, &p->track);
+}
+
 static inline int init_shadow_page(struct intel_vgpu *vgpu,
-                struct intel_vgpu_shadow_page *p, int type)
+                struct intel_vgpu_shadow_page *p, int type, bool hash)
 {
         struct device *kdev = &vgpu->gvt->dev_priv->drm.pdev->dev;
         dma_addr_t daddr;
@@ -634,8 +675,9 @@ static inline int init_shadow_page(struct intel_vgpu *vgpu,
 
         INIT_HLIST_NODE(&p->node);
 
-        p->mfn = daddr >> GTT_PAGE_SHIFT;
-        hash_add(vgpu->gtt.shadow_page_hash_table, &p->node, p->mfn);
+        p->mfn = daddr >> I915_GTT_PAGE_SHIFT;
+        if (hash)
+                hash_add(vgpu->gtt.shadow_page_hash_table, &p->node, p->mfn);
         return 0;
 }
 
@@ -644,7 +686,7 @@ static inline void clean_shadow_page(struct intel_vgpu *vgpu,
 {
         struct device *kdev = &vgpu->gvt->dev_priv->drm.pdev->dev;
 
-        dma_unmap_page(kdev, p->mfn << GTT_PAGE_SHIFT, 4096,
+        dma_unmap_page(kdev, p->mfn << I915_GTT_PAGE_SHIFT, 4096,
                         PCI_DMA_BIDIRECTIONAL);
 
         if (!hlist_unhashed(&p->node))
@@ -664,6 +706,9 @@ static inline struct intel_vgpu_shadow_page *find_shadow_page(
         return NULL;
 }
 
+#define page_track_to_guest_page(ptr) \
+        container_of(ptr, struct intel_vgpu_guest_page, track)
+
 #define guest_page_to_ppgtt_spt(ptr) \
         container_of(ptr, struct intel_vgpu_ppgtt_spt, guest_page)
 
@@ -697,7 +742,7 @@ static void ppgtt_free_shadow_page(struct intel_vgpu_ppgtt_spt *spt)
         trace_spt_free(spt->vgpu->id, spt, spt->shadow_page.type);
 
         clean_shadow_page(spt->vgpu, &spt->shadow_page);
-        intel_vgpu_clean_guest_page(spt->vgpu, &spt->guest_page);
+        clean_guest_page(spt->vgpu, &spt->guest_page);
         list_del_init(&spt->post_shadow_list);
 
         free_spt(spt);
@@ -713,22 +758,24 @@ static void ppgtt_free_all_shadow_page(struct intel_vgpu *vgpu)
                 ppgtt_free_shadow_page(shadow_page_to_ppgtt_spt(sp));
 }
 
-static int ppgtt_handle_guest_write_page_table_bytes(void *gp,
+static int ppgtt_handle_guest_write_page_table_bytes(
+                struct intel_vgpu_guest_page *gpt,
                 u64 pa, void *p_data, int bytes);
 
-static int ppgtt_write_protection_handler(void *gp, u64 pa,
+static int ppgtt_write_protection_handler(void *data, u64 pa,
                 void *p_data, int bytes)
 {
-        struct intel_vgpu_guest_page *gpt = (struct intel_vgpu_guest_page *)gp;
+        struct intel_vgpu_page_track *t = data;
+        struct intel_vgpu_guest_page *p = page_track_to_guest_page(t);
         int ret;
 
         if (bytes != 4 && bytes != 8)
                 return -EINVAL;
 
-        if (!gpt->writeprotection)
+        if (!t->tracked)
                 return -EINVAL;
 
-        ret = ppgtt_handle_guest_write_page_table_bytes(gp,
+        ret = ppgtt_handle_guest_write_page_table_bytes(p,
                 pa, p_data, bytes);
         if (ret)
                 return ret;
@@ -762,13 +809,13 @@ retry:
          * TODO: guest page type may be different with shadow page type,
          *       when we support PSE page in future.
          */
-        ret = init_shadow_page(vgpu, &spt->shadow_page, type);
+        ret = init_shadow_page(vgpu, &spt->shadow_page, type, true);
         if (ret) {
                 gvt_vgpu_err("fail to initialize shadow page for spt\n");
                 goto err;
         }
 
-        ret = intel_vgpu_init_guest_page(vgpu, &spt->guest_page,
+        ret = init_guest_page(vgpu, &spt->guest_page,
                         gfn, ppgtt_write_protection_handler, NULL);
         if (ret) {
                 gvt_vgpu_err("fail to initialize guest page for spt\n");
@@ -798,7 +845,7 @@ static struct intel_vgpu_ppgtt_spt *ppgtt_find_shadow_page(
         ((spt)->vgpu->gvt->device_info.gtt_entry_size_shift)
 
 #define pt_entries(spt) \
-        (GTT_PAGE_SIZE >> pt_entry_size_shift(spt))
+        (I915_GTT_PAGE_SIZE >> pt_entry_size_shift(spt))
 
 #define for_each_present_guest_entry(spt, e, i) \
         for (i = 0; i < pt_entries(spt); i++) \
@@ -856,7 +903,7 @@ static int ppgtt_invalidate_shadow_page(struct intel_vgpu_ppgtt_spt *spt)
         int v = atomic_read(&spt->refcount);
 
         trace_spt_change(spt->vgpu->id, "die", spt,
-                        spt->guest_page.gfn, spt->shadow_page.type);
+                        spt->guest_page.track.gfn, spt->shadow_page.type);
 
         trace_spt_refcount(spt->vgpu->id, "dec", spt, v, (v - 1));
 
@@ -878,7 +925,7 @@ static int ppgtt_invalidate_shadow_page(struct intel_vgpu_ppgtt_spt *spt)
         }
 release:
         trace_spt_change(spt->vgpu->id, "release", spt,
-                        spt->guest_page.gfn, spt->shadow_page.type);
+                        spt->guest_page.track.gfn, spt->shadow_page.type);
         ppgtt_free_shadow_page(spt);
         return 0;
 fail:
@@ -895,6 +942,7 @@ static struct intel_vgpu_ppgtt_spt *ppgtt_populate_shadow_page_by_guest_entry(
         struct intel_gvt_gtt_pte_ops *ops = vgpu->gvt->gtt.pte_ops;
         struct intel_vgpu_ppgtt_spt *s = NULL;
         struct intel_vgpu_guest_page *g;
+        struct intel_vgpu_page_track *t;
         int ret;
 
         if (WARN_ON(!gtt_type_is_pt(get_next_pt_type(we->type)))) {
@@ -902,8 +950,9 @@ static struct intel_vgpu_ppgtt_spt *ppgtt_populate_shadow_page_by_guest_entry(
                 goto fail;
         }
 
-        g = intel_vgpu_find_guest_page(vgpu, ops->get_pfn(we));
-        if (g) {
+        t = intel_vgpu_find_tracked_page(vgpu, ops->get_pfn(we));
+        if (t) {
+                g = page_track_to_guest_page(t);
                 s = guest_page_to_ppgtt_spt(g);
                 ppgtt_get_shadow_page(s);
         } else {
@@ -915,7 +964,8 @@ static struct intel_vgpu_ppgtt_spt *ppgtt_populate_shadow_page_by_guest_entry(
                         goto fail;
                 }
 
-                ret = intel_gvt_hypervisor_set_wp_page(vgpu, &s->guest_page);
+                ret = intel_gvt_hypervisor_enable_page_track(vgpu,
+                                &s->guest_page.track);
                 if (ret)
                         goto fail;
 
@@ -923,7 +973,7 @@ static struct intel_vgpu_ppgtt_spt *ppgtt_populate_shadow_page_by_guest_entry(
                 if (ret)
                         goto fail;
 
-                trace_spt_change(vgpu->id, "new", s, s->guest_page.gfn,
+                trace_spt_change(vgpu->id, "new", s, s->guest_page.track.gfn,
                         s->shadow_page.type);
         }
         return s;
@@ -953,7 +1003,7 @@ static int ppgtt_populate_shadow_page(struct intel_vgpu_ppgtt_spt *spt)
         int ret;
 
         trace_spt_change(spt->vgpu->id, "born", spt,
-                        spt->guest_page.gfn, spt->shadow_page.type);
+                        spt->guest_page.track.gfn, spt->shadow_page.type);
 
         if (gtt_type_is_pte_pt(spt->shadow_page.type)) {
                 for_each_present_guest_entry(spt, &ge, i) {
@@ -1078,11 +1128,11 @@ static int sync_oos_page(struct intel_vgpu *vgpu,
         old.type = new.type = get_entry_type(spt->guest_page_type);
         old.val64 = new.val64 = 0;
 
-        for (index = 0; index < (GTT_PAGE_SIZE >> info->gtt_entry_size_shift);
-                index++) {
+        for (index = 0; index < (I915_GTT_PAGE_SIZE >>
+                                info->gtt_entry_size_shift); index++) {
                 ops->get_entry(oos_page->mem, &old, index, false, 0, vgpu);
                 ops->get_entry(NULL, &new, index, true,
-                        oos_page->guest_page->gfn << PAGE_SHIFT, vgpu);
+                        oos_page->guest_page->track.gfn << PAGE_SHIFT, vgpu);
 
                 if (old.val64 == new.val64
                         && !test_and_clear_bit(index, spt->post_shadow_bitmap))
@@ -1132,8 +1182,9 @@ static int attach_oos_page(struct intel_vgpu *vgpu,
         struct intel_gvt *gvt = vgpu->gvt;
         int ret;
 
-        ret = intel_gvt_hypervisor_read_gpa(vgpu, gpt->gfn << GTT_PAGE_SHIFT,
-                oos_page->mem, GTT_PAGE_SIZE);
+        ret = intel_gvt_hypervisor_read_gpa(vgpu,
+                        gpt->track.gfn << I915_GTT_PAGE_SHIFT,
+                        oos_page->mem, I915_GTT_PAGE_SIZE);
         if (ret)
                 return ret;
 
@@ -1152,7 +1203,7 @@ static int ppgtt_set_guest_page_sync(struct intel_vgpu *vgpu,
 {
         int ret;
 
-        ret = intel_gvt_hypervisor_set_wp_page(vgpu, gpt);
+        ret = intel_gvt_hypervisor_enable_page_track(vgpu, &gpt->track);
         if (ret)
                 return ret;
 
@@ -1200,7 +1251,7 @@ static int ppgtt_set_guest_page_oos(struct intel_vgpu *vgpu,
                         gpt, guest_page_to_ppgtt_spt(gpt)->guest_page_type);
 
         list_add_tail(&oos_page->vm_list, &vgpu->gtt.oos_page_list_head);
-        return intel_gvt_hypervisor_unset_wp_page(vgpu, gpt);
+        return intel_gvt_hypervisor_disable_page_track(vgpu, &gpt->track);
 }
 
 /**
@@ -1335,10 +1386,10 @@ int intel_vgpu_flush_post_shadow(struct intel_vgpu *vgpu)
         return 0;
 }
 
-static int ppgtt_handle_guest_write_page_table_bytes(void *gp,
+static int ppgtt_handle_guest_write_page_table_bytes(
+                struct intel_vgpu_guest_page *gpt,
                 u64 pa, void *p_data, int bytes)
 {
-        struct intel_vgpu_guest_page *gpt = (struct intel_vgpu_guest_page *)gp;
         struct intel_vgpu_ppgtt_spt *spt = guest_page_to_ppgtt_spt(gpt);
         struct intel_vgpu *vgpu = spt->vgpu;
         struct intel_gvt_gtt_pte_ops *ops = vgpu->gvt->gtt.pte_ops;
@@ -1415,7 +1466,7 @@ static int gen8_mm_alloc_page_table(struct intel_vgpu_mm *mm)
                 mm->shadow_page_table = mem + mm->page_table_entry_size;
         } else if (mm->type == INTEL_GVT_MM_GGTT) {
                 mm->page_table_entry_cnt =
-                        (gvt_ggtt_gm_sz(gvt) >> GTT_PAGE_SHIFT);
+                        (gvt_ggtt_gm_sz(gvt) >> I915_GTT_PAGE_SHIFT);
                 mm->page_table_entry_size = mm->page_table_entry_cnt *
                         info->gtt_entry_size;
                 mem = vzalloc(mm->page_table_entry_size);
@@ -1737,8 +1788,8 @@ unsigned long intel_vgpu_gma_to_gpa(struct intel_vgpu_mm *mm, unsigned long gma)
                                 gma_ops->gma_to_ggtt_pte_index(gma));
                 if (ret)
                         goto err;
-                gpa = (pte_ops->get_pfn(&e) << GTT_PAGE_SHIFT)
-                        + (gma & ~GTT_PAGE_MASK);
+                gpa = (pte_ops->get_pfn(&e) << I915_GTT_PAGE_SHIFT)
+                        + (gma & ~I915_GTT_PAGE_MASK);
 
                 trace_gma_translate(vgpu->id, "ggtt", 0, 0, gma, gpa);
                 return gpa;
@@ -1790,8 +1841,8 @@ unsigned long intel_vgpu_gma_to_gpa(struct intel_vgpu_mm *mm, unsigned long gma)
                 }
         }
 
-        gpa = (pte_ops->get_pfn(&e) << GTT_PAGE_SHIFT)
-                + (gma & ~GTT_PAGE_MASK);
+        gpa = (pte_ops->get_pfn(&e) << I915_GTT_PAGE_SHIFT)
+                + (gma & ~I915_GTT_PAGE_MASK);
 
         trace_gma_translate(vgpu->id, "ppgtt", 0,
                         mm->page_table_level, gma, gpa);
@@ -1859,7 +1910,7 @@ static int emulate_gtt_mmio_write(struct intel_vgpu *vgpu, unsigned int off,
         if (bytes != 4 && bytes != 8)
                 return -EINVAL;
 
-        gma = g_gtt_index << GTT_PAGE_SHIFT;
+        gma = g_gtt_index << I915_GTT_PAGE_SHIFT;
 
         /* the VM may configure the whole GM space when ballooning is used */
         if (!vgpu_gmadr_is_valid(vgpu, gma))
@@ -1878,11 +1929,11 @@ static int emulate_gtt_mmio_write(struct intel_vgpu *vgpu, unsigned int off,
                          * update the entry in this situation p2m will fail
                          * settting the shadow entry to point to a scratch page
                          */
-                        ops->set_pfn(&m, gvt->gtt.scratch_ggtt_mfn);
+                        ops->set_pfn(&m, gvt->gtt.scratch_mfn);
                 }
         } else {
                 m = e;
-                ops->set_pfn(&m, gvt->gtt.scratch_ggtt_mfn);
+                ops->set_pfn(&m, gvt->gtt.scratch_mfn);
         }
 
         ggtt_set_shadow_entry(ggtt_mm, &m, g_gtt_index);
@@ -1922,7 +1973,7 @@ static int alloc_scratch_pages(struct intel_vgpu *vgpu,
 {
         struct intel_vgpu_gtt *gtt = &vgpu->gtt;
         struct intel_gvt_gtt_pte_ops *ops = vgpu->gvt->gtt.pte_ops;
-        int page_entry_num = GTT_PAGE_SIZE >>
+        int page_entry_num = I915_GTT_PAGE_SIZE >>
                                 vgpu->gvt->device_info.gtt_entry_size_shift;
         void *scratch_pt;
         int i;
@@ -1946,7 +1997,7 @@ static int alloc_scratch_pages(struct intel_vgpu *vgpu,
                 return -ENOMEM;
         }
         gtt->scratch_pt[type].page_mfn =
-                (unsigned long)(daddr >> GTT_PAGE_SHIFT);
+                (unsigned long)(daddr >> I915_GTT_PAGE_SHIFT);
         gtt->scratch_pt[type].page = virt_to_page(scratch_pt);
         gvt_dbg_mm("vgpu%d create scratch_pt: type %d mfn=0x%lx\n",
                         vgpu->id, type, gtt->scratch_pt[type].page_mfn);
@@ -1989,7 +2040,7 @@ static int release_scratch_page_tree(struct intel_vgpu *vgpu)
         for (i = GTT_TYPE_PPGTT_PTE_PT; i < GTT_TYPE_MAX; i++) {
                 if (vgpu->gtt.scratch_pt[i].page != NULL) {
                         daddr = (dma_addr_t)(vgpu->gtt.scratch_pt[i].page_mfn <<
-                                        GTT_PAGE_SHIFT);
+                                        I915_GTT_PAGE_SHIFT);
                         dma_unmap_page(dev, daddr, 4096, PCI_DMA_BIDIRECTIONAL);
                         __free_page(vgpu->gtt.scratch_pt[i].page);
                         vgpu->gtt.scratch_pt[i].page = NULL;
@@ -2032,7 +2083,7 @@ int intel_vgpu_init_gtt(struct intel_vgpu *vgpu)
         struct intel_vgpu_gtt *gtt = &vgpu->gtt;
         struct intel_vgpu_mm *ggtt_mm;
 
-        hash_init(gtt->guest_page_hash_table);
+        hash_init(gtt->tracked_guest_page_hash_table);
         hash_init(gtt->shadow_page_hash_table);
 
         INIT_LIST_HEAD(&gtt->mm_list_head);
@@ -2285,15 +2336,16 @@ int intel_gvt_init_gtt(struct intel_gvt *gvt)
                 __free_page(virt_to_page(page));
                 return -ENOMEM;
         }
-        gvt->gtt.scratch_ggtt_page = virt_to_page(page);
-        gvt->gtt.scratch_ggtt_mfn = (unsigned long)(daddr >> GTT_PAGE_SHIFT);
+
+        gvt->gtt.scratch_page = virt_to_page(page);
+        gvt->gtt.scratch_mfn = (unsigned long)(daddr >> I915_GTT_PAGE_SHIFT);
 
         if (enable_out_of_sync) {
                 ret = setup_spt_oos(gvt);
                 if (ret) {
                         gvt_err("fail to initialize SPT oos\n");
                         dma_unmap_page(dev, daddr, 4096, PCI_DMA_BIDIRECTIONAL);
-                        __free_page(gvt->gtt.scratch_ggtt_page);
+                        __free_page(gvt->gtt.scratch_page);
                         return ret;
                 }
         }
@@ -2312,12 +2364,12 @@ int intel_gvt_init_gtt(struct intel_gvt *gvt)
 void intel_gvt_clean_gtt(struct intel_gvt *gvt)
 {
         struct device *dev = &gvt->dev_priv->drm.pdev->dev;
-        dma_addr_t daddr = (dma_addr_t)(gvt->gtt.scratch_ggtt_mfn <<
-                                        GTT_PAGE_SHIFT);
+        dma_addr_t daddr = (dma_addr_t)(gvt->gtt.scratch_mfn <<
+                                        I915_GTT_PAGE_SHIFT);
 
         dma_unmap_page(dev, daddr, 4096, PCI_DMA_BIDIRECTIONAL);
 
-        __free_page(gvt->gtt.scratch_ggtt_page);
+        __free_page(gvt->gtt.scratch_page);
 
         if (enable_out_of_sync)
                 clean_spt_oos(gvt);
@@ -2343,7 +2395,7 @@ void intel_vgpu_reset_ggtt(struct intel_vgpu *vgpu)
 
         memset(&e, 0, sizeof(struct intel_gvt_gtt_entry));
         e.type = GTT_TYPE_GGTT_PTE;
-        ops->set_pfn(&e, gvt->gtt.scratch_ggtt_mfn);
+        ops->set_pfn(&e, gvt->gtt.scratch_mfn);
         e.val64 |= _PAGE_PRESENT;
 
         index = vgpu_aperture_gmadr_base(vgpu) >> PAGE_SHIFT;
@@ -2369,8 +2421,6 @@ void intel_vgpu_reset_ggtt(struct intel_vgpu *vgpu)
  */
 void intel_vgpu_reset_gtt(struct intel_vgpu *vgpu)
 {
-        int i;
-
         ppgtt_free_all_shadow_page(vgpu);
 
         /* Shadow pages are only created when there is no page
@@ -2380,11 +2430,4 @@ void intel_vgpu_reset_gtt(struct intel_vgpu *vgpu)
         intel_vgpu_free_mm(vgpu, INTEL_GVT_MM_PPGTT);
 
         intel_vgpu_reset_ggtt(vgpu);
-
-        /* clear scratch page for security */
-        for (i = GTT_TYPE_PPGTT_PTE_PT; i < GTT_TYPE_MAX; i++) {
-                if (vgpu->gtt.scratch_pt[i].page != NULL)
-                        memset(page_address(vgpu->gtt.scratch_pt[i].page),
-                                0, PAGE_SIZE);
-        }
 }
diff --git a/drivers/gpu/drm/i915/gvt/gtt.h b/drivers/gpu/drm/i915/gvt/gtt.h
index 30a4c8d16026..f98c1c19b4cb 100644
--- a/drivers/gpu/drm/i915/gvt/gtt.h
+++ b/drivers/gpu/drm/i915/gvt/gtt.h
@@ -34,9 +34,8 @@
 #ifndef _GVT_GTT_H_
 #define _GVT_GTT_H_
 
-#define GTT_PAGE_SHIFT          12
-#define GTT_PAGE_SIZE           (1UL << GTT_PAGE_SHIFT)
-#define GTT_PAGE_MASK           (~(GTT_PAGE_SIZE-1))
+#define I915_GTT_PAGE_SHIFT         12
+#define I915_GTT_PAGE_MASK              (~(I915_GTT_PAGE_SIZE - 1))
 
 struct intel_vgpu_mm;
 
@@ -63,6 +62,7 @@ struct intel_gvt_gtt_pte_ops {
                          struct intel_vgpu *vgpu);
         bool (*test_present)(struct intel_gvt_gtt_entry *e);
         void (*clear_present)(struct intel_gvt_gtt_entry *e);
+        void (*set_present)(struct intel_gvt_gtt_entry *e);
         bool (*test_pse)(struct intel_gvt_gtt_entry *e);
         void (*set_pfn)(struct intel_gvt_gtt_entry *e, unsigned long pfn);
         unsigned long (*get_pfn)(struct intel_gvt_gtt_entry *e);
@@ -86,8 +86,8 @@ struct intel_gvt_gtt {
         struct list_head oos_page_free_list_head;
         struct list_head mm_lru_list_head;
 
-        struct page *scratch_ggtt_page;
-        unsigned long scratch_ggtt_mfn;
+        struct page *scratch_page;
+        unsigned long scratch_mfn;
 };
 
 enum {
@@ -193,18 +193,16 @@ struct intel_vgpu_scratch_pt {
         unsigned long page_mfn;
 };
 
-
 struct intel_vgpu_gtt {
         struct intel_vgpu_mm *ggtt_mm;
         unsigned long active_ppgtt_mm_bitmap;
         struct list_head mm_list_head;
         DECLARE_HASHTABLE(shadow_page_hash_table, INTEL_GVT_GTT_HASH_BITS);
-        DECLARE_HASHTABLE(guest_page_hash_table, INTEL_GVT_GTT_HASH_BITS);
-        atomic_t n_write_protected_guest_page;
+        DECLARE_HASHTABLE(tracked_guest_page_hash_table, INTEL_GVT_GTT_HASH_BITS);
+        atomic_t n_tracked_guest_page;
         struct list_head oos_page_list_head;
         struct list_head post_shadow_list_head;
         struct intel_vgpu_scratch_pt scratch_pt[GTT_TYPE_MAX];
-
 };
 
 extern int intel_vgpu_init_gtt(struct intel_vgpu *vgpu);
@@ -228,12 +226,16 @@ struct intel_vgpu_shadow_page {
         unsigned long mfn;
 };
 
-struct intel_vgpu_guest_page {
+struct intel_vgpu_page_track {
         struct hlist_node node;
-        bool writeprotection;
+        bool tracked;
         unsigned long gfn;
         int (*handler)(void *, u64, void *, int);
         void *data;
+};
+
+struct intel_vgpu_guest_page {
+        struct intel_vgpu_page_track track;
         unsigned long write_cnt;
         struct intel_vgpu_oos_page *oos_page;
 };
@@ -243,7 +245,7 @@ struct intel_vgpu_oos_page {
         struct list_head list;
         struct list_head vm_list;
         int id;
-        unsigned char mem[GTT_PAGE_SIZE];
+        unsigned char mem[I915_GTT_PAGE_SIZE];
 };
 
 #define GTT_ENTRY_NUM_IN_ONE_PAGE 512
@@ -258,22 +260,16 @@ struct intel_vgpu_ppgtt_spt {
         struct list_head post_shadow_list;
 };
 
-int intel_vgpu_init_guest_page(struct intel_vgpu *vgpu,
-                struct intel_vgpu_guest_page *guest_page,
+int intel_vgpu_init_page_track(struct intel_vgpu *vgpu,
+                struct intel_vgpu_page_track *t,
                 unsigned long gfn,
                 int (*handler)(void *gp, u64, void *, int),
                 void *data);
 
-void intel_vgpu_clean_guest_page(struct intel_vgpu *vgpu,
-                struct intel_vgpu_guest_page *guest_page);
-
-int intel_vgpu_set_guest_page_writeprotection(struct intel_vgpu *vgpu,
-                struct intel_vgpu_guest_page *guest_page);
-
-void intel_vgpu_clear_guest_page_writeprotection(struct intel_vgpu *vgpu,
-                struct intel_vgpu_guest_page *guest_page);
+void intel_vgpu_clean_page_track(struct intel_vgpu *vgpu,
+                struct intel_vgpu_page_track *t);
 
-struct intel_vgpu_guest_page *intel_vgpu_find_guest_page(
+struct intel_vgpu_page_track *intel_vgpu_find_tracked_page(
                 struct intel_vgpu *vgpu, unsigned long gfn);
 
 int intel_vgpu_sync_oos_pages(struct intel_vgpu *vgpu);
diff --git a/drivers/gpu/drm/i915/gvt/gvt.c b/drivers/gpu/drm/i915/gvt/gvt.c
index aaa347f8620c..3a74a408a966 100644
--- a/drivers/gpu/drm/i915/gvt/gvt.c
+++ b/drivers/gpu/drm/i915/gvt/gvt.c
@@ -36,6 +36,8 @@
 
 #include "i915_drv.h"
 #include "gvt.h"
+#include <linux/vfio.h>
+#include <linux/mdev.h>
 
 struct intel_gvt_host intel_gvt_host;
 
@@ -44,6 +46,129 @@ static const char * const supported_hypervisors[] = {
         [INTEL_GVT_HYPERVISOR_KVM] = "KVM",
 };
 
+static struct intel_vgpu_type *intel_gvt_find_vgpu_type(struct intel_gvt *gvt,
+                const char *name)
+{
+        int i;
+        struct intel_vgpu_type *t;
+        const char *driver_name = dev_driver_string(
+                        &gvt->dev_priv->drm.pdev->dev);
+
+        for (i = 0; i < gvt->num_types; i++) {
+                t = &gvt->types[i];
+                if (!strncmp(t->name, name + strlen(driver_name) + 1,
+                        sizeof(t->name)))
+                        return t;
+        }
+
+        return NULL;
+}
+
+static ssize_t available_instances_show(struct kobject *kobj,
+                                        struct device *dev, char *buf)
+{
+        struct intel_vgpu_type *type;
+        unsigned int num = 0;
+        void *gvt = kdev_to_i915(dev)->gvt;
+
+        type = intel_gvt_find_vgpu_type(gvt, kobject_name(kobj));
+        if (!type)
+                num = 0;
+        else
+                num = type->avail_instance;
+
+        return sprintf(buf, "%u\n", num);
+}
+
+static ssize_t device_api_show(struct kobject *kobj, struct device *dev,
+                char *buf)
+{
+        return sprintf(buf, "%s\n", VFIO_DEVICE_API_PCI_STRING);
+}
+
+static ssize_t description_show(struct kobject *kobj, struct device *dev,
+                char *buf)
+{
+        struct intel_vgpu_type *type;
+        void *gvt = kdev_to_i915(dev)->gvt;
+
+        type = intel_gvt_find_vgpu_type(gvt, kobject_name(kobj));
+        if (!type)
+                return 0;
+
+        return sprintf(buf, "low_gm_size: %dMB\nhigh_gm_size: %dMB\n"
+                       "fence: %d\nresolution: %s\n"
+                       "weight: %d\n",
+                       BYTES_TO_MB(type->low_gm_size),
+                       BYTES_TO_MB(type->high_gm_size),
+                       type->fence, vgpu_edid_str(type->resolution),
+                       type->weight);
+}
+
+static MDEV_TYPE_ATTR_RO(available_instances);
+static MDEV_TYPE_ATTR_RO(device_api);
+static MDEV_TYPE_ATTR_RO(description);
+
+static struct attribute *gvt_type_attrs[] = {
+        &mdev_type_attr_available_instances.attr,
+        &mdev_type_attr_device_api.attr,
+        &mdev_type_attr_description.attr,
+        NULL,
+};
+
+static struct attribute_group *gvt_vgpu_type_groups[] = {
+        [0 ... NR_MAX_INTEL_VGPU_TYPES - 1] = NULL,
+};
+
+static bool intel_get_gvt_attrs(struct attribute ***type_attrs,
+                struct attribute_group ***intel_vgpu_type_groups)
+{
+        *type_attrs = gvt_type_attrs;
+        *intel_vgpu_type_groups = gvt_vgpu_type_groups;
+        return true;
+}
+
+static bool intel_gvt_init_vgpu_type_groups(struct intel_gvt *gvt)
+{
+        int i, j;
+        struct intel_vgpu_type *type;
+        struct attribute_group *group;
+
+        for (i = 0; i < gvt->num_types; i++) {
+                type = &gvt->types[i];
+
+                group = kzalloc(sizeof(struct attribute_group), GFP_KERNEL);
+                if (WARN_ON(!group))
+                        goto unwind;
+
+                group->name = type->name;
+                group->attrs = gvt_type_attrs;
+                gvt_vgpu_type_groups[i] = group;
+        }
+
+        return true;
+
+unwind:
+        for (j = 0; j < i; j++) {
+                group = gvt_vgpu_type_groups[j];
+                kfree(group);
+        }
+
+        return false;
+}
+
+static void intel_gvt_cleanup_vgpu_type_groups(struct intel_gvt *gvt)
+{
+        int i;
+        struct attribute_group *group;
+
+        for (i = 0; i < gvt->num_types; i++) {
+                group = gvt_vgpu_type_groups[i];
+                gvt_vgpu_type_groups[i] = NULL;
+                kfree(group);
+        }
+}
+
 static const struct intel_gvt_ops intel_gvt_ops = {
         .emulate_cfg_read = intel_vgpu_emulate_cfg_read,
         .emulate_cfg_write = intel_vgpu_emulate_cfg_write,
@@ -54,6 +179,8 @@ static const struct intel_gvt_ops intel_gvt_ops = {
         .vgpu_reset = intel_gvt_reset_vgpu,
         .vgpu_activate = intel_gvt_activate_vgpu,
         .vgpu_deactivate = intel_gvt_deactivate_vgpu,
+        .gvt_find_vgpu_type = intel_gvt_find_vgpu_type,
+        .get_gvt_attrs = intel_get_gvt_attrs,
 };
 
 /**
@@ -191,17 +318,18 @@ void intel_gvt_clean_device(struct drm_i915_private *dev_priv)
         if (WARN_ON(!gvt))
                 return;
 
+        intel_gvt_debugfs_clean(gvt);
         clean_service_thread(gvt);
         intel_gvt_clean_cmd_parser(gvt);
         intel_gvt_clean_sched_policy(gvt);
         intel_gvt_clean_workload_scheduler(gvt);
-        intel_gvt_clean_opregion(gvt);
         intel_gvt_clean_gtt(gvt);
         intel_gvt_clean_irq(gvt);
         intel_gvt_clean_mmio_info(gvt);
         intel_gvt_free_firmware(gvt);
 
         intel_gvt_hypervisor_host_exit(&dev_priv->drm.pdev->dev, gvt);
+        intel_gvt_cleanup_vgpu_type_groups(gvt);
         intel_gvt_clean_vgpu_types(gvt);
 
         idr_destroy(&gvt->vgpu_idr);
@@ -268,13 +396,9 @@ int intel_gvt_init_device(struct drm_i915_private *dev_priv)
         if (ret)
                 goto out_clean_irq;
 
-        ret = intel_gvt_init_opregion(gvt);
-        if (ret)
-                goto out_clean_gtt;
-
         ret = intel_gvt_init_workload_scheduler(gvt);
         if (ret)
-                goto out_clean_opregion;
+                goto out_clean_gtt;
 
         ret = intel_gvt_init_sched_policy(gvt);
         if (ret)
@@ -292,6 +416,12 @@ int intel_gvt_init_device(struct drm_i915_private *dev_priv)
         if (ret)
                 goto out_clean_thread;
 
+        ret = intel_gvt_init_vgpu_type_groups(gvt);
+        if (ret == false) {
+                gvt_err("failed to init vgpu type groups: %d\n", ret);
+                goto out_clean_types;
+        }
+
         ret = intel_gvt_hypervisor_host_init(&dev_priv->drm.pdev->dev, gvt,
                                 &intel_gvt_ops);
         if (ret) {
@@ -307,6 +437,10 @@ int intel_gvt_init_device(struct drm_i915_private *dev_priv)
         }
         gvt->idle_vgpu = vgpu;
 
+        ret = intel_gvt_debugfs_init(gvt);
+        if (ret)
+                gvt_err("debugfs registeration failed, go on.\n");
+
         gvt_dbg_core("gvt device initialization is done\n");
         dev_priv->gvt = gvt;
         return 0;
@@ -321,8 +455,6 @@ out_clean_sched_policy:
         intel_gvt_clean_sched_policy(gvt);
 out_clean_workload_scheduler:
         intel_gvt_clean_workload_scheduler(gvt);
-out_clean_opregion:
-        intel_gvt_clean_opregion(gvt);
 out_clean_gtt:
         intel_gvt_clean_gtt(gvt);
 out_clean_irq:
diff --git a/drivers/gpu/drm/i915/gvt/gvt.h b/drivers/gpu/drm/i915/gvt/gvt.h
index 9c2e7c0aa38f..393066726993 100644
--- a/drivers/gpu/drm/i915/gvt/gvt.h
+++ b/drivers/gpu/drm/i915/gvt/gvt.h
@@ -125,7 +125,6 @@ struct intel_vgpu_irq {
 struct intel_vgpu_opregion {
         void *va;
         u32 gfn[INTEL_GVT_OPREGION_PAGES];
-        struct page *pages[INTEL_GVT_OPREGION_PAGES];
 };
 
 #define vgpu_opregion(vgpu) (&(vgpu->opregion))
@@ -142,6 +141,33 @@ struct vgpu_sched_ctl {
         int weight;
 };
 
+enum {
+        INTEL_VGPU_EXECLIST_SUBMISSION = 1,
+        INTEL_VGPU_GUC_SUBMISSION,
+};
+
+struct intel_vgpu_submission_ops {
+        const char *name;
+        int (*init)(struct intel_vgpu *vgpu);
+        void (*clean)(struct intel_vgpu *vgpu);
+        void (*reset)(struct intel_vgpu *vgpu, unsigned long engine_mask);
+};
+
+struct intel_vgpu_submission {
+        struct intel_vgpu_execlist execlist[I915_NUM_ENGINES];
+        struct list_head workload_q_head[I915_NUM_ENGINES];
+        struct kmem_cache *workloads;
+        atomic_t running_workload_num;
+        struct i915_gem_context *shadow_ctx;
+        DECLARE_BITMAP(shadow_ctx_desc_updated, I915_NUM_ENGINES);
+        DECLARE_BITMAP(tlb_handle_pending, I915_NUM_ENGINES);
+        void *ring_scan_buffer[I915_NUM_ENGINES];
+        int ring_scan_buffer_size[I915_NUM_ENGINES];
+        const struct intel_vgpu_submission_ops *ops;
+        int virtual_submission_interface;
+        bool active;
+};
+
 struct intel_vgpu {
         struct intel_gvt *gvt;
         int id;
@@ -161,16 +187,10 @@ struct intel_vgpu {
         struct intel_vgpu_gtt gtt;
         struct intel_vgpu_opregion opregion;
         struct intel_vgpu_display display;
-        struct intel_vgpu_execlist execlist[I915_NUM_ENGINES];
-        struct list_head workload_q_head[I915_NUM_ENGINES];
-        struct kmem_cache *workloads;
-        atomic_t running_workload_num;
-        /* 1/2K for each reserve ring buffer */
-        void *reserve_ring_buffer_va[I915_NUM_ENGINES];
-        int reserve_ring_buffer_size[I915_NUM_ENGINES];
-        DECLARE_BITMAP(tlb_handle_pending, I915_NUM_ENGINES);
-        struct i915_gem_context *shadow_ctx;
-        DECLARE_BITMAP(shadow_ctx_desc_updated, I915_NUM_ENGINES);
+        struct intel_vgpu_submission submission;
+        u32 hws_pga[I915_NUM_ENGINES];
+
+        struct dentry *debugfs;
 
 #if IS_ENABLED(CONFIG_DRM_I915_GVT_KVMGT)
         struct {
@@ -190,6 +210,10 @@ struct intel_vgpu {
 #endif
 };
 
+/* validating GM healthy status*/
+#define vgpu_is_vm_unhealthy(ret_val) \
+        (((ret_val) == -EBADRQC) || ((ret_val) == -EFAULT))
+
 struct intel_gvt_gm {
         unsigned long vgpu_allocated_low_gm_size;
         unsigned long vgpu_allocated_high_gm_size;
@@ -231,7 +255,7 @@ struct intel_gvt_mmio {
         unsigned int num_mmio_block;
 
         DECLARE_HASHTABLE(mmio_info_table, INTEL_GVT_MMIO_HASH_BITS);
-        unsigned int num_tracked_mmio;
+        unsigned long num_tracked_mmio;
 };
 
 struct intel_gvt_firmware {
@@ -240,11 +264,6 @@ struct intel_gvt_firmware {
         bool firmware_loaded;
 };
 
-struct intel_gvt_opregion {
-        void *opregion_va;
-        u32 opregion_pa;
-};
-
 #define NR_MAX_INTEL_VGPU_TYPES 20
 struct intel_vgpu_type {
         char name[16];
@@ -268,7 +287,6 @@ struct intel_gvt {
         struct intel_gvt_firmware firmware;
         struct intel_gvt_irq irq;
         struct intel_gvt_gtt gtt;
-        struct intel_gvt_opregion opregion;
         struct intel_gvt_workload_scheduler scheduler;
         struct notifier_block shadow_ctx_notifier_block[I915_NUM_ENGINES];
         DECLARE_HASHTABLE(cmd_table, GVT_CMD_HASH_BITS);
@@ -279,6 +297,8 @@ struct intel_gvt {
         struct task_struct *service_thread;
         wait_queue_head_t service_thread_wq;
         unsigned long service_request;
+
+        struct dentry *debugfs_root;
 };
 
 static inline struct intel_gvt *to_gvt(struct drm_i915_private *i915)
@@ -484,9 +504,6 @@ static inline u64 intel_vgpu_get_bar_gpa(struct intel_vgpu *vgpu, int bar)
                         PCI_BASE_ADDRESS_MEM_MASK;
 }
 
-void intel_gvt_clean_opregion(struct intel_gvt *gvt);
-int intel_gvt_init_opregion(struct intel_gvt *gvt);
-
 void intel_vgpu_clean_opregion(struct intel_vgpu *vgpu);
 int intel_vgpu_init_opregion(struct intel_vgpu *vgpu, u32 gpa);
 
@@ -494,6 +511,7 @@ int intel_vgpu_emulate_opregion_request(struct intel_vgpu *vgpu, u32 swsci);
 void populate_pvinfo_page(struct intel_vgpu *vgpu);
 
 int intel_gvt_scan_and_shadow_workload(struct intel_vgpu_workload *workload);
+void enter_failsafe_mode(struct intel_vgpu *vgpu, int reason);
 
 struct intel_gvt_ops {
         int (*emulate_cfg_read)(struct intel_vgpu *, unsigned int, void *,
@@ -510,12 +528,17 @@ struct intel_gvt_ops {
         void (*vgpu_reset)(struct intel_vgpu *);
         void (*vgpu_activate)(struct intel_vgpu *);
         void (*vgpu_deactivate)(struct intel_vgpu *);
+        struct intel_vgpu_type *(*gvt_find_vgpu_type)(struct intel_gvt *gvt,
+                        const char *name);
+        bool (*get_gvt_attrs)(struct attribute ***type_attrs,
+                        struct attribute_group ***intel_vgpu_type_groups);
 };
 
 
 enum {
         GVT_FAILSAFE_UNSUPPORTED_GUEST,
         GVT_FAILSAFE_INSUFFICIENT_RESOURCE,
+        GVT_FAILSAFE_GUEST_ERR,
 };
 
 static inline void mmio_hw_access_pre(struct drm_i915_private *dev_priv)
@@ -591,6 +614,12 @@ static inline bool intel_gvt_mmio_has_mode_mask(
         return gvt->mmio.mmio_attribute[offset >> 2] & F_MODE_MASK;
 }
 
+int intel_gvt_debugfs_add_vgpu(struct intel_vgpu *vgpu);
+void intel_gvt_debugfs_remove_vgpu(struct intel_vgpu *vgpu);
+int intel_gvt_debugfs_init(struct intel_gvt *gvt);
+void intel_gvt_debugfs_clean(struct intel_gvt *gvt);
+
+
 #include "trace.h"
 #include "mpt.h"
 
diff --git a/drivers/gpu/drm/i915/gvt/handlers.c b/drivers/gpu/drm/i915/gvt/handlers.c
index 44cd5ff5e97d..94fc04210bac 100644
--- a/drivers/gpu/drm/i915/gvt/handlers.c
+++ b/drivers/gpu/drm/i915/gvt/handlers.c
@@ -137,17 +137,26 @@ static int new_mmio_info(struct intel_gvt *gvt,
         return 0;
 }
 
-static int render_mmio_to_ring_id(struct intel_gvt *gvt, unsigned int reg)
+/**
+ * intel_gvt_render_mmio_to_ring_id - convert a mmio offset into ring id
+ * @gvt: a GVT device
+ * @offset: register offset
+ *
+ * Returns:
+ * Ring ID on success, negative error code if failed.
+ */
+int intel_gvt_render_mmio_to_ring_id(struct intel_gvt *gvt,
+                unsigned int offset)
 {
         enum intel_engine_id id;
         struct intel_engine_cs *engine;
 
-        reg &= ~GENMASK(11, 0);
+        offset &= ~GENMASK(11, 0);
         for_each_engine(engine, gvt->dev_priv, id) {
-                if (engine->mmio_base == reg)
+                if (engine->mmio_base == offset)
                         return id;
         }
-        return -1;
+        return -ENODEV;
 }
 
 #define offset_to_fence_num(offset) \
@@ -157,7 +166,7 @@ static int render_mmio_to_ring_id(struct intel_gvt *gvt, unsigned int reg)
         (num * 8 + i915_mmio_reg_offset(FENCE_REG_GEN6_LO(0)))
 
 
-static void enter_failsafe_mode(struct intel_vgpu *vgpu, int reason)
+void enter_failsafe_mode(struct intel_vgpu *vgpu, int reason)
 {
         switch (reason) {
         case GVT_FAILSAFE_UNSUPPORTED_GUEST:
@@ -165,6 +174,8 @@ static void enter_failsafe_mode(struct intel_vgpu *vgpu, int reason)
                 break;
         case GVT_FAILSAFE_INSUFFICIENT_RESOURCE:
                 pr_err("Graphics resource is not enough for the guest\n");
+        case GVT_FAILSAFE_GUEST_ERR:
+                pr_err("GVT Internal error  for the guest\n");
         default:
                 break;
         }
@@ -1369,6 +1380,34 @@ static int mailbox_write(struct intel_vgpu *vgpu, unsigned int offset,
         return intel_vgpu_default_mmio_write(vgpu, offset, &value, bytes);
 }
 
+static int hws_pga_write(struct intel_vgpu *vgpu, unsigned int offset,
+                void *p_data, unsigned int bytes)
+{
+        u32 value = *(u32 *)p_data;
+        int ring_id = intel_gvt_render_mmio_to_ring_id(vgpu->gvt, offset);
+
+        if (!intel_gvt_ggtt_validate_range(vgpu, value, I915_GTT_PAGE_SIZE)) {
+                gvt_vgpu_err("VM(%d) write invalid HWSP address, reg:0x%x, value:0x%x\n",
+                              vgpu->id, offset, value);
+                return -EINVAL;
+        }
+        /*
+         * Need to emulate all the HWSP register write to ensure host can
+         * update the VM CSB status correctly. Here listed registers can
+         * support BDW, SKL or other platforms with same HWSP registers.
+         */
+        if (unlikely(ring_id < 0 || ring_id > I915_NUM_ENGINES)) {
+                gvt_vgpu_err("VM(%d) access unknown hardware status page register:0x%x\n",
+                             vgpu->id, offset);
+                return -EINVAL;
+        }
+        vgpu->hws_pga[ring_id] = value;
+        gvt_dbg_mmio("VM(%d) write: 0x%x to HWSP: 0x%x\n",
+                     vgpu->id, value, offset);
+
+        return intel_vgpu_default_mmio_write(vgpu, offset, &value, bytes);
+}
+
 static int skl_power_well_ctl_write(struct intel_vgpu *vgpu,
                 unsigned int offset, void *p_data, unsigned int bytes)
 {
@@ -1398,18 +1437,36 @@ static int skl_lcpll_write(struct intel_vgpu *vgpu, unsigned int offset,
 static int mmio_read_from_hw(struct intel_vgpu *vgpu,
                 unsigned int offset, void *p_data, unsigned int bytes)
 {
-        struct drm_i915_private *dev_priv = vgpu->gvt->dev_priv;
+        struct intel_gvt *gvt = vgpu->gvt;
+        struct drm_i915_private *dev_priv = gvt->dev_priv;
+        int ring_id;
+        u32 ring_base;
+
+        ring_id = intel_gvt_render_mmio_to_ring_id(gvt, offset);
+        /**
+         * Read HW reg in following case
+         * a. the offset isn't a ring mmio
+         * b. the offset's ring is running on hw.
+         * c. the offset is ring time stamp mmio
+         */
+        if (ring_id >= 0)
+                ring_base = dev_priv->engine[ring_id]->mmio_base;
+
+        if (ring_id < 0 || vgpu  == gvt->scheduler.engine_owner[ring_id] ||
+            offset == i915_mmio_reg_offset(RING_TIMESTAMP(ring_base)) ||
+            offset == i915_mmio_reg_offset(RING_TIMESTAMP_UDW(ring_base))) {
+                mmio_hw_access_pre(dev_priv);
+                vgpu_vreg(vgpu, offset) = I915_READ(_MMIO(offset));
+                mmio_hw_access_post(dev_priv);
+        }
 
-        mmio_hw_access_pre(dev_priv);
-        vgpu_vreg(vgpu, offset) = I915_READ(_MMIO(offset));
-        mmio_hw_access_post(dev_priv);
         return intel_vgpu_default_mmio_read(vgpu, offset, p_data, bytes);
 }
 
 static int elsp_mmio_write(struct intel_vgpu *vgpu, unsigned int offset,
                 void *p_data, unsigned int bytes)
 {
-        int ring_id = render_mmio_to_ring_id(vgpu->gvt, offset);
+        int ring_id = intel_gvt_render_mmio_to_ring_id(vgpu->gvt, offset);
         struct intel_vgpu_execlist *execlist;
         u32 data = *(u32 *)p_data;
         int ret = 0;
@@ -1417,9 +1474,9 @@ static int elsp_mmio_write(struct intel_vgpu *vgpu, unsigned int offset,
         if (WARN_ON(ring_id < 0 || ring_id > I915_NUM_ENGINES - 1))
                 return -EINVAL;
 
-        execlist = &vgpu->execlist[ring_id];
+        execlist = &vgpu->submission.execlist[ring_id];
 
-        execlist->elsp_dwords.data[execlist->elsp_dwords.index] = data;
+        execlist->elsp_dwords.data[3 - execlist->elsp_dwords.index] = data;
         if (execlist->elsp_dwords.index == 3) {
                 ret = intel_vgpu_submit_execlist(vgpu, ring_id);
                 if(ret)
@@ -1435,9 +1492,11 @@ static int elsp_mmio_write(struct intel_vgpu *vgpu, unsigned int offset,
 static int ring_mode_mmio_write(struct intel_vgpu *vgpu, unsigned int offset,
                 void *p_data, unsigned int bytes)
 {
+        struct intel_vgpu_submission *s = &vgpu->submission;
         u32 data = *(u32 *)p_data;
-        int ring_id = render_mmio_to_ring_id(vgpu->gvt, offset);
+        int ring_id = intel_gvt_render_mmio_to_ring_id(vgpu->gvt, offset);
         bool enable_execlist;
+        int ret;
 
         write_vreg(vgpu, offset, p_data, bytes);
 
@@ -1459,8 +1518,18 @@ static int ring_mode_mmio_write(struct intel_vgpu *vgpu, unsigned int offset,
                                 (enable_execlist ? "enabling" : "disabling"),
                                 ring_id);
 
-                if (enable_execlist)
-                        intel_vgpu_start_schedule(vgpu);
+                if (!enable_execlist)
+                        return 0;
+
+                if (s->active)
+                        return 0;
+
+                ret = intel_vgpu_select_submission_ops(vgpu,
+                                INTEL_VGPU_EXECLIST_SUBMISSION);
+                if (ret)
+                        return ret;
+
+                intel_vgpu_start_schedule(vgpu);
         }
         return 0;
 }
@@ -1492,7 +1561,7 @@ static int gvt_reg_tlb_control_handler(struct intel_vgpu *vgpu,
         default:
                 return -EINVAL;
         }
-        set_bit(id, (void *)vgpu->tlb_handle_pending);
+        set_bit(id, (void *)vgpu->submission.tlb_handle_pending);
 
         return 0;
 }
@@ -2478,7 +2547,7 @@ static int init_broadwell_mmio_info(struct intel_gvt *gvt)
         MMIO_RING_F(RING_REG, 32, 0, 0, 0, D_BDW_PLUS, NULL, NULL);
 #undef RING_REG
 
-        MMIO_RING_GM_RDR(RING_HWS_PGA, D_BDW_PLUS, NULL, NULL);
+        MMIO_RING_GM_RDR(RING_HWS_PGA, D_BDW_PLUS, NULL, hws_pga_write);
 
         MMIO_DFH(HDC_CHICKEN0, D_BDW_PLUS, F_MODE_MASK | F_CMD_ACCESS, NULL, NULL);
 
@@ -2879,14 +2948,46 @@ int intel_gvt_setup_mmio_info(struct intel_gvt *gvt)
         gvt->mmio.mmio_block = mmio_blocks;
         gvt->mmio.num_mmio_block = ARRAY_SIZE(mmio_blocks);
 
-        gvt_dbg_mmio("traced %u virtual mmio registers\n",
-                     gvt->mmio.num_tracked_mmio);
         return 0;
 err:
         intel_gvt_clean_mmio_info(gvt);
         return ret;
 }
 
+/**
+ * intel_gvt_for_each_tracked_mmio - iterate each tracked mmio
+ * @gvt: a GVT device
+ * @handler: the handler
+ * @data: private data given to handler
+ *
+ * Returns:
+ * Zero on success, negative error code if failed.
+ */
+int intel_gvt_for_each_tracked_mmio(struct intel_gvt *gvt,
+        int (*handler)(struct intel_gvt *gvt, u32 offset, void *data),
+        void *data)
+{
+        struct gvt_mmio_block *block = gvt->mmio.mmio_block;
+        struct intel_gvt_mmio_info *e;
+        int i, j, ret;
+
+        hash_for_each(gvt->mmio.mmio_info_table, i, e, node) {
+                ret = handler(gvt, e->offset, data);
+                if (ret)
+                        return ret;
+        }
+
+        for (i = 0; i < gvt->mmio.num_mmio_block; i++, block++) {
+                for (j = 0; j < block->size; j += 4) {
+                        ret = handler(gvt,
+                                INTEL_GVT_MMIO_OFFSET(block->offset) + j,
+                                data);
+                        if (ret)
+                                return ret;
+                }
+        }
+        return 0;
+}
 
 /**
  * intel_vgpu_default_mmio_read - default MMIO read handler
diff --git a/drivers/gpu/drm/i915/gvt/kvmgt.c b/drivers/gpu/drm/i915/gvt/kvmgt.c
index 96060920a6fe..110f07e8bcfb 100644
--- a/drivers/gpu/drm/i915/gvt/kvmgt.c
+++ b/drivers/gpu/drm/i915/gvt/kvmgt.c
@@ -248,120 +248,6 @@ static void gvt_cache_destroy(struct intel_vgpu *vgpu)
         }
 }
 
-static struct intel_vgpu_type *intel_gvt_find_vgpu_type(struct intel_gvt *gvt,
-                const char *name)
-{
-        int i;
-        struct intel_vgpu_type *t;
-        const char *driver_name = dev_driver_string(
-                        &gvt->dev_priv->drm.pdev->dev);
-
-        for (i = 0; i < gvt->num_types; i++) {
-                t = &gvt->types[i];
-                if (!strncmp(t->name, name + strlen(driver_name) + 1,
-                        sizeof(t->name)))
-                        return t;
-        }
-
-        return NULL;
-}
-
-static ssize_t available_instances_show(struct kobject *kobj,
-                                        struct device *dev, char *buf)
-{
-        struct intel_vgpu_type *type;
-        unsigned int num = 0;
-        void *gvt = kdev_to_i915(dev)->gvt;
-
-        type = intel_gvt_find_vgpu_type(gvt, kobject_name(kobj));
-        if (!type)
-                num = 0;
-        else
-                num = type->avail_instance;
-
-        return sprintf(buf, "%u\n", num);
-}
-
-static ssize_t device_api_show(struct kobject *kobj, struct device *dev,
-                char *buf)
-{
-        return sprintf(buf, "%s\n", VFIO_DEVICE_API_PCI_STRING);
-}
-
-static ssize_t description_show(struct kobject *kobj, struct device *dev,
-                char *buf)
-{
-        struct intel_vgpu_type *type;
-        void *gvt = kdev_to_i915(dev)->gvt;
-
-        type = intel_gvt_find_vgpu_type(gvt, kobject_name(kobj));
-        if (!type)
-                return 0;
-
-        return sprintf(buf, "low_gm_size: %dMB\nhigh_gm_size: %dMB\n"
-                       "fence: %d\nresolution: %s\n"
-                       "weight: %d\n",
-                       BYTES_TO_MB(type->low_gm_size),
-                       BYTES_TO_MB(type->high_gm_size),
-                       type->fence, vgpu_edid_str(type->resolution),
-                       type->weight);
-}
-
-static MDEV_TYPE_ATTR_RO(available_instances);
-static MDEV_TYPE_ATTR_RO(device_api);
-static MDEV_TYPE_ATTR_RO(description);
-
-static struct attribute *type_attrs[] = {
-        &mdev_type_attr_available_instances.attr,
-        &mdev_type_attr_device_api.attr,
-        &mdev_type_attr_description.attr,
-        NULL,
-};
-
-static struct attribute_group *intel_vgpu_type_groups[] = {
-        [0 ... NR_MAX_INTEL_VGPU_TYPES - 1] = NULL,
-};
-
-static bool intel_gvt_init_vgpu_type_groups(struct intel_gvt *gvt)
-{
-        int i, j;
-        struct intel_vgpu_type *type;
-        struct attribute_group *group;
-
-        for (i = 0; i < gvt->num_types; i++) {
-                type = &gvt->types[i];
-
-                group = kzalloc(sizeof(struct attribute_group), GFP_KERNEL);
-                if (WARN_ON(!group))
-                        goto unwind;
-
-                group->name = type->name;
-                group->attrs = type_attrs;
-                intel_vgpu_type_groups[i] = group;
-        }
-
-        return true;
-
-unwind:
-        for (j = 0; j < i; j++) {
-                group = intel_vgpu_type_groups[j];
-                kfree(group);
-        }
-
-        return false;
-}
-
-static void intel_gvt_cleanup_vgpu_type_groups(struct intel_gvt *gvt)
-{
-        int i;
-        struct attribute_group *group;
-
-        for (i = 0; i < gvt->num_types; i++) {
-                group = intel_vgpu_type_groups[i];
-                kfree(group);
-        }
-}
-
 static void kvmgt_protect_table_init(struct kvmgt_guest_info *info)
 {
         hash_init(info->ptable);
@@ -441,7 +327,7 @@ static int intel_vgpu_create(struct kobject *kobj, struct mdev_device *mdev)
         pdev = mdev_parent_dev(mdev);
         gvt = kdev_to_i915(pdev)->gvt;
 
-        type = intel_gvt_find_vgpu_type(gvt, kobject_name(kobj));
+        type = intel_gvt_ops->gvt_find_vgpu_type(gvt, kobject_name(kobj));
         if (!type) {
                 gvt_vgpu_err("failed to find type %s to create\n",
                                                 kobject_name(kobj));
@@ -1188,7 +1074,7 @@ hw_id_show(struct device *dev, struct device_attribute *attr,
                 struct intel_vgpu *vgpu = (struct intel_vgpu *)
                         mdev_get_drvdata(mdev);
                 return sprintf(buf, "%u\n",
-                               vgpu->shadow_ctx->hw_id);
+                               vgpu->submission.shadow_ctx->hw_id);
         }
         return sprintf(buf, "\n");
 }
@@ -1212,8 +1098,7 @@ static const struct attribute_group *intel_vgpu_groups[] = {
         NULL,
 };
 
-static const struct mdev_parent_ops intel_vgpu_ops = {
-        .supported_type_groups  = intel_vgpu_type_groups,
+static struct mdev_parent_ops intel_vgpu_ops = {
         .mdev_attr_groups       = intel_vgpu_groups,
         .create                 = intel_vgpu_create,
         .remove                 = intel_vgpu_remove,
@@ -1229,17 +1114,20 @@ static const struct mdev_parent_ops intel_vgpu_ops = {
 
 static int kvmgt_host_init(struct device *dev, void *gvt, const void *ops)
 {
-        if (!intel_gvt_init_vgpu_type_groups(gvt))
-                return -EFAULT;
+        struct attribute **kvm_type_attrs;
+        struct attribute_group **kvm_vgpu_type_groups;
 
         intel_gvt_ops = ops;
+        if (!intel_gvt_ops->get_gvt_attrs(&kvm_type_attrs,
+                        &kvm_vgpu_type_groups))
+                return -EFAULT;
+        intel_vgpu_ops.supported_type_groups = kvm_vgpu_type_groups;
 
         return mdev_register_device(dev, &intel_vgpu_ops);
 }
 
 static void kvmgt_host_exit(struct device *dev, void *gvt)
 {
-        intel_gvt_cleanup_vgpu_type_groups(gvt);
         mdev_unregister_device(dev);
 }
 
diff --git a/drivers/gpu/drm/i915/gvt/mmio.c b/drivers/gpu/drm/i915/gvt/mmio.c
index 1e1310f50289..4ea0feb5f04d 100644
--- a/drivers/gpu/drm/i915/gvt/mmio.c
+++ b/drivers/gpu/drm/i915/gvt/mmio.c
@@ -117,18 +117,18 @@ static void failsafe_emulate_mmio_rw(struct intel_vgpu *vgpu, uint64_t pa,
                 else
                         memcpy(pt, p_data, bytes);
 
-        } else if (atomic_read(&vgpu->gtt.n_write_protected_guest_page)) {
-                struct intel_vgpu_guest_page *gp;
+        } else if (atomic_read(&vgpu->gtt.n_tracked_guest_page)) {
+                struct intel_vgpu_page_track *t;
 
                 /* Since we enter the failsafe mode early during guest boot,
                  * guest may not have chance to set up its ppgtt table, so
                  * there should not be any wp pages for guest. Keep the wp
                  * related code here in case we need to handle it in furture.
                  */
-                gp = intel_vgpu_find_guest_page(vgpu, pa >> PAGE_SHIFT);
-                if (gp) {
+                t = intel_vgpu_find_tracked_page(vgpu, pa >> PAGE_SHIFT);
+                if (t) {
                         /* remove write protection to prevent furture traps */
-                        intel_vgpu_clean_guest_page(vgpu, gp);
+                        intel_vgpu_clean_page_track(vgpu, t);
                         if (read)
                                 intel_gvt_hypervisor_read_gpa(vgpu, pa,
                                                 p_data, bytes);
@@ -170,17 +170,17 @@ int intel_vgpu_emulate_mmio_read(struct intel_vgpu *vgpu, uint64_t pa,
                 return ret;
         }
 
-        if (atomic_read(&vgpu->gtt.n_write_protected_guest_page)) {
-                struct intel_vgpu_guest_page *gp;
+        if (atomic_read(&vgpu->gtt.n_tracked_guest_page)) {
+                struct intel_vgpu_page_track *t;
 
-                gp = intel_vgpu_find_guest_page(vgpu, pa >> PAGE_SHIFT);
-                if (gp) {
+                t = intel_vgpu_find_tracked_page(vgpu, pa >> PAGE_SHIFT);
+                if (t) {
                         ret = intel_gvt_hypervisor_read_gpa(vgpu, pa,
                                         p_data, bytes);
                         if (ret) {
                                 gvt_vgpu_err("guest page read error %d, "
                                         "gfn 0x%lx, pa 0x%llx, var 0x%x, len %d\n",
-                                        ret, gp->gfn, pa, *(u32 *)p_data,
+                                        ret, t->gfn, pa, *(u32 *)p_data,
                                         bytes);
                         }
                         mutex_unlock(&gvt->lock);
@@ -267,17 +267,17 @@ int intel_vgpu_emulate_mmio_write(struct intel_vgpu *vgpu, uint64_t pa,
                 return ret;
         }
 
-        if (atomic_read(&vgpu->gtt.n_write_protected_guest_page)) {
-                struct intel_vgpu_guest_page *gp;
+        if (atomic_read(&vgpu->gtt.n_tracked_guest_page)) {
+                struct intel_vgpu_page_track *t;
 
-                gp = intel_vgpu_find_guest_page(vgpu, pa >> PAGE_SHIFT);
-                if (gp) {
-                        ret = gp->handler(gp, pa, p_data, bytes);
+                t = intel_vgpu_find_tracked_page(vgpu, pa >> PAGE_SHIFT);
+                if (t) {
+                        ret = t->handler(t, pa, p_data, bytes);
                         if (ret) {
                                 gvt_err("guest page write error %d, "
                                         "gfn 0x%lx, pa 0x%llx, "
                                         "var 0x%x, len %d\n",
-                                        ret, gp->gfn, pa,
+                                        ret, t->gfn, pa,
                                         *(u32 *)p_data, bytes);
                         }
                         mutex_unlock(&gvt->lock);
diff --git a/drivers/gpu/drm/i915/gvt/mmio.h b/drivers/gpu/drm/i915/gvt/mmio.h
index 32cd64ddad26..62709ac351cd 100644
--- a/drivers/gpu/drm/i915/gvt/mmio.h
+++ b/drivers/gpu/drm/i915/gvt/mmio.h
@@ -65,11 +65,17 @@ struct intel_gvt_mmio_info {
         struct hlist_node node;
 };
 
+int intel_gvt_render_mmio_to_ring_id(struct intel_gvt *gvt,
+                unsigned int reg);
 unsigned long intel_gvt_get_device_type(struct intel_gvt *gvt);
 bool intel_gvt_match_device(struct intel_gvt *gvt, unsigned long device);
 
 int intel_gvt_setup_mmio_info(struct intel_gvt *gvt);
 void intel_gvt_clean_mmio_info(struct intel_gvt *gvt);
+int intel_gvt_for_each_tracked_mmio(struct intel_gvt *gvt,
+        int (*handler)(struct intel_gvt *gvt, u32 offset, void *data),
+        void *data);
+
 
 #define INTEL_GVT_MMIO_OFFSET(reg) ({ \
         typeof(reg) __reg = reg; \
diff --git a/drivers/gpu/drm/i915/gvt/mpt.h b/drivers/gpu/drm/i915/gvt/mpt.h
index f0e5487e6688..c436e20ea59e 100644
--- a/drivers/gpu/drm/i915/gvt/mpt.h
+++ b/drivers/gpu/drm/i915/gvt/mpt.h
@@ -154,51 +154,53 @@ static inline unsigned long intel_gvt_hypervisor_virt_to_mfn(void *p)
 }
 
 /**
- * intel_gvt_hypervisor_set_wp_page - set a guest page to write-protected
+ * intel_gvt_hypervisor_enable - set a guest page to write-protected
  * @vgpu: a vGPU
- * @p: intel_vgpu_guest_page
+ * @t: page track data structure
  *
  * Returns:
  * Zero on success, negative error code if failed.
  */
-static inline int intel_gvt_hypervisor_set_wp_page(struct intel_vgpu *vgpu,
-                struct intel_vgpu_guest_page *p)
+static inline int intel_gvt_hypervisor_enable_page_track(
+                struct intel_vgpu *vgpu,
+                struct intel_vgpu_page_track *t)
 {
         int ret;
 
-        if (p->writeprotection)
+        if (t->tracked)
                 return 0;
 
-        ret = intel_gvt_host.mpt->set_wp_page(vgpu->handle, p->gfn);
+        ret = intel_gvt_host.mpt->set_wp_page(vgpu->handle, t->gfn);
         if (ret)
                 return ret;
-        p->writeprotection = true;
-        atomic_inc(&vgpu->gtt.n_write_protected_guest_page);
+        t->tracked = true;
+        atomic_inc(&vgpu->gtt.n_tracked_guest_page);
         return 0;
 }
 
 /**
- * intel_gvt_hypervisor_unset_wp_page - remove the write-protection of a
+ * intel_gvt_hypervisor_disable_page_track - remove the write-protection of a
  * guest page
  * @vgpu: a vGPU
- * @p: intel_vgpu_guest_page
+ * @t: page track data structure
  *
  * Returns:
  * Zero on success, negative error code if failed.
  */
-static inline int intel_gvt_hypervisor_unset_wp_page(struct intel_vgpu *vgpu,
-                struct intel_vgpu_guest_page *p)
+static inline int intel_gvt_hypervisor_disable_page_track(
+                struct intel_vgpu *vgpu,
+                struct intel_vgpu_page_track *t)
 {
         int ret;
 
-        if (!p->writeprotection)
+        if (!t->tracked)
                 return 0;
 
-        ret = intel_gvt_host.mpt->unset_wp_page(vgpu->handle, p->gfn);
+        ret = intel_gvt_host.mpt->unset_wp_page(vgpu->handle, t->gfn);
         if (ret)
                 return ret;
-        p->writeprotection = false;
-        atomic_dec(&vgpu->gtt.n_write_protected_guest_page);
+        t->tracked = false;
+        atomic_dec(&vgpu->gtt.n_tracked_guest_page);
         return 0;
 }
 
diff --git a/drivers/gpu/drm/i915/gvt/opregion.c b/drivers/gpu/drm/i915/gvt/opregion.c
index 311799136d7f..80720e59723a 100644
--- a/drivers/gpu/drm/i915/gvt/opregion.c
+++ b/drivers/gpu/drm/i915/gvt/opregion.c
@@ -25,36 +25,247 @@
 #include "i915_drv.h"
 #include "gvt.h"
 
-static int init_vgpu_opregion(struct intel_vgpu *vgpu, u32 gpa)
+/*
+ * Note: Only for GVT-g virtual VBT generation, other usage must
+ * not do like this.
+ */
+#define _INTEL_BIOS_PRIVATE
+#include "intel_vbt_defs.h"
+
+#define OPREGION_SIGNATURE "IntelGraphicsMem"
+#define MBOX_VBT      (1<<3)
+
+/* device handle */
+#define DEVICE_TYPE_CRT    0x01
+#define DEVICE_TYPE_EFP1   0x04
+#define DEVICE_TYPE_EFP2   0x40
+#define DEVICE_TYPE_EFP3   0x20
+#define DEVICE_TYPE_EFP4   0x10
+
+#define DEV_SIZE        38
+
+struct opregion_header {
+        u8 signature[16];
+        u32 size;
+        u32 opregion_ver;
+        u8 bios_ver[32];
+        u8 vbios_ver[16];
+        u8 driver_ver[16];
+        u32 mboxes;
+        u32 driver_model;
+        u32 pcon;
+        u8 dver[32];
+        u8 rsvd[124];
+} __packed;
+
+struct bdb_data_header {
+        u8 id;
+        u16 size; /* data size */
+} __packed;
+
+struct efp_child_device_config {
+        u16 handle;
+        u16 device_type;
+        u16 device_class;
+        u8 i2c_speed;
+        u8 dp_onboard_redriver; /* 158 */
+        u8 dp_ondock_redriver; /* 158 */
+        u8 hdmi_level_shifter_value:4; /* 169 */
+        u8 hdmi_max_data_rate:4; /* 204 */
+        u16 dtd_buf_ptr; /* 161 */
+        u8 edidless_efp:1; /* 161 */
+        u8 compression_enable:1; /* 198 */
+        u8 compression_method:1; /* 198 */
+        u8 ganged_edp:1; /* 202 */
+        u8 skip0:4;
+        u8 compression_structure_index:4; /* 198 */
+        u8 skip1:4;
+        u8 slave_port; /*  202 */
+        u8 skip2;
+        u8 dvo_port;
+        u8 i2c_pin; /* for add-in card */
+        u8 slave_addr; /* for add-in card */
+        u8 ddc_pin;
+        u16 edid_ptr;
+        u8 dvo_config;
+        u8 efp_docked_port:1; /* 158 */
+        u8 lane_reversal:1; /* 184 */
+        u8 onboard_lspcon:1; /* 192 */
+        u8 iboost_enable:1; /* 196 */
+        u8 hpd_invert:1; /* BXT 196 */
+        u8 slip3:3;
+        u8 hdmi_compat:1;
+        u8 dp_compat:1;
+        u8 tmds_compat:1;
+        u8 skip4:5;
+        u8 aux_channel;
+        u8 dongle_detect;
+        u8 pipe_cap:2;
+        u8 sdvo_stall:1; /* 158 */
+        u8 hpd_status:2;
+        u8 integrated_encoder:1;
+        u8 skip5:2;
+        u8 dvo_wiring;
+        u8 mipi_bridge_type; /* 171 */
+        u16 device_class_ext;
+        u8 dvo_function;
+        u8 dp_usb_type_c:1; /* 195 */
+        u8 skip6:7;
+        u8 dp_usb_type_c_2x_gpio_index; /* 195 */
+        u16 dp_usb_type_c_2x_gpio_pin; /* 195 */
+        u8 iboost_dp:4; /* 196 */
+        u8 iboost_hdmi:4; /* 196 */
+} __packed;
+
+struct vbt {
+        /* header->bdb_offset point to bdb_header offset */
+        struct vbt_header header;
+        struct bdb_header bdb_header;
+
+        struct bdb_data_header general_features_header;
+        struct bdb_general_features general_features;
+
+        struct bdb_data_header general_definitions_header;
+        struct bdb_general_definitions general_definitions;
+
+        struct efp_child_device_config child0;
+        struct efp_child_device_config child1;
+        struct efp_child_device_config child2;
+        struct efp_child_device_config child3;
+
+        struct bdb_data_header driver_features_header;
+        struct bdb_driver_features driver_features;
+};
+
+static void virt_vbt_generation(struct vbt *v)
 {
-        u8 *buf;
-        int i;
+        int num_child;
+
+        memset(v, 0, sizeof(struct vbt));
+
+        v->header.signature[0] = '$';
+        v->header.signature[1] = 'V';
+        v->header.signature[2] = 'B';
+        v->header.signature[3] = 'T';
+
+        /* there's features depending on version! */
+        v->header.version = 155;
+        v->header.header_size = sizeof(v->header);
+        v->header.vbt_size = sizeof(struct vbt) - sizeof(v->header);
+        v->header.bdb_offset = offsetof(struct vbt, bdb_header);
+
+        strcpy(&v->bdb_header.signature[0], "BIOS_DATA_BLOCK");
+        v->bdb_header.version = 186; /* child_dev_size = 38 */
+        v->bdb_header.header_size = sizeof(v->bdb_header);
+
+        v->bdb_header.bdb_size = sizeof(struct vbt) - sizeof(struct vbt_header)
+                - sizeof(struct bdb_header);
+
+        /* general features */
+        v->general_features_header.id = BDB_GENERAL_FEATURES;
+        v->general_features_header.size = sizeof(struct bdb_general_features);
+        v->general_features.int_crt_support = 0;
+        v->general_features.int_tv_support = 0;
+
+        /* child device */
+        num_child = 4; /* each port has one child */
+        v->general_definitions_header.id = BDB_GENERAL_DEFINITIONS;
+        /* size will include child devices */
+        v->general_definitions_header.size =
+                sizeof(struct bdb_general_definitions) + num_child * DEV_SIZE;
+        v->general_definitions.child_dev_size = DEV_SIZE;
+
+        /* portA */
+        v->child0.handle = DEVICE_TYPE_EFP1;
+        v->child0.device_type = DEVICE_TYPE_DP;
+        v->child0.dvo_port = DVO_PORT_DPA;
+        v->child0.aux_channel = DP_AUX_A;
+        v->child0.dp_compat = true;
+        v->child0.integrated_encoder = true;
+
+        /* portB */
+        v->child1.handle = DEVICE_TYPE_EFP2;
+        v->child1.device_type = DEVICE_TYPE_DP;
+        v->child1.dvo_port = DVO_PORT_DPB;
+        v->child1.aux_channel = DP_AUX_B;
+        v->child1.dp_compat = true;
+        v->child1.integrated_encoder = true;
+
+        /* portC */
+        v->child2.handle = DEVICE_TYPE_EFP3;
+        v->child2.device_type = DEVICE_TYPE_DP;
+        v->child2.dvo_port = DVO_PORT_DPC;
+        v->child2.aux_channel = DP_AUX_C;
+        v->child2.dp_compat = true;
+        v->child2.integrated_encoder = true;
+
+        /* portD */
+        v->child3.handle = DEVICE_TYPE_EFP4;
+        v->child3.device_type = DEVICE_TYPE_DP;
+        v->child3.dvo_port = DVO_PORT_DPD;
+        v->child3.aux_channel = DP_AUX_D;
+        v->child3.dp_compat = true;
+        v->child3.integrated_encoder = true;
+
+        /* driver features */
+        v->driver_features_header.id = BDB_DRIVER_FEATURES;
+        v->driver_features_header.size = sizeof(struct bdb_driver_features);
+        v->driver_features.lvds_config = BDB_DRIVER_FEATURE_NO_LVDS;
+}
 
-        if (WARN((vgpu_opregion(vgpu)->va),
-                        "vgpu%d: opregion has been initialized already.\n",
-                        vgpu->id))
-                return -EINVAL;
+static int alloc_and_init_virt_opregion(struct intel_vgpu *vgpu)
+{
+        u8 *buf;
+        struct opregion_header *header;
+        struct vbt v;
 
+        gvt_dbg_core("init vgpu%d opregion\n", vgpu->id);
         vgpu_opregion(vgpu)->va = (void *)__get_free_pages(GFP_KERNEL |
                         __GFP_ZERO,
                         get_order(INTEL_GVT_OPREGION_SIZE));
-
-        if (!vgpu_opregion(vgpu)->va)
+        if (!vgpu_opregion(vgpu)->va) {
+                gvt_err("fail to get memory for vgpu virt opregion\n");
                 return -ENOMEM;
+        }
 
-        memcpy(vgpu_opregion(vgpu)->va, vgpu->gvt->opregion.opregion_va,
-               INTEL_GVT_OPREGION_SIZE);
-
-        for (i = 0; i < INTEL_GVT_OPREGION_PAGES; i++)
-                vgpu_opregion(vgpu)->gfn[i] = (gpa >> PAGE_SHIFT) + i;
+        /* emulated opregion with VBT mailbox only */
+        buf = (u8 *)vgpu_opregion(vgpu)->va;
+        header = (struct opregion_header *)buf;
+        memcpy(header->signature, OPREGION_SIGNATURE,
+                        sizeof(OPREGION_SIGNATURE));
+        header->size = 0x8;
+        header->opregion_ver = 0x02000000;
+        header->mboxes = MBOX_VBT;
 
         /* for unknown reason, the value in LID field is incorrect
          * which block the windows guest, so workaround it by force
          * setting it to "OPEN"
          */
-        buf = (u8 *)vgpu_opregion(vgpu)->va;
         buf[INTEL_GVT_OPREGION_CLID] = 0x3;
 
+        /* emulated vbt from virt vbt generation */
+        virt_vbt_generation(&v);
+        memcpy(buf + INTEL_GVT_OPREGION_VBT_OFFSET, &v, sizeof(struct vbt));
+
+        return 0;
+}
+
+static int init_vgpu_opregion(struct intel_vgpu *vgpu, u32 gpa)
+{
+        int i, ret;
+
+        if (WARN((vgpu_opregion(vgpu)->va),
+                        "vgpu%d: opregion has been initialized already.\n",
+                        vgpu->id))
+                return -EINVAL;
+
+        ret = alloc_and_init_virt_opregion(vgpu);
+        if (ret < 0)
+                return ret;
+
+        for (i = 0; i < INTEL_GVT_OPREGION_PAGES; i++)
+                vgpu_opregion(vgpu)->gfn[i] = (gpa >> PAGE_SHIFT) + i;
+
         return 0;
 }
 
@@ -132,40 +343,6 @@ int intel_vgpu_init_opregion(struct intel_vgpu *vgpu, u32 gpa)
         return 0;
 }
 
-/**
- * intel_gvt_clean_opregion - clean host opergion related stuffs
- * @gvt: a GVT device
- *
- */
-void intel_gvt_clean_opregion(struct intel_gvt *gvt)
-{
-        memunmap(gvt->opregion.opregion_va);
-        gvt->opregion.opregion_va = NULL;
-}
-
-/**
- * intel_gvt_init_opregion - initialize host opergion related stuffs
- * @gvt: a GVT device
- *
- * Returns:
- * Zero on success, negative error code if failed.
- */
-int intel_gvt_init_opregion(struct intel_gvt *gvt)
-{
-        gvt_dbg_core("init host opregion\n");
-
-        pci_read_config_dword(gvt->dev_priv->drm.pdev, INTEL_GVT_PCI_OPREGION,
-                        &gvt->opregion.opregion_pa);
-
-        gvt->opregion.opregion_va = memremap(gvt->opregion.opregion_pa,
-                                             INTEL_GVT_OPREGION_SIZE, MEMREMAP_WB);
-        if (!gvt->opregion.opregion_va) {
-                gvt_err("fail to map host opregion\n");
-                return -EFAULT;
-        }
-        return 0;
-}
-
 #define GVT_OPREGION_FUNC(scic)                                 \
         ({                                                      \
          u32 __ret;                                             \
diff --git a/drivers/gpu/drm/i915/gvt/reg.h b/drivers/gpu/drm/i915/gvt/reg.h
index 7d01c77a0f7a..d4f7ce6dc1d7 100644
--- a/drivers/gpu/drm/i915/gvt/reg.h
+++ b/drivers/gpu/drm/i915/gvt/reg.h
@@ -51,6 +51,9 @@
 
 #define INTEL_GVT_OPREGION_PAGES        2
 #define INTEL_GVT_OPREGION_SIZE         (INTEL_GVT_OPREGION_PAGES * PAGE_SIZE)
+#define INTEL_GVT_OPREGION_VBT_OFFSET   0x400
+#define INTEL_GVT_OPREGION_VBT_SIZE     \
+                (INTEL_GVT_OPREGION_SIZE - INTEL_GVT_OPREGION_VBT_OFFSET)
 
 #define VGT_SPRSTRIDE(pipe)     _PIPE(pipe, _SPRA_STRIDE, _PLANE_STRIDE_2_B)
 
@@ -71,6 +74,7 @@
 #define RB_HEAD_OFF_MASK        ((1U << 21) - (1U << 2))
 #define RB_TAIL_OFF_MASK        ((1U << 21) - (1U << 3))
 #define RB_TAIL_SIZE_MASK       ((1U << 21) - (1U << 12))
-#define _RING_CTL_BUF_SIZE(ctl) (((ctl) & RB_TAIL_SIZE_MASK) + GTT_PAGE_SIZE)
+#define _RING_CTL_BUF_SIZE(ctl) (((ctl) & RB_TAIL_SIZE_MASK) + \
+                I915_GTT_PAGE_SIZE)
 
 #endif
diff --git a/drivers/gpu/drm/i915/gvt/render.c b/drivers/gpu/drm/i915/gvt/render.c
index 6d066cf35478..0672178548ef 100644
--- a/drivers/gpu/drm/i915/gvt/render.c
+++ b/drivers/gpu/drm/i915/gvt/render.c
@@ -147,6 +147,7 @@ static u32 gen9_render_mocs_L3[32];
 static void handle_tlb_pending_event(struct intel_vgpu *vgpu, int ring_id)
 {
         struct drm_i915_private *dev_priv = vgpu->gvt->dev_priv;
+        struct intel_vgpu_submission *s = &vgpu->submission;
         enum forcewake_domains fw;
         i915_reg_t reg;
         u32 regs[] = {
@@ -160,7 +161,7 @@ static void handle_tlb_pending_event(struct intel_vgpu *vgpu, int ring_id)
         if (WARN_ON(ring_id >= ARRAY_SIZE(regs)))
                 return;
 
-        if (!test_and_clear_bit(ring_id, (void *)vgpu->tlb_handle_pending))
+        if (!test_and_clear_bit(ring_id, (void *)s->tlb_handle_pending))
                 return;
 
         reg = _MMIO(regs[ring_id]);
@@ -208,7 +209,7 @@ static void load_mocs(struct intel_vgpu *vgpu, int ring_id)
         offset.reg = regs[ring_id];
         for (i = 0; i < 64; i++) {
                 gen9_render_mocs[ring_id][i] = I915_READ_FW(offset);
-                I915_WRITE(offset, vgpu_vreg(vgpu, offset));
+                I915_WRITE_FW(offset, vgpu_vreg(vgpu, offset));
                 offset.reg += 4;
         }
 
@@ -261,14 +262,15 @@ static void restore_mocs(struct intel_vgpu *vgpu, int ring_id)
 static void switch_mmio_to_vgpu(struct intel_vgpu *vgpu, int ring_id)
 {
         struct drm_i915_private *dev_priv = vgpu->gvt->dev_priv;
-        struct render_mmio *mmio;
-        u32 v;
-        int i, array_size;
-        u32 *reg_state = vgpu->shadow_ctx->engine[ring_id].lrc_reg_state;
+        struct intel_vgpu_submission *s = &vgpu->submission;
+        u32 *reg_state = s->shadow_ctx->engine[ring_id].lrc_reg_state;
         u32 ctx_ctrl = reg_state[CTX_CONTEXT_CONTROL_VAL];
         u32 inhibit_mask =
                 _MASKED_BIT_ENABLE(CTX_CTRL_ENGINE_CTX_RESTORE_INHIBIT);
         i915_reg_t last_reg = _MMIO(0);
+        struct render_mmio *mmio;
+        u32 v;
+        int i, array_size;
 
         if (IS_SKYLAKE(vgpu->gvt->dev_priv)
                 || IS_KABYLAKE(vgpu->gvt->dev_priv)) {
diff --git a/drivers/gpu/drm/i915/gvt/scheduler.c b/drivers/gpu/drm/i915/gvt/scheduler.c
index 3ac1dc97a7a0..d6177a0baeec 100644
--- a/drivers/gpu/drm/i915/gvt/scheduler.c
+++ b/drivers/gpu/drm/i915/gvt/scheduler.c
@@ -57,7 +57,7 @@ static int populate_shadow_context(struct intel_vgpu_workload *workload)
         struct intel_vgpu *vgpu = workload->vgpu;
         struct intel_gvt *gvt = vgpu->gvt;
         int ring_id = workload->ring_id;
-        struct i915_gem_context *shadow_ctx = workload->vgpu->shadow_ctx;
+        struct i915_gem_context *shadow_ctx = vgpu->submission.shadow_ctx;
         struct drm_i915_gem_object *ctx_obj =
                 shadow_ctx->engine[ring_id].state->obj;
         struct execlist_ring_context *shadow_ring_context;
@@ -81,16 +81,16 @@ static int populate_shadow_context(struct intel_vgpu_workload *workload)
         while (i < context_page_num) {
                 context_gpa = intel_vgpu_gma_to_gpa(vgpu->gtt.ggtt_mm,
                                 (u32)((workload->ctx_desc.lrca + i) <<
-                                GTT_PAGE_SHIFT));
+                                I915_GTT_PAGE_SHIFT));
                 if (context_gpa == INTEL_GVT_INVALID_ADDR) {
                         gvt_vgpu_err("Invalid guest context descriptor\n");
-                        return -EINVAL;
+                        return -EFAULT;
                 }
 
                 page = i915_gem_object_get_page(ctx_obj, LRC_HEADER_PAGES + i);
                 dst = kmap(page);
                 intel_gvt_hypervisor_read_gpa(vgpu, context_gpa, dst,
-                                GTT_PAGE_SIZE);
+                                I915_GTT_PAGE_SIZE);
                 kunmap(page);
                 i++;
         }
@@ -120,7 +120,7 @@ static int populate_shadow_context(struct intel_vgpu_workload *workload)
                         sizeof(*shadow_ring_context),
                         (void *)shadow_ring_context +
                         sizeof(*shadow_ring_context),
-                        GTT_PAGE_SIZE - sizeof(*shadow_ring_context));
+                        I915_GTT_PAGE_SIZE - sizeof(*shadow_ring_context));
 
         kunmap(page);
         return 0;
@@ -131,6 +131,20 @@ static inline bool is_gvt_request(struct drm_i915_gem_request *req)
         return i915_gem_context_force_single_submission(req->ctx);
 }
 
+static void save_ring_hw_state(struct intel_vgpu *vgpu, int ring_id)
+{
+        struct drm_i915_private *dev_priv = vgpu->gvt->dev_priv;
+        u32 ring_base = dev_priv->engine[ring_id]->mmio_base;
+        i915_reg_t reg;
+
+        reg = RING_INSTDONE(ring_base);
+        vgpu_vreg(vgpu, i915_mmio_reg_offset(reg)) = I915_READ_FW(reg);
+        reg = RING_ACTHD(ring_base);
+        vgpu_vreg(vgpu, i915_mmio_reg_offset(reg)) = I915_READ_FW(reg);
+        reg = RING_ACTHD_UDW(ring_base);
+        vgpu_vreg(vgpu, i915_mmio_reg_offset(reg)) = I915_READ_FW(reg);
+}
+
 static int shadow_context_status_change(struct notifier_block *nb,
                 unsigned long action, void *data)
 {
@@ -176,6 +190,7 @@ static int shadow_context_status_change(struct notifier_block *nb,
                 break;
         case INTEL_CONTEXT_SCHEDULE_OUT:
         case INTEL_CONTEXT_SCHEDULE_PREEMPTED:
+                save_ring_hw_state(workload->vgpu, ring_id);
                 atomic_set(&workload->shadow_ctx_active, 0);
                 break;
         default:
@@ -250,11 +265,12 @@ void release_shadow_wa_ctx(struct intel_shadow_wa_ctx *wa_ctx)
  */
 int intel_gvt_scan_and_shadow_workload(struct intel_vgpu_workload *workload)
 {
+        struct intel_vgpu *vgpu = workload->vgpu;
+        struct intel_vgpu_submission *s = &vgpu->submission;
+        struct i915_gem_context *shadow_ctx = s->shadow_ctx;
+        struct drm_i915_private *dev_priv = vgpu->gvt->dev_priv;
         int ring_id = workload->ring_id;
-        struct i915_gem_context *shadow_ctx = workload->vgpu->shadow_ctx;
-        struct drm_i915_private *dev_priv = workload->vgpu->gvt->dev_priv;
         struct intel_engine_cs *engine = dev_priv->engine[ring_id];
-        struct intel_vgpu *vgpu = workload->vgpu;
         struct intel_ring *ring;
         int ret;
 
@@ -267,7 +283,7 @@ int intel_gvt_scan_and_shadow_workload(struct intel_vgpu_workload *workload)
         shadow_ctx->desc_template |= workload->ctx_desc.addressing_mode <<
                                     GEN8_CTX_ADDRESSING_MODE_SHIFT;
 
-        if (!test_and_set_bit(ring_id, vgpu->shadow_ctx_desc_updated))
+        if (!test_and_set_bit(ring_id, s->shadow_ctx_desc_updated))
                 shadow_context_descriptor_update(shadow_ctx,
                                         dev_priv->engine[ring_id]);
 
@@ -310,14 +326,15 @@ err_scan:
         return ret;
 }
 
-int intel_gvt_generate_request(struct intel_vgpu_workload *workload)
+static int intel_gvt_generate_request(struct intel_vgpu_workload *workload)
 {
         int ring_id = workload->ring_id;
         struct drm_i915_private *dev_priv = workload->vgpu->gvt->dev_priv;
         struct intel_engine_cs *engine = dev_priv->engine[ring_id];
         struct drm_i915_gem_request *rq;
         struct intel_vgpu *vgpu = workload->vgpu;
-        struct i915_gem_context *shadow_ctx = vgpu->shadow_ctx;
+        struct intel_vgpu_submission *s = &vgpu->submission;
+        struct i915_gem_context *shadow_ctx = s->shadow_ctx;
         int ret;
 
         rq = i915_gem_request_alloc(dev_priv->engine[ring_id], shadow_ctx);
@@ -341,11 +358,203 @@ err_unpin:
         return ret;
 }
 
+static void release_shadow_batch_buffer(struct intel_vgpu_workload *workload);
+
+static int prepare_shadow_batch_buffer(struct intel_vgpu_workload *workload)
+{
+        struct intel_gvt *gvt = workload->vgpu->gvt;
+        const int gmadr_bytes = gvt->device_info.gmadr_bytes_in_cmd;
+        struct intel_vgpu_shadow_bb *bb;
+        int ret;
+
+        list_for_each_entry(bb, &workload->shadow_bb, list) {
+                bb->vma = i915_gem_object_ggtt_pin(bb->obj, NULL, 0, 0, 0);
+                if (IS_ERR(bb->vma)) {
+                        ret = PTR_ERR(bb->vma);
+                        goto err;
+                }
+
+                /* relocate shadow batch buffer */
+                bb->bb_start_cmd_va[1] = i915_ggtt_offset(bb->vma);
+                if (gmadr_bytes == 8)
+                        bb->bb_start_cmd_va[2] = 0;
+
+                /* No one is going to touch shadow bb from now on. */
+                if (bb->clflush & CLFLUSH_AFTER) {
+                        drm_clflush_virt_range(bb->va, bb->obj->base.size);
+                        bb->clflush &= ~CLFLUSH_AFTER;
+                }
+
+                ret = i915_gem_object_set_to_gtt_domain(bb->obj, false);
+                if (ret)
+                        goto err;
+
+                i915_gem_obj_finish_shmem_access(bb->obj);
+                bb->accessing = false;
+
+                i915_vma_move_to_active(bb->vma, workload->req, 0);
+        }
+        return 0;
+err:
+        release_shadow_batch_buffer(workload);
+        return ret;
+}
+
+static int update_wa_ctx_2_shadow_ctx(struct intel_shadow_wa_ctx *wa_ctx)
+{
+        struct intel_vgpu_workload *workload = container_of(wa_ctx,
+                                        struct intel_vgpu_workload,
+                                        wa_ctx);
+        int ring_id = workload->ring_id;
+        struct intel_vgpu_submission *s = &workload->vgpu->submission;
+        struct i915_gem_context *shadow_ctx = s->shadow_ctx;
+        struct drm_i915_gem_object *ctx_obj =
+                shadow_ctx->engine[ring_id].state->obj;
+        struct execlist_ring_context *shadow_ring_context;
+        struct page *page;
+
+        page = i915_gem_object_get_page(ctx_obj, LRC_STATE_PN);
+        shadow_ring_context = kmap_atomic(page);
+
+        shadow_ring_context->bb_per_ctx_ptr.val =
+                (shadow_ring_context->bb_per_ctx_ptr.val &
+                (~PER_CTX_ADDR_MASK)) | wa_ctx->per_ctx.shadow_gma;
+        shadow_ring_context->rcs_indirect_ctx.val =
+                (shadow_ring_context->rcs_indirect_ctx.val &
+                (~INDIRECT_CTX_ADDR_MASK)) | wa_ctx->indirect_ctx.shadow_gma;
+
+        kunmap_atomic(shadow_ring_context);
+        return 0;
+}
+
+static int prepare_shadow_wa_ctx(struct intel_shadow_wa_ctx *wa_ctx)
+{
+        struct i915_vma *vma;
+        unsigned char *per_ctx_va =
+                (unsigned char *)wa_ctx->indirect_ctx.shadow_va +
+                wa_ctx->indirect_ctx.size;
+
+        if (wa_ctx->indirect_ctx.size == 0)
+                return 0;
+
+        vma = i915_gem_object_ggtt_pin(wa_ctx->indirect_ctx.obj, NULL,
+                                       0, CACHELINE_BYTES, 0);
+        if (IS_ERR(vma))
+                return PTR_ERR(vma);
+
+        /* FIXME: we are not tracking our pinned VMA leaving it
+         * up to the core to fix up the stray pin_count upon
+         * free.
+         */
+
+        wa_ctx->indirect_ctx.shadow_gma = i915_ggtt_offset(vma);
+
+        wa_ctx->per_ctx.shadow_gma = *((unsigned int *)per_ctx_va + 1);
+        memset(per_ctx_va, 0, CACHELINE_BYTES);
+
+        update_wa_ctx_2_shadow_ctx(wa_ctx);
+        return 0;
+}
+
+static void release_shadow_batch_buffer(struct intel_vgpu_workload *workload)
+{
+        struct intel_vgpu *vgpu = workload->vgpu;
+        struct drm_i915_private *dev_priv = vgpu->gvt->dev_priv;
+        struct intel_vgpu_shadow_bb *bb, *pos;
+
+        if (list_empty(&workload->shadow_bb))
+                return;
+
+        bb = list_first_entry(&workload->shadow_bb,
+                        struct intel_vgpu_shadow_bb, list);
+
+        mutex_lock(&dev_priv->drm.struct_mutex);
+
+        list_for_each_entry_safe(bb, pos, &workload->shadow_bb, list) {
+                if (bb->obj) {
+                        if (bb->accessing)
+                                i915_gem_obj_finish_shmem_access(bb->obj);
+
+                        if (bb->va && !IS_ERR(bb->va))
+                                i915_gem_object_unpin_map(bb->obj);
+
+                        if (bb->vma && !IS_ERR(bb->vma)) {
+                                i915_vma_unpin(bb->vma);
+                                i915_vma_close(bb->vma);
+                        }
+                        __i915_gem_object_release_unless_active(bb->obj);
+                }
+                list_del(&bb->list);
+                kfree(bb);
+        }
+
+        mutex_unlock(&dev_priv->drm.struct_mutex);
+}
+
+static int prepare_workload(struct intel_vgpu_workload *workload)
+{
+        struct intel_vgpu *vgpu = workload->vgpu;
+        int ret = 0;
+
+        ret = intel_vgpu_pin_mm(workload->shadow_mm);
+        if (ret) {
+                gvt_vgpu_err("fail to vgpu pin mm\n");
+                return ret;
+        }
+
+        ret = intel_vgpu_sync_oos_pages(workload->vgpu);
+        if (ret) {
+                gvt_vgpu_err("fail to vgpu sync oos pages\n");
+                goto err_unpin_mm;
+        }
+
+        ret = intel_vgpu_flush_post_shadow(workload->vgpu);
+        if (ret) {
+                gvt_vgpu_err("fail to flush post shadow\n");
+                goto err_unpin_mm;
+        }
+
+        ret = intel_gvt_generate_request(workload);
+        if (ret) {
+                gvt_vgpu_err("fail to generate request\n");
+                goto err_unpin_mm;
+        }
+
+        ret = prepare_shadow_batch_buffer(workload);
+        if (ret) {
+                gvt_vgpu_err("fail to prepare_shadow_batch_buffer\n");
+                goto err_unpin_mm;
+        }
+
+        ret = prepare_shadow_wa_ctx(&workload->wa_ctx);
+        if (ret) {
+                gvt_vgpu_err("fail to prepare_shadow_wa_ctx\n");
+                goto err_shadow_batch;
+        }
+
+        if (workload->prepare) {
+                ret = workload->prepare(workload);
+                if (ret)
+                        goto err_shadow_wa_ctx;
+        }
+
+        return 0;
+err_shadow_wa_ctx:
+        release_shadow_wa_ctx(&workload->wa_ctx);
+err_shadow_batch:
+        release_shadow_batch_buffer(workload);
+err_unpin_mm:
+        intel_vgpu_unpin_mm(workload->shadow_mm);
+        return ret;
+}
+
 static int dispatch_workload(struct intel_vgpu_workload *workload)
 {
+        struct intel_vgpu *vgpu = workload->vgpu;
+        struct intel_vgpu_submission *s = &vgpu->submission;
+        struct i915_gem_context *shadow_ctx = s->shadow_ctx;
+        struct drm_i915_private *dev_priv = vgpu->gvt->dev_priv;
         int ring_id = workload->ring_id;
-        struct i915_gem_context *shadow_ctx = workload->vgpu->shadow_ctx;
-        struct drm_i915_private *dev_priv = workload->vgpu->gvt->dev_priv;
         struct intel_engine_cs *engine = dev_priv->engine[ring_id];
         int ret = 0;
 
@@ -358,12 +567,10 @@ static int dispatch_workload(struct intel_vgpu_workload *workload)
         if (ret)
                 goto out;
 
-        if (workload->prepare) {
-                ret = workload->prepare(workload);
-                if (ret) {
-                        engine->context_unpin(engine, shadow_ctx);
-                        goto out;
-                }
+        ret = prepare_workload(workload);
+        if (ret) {
+                engine->context_unpin(engine, shadow_ctx);
+                goto out;
         }
 
 out:
@@ -431,7 +638,7 @@ static struct intel_vgpu_workload *pick_next_workload(
 
         gvt_dbg_sched("ring id %d pick new workload %p\n", ring_id, workload);
 
-        atomic_inc(&workload->vgpu->running_workload_num);
+        atomic_inc(&workload->vgpu->submission.running_workload_num);
 out:
         mutex_unlock(&gvt->lock);
         return workload;
@@ -441,8 +648,9 @@ static void update_guest_context(struct intel_vgpu_workload *workload)
 {
         struct intel_vgpu *vgpu = workload->vgpu;
         struct intel_gvt *gvt = vgpu->gvt;
+        struct intel_vgpu_submission *s = &vgpu->submission;
+        struct i915_gem_context *shadow_ctx = s->shadow_ctx;
         int ring_id = workload->ring_id;
-        struct i915_gem_context *shadow_ctx = workload->vgpu->shadow_ctx;
         struct drm_i915_gem_object *ctx_obj =
                 shadow_ctx->engine[ring_id].state->obj;
         struct execlist_ring_context *shadow_ring_context;
@@ -466,7 +674,7 @@ static void update_guest_context(struct intel_vgpu_workload *workload)
         while (i < context_page_num) {
                 context_gpa = intel_vgpu_gma_to_gpa(vgpu->gtt.ggtt_mm,
                                 (u32)((workload->ctx_desc.lrca + i) <<
-                                        GTT_PAGE_SHIFT));
+                                        I915_GTT_PAGE_SHIFT));
                 if (context_gpa == INTEL_GVT_INVALID_ADDR) {
                         gvt_vgpu_err("invalid guest context descriptor\n");
                         return;
@@ -475,7 +683,7 @@ static void update_guest_context(struct intel_vgpu_workload *workload)
                 page = i915_gem_object_get_page(ctx_obj, LRC_HEADER_PAGES + i);
                 src = kmap(page);
                 intel_gvt_hypervisor_write_gpa(vgpu, context_gpa, src,
-                                GTT_PAGE_SIZE);
+                                I915_GTT_PAGE_SIZE);
                 kunmap(page);
                 i++;
         }
@@ -500,23 +708,41 @@ static void update_guest_context(struct intel_vgpu_workload *workload)
                         sizeof(*shadow_ring_context),
                         (void *)shadow_ring_context +
                         sizeof(*shadow_ring_context),
-                        GTT_PAGE_SIZE - sizeof(*shadow_ring_context));
+                        I915_GTT_PAGE_SIZE - sizeof(*shadow_ring_context));
 
         kunmap(page);
 }
 
+static void clean_workloads(struct intel_vgpu *vgpu, unsigned long engine_mask)
+{
+        struct intel_vgpu_submission *s = &vgpu->submission;
+        struct drm_i915_private *dev_priv = vgpu->gvt->dev_priv;
+        struct intel_engine_cs *engine;
+        struct intel_vgpu_workload *pos, *n;
+        unsigned int tmp;
+
+        /* free the unsubmited workloads in the queues. */
+        for_each_engine_masked(engine, dev_priv, engine_mask, tmp) {
+                list_for_each_entry_safe(pos, n,
+                        &s->workload_q_head[engine->id], list) {
+                        list_del_init(&pos->list);
+                        intel_vgpu_destroy_workload(pos);
+                }
+                clear_bit(engine->id, s->shadow_ctx_desc_updated);
+        }
+}
+
 static void complete_current_workload(struct intel_gvt *gvt, int ring_id)
 {
         struct intel_gvt_workload_scheduler *scheduler = &gvt->scheduler;
-        struct intel_vgpu_workload *workload;
-        struct intel_vgpu *vgpu;
+        struct intel_vgpu_workload *workload =
+                scheduler->current_workload[ring_id];
+        struct intel_vgpu *vgpu = workload->vgpu;
+        struct intel_vgpu_submission *s = &vgpu->submission;
         int event;
 
         mutex_lock(&gvt->lock);
 
-        workload = scheduler->current_workload[ring_id];
-        vgpu = workload->vgpu;
-
         /* For the workload w/ request, needs to wait for the context
          * switch to make sure request is completed.
          * For the workload w/o request, directly complete the workload.
@@ -553,7 +779,7 @@ static void complete_current_workload(struct intel_gvt *gvt, int ring_id)
                 }
                 mutex_lock(&dev_priv->drm.struct_mutex);
                 /* unpin shadow ctx as the shadow_ctx update is done */
-                engine->context_unpin(engine, workload->vgpu->shadow_ctx);
+                engine->context_unpin(engine, s->shadow_ctx);
                 mutex_unlock(&dev_priv->drm.struct_mutex);
         }
 
@@ -563,9 +789,32 @@ static void complete_current_workload(struct intel_gvt *gvt, int ring_id)
         scheduler->current_workload[ring_id] = NULL;
 
         list_del_init(&workload->list);
+
+        if (!workload->status) {
+                release_shadow_batch_buffer(workload);
+                release_shadow_wa_ctx(&workload->wa_ctx);
+        }
+
+        if (workload->status || (vgpu->resetting_eng & ENGINE_MASK(ring_id))) {
+                /* if workload->status is not successful means HW GPU
+                 * has occurred GPU hang or something wrong with i915/GVT,
+                 * and GVT won't inject context switch interrupt to guest.
+                 * So this error is a vGPU hang actually to the guest.
+                 * According to this we should emunlate a vGPU hang. If
+                 * there are pending workloads which are already submitted
+                 * from guest, we should clean them up like HW GPU does.
+                 *
+                 * if it is in middle of engine resetting, the pending
+                 * workloads won't be submitted to HW GPU and will be
+                 * cleaned up during the resetting process later, so doing
+                 * the workload clean up here doesn't have any impact.
+                 **/
+                clean_workloads(vgpu, ENGINE_MASK(ring_id));
+        }
+
         workload->complete(workload);
 
-        atomic_dec(&vgpu->running_workload_num);
+        atomic_dec(&s->running_workload_num);
         wake_up(&scheduler->workload_complete_wq);
 
         if (gvt->scheduler.need_reschedule)
@@ -648,20 +897,23 @@ complete:
                                         FORCEWAKE_ALL);
 
                 intel_runtime_pm_put(gvt->dev_priv);
+                if (ret && (vgpu_is_vm_unhealthy(ret)))
+                        enter_failsafe_mode(vgpu, GVT_FAILSAFE_GUEST_ERR);
         }
         return 0;
 }
 
 void intel_gvt_wait_vgpu_idle(struct intel_vgpu *vgpu)
 {
+        struct intel_vgpu_submission *s = &vgpu->submission;
         struct intel_gvt *gvt = vgpu->gvt;
         struct intel_gvt_workload_scheduler *scheduler = &gvt->scheduler;
 
-        if (atomic_read(&vgpu->running_workload_num)) {
+        if (atomic_read(&s->running_workload_num)) {
                 gvt_dbg_sched("wait vgpu idle\n");
 
                 wait_event(scheduler->workload_complete_wq,
-                                !atomic_read(&vgpu->running_workload_num));
+                                !atomic_read(&s->running_workload_num));
         }
 }
 
@@ -726,23 +978,354 @@ err:
         return ret;
 }
 
-void intel_vgpu_clean_gvt_context(struct intel_vgpu *vgpu)
+/**
+ * intel_vgpu_clean_submission - free submission-related resource for vGPU
+ * @vgpu: a vGPU
+ *
+ * This function is called when a vGPU is being destroyed.
+ *
+ */
+void intel_vgpu_clean_submission(struct intel_vgpu *vgpu)
+{
+        struct intel_vgpu_submission *s = &vgpu->submission;
+
+        intel_vgpu_select_submission_ops(vgpu, 0);
+        i915_gem_context_put(s->shadow_ctx);
+        kmem_cache_destroy(s->workloads);
+}
+
+
+/**
+ * intel_vgpu_reset_submission - reset submission-related resource for vGPU
+ * @vgpu: a vGPU
+ * @engine_mask: engines expected to be reset
+ *
+ * This function is called when a vGPU is being destroyed.
+ *
+ */
+void intel_vgpu_reset_submission(struct intel_vgpu *vgpu,
+                unsigned long engine_mask)
 {
-        i915_gem_context_put(vgpu->shadow_ctx);
+        struct intel_vgpu_submission *s = &vgpu->submission;
+
+        if (!s->active)
+                return;
+
+        clean_workloads(vgpu, engine_mask);
+        s->ops->reset(vgpu, engine_mask);
 }
 
-int intel_vgpu_init_gvt_context(struct intel_vgpu *vgpu)
+/**
+ * intel_vgpu_setup_submission - setup submission-related resource for vGPU
+ * @vgpu: a vGPU
+ *
+ * This function is called when a vGPU is being created.
+ *
+ * Returns:
+ * Zero on success, negative error code if failed.
+ *
+ */
+int intel_vgpu_setup_submission(struct intel_vgpu *vgpu)
 {
-        atomic_set(&vgpu->running_workload_num, 0);
+        struct intel_vgpu_submission *s = &vgpu->submission;
+        enum intel_engine_id i;
+        struct intel_engine_cs *engine;
+        int ret;
 
-        vgpu->shadow_ctx = i915_gem_context_create_gvt(
+        s->shadow_ctx = i915_gem_context_create_gvt(
                         &vgpu->gvt->dev_priv->drm);
-        if (IS_ERR(vgpu->shadow_ctx))
-                return PTR_ERR(vgpu->shadow_ctx);
+        if (IS_ERR(s->shadow_ctx))
+                return PTR_ERR(s->shadow_ctx);
+
+        bitmap_zero(s->shadow_ctx_desc_updated, I915_NUM_ENGINES);
+
+        s->workloads = kmem_cache_create("gvt-g_vgpu_workload",
+                        sizeof(struct intel_vgpu_workload), 0,
+                        SLAB_HWCACHE_ALIGN,
+                        NULL);
+
+        if (!s->workloads) {
+                ret = -ENOMEM;
+                goto out_shadow_ctx;
+        }
+
+        for_each_engine(engine, vgpu->gvt->dev_priv, i)
+                INIT_LIST_HEAD(&s->workload_q_head[i]);
+
+        atomic_set(&s->running_workload_num, 0);
+        bitmap_zero(s->tlb_handle_pending, I915_NUM_ENGINES);
+
+        return 0;
+
+out_shadow_ctx:
+        i915_gem_context_put(s->shadow_ctx);
+        return ret;
+}
+
+/**
+ * intel_vgpu_select_submission_ops - select virtual submission interface
+ * @vgpu: a vGPU
+ * @interface: expected vGPU virtual submission interface
+ *
+ * This function is called when guest configures submission interface.
+ *
+ * Returns:
+ * Zero on success, negative error code if failed.
+ *
+ */
+int intel_vgpu_select_submission_ops(struct intel_vgpu *vgpu,
+                                     unsigned int interface)
+{
+        struct intel_vgpu_submission *s = &vgpu->submission;
+        const struct intel_vgpu_submission_ops *ops[] = {
+                [INTEL_VGPU_EXECLIST_SUBMISSION] =
+                        &intel_vgpu_execlist_submission_ops,
+        };
+        int ret;
+
+        if (WARN_ON(interface >= ARRAY_SIZE(ops)))
+                return -EINVAL;
+
+        if (s->active) {
+                s->ops->clean(vgpu);
+                s->active = false;
+                gvt_dbg_core("vgpu%d: de-select ops [ %s ] \n",
+                                vgpu->id, s->ops->name);
+        }
+
+        if (interface == 0) {
+                s->ops = NULL;
+                s->virtual_submission_interface = 0;
+                gvt_dbg_core("vgpu%d: no submission ops\n", vgpu->id);
+                return 0;
+        }
+
+        ret = ops[interface]->init(vgpu);
+        if (ret)
+                return ret;
+
+        s->ops = ops[interface];
+        s->virtual_submission_interface = interface;
+        s->active = true;
+
+        gvt_dbg_core("vgpu%d: activate ops [ %s ]\n",
+                        vgpu->id, s->ops->name);
+
+        return 0;
+}
+
+/**
+ * intel_vgpu_destroy_workload - destroy a vGPU workload
+ * @vgpu: a vGPU
+ *
+ * This function is called when destroy a vGPU workload.
+ *
+ */
+void intel_vgpu_destroy_workload(struct intel_vgpu_workload *workload)
+{
+        struct intel_vgpu_submission *s = &workload->vgpu->submission;
+
+        if (workload->shadow_mm)
+                intel_gvt_mm_unreference(workload->shadow_mm);
+
+        kmem_cache_free(s->workloads, workload);
+}
+
+static struct intel_vgpu_workload *
+alloc_workload(struct intel_vgpu *vgpu)
+{
+        struct intel_vgpu_submission *s = &vgpu->submission;
+        struct intel_vgpu_workload *workload;
 
-        vgpu->shadow_ctx->engine[RCS].initialised = true;
+        workload = kmem_cache_zalloc(s->workloads, GFP_KERNEL);
+        if (!workload)
+                return ERR_PTR(-ENOMEM);
 
-        bitmap_zero(vgpu->shadow_ctx_desc_updated, I915_NUM_ENGINES);
+        INIT_LIST_HEAD(&workload->list);
+        INIT_LIST_HEAD(&workload->shadow_bb);
 
+        init_waitqueue_head(&workload->shadow_ctx_status_wq);
+        atomic_set(&workload->shadow_ctx_active, 0);
+
+        workload->status = -EINPROGRESS;
+        workload->shadowed = false;
+        workload->vgpu = vgpu;
+
+        return workload;
+}
+
+#define RING_CTX_OFF(x) \
+        offsetof(struct execlist_ring_context, x)
+
+static void read_guest_pdps(struct intel_vgpu *vgpu,
+                u64 ring_context_gpa, u32 pdp[8])
+{
+        u64 gpa;
+        int i;
+
+        gpa = ring_context_gpa + RING_CTX_OFF(pdp3_UDW.val);
+
+        for (i = 0; i < 8; i++)
+                intel_gvt_hypervisor_read_gpa(vgpu,
+                                gpa + i * 8, &pdp[7 - i], 4);
+}
+
+static int prepare_mm(struct intel_vgpu_workload *workload)
+{
+        struct execlist_ctx_descriptor_format *desc = &workload->ctx_desc;
+        struct intel_vgpu_mm *mm;
+        struct intel_vgpu *vgpu = workload->vgpu;
+        int page_table_level;
+        u32 pdp[8];
+
+        if (desc->addressing_mode == 1) { /* legacy 32-bit */
+                page_table_level = 3;
+        } else if (desc->addressing_mode == 3) { /* legacy 64 bit */
+                page_table_level = 4;
+        } else {
+                gvt_vgpu_err("Advanced Context mode(SVM) is not supported!\n");
+                return -EINVAL;
+        }
+
+        read_guest_pdps(workload->vgpu, workload->ring_context_gpa, pdp);
+
+        mm = intel_vgpu_find_ppgtt_mm(workload->vgpu, page_table_level, pdp);
+        if (mm) {
+                intel_gvt_mm_reference(mm);
+        } else {
+
+                mm = intel_vgpu_create_mm(workload->vgpu, INTEL_GVT_MM_PPGTT,
+                                pdp, page_table_level, 0);
+                if (IS_ERR(mm)) {
+                        gvt_vgpu_err("fail to create mm object.\n");
+                        return PTR_ERR(mm);
+                }
+        }
+        workload->shadow_mm = mm;
         return 0;
 }
+
+#define same_context(a, b) (((a)->context_id == (b)->context_id) && \
+                ((a)->lrca == (b)->lrca))
+
+#define get_last_workload(q) \
+        (list_empty(q) ? NULL : container_of(q->prev, \
+        struct intel_vgpu_workload, list))
+/**
+ * intel_vgpu_create_workload - create a vGPU workload
+ * @vgpu: a vGPU
+ * @desc: a guest context descriptor
+ *
+ * This function is called when creating a vGPU workload.
+ *
+ * Returns:
+ * struct intel_vgpu_workload * on success, negative error code in
+ * pointer if failed.
+ *
+ */
+struct intel_vgpu_workload *
+intel_vgpu_create_workload(struct intel_vgpu *vgpu, int ring_id,
+                           struct execlist_ctx_descriptor_format *desc)
+{
+        struct intel_vgpu_submission *s = &vgpu->submission;
+        struct list_head *q = workload_q_head(vgpu, ring_id);
+        struct intel_vgpu_workload *last_workload = get_last_workload(q);
+        struct intel_vgpu_workload *workload = NULL;
+        struct drm_i915_private *dev_priv = vgpu->gvt->dev_priv;
+        u64 ring_context_gpa;
+        u32 head, tail, start, ctl, ctx_ctl, per_ctx, indirect_ctx;
+        int ret;
+
+        ring_context_gpa = intel_vgpu_gma_to_gpa(vgpu->gtt.ggtt_mm,
+                        (u32)((desc->lrca + 1) << I915_GTT_PAGE_SHIFT));
+        if (ring_context_gpa == INTEL_GVT_INVALID_ADDR) {
+                gvt_vgpu_err("invalid guest context LRCA: %x\n", desc->lrca);
+                return ERR_PTR(-EINVAL);
+        }
+
+        intel_gvt_hypervisor_read_gpa(vgpu, ring_context_gpa +
+                        RING_CTX_OFF(ring_header.val), &head, 4);
+
+        intel_gvt_hypervisor_read_gpa(vgpu, ring_context_gpa +
+                        RING_CTX_OFF(ring_tail.val), &tail, 4);
+
+        head &= RB_HEAD_OFF_MASK;
+        tail &= RB_TAIL_OFF_MASK;
+
+        if (last_workload && same_context(&last_workload->ctx_desc, desc)) {
+                gvt_dbg_el("ring id %d cur workload == last\n", ring_id);
+                gvt_dbg_el("ctx head %x real head %lx\n", head,
+                                last_workload->rb_tail);
+                /*
+                 * cannot use guest context head pointer here,
+                 * as it might not be updated at this time
+                 */
+                head = last_workload->rb_tail;
+        }
+
+        gvt_dbg_el("ring id %d begin a new workload\n", ring_id);
+
+        /* record some ring buffer register values for scan and shadow */
+        intel_gvt_hypervisor_read_gpa(vgpu, ring_context_gpa +
+                        RING_CTX_OFF(rb_start.val), &start, 4);
+        intel_gvt_hypervisor_read_gpa(vgpu, ring_context_gpa +
+                        RING_CTX_OFF(rb_ctrl.val), &ctl, 4);
+        intel_gvt_hypervisor_read_gpa(vgpu, ring_context_gpa +
+                        RING_CTX_OFF(ctx_ctrl.val), &ctx_ctl, 4);
+
+        workload = alloc_workload(vgpu);
+        if (IS_ERR(workload))
+                return workload;
+
+        workload->ring_id = ring_id;
+        workload->ctx_desc = *desc;
+        workload->ring_context_gpa = ring_context_gpa;
+        workload->rb_head = head;
+        workload->rb_tail = tail;
+        workload->rb_start = start;
+        workload->rb_ctl = ctl;
+
+        if (ring_id == RCS) {
+                intel_gvt_hypervisor_read_gpa(vgpu, ring_context_gpa +
+                        RING_CTX_OFF(bb_per_ctx_ptr.val), &per_ctx, 4);
+                intel_gvt_hypervisor_read_gpa(vgpu, ring_context_gpa +
+                        RING_CTX_OFF(rcs_indirect_ctx.val), &indirect_ctx, 4);
+
+                workload->wa_ctx.indirect_ctx.guest_gma =
+                        indirect_ctx & INDIRECT_CTX_ADDR_MASK;
+                workload->wa_ctx.indirect_ctx.size =
+                        (indirect_ctx & INDIRECT_CTX_SIZE_MASK) *
+                        CACHELINE_BYTES;
+                workload->wa_ctx.per_ctx.guest_gma =
+                        per_ctx & PER_CTX_ADDR_MASK;
+                workload->wa_ctx.per_ctx.valid = per_ctx & 1;
+        }
+
+        gvt_dbg_el("workload %p ring id %d head %x tail %x start %x ctl %x\n",
+                        workload, ring_id, head, tail, start, ctl);
+
+        ret = prepare_mm(workload);
+        if (ret) {
+                kmem_cache_free(s->workloads, workload);
+                return ERR_PTR(ret);
+        }
+
+        /* Only scan and shadow the first workload in the queue
+         * as there is only one pre-allocated buf-obj for shadow.
+         */
+        if (list_empty(workload_q_head(vgpu, ring_id))) {
+                intel_runtime_pm_get(dev_priv);
+                mutex_lock(&dev_priv->drm.struct_mutex);
+                ret = intel_gvt_scan_and_shadow_workload(workload);
+                mutex_unlock(&dev_priv->drm.struct_mutex);
+                intel_runtime_pm_put(dev_priv);
+        }
+
+        if (ret && (vgpu_is_vm_unhealthy(ret))) {
+                enter_failsafe_mode(vgpu, GVT_FAILSAFE_GUEST_ERR);
+                intel_vgpu_destroy_workload(workload);
+                return ERR_PTR(ret);
+        }
+
+        return workload;
+}
diff --git a/drivers/gpu/drm/i915/gvt/scheduler.h b/drivers/gpu/drm/i915/gvt/scheduler.h
index b9f872204d7e..e4a9f9acd4a9 100644
--- a/drivers/gpu/drm/i915/gvt/scheduler.h
+++ b/drivers/gpu/drm/i915/gvt/scheduler.h
@@ -112,17 +112,18 @@ struct intel_vgpu_workload {
         struct intel_shadow_wa_ctx wa_ctx;
 };
 
-/* Intel shadow batch buffer is a i915 gem object */
-struct intel_shadow_bb_entry {
+struct intel_vgpu_shadow_bb {
         struct list_head list;
         struct drm_i915_gem_object *obj;
+        struct i915_vma *vma;
         void *va;
-        unsigned long len;
         u32 *bb_start_cmd_va;
+        unsigned int clflush;
+        bool accessing;
 };
 
 #define workload_q_head(vgpu, ring_id) \
-        (&(vgpu->workload_q_head[ring_id]))
+        (&(vgpu->submission.workload_q_head[ring_id]))
 
 #define queue_workload(workload) do { \
         list_add_tail(&workload->list, \
@@ -137,12 +138,23 @@ void intel_gvt_clean_workload_scheduler(struct intel_gvt *gvt);
 
 void intel_gvt_wait_vgpu_idle(struct intel_vgpu *vgpu);
 
-int intel_vgpu_init_gvt_context(struct intel_vgpu *vgpu);
+int intel_vgpu_setup_submission(struct intel_vgpu *vgpu);
 
-void intel_vgpu_clean_gvt_context(struct intel_vgpu *vgpu);
+void intel_vgpu_reset_submission(struct intel_vgpu *vgpu,
+                                 unsigned long engine_mask);
 
-void release_shadow_wa_ctx(struct intel_shadow_wa_ctx *wa_ctx);
+void intel_vgpu_clean_submission(struct intel_vgpu *vgpu);
 
-int intel_gvt_generate_request(struct intel_vgpu_workload *workload);
+int intel_vgpu_select_submission_ops(struct intel_vgpu *vgpu,
+                                     unsigned int interface);
+
+extern const struct intel_vgpu_submission_ops
+intel_vgpu_execlist_submission_ops;
+
+struct intel_vgpu_workload *
+intel_vgpu_create_workload(struct intel_vgpu *vgpu, int ring_id,
+                           struct execlist_ctx_descriptor_format *desc);
+
+void intel_vgpu_destroy_workload(struct intel_vgpu_workload *workload);
 
 #endif
diff --git a/drivers/gpu/drm/i915/gvt/vgpu.c b/drivers/gpu/drm/i915/gvt/vgpu.c
index 02c61a1ad56a..c6b82d1ba7de 100644
--- a/drivers/gpu/drm/i915/gvt/vgpu.c
+++ b/drivers/gpu/drm/i915/gvt/vgpu.c
@@ -43,7 +43,10 @@ void populate_pvinfo_page(struct intel_vgpu *vgpu)
         vgpu_vreg(vgpu, vgtif_reg(version_minor)) = 0;
         vgpu_vreg(vgpu, vgtif_reg(display_ready)) = 0;
         vgpu_vreg(vgpu, vgtif_reg(vgt_id)) = vgpu->id;
+
         vgpu_vreg(vgpu, vgtif_reg(vgt_caps)) = VGT_CAPS_FULL_48BIT_PPGTT;
+        vgpu_vreg(vgpu, vgtif_reg(vgt_caps)) |= VGT_CAPS_HWSP_EMULATION;
+
         vgpu_vreg(vgpu, vgtif_reg(avail_rs.mappable_gmadr.base)) =
                 vgpu_aperture_gmadr_base(vgpu);
         vgpu_vreg(vgpu, vgtif_reg(avail_rs.mappable_gmadr.size)) =
@@ -226,7 +229,7 @@ void intel_gvt_deactivate_vgpu(struct intel_vgpu *vgpu)
 
         vgpu->active = false;
 
-        if (atomic_read(&vgpu->running_workload_num)) {
+        if (atomic_read(&vgpu->submission.running_workload_num)) {
                 mutex_unlock(&gvt->lock);
                 intel_gvt_wait_vgpu_idle(vgpu);
                 mutex_lock(&gvt->lock);
@@ -252,10 +255,10 @@ void intel_gvt_destroy_vgpu(struct intel_vgpu *vgpu)
 
         WARN(vgpu->active, "vGPU is still active!\n");
 
+        intel_gvt_debugfs_remove_vgpu(vgpu);
         idr_remove(&gvt->vgpu_idr, vgpu->id);
         intel_vgpu_clean_sched_policy(vgpu);
-        intel_vgpu_clean_gvt_context(vgpu);
-        intel_vgpu_clean_execlist(vgpu);
+        intel_vgpu_clean_submission(vgpu);
         intel_vgpu_clean_display(vgpu);
         intel_vgpu_clean_opregion(vgpu);
         intel_vgpu_clean_gtt(vgpu);
@@ -293,7 +296,7 @@ struct intel_vgpu *intel_gvt_create_idle_vgpu(struct intel_gvt *gvt)
         vgpu->gvt = gvt;
 
         for (i = 0; i < I915_NUM_ENGINES; i++)
-                INIT_LIST_HEAD(&vgpu->workload_q_head[i]);
+                INIT_LIST_HEAD(&vgpu->submission.workload_q_head[i]);
 
         ret = intel_vgpu_init_sched_policy(vgpu);
         if (ret)
@@ -346,7 +349,6 @@ static struct intel_vgpu *__intel_gvt_create_vgpu(struct intel_gvt *gvt,
         vgpu->handle = param->handle;
         vgpu->gvt = gvt;
         vgpu->sched_ctl.weight = param->weight;
-        bitmap_zero(vgpu->tlb_handle_pending, I915_NUM_ENGINES);
 
         intel_vgpu_init_cfg_space(vgpu, param->primary);
 
@@ -372,26 +374,26 @@ static struct intel_vgpu *__intel_gvt_create_vgpu(struct intel_gvt *gvt,
         if (ret)
                 goto out_clean_gtt;
 
-        ret = intel_vgpu_init_execlist(vgpu);
+        ret = intel_vgpu_setup_submission(vgpu);
         if (ret)
                 goto out_clean_display;
 
-        ret = intel_vgpu_init_gvt_context(vgpu);
+        ret = intel_vgpu_init_sched_policy(vgpu);
         if (ret)
-                goto out_clean_execlist;
+                goto out_clean_submission;
 
-        ret = intel_vgpu_init_sched_policy(vgpu);
+        ret = intel_gvt_debugfs_add_vgpu(vgpu);
         if (ret)
-                goto out_clean_shadow_ctx;
+                goto out_clean_sched_policy;
 
         mutex_unlock(&gvt->lock);
 
         return vgpu;
 
-out_clean_shadow_ctx:
-        intel_vgpu_clean_gvt_context(vgpu);
-out_clean_execlist:
-        intel_vgpu_clean_execlist(vgpu);
+out_clean_sched_policy:
+        intel_vgpu_clean_sched_policy(vgpu);
+out_clean_submission:
+        intel_vgpu_clean_submission(vgpu);
 out_clean_display:
         intel_vgpu_clean_display(vgpu);
 out_clean_gtt:
@@ -500,10 +502,10 @@ void intel_gvt_reset_vgpu_locked(struct intel_vgpu *vgpu, bool dmlr,
                 mutex_lock(&gvt->lock);
         }
 
-        intel_vgpu_reset_execlist(vgpu, resetting_eng);
-
+        intel_vgpu_reset_submission(vgpu, resetting_eng);
         /* full GPU reset or device model level reset */
         if (engine_mask == ALL_ENGINES || dmlr) {
+                intel_vgpu_select_submission_ops(vgpu, 0);
 
                 /*fence will not be reset during virtual reset */
                 if (dmlr) {
diff --git a/drivers/gpu/drm/i915/i915_cmd_parser.c b/drivers/gpu/drm/i915/i915_cmd_parser.c
index 8ba932b22f7c..b11629beeb63 100644
--- a/drivers/gpu/drm/i915/i915_cmd_parser.c
+++ b/drivers/gpu/drm/i915/i915_cmd_parser.c
@@ -798,22 +798,15 @@ struct cmd_node {
  */
 static inline u32 cmd_header_key(u32 x)
 {
-        u32 shift;
-
         switch (x >> INSTR_CLIENT_SHIFT) {
         default:
         case INSTR_MI_CLIENT:
-                shift = STD_MI_OPCODE_SHIFT;
-                break;
+                return x >> STD_MI_OPCODE_SHIFT;
         case INSTR_RC_CLIENT:
-                shift = STD_3D_OPCODE_SHIFT;
-                break;
+                return x >> STD_3D_OPCODE_SHIFT;
         case INSTR_BC_CLIENT:
-                shift = STD_2D_OPCODE_SHIFT;
-                break;
+                return x >> STD_2D_OPCODE_SHIFT;
         }
-
-        return x >> shift;
 }
 
 static int init_hash_table(struct intel_engine_cs *engine,
diff --git a/drivers/gpu/drm/i915/i915_debugfs.c b/drivers/gpu/drm/i915/i915_debugfs.c
index c65e381b85f3..df3852c02a35 100644
--- a/drivers/gpu/drm/i915/i915_debugfs.c
+++ b/drivers/gpu/drm/i915/i915_debugfs.c
@@ -30,7 +30,7 @@
 #include <linux/sort.h>
 #include <linux/sched/mm.h>
 #include "intel_drv.h"
-#include "i915_guc_submission.h"
+#include "intel_guc_submission.h"
 
 static inline struct drm_i915_private *node_to_i915(struct drm_info_node *node)
 {
@@ -1974,7 +1974,6 @@ static int i915_context_status(struct seq_file *m, void *unused)
                         struct intel_context *ce = &ctx->engine[engine->id];
 
                         seq_printf(m, "%s: ", engine->name);
-                        seq_putc(m, ce->initialised ? 'I' : 'i');
                         if (ce->state)
                                 describe_obj(m, ce->state->obj);
                         if (ce->ring)
@@ -2434,7 +2433,7 @@ static void i915_guc_log_info(struct seq_file *m,
 
 static void i915_guc_client_info(struct seq_file *m,
                                  struct drm_i915_private *dev_priv,
-                                 struct i915_guc_client *client)
+                                 struct intel_guc_client *client)
 {
         struct intel_engine_cs *engine;
         enum intel_engine_id id;
@@ -2484,6 +2483,8 @@ static int i915_guc_info(struct seq_file *m, void *data)
 
         seq_printf(m, "\nGuC execbuf client @ %p:\n", guc->execbuf_client);
         i915_guc_client_info(m, dev_priv, guc->execbuf_client);
+        seq_printf(m, "\nGuC preempt client @ %p:\n", guc->preempt_client);
+        i915_guc_client_info(m, dev_priv, guc->preempt_client);
 
         i915_guc_log_info(m, dev_priv);
 
@@ -2497,7 +2498,7 @@ static int i915_guc_stage_pool(struct seq_file *m, void *data)
         struct drm_i915_private *dev_priv = node_to_i915(m->private);
         const struct intel_guc *guc = &dev_priv->guc;
         struct guc_stage_desc *desc = guc->stage_desc_pool_vaddr;
-        struct i915_guc_client *client = guc->execbuf_client;
+        struct intel_guc_client *client = guc->execbuf_client;
         unsigned int tmp;
         int index;
 
@@ -2734,39 +2735,76 @@ static int i915_sink_crc(struct seq_file *m, void *data)
         struct intel_connector *connector;
         struct drm_connector_list_iter conn_iter;
         struct intel_dp *intel_dp = NULL;
+        struct drm_modeset_acquire_ctx ctx;
         int ret;
         u8 crc[6];
 
-        drm_modeset_lock_all(dev);
+        drm_modeset_acquire_init(&ctx, DRM_MODESET_ACQUIRE_INTERRUPTIBLE);
+
         drm_connector_list_iter_begin(dev, &conn_iter);
+
         for_each_intel_connector_iter(connector, &conn_iter) {
                 struct drm_crtc *crtc;
+                struct drm_connector_state *state;
+                struct intel_crtc_state *crtc_state;
 
-                if (!connector->base.state->best_encoder)
+                if (connector->base.connector_type != DRM_MODE_CONNECTOR_eDP)
                         continue;
 
-                crtc = connector->base.state->crtc;
-                if (!crtc->state->active)
+retry:
+                ret = drm_modeset_lock(&dev->mode_config.connection_mutex, &ctx);
+                if (ret)
+                        goto err;
+
+                state = connector->base.state;
+                if (!state->best_encoder)
                         continue;
 
-                if (connector->base.connector_type != DRM_MODE_CONNECTOR_eDP)
+                crtc = state->crtc;
+                ret = drm_modeset_lock(&crtc->mutex, &ctx);
+                if (ret)
+                        goto err;
+
+                crtc_state = to_intel_crtc_state(crtc->state);
+                if (!crtc_state->base.active)
                         continue;
 
-                intel_dp = enc_to_intel_dp(connector->base.state->best_encoder);
+                /*
+                 * We need to wait for all crtc updates to complete, to make
+                 * sure any pending modesets and plane updates are completed.
+                 */
+                if (crtc_state->base.commit) {
+                        ret = wait_for_completion_interruptible(&crtc_state->base.commit->hw_done);
 
-                ret = intel_dp_sink_crc(intel_dp, crc);
+                        if (ret)
+                                goto err;
+                }
+
+                intel_dp = enc_to_intel_dp(state->best_encoder);
+
+                ret = intel_dp_sink_crc(intel_dp, crtc_state, crc);
                 if (ret)
-                        goto out;
+                        goto err;
 
                 seq_printf(m, "%02x%02x%02x%02x%02x%02x\n",
                            crc[0], crc[1], crc[2],
                            crc[3], crc[4], crc[5]);
                 goto out;
+
+err:
+                if (ret == -EDEADLK) {
+                        ret = drm_modeset_backoff(&ctx);
+                        if (!ret)
+                                goto retry;
+                }
+                goto out;
         }
         ret = -ENODEV;
 out:
         drm_connector_list_iter_end(&conn_iter);
-        drm_modeset_unlock_all(dev);
+        drm_modeset_drop_locks(&ctx);
+        drm_modeset_acquire_fini(&ctx);
+
         return ret;
 }
 
@@ -3049,7 +3087,7 @@ static void intel_connector_info(struct seq_file *m,
                 break;
         case DRM_MODE_CONNECTOR_HDMIA:
                 if (intel_encoder->type == INTEL_OUTPUT_HDMI ||
-                    intel_encoder->type == INTEL_OUTPUT_UNKNOWN)
+                    intel_encoder->type == INTEL_OUTPUT_DDI)
                         intel_hdmi_info(m, intel_connector);
                 break;
         default:
@@ -3244,6 +3282,8 @@ static int i915_engine_info(struct seq_file *m, void *unused)
                    yesno(dev_priv->gt.awake));
         seq_printf(m, "Global active requests: %d\n",
                    dev_priv->gt.active_requests);
+        seq_printf(m, "CS timestamp frequency: %u kHz\n",
+                   dev_priv->info.cs_timestamp_frequency_khz);
 
         p = drm_seq_file_printer(m);
         for_each_engine(engine, dev_priv, id)
@@ -3601,7 +3641,7 @@ static int i915_dp_mst_info(struct seq_file *m, void *unused)
                         continue;
 
                 seq_printf(m, "MST Source Port %c\n",
-                           port_name(intel_dig_port->port));
+                           port_name(intel_dig_port->base.port));
                 drm_dp_mst_dump_topology(m, &intel_dig_port->dp.mst_mgr);
         }
         drm_connector_list_iter_end(&conn_iter);
@@ -4448,6 +4488,61 @@ static void cherryview_sseu_device_status(struct drm_i915_private *dev_priv,
         }
 }
 
+static void gen10_sseu_device_status(struct drm_i915_private *dev_priv,
+                                     struct sseu_dev_info *sseu)
+{
+        const struct intel_device_info *info = INTEL_INFO(dev_priv);
+        int s_max = 6, ss_max = 4;
+        int s, ss;
+        u32 s_reg[s_max], eu_reg[2 * s_max], eu_mask[2];
+
+        for (s = 0; s < s_max; s++) {
+                /*
+                 * FIXME: Valid SS Mask respects the spec and read
+                 * only valid bits for those registers, excluding reserverd
+                 * although this seems wrong because it would leave many
+                 * subslices without ACK.
+                 */
+                s_reg[s] = I915_READ(GEN10_SLICE_PGCTL_ACK(s)) &
+                        GEN10_PGCTL_VALID_SS_MASK(s);
+                eu_reg[2 * s] = I915_READ(GEN10_SS01_EU_PGCTL_ACK(s));
+                eu_reg[2 * s + 1] = I915_READ(GEN10_SS23_EU_PGCTL_ACK(s));
+        }
+
+        eu_mask[0] = GEN9_PGCTL_SSA_EU08_ACK |
+                     GEN9_PGCTL_SSA_EU19_ACK |
+                     GEN9_PGCTL_SSA_EU210_ACK |
+                     GEN9_PGCTL_SSA_EU311_ACK;
+        eu_mask[1] = GEN9_PGCTL_SSB_EU08_ACK |
+                     GEN9_PGCTL_SSB_EU19_ACK |
+                     GEN9_PGCTL_SSB_EU210_ACK |
+                     GEN9_PGCTL_SSB_EU311_ACK;
+
+        for (s = 0; s < s_max; s++) {
+                if ((s_reg[s] & GEN9_PGCTL_SLICE_ACK) == 0)
+                        /* skip disabled slice */
+                        continue;
+
+                sseu->slice_mask |= BIT(s);
+                sseu->subslice_mask = info->sseu.subslice_mask;
+
+                for (ss = 0; ss < ss_max; ss++) {
+                        unsigned int eu_cnt;
+
+                        if (!(s_reg[s] & (GEN9_PGCTL_SS_ACK(ss))))
+                                /* skip disabled subslice */
+                                continue;
+
+                        eu_cnt = 2 * hweight32(eu_reg[2 * s + ss / 2] &
+                                               eu_mask[ss % 2]);
+                        sseu->eu_total += eu_cnt;
+                        sseu->eu_per_subslice = max_t(unsigned int,
+                                                      sseu->eu_per_subslice,
+                                                      eu_cnt);
+                }
+        }
+}
+
 static void gen9_sseu_device_status(struct drm_i915_private *dev_priv,
                                     struct sseu_dev_info *sseu)
 {
@@ -4483,7 +4578,7 @@ static void gen9_sseu_device_status(struct drm_i915_private *dev_priv,
 
                 sseu->slice_mask |= BIT(s);
 
-                if (IS_GEN9_BC(dev_priv) || IS_CANNONLAKE(dev_priv))
+                if (IS_GEN9_BC(dev_priv))
                         sseu->subslice_mask =
                                 INTEL_INFO(dev_priv)->sseu.subslice_mask;
 
@@ -4589,8 +4684,10 @@ static int i915_sseu_status(struct seq_file *m, void *unused)
                 cherryview_sseu_device_status(dev_priv, &sseu);
         } else if (IS_BROADWELL(dev_priv)) {
                 broadwell_sseu_device_status(dev_priv, &sseu);
-        } else if (INTEL_GEN(dev_priv) >= 9) {
+        } else if (IS_GEN9(dev_priv)) {
                 gen9_sseu_device_status(dev_priv, &sseu);
+        } else if (INTEL_GEN(dev_priv) >= 10) {
+                gen10_sseu_device_status(dev_priv, &sseu);
         }
 
         intel_runtime_pm_put(dev_priv);
diff --git a/drivers/gpu/drm/i915/i915_drv.c b/drivers/gpu/drm/i915/i915_drv.c
index 2cf10d17acfb..5170a8ea83d4 100644
--- a/drivers/gpu/drm/i915/i915_drv.c
+++ b/drivers/gpu/drm/i915/i915_drv.c
@@ -372,9 +372,8 @@ static int i915_getparam(struct drm_device *dev, void *data,
                         value |= I915_SCHEDULER_CAP_ENABLED;
                         value |= I915_SCHEDULER_CAP_PRIORITY;
 
-                        if (INTEL_INFO(dev_priv)->has_logical_ring_preemption &&
-                            i915_modparams.enable_execlists &&
-                            !i915_modparams.enable_guc_submission)
+                        if (HAS_LOGICAL_RING_PREEMPTION(dev_priv) &&
+                            i915_modparams.enable_execlists)
                                 value |= I915_SCHEDULER_CAP_PREEMPTION;
                 }
                 break;
@@ -407,6 +406,9 @@ static int i915_getparam(struct drm_device *dev, void *data,
                  */
                 value = 1;
                 break;
+        case I915_PARAM_HAS_CONTEXT_ISOLATION:
+                value = intel_engines_has_context_isolation(dev_priv);
+                break;
         case I915_PARAM_SLICE_MASK:
                 value = INTEL_INFO(dev_priv)->sseu.slice_mask;
                 if (!value)
@@ -417,6 +419,9 @@ static int i915_getparam(struct drm_device *dev, void *data,
                 if (!value)
                         return -ENODEV;
                 break;
+        case I915_PARAM_CS_TIMESTAMP_FREQUENCY:
+                value = 1000 * INTEL_INFO(dev_priv)->cs_timestamp_frequency_khz;
+                break;
         default:
                 DRM_DEBUG("Unknown parameter %d\n", param->param);
                 return -EINVAL;
@@ -677,7 +682,7 @@ static int i915_load_modeset_init(struct drm_device *dev)
         if (ret)
                 goto cleanup_uc;
 
-        intel_modeset_gem_init(dev);
+        intel_setup_overlay(dev_priv);
 
         if (INTEL_INFO(dev_priv)->num_pipes == 0)
                 return 0;
@@ -838,6 +843,11 @@ static void i915_workqueues_cleanup(struct drm_i915_private *dev_priv)
  * We don't keep the workarounds for pre-production hardware, so we expect our
  * driver to fail on these machines in one way or another. A little warning on
  * dmesg may help both the user and the bug triagers.
+ *
+ * Our policy for removing pre-production workarounds is to keep the
+ * current gen workarounds as a guide to the bring-up of the next gen
+ * (workarounds have a habit of persisting!). Anything older than that
+ * should be removed along with the complications they introduce.
  */
 static void intel_detect_preproduction_hw(struct drm_i915_private *dev_priv)
 {
@@ -892,7 +902,6 @@ static int i915_driver_init_early(struct drm_i915_private *dev_priv,
         mutex_init(&dev_priv->backlight_lock);
         spin_lock_init(&dev_priv->uncore.lock);
 
-        spin_lock_init(&dev_priv->mm.object_stat_lock);
         mutex_init(&dev_priv->sb_lock);
         mutex_init(&dev_priv->modeset_restore_lock);
         mutex_init(&dev_priv->av_mutex);
@@ -1682,8 +1691,6 @@ static int i915_drm_resume(struct drm_device *dev)
 
         intel_csr_ucode_resume(dev_priv);
 
-        i915_gem_resume(dev_priv);
-
         i915_restore_state(dev_priv);
         intel_pps_unlock_regs_wa(dev_priv);
         intel_opregion_setup(dev_priv);
@@ -1704,14 +1711,7 @@ static int i915_drm_resume(struct drm_device *dev)
 
         drm_mode_config_reset(dev);
 
-        mutex_lock(&dev->struct_mutex);
-        if (i915_gem_init_hw(dev_priv)) {
-                DRM_ERROR("failed to re-initialize GPU, declaring wedged!\n");
-                i915_gem_set_wedged(dev_priv);
-        }
-        mutex_unlock(&dev->struct_mutex);
-
-        intel_guc_resume(dev_priv);
+        i915_gem_resume(dev_priv);
 
         intel_modeset_init_hw(dev);
         intel_init_clock_gating(dev_priv);
@@ -1745,8 +1745,6 @@ static int i915_drm_resume(struct drm_device *dev)
 
         intel_opregion_notify_adapter(dev_priv, PCI_D0);
 
-        intel_autoenable_gt_powersave(dev_priv);
-
         enable_rpm_wakeref_asserts(dev_priv);
 
         return 0;
@@ -1952,6 +1950,12 @@ error:
         goto finish;
 }
 
+static inline int intel_gt_reset_engine(struct drm_i915_private *dev_priv,
+                                        struct intel_engine_cs *engine)
+{
+        return intel_gpu_reset(dev_priv, intel_engine_flag(engine));
+}
+
 /**
  * i915_reset_engine - reset GPU engine to recover from a hang
  * @engine: engine to reset
@@ -1986,10 +1990,14 @@ int i915_reset_engine(struct intel_engine_cs *engine, unsigned int flags)
                 goto out;
         }
 
-        ret = intel_gpu_reset(engine->i915, intel_engine_flag(engine));
+        if (!engine->i915->guc.execbuf_client)
+                ret = intel_gt_reset_engine(engine->i915, engine);
+        else
+                ret = intel_guc_reset_engine(&engine->i915->guc, engine);
         if (ret) {
                 /* If we fail here, we expect to fallback to a global reset */
-                DRM_DEBUG_DRIVER("Failed to reset %s, ret=%d\n",
+                DRM_DEBUG_DRIVER("%sFailed to reset %s, ret=%d\n",
+                                 engine->i915->guc.execbuf_client ? "GuC " : "",
                                  engine->name, ret);
                 goto out;
         }
@@ -2524,6 +2532,8 @@ static int intel_runtime_suspend(struct device *kdev)
 
         intel_runtime_pm_disable_interrupts(dev_priv);
 
+        intel_uncore_suspend(dev_priv);
+
         ret = 0;
         if (IS_GEN9_LP(dev_priv)) {
                 bxt_display_core_uninit(dev_priv);
@@ -2536,6 +2546,8 @@ static int intel_runtime_suspend(struct device *kdev)
 
         if (ret) {
                 DRM_ERROR("Runtime suspend failed, disabling it (%d)\n", ret);
+                intel_uncore_runtime_resume(dev_priv);
+
                 intel_runtime_pm_enable_interrupts(dev_priv);
 
                 enable_rpm_wakeref_asserts(dev_priv);
@@ -2543,8 +2555,6 @@ static int intel_runtime_suspend(struct device *kdev)
                 return ret;
         }
 
-        intel_uncore_suspend(dev_priv);
-
         enable_rpm_wakeref_asserts(dev_priv);
         WARN_ON_ONCE(atomic_read(&dev_priv->runtime_pm.wakeref_count));
 
diff --git a/drivers/gpu/drm/i915/i915_drv.h b/drivers/gpu/drm/i915/i915_drv.h
index 54b5d4c582b6..36bb4927484a 100644
--- a/drivers/gpu/drm/i915/i915_drv.h
+++ b/drivers/gpu/drm/i915/i915_drv.h
@@ -67,7 +67,6 @@
 #include "i915_gem_fence_reg.h"
 #include "i915_gem_object.h"
 #include "i915_gem_gtt.h"
-#include "i915_gem_render_state.h"
 #include "i915_gem_request.h"
 #include "i915_gem_timeline.h"
 
@@ -80,8 +79,8 @@
 
 #define DRIVER_NAME             "i915"
 #define DRIVER_DESC             "Intel Graphics"
-#define DRIVER_DATE             "20171023"
-#define DRIVER_TIMESTAMP        1508748913
+#define DRIVER_DATE             "20171117"
+#define DRIVER_TIMESTAMP        1510958822
 
 /* Use I915_STATE_WARN(x) and I915_STATE_WARN_ON() (rather than WARN() and
  * WARN_ON()) for hw state sanity checks to check for unexpected conditions
@@ -726,10 +725,12 @@ struct drm_i915_display_funcs {
         void (*crtc_disable)(struct intel_crtc_state *old_crtc_state,
                              struct drm_atomic_state *old_state);
         void (*update_crtcs)(struct drm_atomic_state *state);
-        void (*audio_codec_enable)(struct drm_connector *connector,
-                                   struct intel_encoder *encoder,
-                                   const struct drm_display_mode *adjusted_mode);
-        void (*audio_codec_disable)(struct intel_encoder *encoder);
+        void (*audio_codec_enable)(struct intel_encoder *encoder,
+                                   const struct intel_crtc_state *crtc_state,
+                                   const struct drm_connector_state *conn_state);
+        void (*audio_codec_disable)(struct intel_encoder *encoder,
+                                    const struct intel_crtc_state *old_crtc_state,
+                                    const struct drm_connector_state *old_conn_state);
         void (*fdi_link_train)(struct intel_crtc *crtc,
                                const struct intel_crtc_state *crtc_state);
         void (*init_clock_gating)(struct drm_i915_private *dev_priv);
@@ -884,6 +885,8 @@ struct intel_device_info {
         /* Slice/subslice/EU info */
         struct sseu_dev_info sseu;
 
+        u32 cs_timestamp_frequency_khz;
+
         struct color_luts {
                 u16 degamma_lut_size;
                 u16 gamma_lut_size;
@@ -911,6 +914,12 @@ struct i915_gpu_state {
         struct intel_device_info device_info;
         struct i915_params params;
 
+        struct i915_error_uc {
+                struct intel_uc_fw guc_fw;
+                struct intel_uc_fw huc_fw;
+                struct drm_i915_error_object *guc_log;
+        } uc;
+
         /* Generic register state */
         u32 eir;
         u32 pgtbl_er;
@@ -934,7 +943,6 @@ struct i915_gpu_state {
         struct intel_overlay_error_state *overlay;
         struct intel_display_error_state *display;
         struct drm_i915_error_object *semaphore;
-        struct drm_i915_error_object *guc_log;
 
         struct drm_i915_error_engine {
                 int engine_id;
@@ -1386,7 +1394,6 @@ struct intel_gen6_power_mgmt {
         struct intel_rps rps;
         struct intel_rc6 rc6;
         struct intel_llc_pstate llc_pstate;
-        struct delayed_work autoenable_work;
 };
 
 /* defined intel_pm.c */
@@ -1698,6 +1705,8 @@ enum modeset_restore {
 #define DDC_PIN_D  0x06
 
 struct ddi_vbt_port_info {
+        int max_tmds_clock;
+
         /*
          * This is an index in the HDMI/DVI DDI buffer translation table.
          * The special value HDMI_LEVEL_SHIFT_UNKNOWN means the VBT didn't
@@ -2228,6 +2237,7 @@ struct i915_oa_ops {
 
 struct intel_cdclk_state {
         unsigned int cdclk, vco, ref;
+        u8 voltage_level;
 };
 
 struct drm_i915_private {
@@ -2339,6 +2349,7 @@ struct drm_i915_private {
         unsigned int max_dotclk_freq;
         unsigned int rawclk_freq;
         unsigned int hpll_freq;
+        unsigned int fdi_pll_freq;
         unsigned int czclk_freq;
 
         struct {
@@ -2415,6 +2426,8 @@ struct drm_i915_private {
         unsigned int active_crtcs;
         /* minimum acceptable cdclk for each pipe */
         int min_cdclk[I915_MAX_PIPES];
+        /* minimum acceptable voltage level for each pipe */
+        u8 min_voltage_level[I915_MAX_PIPES];
 
         int dpio_phy_iosf_port[I915_NUM_PHYS_VLV];
 
@@ -3046,6 +3059,8 @@ intel_info(const struct drm_i915_private *dev_priv)
                                  (INTEL_DEVID(dev_priv) & 0x00F0) == 0x00A0)
 #define IS_CFL_GT2(dev_priv)    (IS_COFFEELAKE(dev_priv) && \
                                  (dev_priv)->info.gt == 2)
+#define IS_CFL_GT3(dev_priv)    (IS_COFFEELAKE(dev_priv) && \
+                                 (dev_priv)->info.gt == 3)
 
 #define IS_ALPHA_SUPPORT(intel_info) ((intel_info)->is_alpha_support)
 
@@ -3137,6 +3152,8 @@ intel_info(const struct drm_i915_private *dev_priv)
 
 #define HAS_LOGICAL_RING_CONTEXTS(dev_priv) \
                 ((dev_priv)->info.has_logical_ring_contexts)
+#define HAS_LOGICAL_RING_PREEMPTION(dev_priv) \
+                ((dev_priv)->info.has_logical_ring_preemption)
 #define USES_PPGTT(dev_priv)            (i915_modparams.enable_ppgtt)
 #define USES_FULL_PPGTT(dev_priv)       (i915_modparams.enable_ppgtt >= 2)
 #define USES_FULL_48BIT_PPGTT(dev_priv) (i915_modparams.enable_ppgtt == 3)
@@ -3315,7 +3332,9 @@ extern int i915_reset_engine(struct intel_engine_cs *engine,
                              unsigned int flags);
 
 extern bool intel_has_reset_engine(struct drm_i915_private *dev_priv);
-extern int intel_guc_reset(struct drm_i915_private *dev_priv);
+extern int intel_reset_guc(struct drm_i915_private *dev_priv);
+extern int intel_guc_reset_engine(struct intel_guc *guc,
+                                  struct intel_engine_cs *engine);
 extern void intel_engine_init_hangcheck(struct intel_engine_cs *engine);
 extern void intel_hangcheck_init(struct drm_i915_private *dev_priv);
 extern unsigned long i915_chipset_val(struct drm_i915_private *dev_priv);
@@ -4107,7 +4126,6 @@ void intel_device_info_dump(struct drm_i915_private *dev_priv);
 /* modesetting */
 extern void intel_modeset_init_hw(struct drm_device *dev);
 extern int intel_modeset_init(struct drm_device *dev);
-extern void intel_modeset_gem_init(struct drm_device *dev);
 extern void intel_modeset_cleanup(struct drm_device *dev);
 extern int intel_connector_register(struct drm_connector *);
 extern void intel_connector_unregister(struct drm_connector *);
@@ -4174,8 +4192,7 @@ bool bxt_ddi_phy_is_enabled(struct drm_i915_private *dev_priv,
                             enum dpio_phy phy);
 bool bxt_ddi_phy_verify_state(struct drm_i915_private *dev_priv,
                               enum dpio_phy phy);
-uint8_t bxt_ddi_phy_calc_lane_lat_optim_mask(struct intel_encoder *encoder,
-                                             uint8_t lane_count);
+uint8_t bxt_ddi_phy_calc_lane_lat_optim_mask(uint8_t lane_count);
 void bxt_ddi_phy_set_lane_optim_mask(struct intel_encoder *encoder,
                                      uint8_t lane_lat_optim_mask);
 uint8_t bxt_ddi_phy_get_lane_lat_optim_mask(struct intel_encoder *encoder);
@@ -4184,18 +4201,25 @@ void chv_set_phy_signal_level(struct intel_encoder *encoder,
                               u32 deemph_reg_value, u32 margin_reg_value,
                               bool uniq_trans_scale);
 void chv_data_lane_soft_reset(struct intel_encoder *encoder,
+                              const struct intel_crtc_state *crtc_state,
                               bool reset);
-void chv_phy_pre_pll_enable(struct intel_encoder *encoder);
-void chv_phy_pre_encoder_enable(struct intel_encoder *encoder);
+void chv_phy_pre_pll_enable(struct intel_encoder *encoder,
+                            const struct intel_crtc_state *crtc_state);
+void chv_phy_pre_encoder_enable(struct intel_encoder *encoder,
+                                const struct intel_crtc_state *crtc_state);
 void chv_phy_release_cl2_override(struct intel_encoder *encoder);
-void chv_phy_post_pll_disable(struct intel_encoder *encoder);
+void chv_phy_post_pll_disable(struct intel_encoder *encoder,
+                              const struct intel_crtc_state *old_crtc_state);
 
 void vlv_set_phy_signal_level(struct intel_encoder *encoder,
                               u32 demph_reg_value, u32 preemph_reg_value,
                               u32 uniqtranscale_reg_value, u32 tx3_demph);
-void vlv_phy_pre_pll_enable(struct intel_encoder *encoder);
-void vlv_phy_pre_encoder_enable(struct intel_encoder *encoder);
-void vlv_phy_reset_lanes(struct intel_encoder *encoder);
+void vlv_phy_pre_pll_enable(struct intel_encoder *encoder,
+                            const struct intel_crtc_state *crtc_state);
+void vlv_phy_pre_encoder_enable(struct intel_encoder *encoder,
+                                const struct intel_crtc_state *crtc_state);
+void vlv_phy_reset_lanes(struct intel_encoder *encoder,
+                         const struct intel_crtc_state *old_crtc_state);
 
 int intel_gpu_freq(struct drm_i915_private *dev_priv, int val);
 int intel_freq_opcode(struct drm_i915_private *dev_priv, int val);
diff --git a/drivers/gpu/drm/i915/i915_gem.c b/drivers/gpu/drm/i915/i915_gem.c
index 3a140eedfc83..61ba321e9970 100644
--- a/drivers/gpu/drm/i915/i915_gem.c
+++ b/drivers/gpu/drm/i915/i915_gem.c
@@ -538,7 +538,7 @@ i915_gem_object_wait_priority(struct drm_i915_gem_object *obj,
  * @obj: i915 gem object
  * @flags: how to wait (under a lock, for all rendering or just for writes etc)
  * @timeout: how long to wait
- * @rps: client (user process) to charge for any waitboosting
+ * @rps_client: client (user process) to charge for any waitboosting
  */
 int
 i915_gem_object_wait(struct drm_i915_gem_object *obj,
@@ -1619,7 +1619,19 @@ i915_gem_set_domain_ioctl(struct drm_device *dev, void *data,
         if (err)
                 goto out;
 
-        /* Flush and acquire obj->pages so that we are coherent through
+        /*
+         * Proxy objects do not control access to the backing storage, ergo
+         * they cannot be used as a means to manipulate the cache domain
+         * tracking for that backing storage. The proxy object is always
+         * considered to be outside of any cache domain.
+         */
+        if (i915_gem_object_is_proxy(obj)) {
+                err = -ENXIO;
+                goto out;
+        }
+
+        /*
+         * Flush and acquire obj->pages so that we are coherent through
          * direct access in memory with previous cached writes through
          * shmemfs and that our cache domain tracking remains valid.
          * For example, if the obj->filp was moved to swap without us
@@ -1675,6 +1687,11 @@ i915_gem_sw_finish_ioctl(struct drm_device *dev, void *data,
         if (!obj)
                 return -ENOENT;
 
+        /*
+         * Proxy objects are barred from CPU access, so there is no
+         * need to ban sw_finish as it is a nop.
+         */
+
         /* Pinned buffers may be scanout, so flush the cache */
         i915_gem_object_flush_if_display(obj);
         i915_gem_object_put(obj);
@@ -1725,7 +1742,7 @@ i915_gem_mmap_ioctl(struct drm_device *dev, void *data,
          */
         if (!obj->base.filp) {
                 i915_gem_object_put(obj);
-                return -EINVAL;
+                return -ENXIO;
         }
 
         addr = vm_mmap(obj->base.filp, 0, args->size,
@@ -2669,7 +2686,8 @@ void *i915_gem_object_pin_map(struct drm_i915_gem_object *obj,
         void *ptr;
         int ret;
 
-        GEM_BUG_ON(!i915_gem_object_has_struct_page(obj));
+        if (unlikely(!i915_gem_object_has_struct_page(obj)))
+                return ERR_PTR(-ENXIO);
 
         ret = mutex_lock_interruptible(&obj->mm.lock);
         if (ret)
@@ -2915,13 +2933,23 @@ i915_gem_reset_prepare_engine(struct intel_engine_cs *engine)
          * Prevent request submission to the hardware until we have
          * completed the reset in i915_gem_reset_finish(). If a request
          * is completed by one engine, it may then queue a request
-         * to a second via its engine->irq_tasklet *just* as we are
+         * to a second via its execlists->tasklet *just* as we are
          * calling engine->init_hw() and also writing the ELSP.
-         * Turning off the engine->irq_tasklet until the reset is over
+         * Turning off the execlists->tasklet until the reset is over
          * prevents the race.
          */
-        tasklet_kill(&engine->execlists.irq_tasklet);
-        tasklet_disable(&engine->execlists.irq_tasklet);
+        tasklet_kill(&engine->execlists.tasklet);
+        tasklet_disable(&engine->execlists.tasklet);
+
+        /*
+         * We're using worker to queue preemption requests from the tasklet in
+         * GuC submission mode.
+         * Even though tasklet was disabled, we may still have a worker queued.
+         * Let's make sure that all workers scheduled before disabling the
+         * tasklet are completed before continuing with the reset.
+         */
+        if (engine->i915->guc.preempt_wq)
+                flush_workqueue(engine->i915->guc.preempt_wq);
 
         if (engine->irq_seqno_barrier)
                 engine->irq_seqno_barrier(engine);
@@ -3100,7 +3128,7 @@ void i915_gem_reset(struct drm_i915_private *dev_priv)
 
 void i915_gem_reset_finish_engine(struct intel_engine_cs *engine)
 {
-        tasklet_enable(&engine->execlists.irq_tasklet);
+        tasklet_enable(&engine->execlists.tasklet);
         kthread_unpark(engine->breadcrumbs.signaler);
 
         intel_uncore_forcewake_put(engine->i915, FORCEWAKE_ALL);
@@ -3278,13 +3306,20 @@ i915_gem_retire_work_handler(struct work_struct *work)
         }
 }
 
+static inline bool
+new_requests_since_last_retire(const struct drm_i915_private *i915)
+{
+        return (READ_ONCE(i915->gt.active_requests) ||
+                work_pending(&i915->gt.idle_work.work));
+}
+
 static void
 i915_gem_idle_work_handler(struct work_struct *work)
 {
         struct drm_i915_private *dev_priv =
                 container_of(work, typeof(*dev_priv), gt.idle_work.work);
-        struct drm_device *dev = &dev_priv->drm;
         bool rearm_hangcheck;
+        ktime_t end;
 
         if (!READ_ONCE(dev_priv->gt.awake))
                 return;
@@ -3293,14 +3328,21 @@ i915_gem_idle_work_handler(struct work_struct *work)
          * Wait for last execlists context complete, but bail out in case a
          * new request is submitted.
          */
-        wait_for(intel_engines_are_idle(dev_priv), 10);
-        if (READ_ONCE(dev_priv->gt.active_requests))
-                return;
+        end = ktime_add_ms(ktime_get(), 200);
+        do {
+                if (new_requests_since_last_retire(dev_priv))
+                        return;
+
+                if (intel_engines_are_idle(dev_priv))
+                        break;
+
+                usleep_range(100, 500);
+        } while (ktime_before(ktime_get(), end));
 
         rearm_hangcheck =
                 cancel_delayed_work_sync(&dev_priv->gpu_error.hangcheck_work);
 
-        if (!mutex_trylock(&dev->struct_mutex)) {
+        if (!mutex_trylock(&dev_priv->drm.struct_mutex)) {
                 /* Currently busy, come back later */
                 mod_delayed_work(dev_priv->wq,
                                  &dev_priv->gt.idle_work,
@@ -3312,16 +3354,23 @@ i915_gem_idle_work_handler(struct work_struct *work)
          * New request retired after this work handler started, extend active
          * period until next instance of the work.
          */
-        if (work_pending(work))
-                goto out_unlock;
-
-        if (dev_priv->gt.active_requests)
+        if (new_requests_since_last_retire(dev_priv))
                 goto out_unlock;
 
-        if (wait_for(intel_engines_are_idle(dev_priv), 10))
-                DRM_ERROR("Timeout waiting for engines to idle\n");
+        /*
+         * Be paranoid and flush a concurrent interrupt to make sure
+         * we don't reactivate any irq tasklets after parking.
+         *
+         * FIXME: Note that even though we have waited for execlists to be idle,
+         * there may still be an in-flight interrupt even though the CSB
+         * is now empty. synchronize_irq() makes sure that a residual interrupt
+         * is completed before we continue, but it doesn't prevent the HW from
+         * raising a spurious interrupt later. To complete the shield we should
+         * coordinate disabling the CS irq with flushing the interrupts.
+         */
+        synchronize_irq(dev_priv->drm.irq);
 
-        intel_engines_mark_idle(dev_priv);
+        intel_engines_park(dev_priv);
         i915_gem_timelines_mark_idle(dev_priv);
 
         GEM_BUG_ON(!dev_priv->gt.awake);
@@ -3332,7 +3381,7 @@ i915_gem_idle_work_handler(struct work_struct *work)
                 gen6_rps_idle(dev_priv);
         intel_runtime_pm_put(dev_priv);
 out_unlock:
-        mutex_unlock(&dev->struct_mutex);
+        mutex_unlock(&dev_priv->drm.struct_mutex);
 
 out_rearm:
         if (rearm_hangcheck) {
@@ -3857,6 +3906,15 @@ int i915_gem_set_caching_ioctl(struct drm_device *dev, void *data,
         if (!obj)
                 return -ENOENT;
 
+        /*
+         * The caching mode of proxy object is handled by its generator, and
+         * not allowed to be changed by userspace.
+         */
+        if (i915_gem_object_is_proxy(obj)) {
+                ret = -ENXIO;
+                goto out;
+        }
+
         if (obj->cache_level == level)
                 goto out;
 
@@ -4662,14 +4720,16 @@ void __i915_gem_object_release_unless_active(struct drm_i915_gem_object *obj)
                 i915_gem_object_put(obj);
 }
 
-static void assert_kernel_context_is_current(struct drm_i915_private *dev_priv)
+static void assert_kernel_context_is_current(struct drm_i915_private *i915)
 {
+        struct i915_gem_context *kernel_context = i915->kernel_context;
         struct intel_engine_cs *engine;
         enum intel_engine_id id;
 
-        for_each_engine(engine, dev_priv, id)
-                GEM_BUG_ON(engine->last_retired_context &&
-                           !i915_gem_context_is_kernel(engine->last_retired_context));
+        for_each_engine(engine, i915, id) {
+                GEM_BUG_ON(__i915_gem_active_peek(&engine->timeline->last_request));
+                GEM_BUG_ON(engine->last_retired_context != kernel_context);
+        }
 }
 
 void i915_gem_sanitize(struct drm_i915_private *i915)
@@ -4773,23 +4833,40 @@ err_unlock:
         return ret;
 }
 
-void i915_gem_resume(struct drm_i915_private *dev_priv)
+void i915_gem_resume(struct drm_i915_private *i915)
 {
-        struct drm_device *dev = &dev_priv->drm;
+        WARN_ON(i915->gt.awake);
 
-        WARN_ON(dev_priv->gt.awake);
+        mutex_lock(&i915->drm.struct_mutex);
+        intel_uncore_forcewake_get(i915, FORCEWAKE_ALL);
 
-        mutex_lock(&dev->struct_mutex);
-        i915_gem_restore_gtt_mappings(dev_priv);
-        i915_gem_restore_fences(dev_priv);
+        i915_gem_restore_gtt_mappings(i915);
+        i915_gem_restore_fences(i915);
 
         /* As we didn't flush the kernel context before suspend, we cannot
          * guarantee that the context image is complete. So let's just reset
          * it and start again.
          */
-        dev_priv->gt.resume(dev_priv);
+        i915->gt.resume(i915);
 
-        mutex_unlock(&dev->struct_mutex);
+        if (i915_gem_init_hw(i915))
+                goto err_wedged;
+
+        intel_guc_resume(i915);
+
+        /* Always reload a context for powersaving. */
+        if (i915_gem_switch_to_kernel_context(i915))
+                goto err_wedged;
+
+out_unlock:
+        intel_uncore_forcewake_put(i915, FORCEWAKE_ALL);
+        mutex_unlock(&i915->drm.struct_mutex);
+        return;
+
+err_wedged:
+        DRM_ERROR("failed to re-initialize GPU, declaring wedged!\n");
+        i915_gem_set_wedged(i915);
+        goto out_unlock;
 }
 
 void i915_gem_init_swizzling(struct drm_i915_private *dev_priv)
@@ -4906,18 +4983,15 @@ int i915_gem_init_hw(struct drm_i915_private *dev_priv)
                 goto out;
         }
 
-        /* Need to do basic initialisation of all rings first: */
-        ret = __i915_gem_restart_engines(dev_priv);
-        if (ret)
-                goto out;
-
-        intel_mocs_init_l3cc_table(dev_priv);
-
         /* We can't enable contexts until all firmware is loaded */
         ret = intel_uc_init_hw(dev_priv);
         if (ret)
                 goto out;
 
+        intel_mocs_init_l3cc_table(dev_priv);
+
+        /* Only when the HW is re-initialised, can we replay the requests */
+        ret = __i915_gem_restart_engines(dev_priv);
 out:
         intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
         return ret;
@@ -4942,6 +5016,120 @@ bool intel_sanitize_semaphores(struct drm_i915_private *dev_priv, int value)
         return true;
 }
 
+static int __intel_engines_record_defaults(struct drm_i915_private *i915)
+{
+        struct i915_gem_context *ctx;
+        struct intel_engine_cs *engine;
+        enum intel_engine_id id;
+        int err;
+
+        /*
+         * As we reset the gpu during very early sanitisation, the current
+         * register state on the GPU should reflect its defaults values.
+         * We load a context onto the hw (with restore-inhibit), then switch
+         * over to a second context to save that default register state. We
+         * can then prime every new context with that state so they all start
+         * from the same default HW values.
+         */
+
+        ctx = i915_gem_context_create_kernel(i915, 0);
+        if (IS_ERR(ctx))
+                return PTR_ERR(ctx);
+
+        for_each_engine(engine, i915, id) {
+                struct drm_i915_gem_request *rq;
+
+                rq = i915_gem_request_alloc(engine, ctx);
+                if (IS_ERR(rq)) {
+                        err = PTR_ERR(rq);
+                        goto out_ctx;
+                }
+
+                err = i915_switch_context(rq);
+                if (engine->init_context)
+                        err = engine->init_context(rq);
+
+                __i915_add_request(rq, true);
+                if (err)
+                        goto err_active;
+        }
+
+        err = i915_gem_switch_to_kernel_context(i915);
+        if (err)
+                goto err_active;
+
+        err = i915_gem_wait_for_idle(i915, I915_WAIT_LOCKED);
+        if (err)
+                goto err_active;
+
+        assert_kernel_context_is_current(i915);
+
+        for_each_engine(engine, i915, id) {
+                struct i915_vma *state;
+
+                state = ctx->engine[id].state;
+                if (!state)
+                        continue;
+
+                /*
+                 * As we will hold a reference to the logical state, it will
+                 * not be torn down with the context, and importantly the
+                 * object will hold onto its vma (making it possible for a
+                 * stray GTT write to corrupt our defaults). Unmap the vma
+                 * from the GTT to prevent such accidents and reclaim the
+                 * space.
+                 */
+                err = i915_vma_unbind(state);
+                if (err)
+                        goto err_active;
+
+                err = i915_gem_object_set_to_cpu_domain(state->obj, false);
+                if (err)
+                        goto err_active;
+
+                engine->default_state = i915_gem_object_get(state->obj);
+        }
+
+        if (IS_ENABLED(CONFIG_DRM_I915_DEBUG_GEM)) {
+                unsigned int found = intel_engines_has_context_isolation(i915);
+
+                /*
+                 * Make sure that classes with multiple engine instances all
+                 * share the same basic configuration.
+                 */
+                for_each_engine(engine, i915, id) {
+                        unsigned int bit = BIT(engine->uabi_class);
+                        unsigned int expected = engine->default_state ? bit : 0;
+
+                        if ((found & bit) != expected) {
+                                DRM_ERROR("mismatching default context state for class %d on engine %s\n",
+                                          engine->uabi_class, engine->name);
+                        }
+                }
+        }
+
+out_ctx:
+        i915_gem_context_set_closed(ctx);
+        i915_gem_context_put(ctx);
+        return err;
+
+err_active:
+        /*
+         * If we have to abandon now, we expect the engines to be idle
+         * and ready to be torn-down. First try to flush any remaining
+         * request, ensure we are pointing at the kernel context and
+         * then remove it.
+         */
+        if (WARN_ON(i915_gem_switch_to_kernel_context(i915)))
+                goto out_ctx;
+
+        if (WARN_ON(i915_gem_wait_for_idle(i915, I915_WAIT_LOCKED)))
+                goto out_ctx;
+
+        i915_gem_contexts_lost(i915);
+        goto out_ctx;
+}
+
 int i915_gem_init(struct drm_i915_private *dev_priv)
 {
         int ret;
@@ -4991,7 +5179,25 @@ int i915_gem_init(struct drm_i915_private *dev_priv)
         if (ret)
                 goto out_unlock;
 
+        intel_init_gt_powersave(dev_priv);
+
         ret = i915_gem_init_hw(dev_priv);
+        if (ret)
+                goto out_unlock;
+
+        /*
+         * Despite its name intel_init_clock_gating applies both display
+         * clock gating workarounds; GT mmio workarounds and the occasional
+         * GT power context workaround. Worse, sometimes it includes a context
+         * register workaround which we need to apply before we record the
+         * default HW state for all contexts.
+         *
+         * FIXME: break up the workarounds and apply them at the right time!
+         */
+        intel_init_clock_gating(dev_priv);
+
+        ret = __intel_engines_record_defaults(dev_priv);
+out_unlock:
         if (ret == -EIO) {
                 /* Allow engine initialisation to fail by marking the GPU as
                  * wedged. But we only want to do this where the GPU is angry,
@@ -5003,8 +5209,6 @@ int i915_gem_init(struct drm_i915_private *dev_priv)
                 }
                 ret = 0;
         }
-
-out_unlock:
         intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
         mutex_unlock(&dev_priv->drm.struct_mutex);
 
@@ -5058,6 +5262,22 @@ i915_gem_load_init_fences(struct drm_i915_private *dev_priv)
         i915_gem_detect_bit_6_swizzle(dev_priv);
 }
 
+static void i915_gem_init__mm(struct drm_i915_private *i915)
+{
+        spin_lock_init(&i915->mm.object_stat_lock);
+        spin_lock_init(&i915->mm.obj_lock);
+        spin_lock_init(&i915->mm.free_lock);
+
+        init_llist_head(&i915->mm.free_list);
+
+        INIT_LIST_HEAD(&i915->mm.unbound_list);
+        INIT_LIST_HEAD(&i915->mm.bound_list);
+        INIT_LIST_HEAD(&i915->mm.fence_list);
+        INIT_LIST_HEAD(&i915->mm.userfault_list);
+
+        INIT_WORK(&i915->mm.free_work, __i915_gem_free_work);
+}
+
 int
 i915_gem_load_init(struct drm_i915_private *dev_priv)
 {
@@ -5099,15 +5319,7 @@ i915_gem_load_init(struct drm_i915_private *dev_priv)
         if (err)
                 goto err_priorities;
 
-        INIT_WORK(&dev_priv->mm.free_work, __i915_gem_free_work);
-
-        spin_lock_init(&dev_priv->mm.obj_lock);
-        spin_lock_init(&dev_priv->mm.free_lock);
-        init_llist_head(&dev_priv->mm.free_list);
-        INIT_LIST_HEAD(&dev_priv->mm.unbound_list);
-        INIT_LIST_HEAD(&dev_priv->mm.bound_list);
-        INIT_LIST_HEAD(&dev_priv->mm.fence_list);
-        INIT_LIST_HEAD(&dev_priv->mm.userfault_list);
+        i915_gem_init__mm(dev_priv);
 
         INIT_DELAYED_WORK(&dev_priv->gt.retire_work,
                           i915_gem_retire_work_handler);
diff --git a/drivers/gpu/drm/i915/i915_gem.h b/drivers/gpu/drm/i915/i915_gem.h
index ee54597465b6..e920dab7f1b8 100644
--- a/drivers/gpu/drm/i915/i915_gem.h
+++ b/drivers/gpu/drm/i915/i915_gem.h
@@ -28,7 +28,11 @@
 #include <linux/bug.h>
 
 #ifdef CONFIG_DRM_I915_DEBUG_GEM
-#define GEM_BUG_ON(expr) BUG_ON(expr)
+#define GEM_BUG_ON(condition) do { if (unlikely((condition))) { \
+                printk(KERN_ERR "GEM_BUG_ON(%s)\n", __stringify(condition)); \
+                BUG(); \
+                } \
+        } while(0)
 #define GEM_WARN_ON(expr) WARN_ON(expr)
 
 #define GEM_DEBUG_DECL(var) var
@@ -44,6 +48,12 @@
 #define GEM_DEBUG_BUG_ON(expr)
 #endif
 
+#if IS_ENABLED(CONFIG_DRM_I915_TRACE_GEM)
+#define GEM_TRACE(...) trace_printk(__VA_ARGS__)
+#else
+#define GEM_TRACE(...) do { } while (0)
+#endif
+
 #define I915_NUM_ENGINES 5
 
 #endif /* __I915_GEM_H__ */
diff --git a/drivers/gpu/drm/i915/i915_gem_context.c b/drivers/gpu/drm/i915/i915_gem_context.c
index f782cf2069c1..2db040695035 100644
--- a/drivers/gpu/drm/i915/i915_gem_context.c
+++ b/drivers/gpu/drm/i915/i915_gem_context.c
@@ -418,8 +418,8 @@ out:
         return ctx;
 }
 
-static struct i915_gem_context *
-create_kernel_context(struct drm_i915_private *i915, int prio)
+struct i915_gem_context *
+i915_gem_context_create_kernel(struct drm_i915_private *i915, int prio)
 {
         struct i915_gem_context *ctx;
 
@@ -473,7 +473,7 @@ int i915_gem_contexts_init(struct drm_i915_private *dev_priv)
         ida_init(&dev_priv->contexts.hw_ida);
 
         /* lowest priority; idle task */
-        ctx = create_kernel_context(dev_priv, I915_PRIORITY_MIN);
+        ctx = i915_gem_context_create_kernel(dev_priv, I915_PRIORITY_MIN);
         if (IS_ERR(ctx)) {
                 DRM_ERROR("Failed to create default global context\n");
                 err = PTR_ERR(ctx);
@@ -487,7 +487,7 @@ int i915_gem_contexts_init(struct drm_i915_private *dev_priv)
         dev_priv->kernel_context = ctx;
 
         /* highest priority; preempting task */
-        ctx = create_kernel_context(dev_priv, INT_MAX);
+        ctx = i915_gem_context_create_kernel(dev_priv, INT_MAX);
         if (IS_ERR(ctx)) {
                 DRM_ERROR("Failed to create default preempt context\n");
                 err = PTR_ERR(ctx);
@@ -522,28 +522,6 @@ void i915_gem_contexts_lost(struct drm_i915_private *dev_priv)
                 engine->context_unpin(engine, engine->last_retired_context);
                 engine->last_retired_context = NULL;
         }
-
-        /* Force the GPU state to be restored on enabling */
-        if (!i915_modparams.enable_execlists) {
-                struct i915_gem_context *ctx;
-
-                list_for_each_entry(ctx, &dev_priv->contexts.list, link) {
-                        if (!i915_gem_context_is_default(ctx))
-                                continue;
-
-                        for_each_engine(engine, dev_priv, id)
-                                ctx->engine[engine->id].initialised = false;
-
-                        ctx->remap_slice = ALL_L3_SLICES(dev_priv);
-                }
-
-                for_each_engine(engine, dev_priv, id) {
-                        struct intel_context *kce =
-                                &dev_priv->kernel_context->engine[engine->id];
-
-                        kce->initialised = true;
-                }
-        }
 }
 
 void i915_gem_contexts_fini(struct drm_i915_private *i915)
@@ -718,9 +696,6 @@ static inline bool skip_rcs_switch(struct i915_hw_ppgtt *ppgtt,
         if (to->remap_slice)
                 return false;
 
-        if (!to->engine[RCS].initialised)
-                return false;
-
         if (ppgtt && (intel_engine_flag(engine) & ppgtt->pd_dirty_rings))
                 return false;
 
@@ -795,11 +770,14 @@ static int do_rcs_switch(struct drm_i915_gem_request *req)
                         return ret;
         }
 
-        if (!to->engine[RCS].initialised || i915_gem_context_is_default(to))
-                /* NB: If we inhibit the restore, the context is not allowed to
-                 * die because future work may end up depending on valid address
-                 * space. This means we must enforce that a page table load
-                 * occur when this occurs. */
+        if (i915_gem_context_is_kernel(to))
+                /*
+                 * The kernel context(s) is treated as pure scratch and is not
+                 * expected to retain any state (as we sacrifice it during
+                 * suspend and on resume it may be corrupted). This is ok,
+                 * as nothing actually executes using the kernel context; it
+                 * is purely used for flushing user contexts.
+                 */
                 hw_flags = MI_RESTORE_INHIBIT;
         else if (ppgtt && intel_engine_flag(engine) & ppgtt->pd_dirty_rings)
                 hw_flags = MI_FORCE_RESTORE;
@@ -843,15 +821,6 @@ static int do_rcs_switch(struct drm_i915_gem_request *req)
                 to->remap_slice &= ~(1<<i);
         }
 
-        if (!to->engine[RCS].initialised) {
-                if (engine->init_context) {
-                        ret = engine->init_context(req);
-                        if (ret)
-                                return ret;
-                }
-                to->engine[RCS].initialised = true;
-        }
-
         return 0;
 }
 
@@ -899,7 +868,7 @@ int i915_switch_context(struct drm_i915_gem_request *req)
         return do_rcs_switch(req);
 }
 
-static bool engine_has_kernel_context(struct intel_engine_cs *engine)
+static bool engine_has_idle_kernel_context(struct intel_engine_cs *engine)
 {
         struct i915_gem_timeline *timeline;
 
@@ -915,8 +884,7 @@ static bool engine_has_kernel_context(struct intel_engine_cs *engine)
                         return false;
         }
 
-        return (!engine->last_retired_context ||
-                i915_gem_context_is_kernel(engine->last_retired_context));
+        return intel_engine_has_kernel_context(engine);
 }
 
 int i915_gem_switch_to_kernel_context(struct drm_i915_private *dev_priv)
@@ -933,7 +901,7 @@ int i915_gem_switch_to_kernel_context(struct drm_i915_private *dev_priv)
                 struct drm_i915_gem_request *req;
                 int ret;
 
-                if (engine_has_kernel_context(engine))
+                if (engine_has_idle_kernel_context(engine))
                         continue;
 
                 req = i915_gem_request_alloc(engine, dev_priv->kernel_context);
diff --git a/drivers/gpu/drm/i915/i915_gem_context.h b/drivers/gpu/drm/i915/i915_gem_context.h
index 44688e22a5c2..4bfb72f8e1cb 100644
--- a/drivers/gpu/drm/i915/i915_gem_context.h
+++ b/drivers/gpu/drm/i915/i915_gem_context.h
@@ -157,7 +157,6 @@ struct i915_gem_context {
                 u32 *lrc_reg_state;
                 u64 lrc_desc;
                 int pin_count;
-                bool initialised;
         } engine[I915_NUM_ENGINES];
 
         /** ring_size: size for allocating the per-engine ring buffer */
@@ -292,6 +291,9 @@ int i915_gem_context_setparam_ioctl(struct drm_device *dev, void *data,
 int i915_gem_context_reset_stats_ioctl(struct drm_device *dev, void *data,
                                        struct drm_file *file);
 
+struct i915_gem_context *
+i915_gem_context_create_kernel(struct drm_i915_private *i915, int prio);
+
 static inline struct i915_gem_context *
 i915_gem_context_get(struct i915_gem_context *ctx)
 {
diff --git a/drivers/gpu/drm/i915/i915_gem_evict.c b/drivers/gpu/drm/i915/i915_gem_evict.c
index 8daa8a78cdc0..60ca4f05ae94 100644
--- a/drivers/gpu/drm/i915/i915_gem_evict.c
+++ b/drivers/gpu/drm/i915/i915_gem_evict.c
@@ -46,7 +46,7 @@ static bool ggtt_is_idle(struct drm_i915_private *i915)
                return false;
 
        for_each_engine(engine, i915, id) {
-               if (engine->last_retired_context != i915->kernel_context)
+               if (!intel_engine_has_kernel_context(engine))
                        return false;
        }
 
@@ -73,6 +73,7 @@ static int ggtt_flush(struct drm_i915_private *i915)
         if (err)
                 return err;
 
+        GEM_BUG_ON(!ggtt_is_idle(i915));
         return 0;
 }
 
@@ -216,6 +217,7 @@ search_again:
                 if (ret)
                         return ret;
 
+                cond_resched();
                 goto search_again;
         }
 
diff --git a/drivers/gpu/drm/i915/i915_gem_execbuffer.c b/drivers/gpu/drm/i915/i915_gem_execbuffer.c
index 435ed95df144..53ccb27bfe91 100644
--- a/drivers/gpu/drm/i915/i915_gem_execbuffer.c
+++ b/drivers/gpu/drm/i915/i915_gem_execbuffer.c
@@ -2074,23 +2074,27 @@ static struct drm_syncobj **
 get_fence_array(struct drm_i915_gem_execbuffer2 *args,
                 struct drm_file *file)
 {
-        const unsigned int nfences = args->num_cliprects;
+        const unsigned long nfences = args->num_cliprects;
         struct drm_i915_gem_exec_fence __user *user;
         struct drm_syncobj **fences;
-        unsigned int n;
+        unsigned long n;
         int err;
 
         if (!(args->flags & I915_EXEC_FENCE_ARRAY))
                 return NULL;
 
-        if (nfences > SIZE_MAX / sizeof(*fences))
+        /* Check multiplication overflow for access_ok() and kvmalloc_array() */
+        BUILD_BUG_ON(sizeof(size_t) > sizeof(unsigned long));
+        if (nfences > min_t(unsigned long,
+                            ULONG_MAX / sizeof(*user),
+                            SIZE_MAX / sizeof(*fences)))
                 return ERR_PTR(-EINVAL);
 
         user = u64_to_user_ptr(args->cliprects_ptr);
-        if (!access_ok(VERIFY_READ, user, nfences * 2 * sizeof(u32)))
+        if (!access_ok(VERIFY_READ, user, nfences * sizeof(*user)))
                 return ERR_PTR(-EFAULT);
 
-        fences = kvmalloc_array(args->num_cliprects, sizeof(*fences),
+        fences = kvmalloc_array(nfences, sizeof(*fences),
                                 __GFP_NOWARN | GFP_KERNEL);
         if (!fences)
                 return ERR_PTR(-ENOMEM);
@@ -2447,6 +2451,26 @@ err_in_fence:
         return err;
 }
 
+static size_t eb_element_size(void)
+{
+        return (sizeof(struct drm_i915_gem_exec_object2) +
+                sizeof(struct i915_vma *) +
+                sizeof(unsigned int));
+}
+
+static bool check_buffer_count(size_t count)
+{
+        const size_t sz = eb_element_size();
+
+        /*
+         * When using LUT_HANDLE, we impose a limit of INT_MAX for the lookup
+         * array size (see eb_create()). Otherwise, we can accept an array as
+         * large as can be addressed (though use large arrays at your peril)!
+         */
+
+        return !(count < 1 || count > INT_MAX || count > SIZE_MAX / sz - 1);
+}
+
 /*
  * Legacy execbuffer just creates an exec2 list from the original exec object
  * list array and passes it to the real function.
@@ -2455,18 +2479,16 @@ int
 i915_gem_execbuffer(struct drm_device *dev, void *data,
                     struct drm_file *file)
 {
-        const size_t sz = (sizeof(struct drm_i915_gem_exec_object2) +
-                           sizeof(struct i915_vma *) +
-                           sizeof(unsigned int));
         struct drm_i915_gem_execbuffer *args = data;
         struct drm_i915_gem_execbuffer2 exec2;
         struct drm_i915_gem_exec_object *exec_list = NULL;
         struct drm_i915_gem_exec_object2 *exec2_list = NULL;
+        const size_t count = args->buffer_count;
         unsigned int i;
         int err;
 
-        if (args->buffer_count < 1 || args->buffer_count > SIZE_MAX / sz - 1) {
-                DRM_DEBUG("execbuf2 with %d buffers\n", args->buffer_count);
+        if (!check_buffer_count(count)) {
+                DRM_DEBUG("execbuf2 with %zd buffers\n", count);
                 return -EINVAL;
         }
 
@@ -2485,9 +2507,9 @@ i915_gem_execbuffer(struct drm_device *dev, void *data,
                 return -EINVAL;
 
         /* Copy in the exec list from userland */
-        exec_list = kvmalloc_array(args->buffer_count, sizeof(*exec_list),
+        exec_list = kvmalloc_array(count, sizeof(*exec_list),
                                    __GFP_NOWARN | GFP_KERNEL);
-        exec2_list = kvmalloc_array(args->buffer_count + 1, sz,
+        exec2_list = kvmalloc_array(count + 1, eb_element_size(),
                                     __GFP_NOWARN | GFP_KERNEL);
         if (exec_list == NULL || exec2_list == NULL) {
                 DRM_DEBUG("Failed to allocate exec list for %d buffers\n",
@@ -2498,7 +2520,7 @@ i915_gem_execbuffer(struct drm_device *dev, void *data,
         }
         err = copy_from_user(exec_list,
                              u64_to_user_ptr(args->buffers_ptr),
-                             sizeof(*exec_list) * args->buffer_count);
+                             sizeof(*exec_list) * count);
         if (err) {
                 DRM_DEBUG("copy %d exec entries failed %d\n",
                           args->buffer_count, err);
@@ -2548,16 +2570,14 @@ int
 i915_gem_execbuffer2(struct drm_device *dev, void *data,
                      struct drm_file *file)
 {
-        const size_t sz = (sizeof(struct drm_i915_gem_exec_object2) +
-                           sizeof(struct i915_vma *) +
-                           sizeof(unsigned int));
         struct drm_i915_gem_execbuffer2 *args = data;
         struct drm_i915_gem_exec_object2 *exec2_list;
         struct drm_syncobj **fences = NULL;
+        const size_t count = args->buffer_count;
         int err;
 
-        if (args->buffer_count < 1 || args->buffer_count > SIZE_MAX / sz - 1) {
-                DRM_DEBUG("execbuf2 with %d buffers\n", args->buffer_count);
+        if (!check_buffer_count(count)) {
+                DRM_DEBUG("execbuf2 with %zd buffers\n", count);
                 return -EINVAL;
         }
 
@@ -2565,17 +2585,17 @@ i915_gem_execbuffer2(struct drm_device *dev, void *data,
                 return -EINVAL;
 
         /* Allocate an extra slot for use by the command parser */
-        exec2_list = kvmalloc_array(args->buffer_count + 1, sz,
+        exec2_list = kvmalloc_array(count + 1, eb_element_size(),
                                     __GFP_NOWARN | GFP_KERNEL);
         if (exec2_list == NULL) {
-                DRM_DEBUG("Failed to allocate exec list for %d buffers\n",
-                          args->buffer_count);
+                DRM_DEBUG("Failed to allocate exec list for %zd buffers\n",
+                          count);
                 return -ENOMEM;
         }
         if (copy_from_user(exec2_list,
                            u64_to_user_ptr(args->buffers_ptr),
-                           sizeof(*exec2_list) * args->buffer_count)) {
-                DRM_DEBUG("copy %d exec entries failed\n", args->buffer_count);
+                           sizeof(*exec2_list) * count)) {
+                DRM_DEBUG("copy %zd exec entries failed\n", count);
                 kvfree(exec2_list);
                 return -EFAULT;
         }
diff --git a/drivers/gpu/drm/i915/i915_gem_gtt.c b/drivers/gpu/drm/i915/i915_gem_gtt.c
index 2af65ecf2df8..64e8ae1fd832 100644
--- a/drivers/gpu/drm/i915/i915_gem_gtt.c
+++ b/drivers/gpu/drm/i915/i915_gem_gtt.c
@@ -454,6 +454,14 @@ static void vm_free_pages_release(struct i915_address_space *vm,
 
 static void vm_free_page(struct i915_address_space *vm, struct page *page)
 {
+        /*
+         * On !llc, we need to change the pages back to WB. We only do so
+         * in bulk, so we rarely need to change the page attributes here,
+         * but doing so requires a stop_machine() from deep inside arch/x86/mm.
+         * To make detection of the possible sleep more likely, use an
+         * unconditional might_sleep() for everybody.
+         */
+        might_sleep();
         if (!pagevec_add(&vm->free_pages, page))
                 vm_free_pages_release(vm, false);
 }
@@ -2248,35 +2256,62 @@ static bool needs_idle_maps(struct drm_i915_private *dev_priv)
         return IS_GEN5(dev_priv) && IS_MOBILE(dev_priv) && intel_vtd_active();
 }
 
-void i915_check_and_clear_faults(struct drm_i915_private *dev_priv)
+static void gen6_check_and_clear_faults(struct drm_i915_private *dev_priv)
 {
         struct intel_engine_cs *engine;
         enum intel_engine_id id;
-
-        if (INTEL_INFO(dev_priv)->gen < 6)
-                return;
+        u32 fault;
 
         for_each_engine(engine, dev_priv, id) {
-                u32 fault_reg;
-                fault_reg = I915_READ(RING_FAULT_REG(engine));
-                if (fault_reg & RING_FAULT_VALID) {
+                fault = I915_READ(RING_FAULT_REG(engine));
+                if (fault & RING_FAULT_VALID) {
                         DRM_DEBUG_DRIVER("Unexpected fault\n"
                                          "\tAddr: 0x%08lx\n"
                                          "\tAddress space: %s\n"
                                          "\tSource ID: %d\n"
                                          "\tType: %d\n",
-                                         fault_reg & PAGE_MASK,
-                                         fault_reg & RING_FAULT_GTTSEL_MASK ? "GGTT" : "PPGTT",
-                                         RING_FAULT_SRCID(fault_reg),
-                                         RING_FAULT_FAULT_TYPE(fault_reg));
+                                         fault & PAGE_MASK,
+                                         fault & RING_FAULT_GTTSEL_MASK ? "GGTT" : "PPGTT",
+                                         RING_FAULT_SRCID(fault),
+                                         RING_FAULT_FAULT_TYPE(fault));
                         I915_WRITE(RING_FAULT_REG(engine),
-                                   fault_reg & ~RING_FAULT_VALID);
+                                   fault & ~RING_FAULT_VALID);
                 }
         }
 
-        /* Engine specific init may not have been done till this point. */
-        if (dev_priv->engine[RCS])
-                POSTING_READ(RING_FAULT_REG(dev_priv->engine[RCS]));
+        POSTING_READ(RING_FAULT_REG(dev_priv->engine[RCS]));
+}
+
+static void gen8_check_and_clear_faults(struct drm_i915_private *dev_priv)
+{
+        u32 fault = I915_READ(GEN8_RING_FAULT_REG);
+
+        if (fault & RING_FAULT_VALID) {
+                DRM_DEBUG_DRIVER("Unexpected fault\n"
+                                 "\tAddr: 0x%08lx\n"
+                                 "\tEngine ID: %d\n"
+                                 "\tSource ID: %d\n"
+                                 "\tType: %d\n",
+                                 fault & PAGE_MASK,
+                                 GEN8_RING_FAULT_ENGINE_ID(fault),
+                                 RING_FAULT_SRCID(fault),
+                                 RING_FAULT_FAULT_TYPE(fault));
+                I915_WRITE(GEN8_RING_FAULT_REG,
+                           fault & ~RING_FAULT_VALID);
+        }
+
+        POSTING_READ(GEN8_RING_FAULT_REG);
+}
+
+void i915_check_and_clear_faults(struct drm_i915_private *dev_priv)
+{
+        /* From GEN8 onwards we only have one 'All Engine Fault Register' */
+        if (INTEL_GEN(dev_priv) >= 8)
+                gen8_check_and_clear_faults(dev_priv);
+        else if (INTEL_GEN(dev_priv) >= 6)
+                gen6_check_and_clear_faults(dev_priv);
+        else
+                return;
 }
 
 void i915_gem_suspend_gtt_mappings(struct drm_i915_private *dev_priv)
@@ -3041,7 +3076,7 @@ const struct intel_ppat_entry *
 intel_ppat_get(struct drm_i915_private *i915, u8 value)
 {
         struct intel_ppat *ppat = &i915->ppat;
-        struct intel_ppat_entry *entry;
+        struct intel_ppat_entry *entry = NULL;
         unsigned int scanned, best_score;
         int i;
 
@@ -3064,7 +3099,7 @@ intel_ppat_get(struct drm_i915_private *i915, u8 value)
         }
 
         if (scanned == ppat->max_entries) {
-                if (!best_score)
+                if (!entry)
                         return ERR_PTR(-ENOSPC);
 
                 kref_get(&entry->ref);
@@ -3171,12 +3206,6 @@ static void cnl_setup_private_ppat(struct intel_ppat *ppat)
         ppat->match = bdw_private_pat_match;
         ppat->clear_value = GEN8_PPAT_WB | GEN8_PPAT_LLCELLC | GEN8_PPAT_AGE(3);
 
-        /* XXX: spec is unclear if this is still needed for CNL+ */
-        if (!USES_PPGTT(ppat->i915)) {
-                __alloc_ppat_entry(ppat, 0, GEN8_PPAT_UC);
-                return;
-        }
-
         __alloc_ppat_entry(ppat, 0, GEN8_PPAT_WB | GEN8_PPAT_LLC);
         __alloc_ppat_entry(ppat, 1, GEN8_PPAT_WC | GEN8_PPAT_LLCELLC);
         __alloc_ppat_entry(ppat, 2, GEN8_PPAT_WT | GEN8_PPAT_LLCELLC);
diff --git a/drivers/gpu/drm/i915/i915_gem_object.h b/drivers/gpu/drm/i915/i915_gem_object.h
index 63ce38c1cce9..19fb28c177d8 100644
--- a/drivers/gpu/drm/i915/i915_gem_object.h
+++ b/drivers/gpu/drm/i915/i915_gem_object.h
@@ -53,8 +53,9 @@ struct i915_lut_handle {
 
 struct drm_i915_gem_object_ops {
         unsigned int flags;
-#define I915_GEM_OBJECT_HAS_STRUCT_PAGE BIT(0)
-#define I915_GEM_OBJECT_IS_SHRINKABLE   BIT(1)
+#define I915_GEM_OBJECT_HAS_STRUCT_PAGE BIT(0)
+#define I915_GEM_OBJECT_IS_SHRINKABLE   BIT(1)
+#define I915_GEM_OBJECT_IS_PROXY        BIT(2)
 
         /* Interface between the GEM object and its backing storage.
          * get_pages() is called once prior to the use of the associated set
@@ -362,6 +363,12 @@ i915_gem_object_is_shrinkable(const struct drm_i915_gem_object *obj)
 }
 
 static inline bool
+i915_gem_object_is_proxy(const struct drm_i915_gem_object *obj)
+{
+        return obj->ops->flags & I915_GEM_OBJECT_IS_PROXY;
+}
+
+static inline bool
 i915_gem_object_is_active(const struct drm_i915_gem_object *obj)
 {
         return obj->active_count;
diff --git a/drivers/gpu/drm/i915/i915_gem_render_state.c b/drivers/gpu/drm/i915/i915_gem_render_state.c
index 3703dc91eeda..c2723a06fbb4 100644
--- a/drivers/gpu/drm/i915/i915_gem_render_state.c
+++ b/drivers/gpu/drm/i915/i915_gem_render_state.c
@@ -26,10 +26,12 @@
  */
 
 #include "i915_drv.h"
+#include "i915_gem_render_state.h"
 #include "intel_renderstate.h"
 
 struct intel_render_state {
         const struct intel_renderstate_rodata *rodata;
+        struct drm_i915_gem_object *obj;
         struct i915_vma *vma;
         u32 batch_offset;
         u32 batch_size;
@@ -40,6 +42,9 @@ struct intel_render_state {
 static const struct intel_renderstate_rodata *
 render_state_get_rodata(const struct intel_engine_cs *engine)
 {
+        if (engine->id != RCS)
+                return NULL;
+
         switch (INTEL_GEN(engine->i915)) {
         case 6:
                 return &gen6_null_state;
@@ -74,17 +79,16 @@ static int render_state_setup(struct intel_render_state *so,
                               struct drm_i915_private *i915)
 {
         const struct intel_renderstate_rodata *rodata = so->rodata;
-        struct drm_i915_gem_object *obj = so->vma->obj;
         unsigned int i = 0, reloc_index = 0;
         unsigned int needs_clflush;
         u32 *d;
         int ret;
 
-        ret = i915_gem_obj_prepare_shmem_write(obj, &needs_clflush);
+        ret = i915_gem_obj_prepare_shmem_write(so->obj, &needs_clflush);
         if (ret)
                 return ret;
 
-        d = kmap_atomic(i915_gem_object_get_dirty_page(obj, 0));
+        d = kmap_atomic(i915_gem_object_get_dirty_page(so->obj, 0));
 
         while (i < rodata->batch_items) {
                 u32 s = rodata->batch[i];
@@ -112,7 +116,7 @@ static int render_state_setup(struct intel_render_state *so,
                 goto err;
         }
 
-        so->batch_offset = so->vma->node.start;
+        so->batch_offset = i915_ggtt_offset(so->vma);
         so->batch_size = rodata->batch_items * sizeof(u32);
 
         while (i % CACHELINE_DWORDS)
@@ -160,9 +164,9 @@ static int render_state_setup(struct intel_render_state *so,
                 drm_clflush_virt_range(d, i * sizeof(u32));
         kunmap_atomic(d);
 
-        ret = i915_gem_object_set_to_gtt_domain(obj, false);
+        ret = i915_gem_object_set_to_gtt_domain(so->obj, false);
 out:
-        i915_gem_obj_finish_shmem_access(obj);
+        i915_gem_obj_finish_shmem_access(so->obj);
         return ret;
 
 err:
@@ -173,112 +177,61 @@ err:
 
 #undef OUT_BATCH
 
-int i915_gem_render_state_init(struct intel_engine_cs *engine)
+int i915_gem_render_state_emit(struct drm_i915_gem_request *rq)
 {
-        struct intel_render_state *so;
-        const struct intel_renderstate_rodata *rodata;
-        struct drm_i915_gem_object *obj;
-        int ret;
+        struct intel_engine_cs *engine = rq->engine;
+        struct intel_render_state so = {}; /* keep the compiler happy */
+        int err;
 
-        if (engine->id != RCS)
+        so.rodata = render_state_get_rodata(engine);
+        if (!so.rodata)
                 return 0;
 
-        rodata = render_state_get_rodata(engine);
-        if (!rodata)
-                return 0;
-
-        if (rodata->batch_items * 4 > PAGE_SIZE)
+        if (so.rodata->batch_items * 4 > PAGE_SIZE)
                 return -EINVAL;
 
-        so = kmalloc(sizeof(*so), GFP_KERNEL);
-        if (!so)
-                return -ENOMEM;
-
-        obj = i915_gem_object_create_internal(engine->i915, PAGE_SIZE);
-        if (IS_ERR(obj)) {
-                ret = PTR_ERR(obj);
-                goto err_free;
-        }
+        so.obj = i915_gem_object_create_internal(engine->i915, PAGE_SIZE);
+        if (IS_ERR(so.obj))
+                return PTR_ERR(so.obj);
 
-        so->vma = i915_vma_instance(obj, &engine->i915->ggtt.base, NULL);
-        if (IS_ERR(so->vma)) {
-                ret = PTR_ERR(so->vma);
+        so.vma = i915_vma_instance(so.obj, &engine->i915->ggtt.base, NULL);
+        if (IS_ERR(so.vma)) {
+                err = PTR_ERR(so.vma);
                 goto err_obj;
         }
 
-        so->rodata = rodata;
-        engine->render_state = so;
-        return 0;
-
-err_obj:
-        i915_gem_object_put(obj);
-err_free:
-        kfree(so);
-        return ret;
-}
-
-int i915_gem_render_state_emit(struct drm_i915_gem_request *req)
-{
-        struct intel_render_state *so;
-        int ret;
-
-        lockdep_assert_held(&req->i915->drm.struct_mutex);
-
-        so = req->engine->render_state;
-        if (!so)
-                return 0;
-
-        /* Recreate the page after shrinking */
-        if (!i915_gem_object_has_pages(so->vma->obj))
-                so->batch_offset = -1;
-
-        ret = i915_vma_pin(so->vma, 0, 0, PIN_GLOBAL | PIN_HIGH);
-        if (ret)
-                return ret;
+        err = i915_vma_pin(so.vma, 0, 0, PIN_GLOBAL | PIN_HIGH);
+        if (err)
+                goto err_vma;
 
-        if (so->vma->node.start != so->batch_offset) {
-                ret = render_state_setup(so, req->i915);
-                if (ret)
-                        goto err_unpin;
-        }
+        err = render_state_setup(&so, rq->i915);
+        if (err)
+                goto err_unpin;
 
-        ret = req->engine->emit_flush(req, EMIT_INVALIDATE);
-        if (ret)
+        err = engine->emit_flush(rq, EMIT_INVALIDATE);
+        if (err)
                 goto err_unpin;
 
-        ret = req->engine->emit_bb_start(req,
-                                         so->batch_offset, so->batch_size,
-                                         I915_DISPATCH_SECURE);
-        if (ret)
+        err = engine->emit_bb_start(rq,
+                                    so.batch_offset, so.batch_size,
+                                    I915_DISPATCH_SECURE);
+        if (err)
                 goto err_unpin;
 
-        if (so->aux_size > 8) {
-                ret = req->engine->emit_bb_start(req,
-                                                 so->aux_offset, so->aux_size,
-                                                 I915_DISPATCH_SECURE);
-                if (ret)
+        if (so.aux_size > 8) {
+                err = engine->emit_bb_start(rq,
+                                            so.aux_offset, so.aux_size,
+                                            I915_DISPATCH_SECURE);
+                if (err)
                         goto err_unpin;
         }
 
-        i915_vma_move_to_active(so->vma, req, 0);
+        i915_vma_move_to_active(so.vma, rq, 0);
 err_unpin:
-        i915_vma_unpin(so->vma);
-        return ret;
-}
-
-void i915_gem_render_state_fini(struct intel_engine_cs *engine)
-{
-        struct intel_render_state *so;
-        struct drm_i915_gem_object *obj;
-
-        so = fetch_and_zero(&engine->render_state);
-        if (!so)
-                return;
-
-        obj = so->vma->obj;
-
-        i915_vma_close(so->vma);
-        __i915_gem_object_release_unless_active(obj);
-
-        kfree(so);
+        i915_vma_unpin(so.vma);
+err_vma:
+        i915_vma_close(so.vma);
+err_obj:
+        __i915_gem_object_release_unless_active(so.obj);
+        return err;
 }
diff --git a/drivers/gpu/drm/i915/i915_gem_render_state.h b/drivers/gpu/drm/i915/i915_gem_render_state.h
index 87481845799d..86369520482e 100644
--- a/drivers/gpu/drm/i915/i915_gem_render_state.h
+++ b/drivers/gpu/drm/i915/i915_gem_render_state.h
@@ -26,8 +26,6 @@
 
 struct drm_i915_gem_request;
 
-int i915_gem_render_state_init(struct intel_engine_cs *engine);
-int i915_gem_render_state_emit(struct drm_i915_gem_request *req);
-void i915_gem_render_state_fini(struct intel_engine_cs *engine);
+int i915_gem_render_state_emit(struct drm_i915_gem_request *rq);
 
 #endif /* _I915_GEM_RENDER_STATE_H_ */
diff --git a/drivers/gpu/drm/i915/i915_gem_request.c b/drivers/gpu/drm/i915/i915_gem_request.c
index d140fcf5c6a3..e0d6221022a8 100644
--- a/drivers/gpu/drm/i915/i915_gem_request.c
+++ b/drivers/gpu/drm/i915/i915_gem_request.c
@@ -259,6 +259,8 @@ static void mark_busy(struct drm_i915_private *i915)
         if (INTEL_GEN(i915) >= 6)
                 gen6_rps_busy(i915);
 
+        intel_engines_unpark(i915);
+
         queue_delayed_work(i915->wq,
                            &i915->gt.retire_work,
                            round_jiffies_up_relative(HZ));
diff --git a/drivers/gpu/drm/i915/i915_gem_stolen.c b/drivers/gpu/drm/i915/i915_gem_stolen.c
index 03e7abc7e043..1877ae9a1d9b 100644
--- a/drivers/gpu/drm/i915/i915_gem_stolen.c
+++ b/drivers/gpu/drm/i915/i915_gem_stolen.c
@@ -294,6 +294,18 @@ static void g4x_get_stolen_reserved(struct drm_i915_private *dev_priv,
                                      ELK_STOLEN_RESERVED);
         dma_addr_t stolen_top = dev_priv->mm.stolen_base + ggtt->stolen_size;
 
+        if ((reg_val & G4X_STOLEN_RESERVED_ENABLE) == 0) {
+                *base = 0;
+                *size = 0;
+                return;
+        }
+
+        /*
+         * Whether ILK really reuses the ELK register for this is unclear.
+         * Let's see if we catch anyone with this supposedly enabled on ILK.
+         */
+        WARN(IS_GEN5(dev_priv), "ILK stolen reserved found? 0x%08x\n", reg_val);
+
         *base = (reg_val & G4X_STOLEN_RESERVED_ADDR2_MASK) << 16;
 
         WARN_ON((reg_val & G4X_STOLEN_RESERVED_ADDR1_MASK) < *base);
@@ -313,6 +325,12 @@ static void gen6_get_stolen_reserved(struct drm_i915_private *dev_priv,
 {
         uint32_t reg_val = I915_READ(GEN6_STOLEN_RESERVED);
 
+        if ((reg_val & GEN6_STOLEN_RESERVED_ENABLE) == 0) {
+                *base = 0;
+                *size = 0;
+                return;
+        }
+
         *base = reg_val & GEN6_STOLEN_RESERVED_ADDR_MASK;
 
         switch (reg_val & GEN6_STOLEN_RESERVED_SIZE_MASK) {
@@ -339,6 +357,12 @@ static void gen7_get_stolen_reserved(struct drm_i915_private *dev_priv,
 {
         uint32_t reg_val = I915_READ(GEN6_STOLEN_RESERVED);
 
+        if ((reg_val & GEN6_STOLEN_RESERVED_ENABLE) == 0) {
+                *base = 0;
+                *size = 0;
+                return;
+        }
+
         *base = reg_val & GEN7_STOLEN_RESERVED_ADDR_MASK;
 
         switch (reg_val & GEN7_STOLEN_RESERVED_SIZE_MASK) {
@@ -359,6 +383,12 @@ static void chv_get_stolen_reserved(struct drm_i915_private *dev_priv,
 {
         uint32_t reg_val = I915_READ(GEN6_STOLEN_RESERVED);
 
+        if ((reg_val & GEN6_STOLEN_RESERVED_ENABLE) == 0) {
+                *base = 0;
+                *size = 0;
+                return;
+        }
+
         *base = reg_val & GEN6_STOLEN_RESERVED_ADDR_MASK;
 
         switch (reg_val & GEN8_STOLEN_RESERVED_SIZE_MASK) {
@@ -387,6 +417,12 @@ static void bdw_get_stolen_reserved(struct drm_i915_private *dev_priv,
         uint32_t reg_val = I915_READ(GEN6_STOLEN_RESERVED);
         dma_addr_t stolen_top;
 
+        if ((reg_val & GEN6_STOLEN_RESERVED_ENABLE) == 0) {
+                *base = 0;
+                *size = 0;
+                return;
+        }
+
         stolen_top = dev_priv->mm.stolen_base + ggtt->stolen_size;
 
         *base = reg_val & GEN6_STOLEN_RESERVED_ADDR_MASK;
@@ -436,14 +472,12 @@ int i915_gem_init_stolen(struct drm_i915_private *dev_priv)
         case 3:
                 break;
         case 4:
-                if (IS_G4X(dev_priv))
-                        g4x_get_stolen_reserved(dev_priv,
-                                                &reserved_base, &reserved_size);
-                break;
+                if (!IS_G4X(dev_priv))
+                        break;
+                /* fall through */
         case 5:
-                /* Assume the gen6 maximum for the older platforms. */
-                reserved_size = 1024 * 1024;
-                reserved_base = stolen_top - reserved_size;
+                g4x_get_stolen_reserved(dev_priv,
+                                        &reserved_base, &reserved_size);
                 break;
         case 6:
                 gen6_get_stolen_reserved(dev_priv,
@@ -473,9 +507,9 @@ int i915_gem_init_stolen(struct drm_i915_private *dev_priv)
         if (reserved_base < dev_priv->mm.stolen_base ||
             reserved_base + reserved_size > stolen_top) {
                 dma_addr_t reserved_top = reserved_base + reserved_size;
-                DRM_DEBUG_KMS("Stolen reserved area [%pad - %pad] outside stolen memory [%pad - %pad]\n",
-                              &reserved_base, &reserved_top,
-                              &dev_priv->mm.stolen_base, &stolen_top);
+                DRM_ERROR("Stolen reserved area [%pad - %pad] outside stolen memory [%pad - %pad]\n",
+                          &reserved_base, &reserved_top,
+                          &dev_priv->mm.stolen_base, &stolen_top);
                 return 0;
         }
 
diff --git a/drivers/gpu/drm/i915/i915_gem_tiling.c b/drivers/gpu/drm/i915/i915_gem_tiling.c
index 1294cf695df0..b85d7ebd9bee 100644
--- a/drivers/gpu/drm/i915/i915_gem_tiling.c
+++ b/drivers/gpu/drm/i915/i915_gem_tiling.c
@@ -345,6 +345,15 @@ i915_gem_set_tiling_ioctl(struct drm_device *dev, void *data,
         if (!obj)
                 return -ENOENT;
 
+        /*
+         * The tiling mode of proxy objects is handled by its generator, and
+         * not allowed to be changed by userspace.
+         */
+        if (i915_gem_object_is_proxy(obj)) {
+                err = -ENXIO;
+                goto err;
+        }
+
         if (!i915_tiling_ok(obj, args->tiling_mode, args->stride)) {
                 err = -EINVAL;
                 goto err;
diff --git a/drivers/gpu/drm/i915/i915_gpu_error.c b/drivers/gpu/drm/i915/i915_gpu_error.c
index 653fb69e7ecb..7481c8e1b5a8 100644
--- a/drivers/gpu/drm/i915/i915_gpu_error.c
+++ b/drivers/gpu/drm/i915/i915_gpu_error.c
@@ -30,6 +30,8 @@
 #include <generated/utsrelease.h>
 #include <linux/stop_machine.h>
 #include <linux/zlib.h>
+#include <drm/drm_print.h>
+
 #include "i915_drv.h"
 
 static const char *engine_str(int engine)
@@ -175,6 +177,21 @@ static void i915_error_puts(struct drm_i915_error_state_buf *e,
 #define err_printf(e, ...) i915_error_printf(e, __VA_ARGS__)
 #define err_puts(e, s) i915_error_puts(e, s)
 
+static void __i915_printfn_error(struct drm_printer *p, struct va_format *vaf)
+{
+        i915_error_vprintf(p->arg, vaf->fmt, *vaf->va);
+}
+
+static inline struct drm_printer
+i915_error_printer(struct drm_i915_error_state_buf *e)
+{
+        struct drm_printer p = {
+                .printfn = __i915_printfn_error,
+                .arg = e,
+        };
+        return p;
+}
+
 #ifdef CONFIG_DRM_I915_COMPRESS_ERROR
 
 struct compress {
@@ -589,6 +606,21 @@ static void err_print_pciid(struct drm_i915_error_state_buf *m,
                    pdev->subsystem_device);
 }
 
+static void err_print_uc(struct drm_i915_error_state_buf *m,
+                         const struct i915_error_uc *error_uc)
+{
+        struct drm_printer p = i915_error_printer(m);
+        const struct i915_gpu_state *error =
+                container_of(error_uc, typeof(*error), uc);
+
+        if (!error->device_info.has_guc)
+                return;
+
+        intel_uc_fw_dump(&error_uc->guc_fw, &p);
+        intel_uc_fw_dump(&error_uc->huc_fw, &p);
+        print_error_obj(m, NULL, "GuC log buffer", error_uc->guc_log);
+}
+
 int i915_error_state_to_str(struct drm_i915_error_state_buf *m,
                             const struct i915_gpu_state *error)
 {
@@ -763,8 +795,6 @@ int i915_error_state_to_str(struct drm_i915_error_state_buf *m,
 
         print_error_obj(m, NULL, "Semaphores", error->semaphore);
 
-        print_error_obj(m, NULL, "GuC log buffer", error->guc_log);
-
         if (error->overlay)
                 intel_overlay_print_error_state(m, error->overlay);
 
@@ -773,6 +803,7 @@ int i915_error_state_to_str(struct drm_i915_error_state_buf *m,
 
         err_print_capabilities(m, &error->device_info);
         err_print_params(m, &error->params);
+        err_print_uc(m, &error->uc);
 
         if (m->bytes == 0 && m->err)
                 return m->err;
@@ -831,6 +862,22 @@ static __always_inline void free_param(const char *type, void *x)
                 kfree(*(void **)x);
 }
 
+static void cleanup_params(struct i915_gpu_state *error)
+{
+#define FREE(T, x, ...) free_param(#T, &error->params.x);
+        I915_PARAMS_FOR_EACH(FREE);
+#undef FREE
+}
+
+static void cleanup_uc_state(struct i915_gpu_state *error)
+{
+        struct i915_error_uc *error_uc = &error->uc;
+
+        kfree(error_uc->guc_fw.path);
+        kfree(error_uc->huc_fw.path);
+        i915_error_object_free(error_uc->guc_log);
+}
+
 void __i915_gpu_state_free(struct kref *error_ref)
 {
         struct i915_gpu_state *error =
@@ -857,7 +904,6 @@ void __i915_gpu_state_free(struct kref *error_ref)
         }
 
         i915_error_object_free(error->semaphore);
-        i915_error_object_free(error->guc_log);
 
         for (i = 0; i < ARRAY_SIZE(error->active_bo); i++)
                 kfree(error->active_bo[i]);
@@ -866,9 +912,8 @@ void __i915_gpu_state_free(struct kref *error_ref)
         kfree(error->overlay);
         kfree(error->display);
 
-#define FREE(T, x, ...) free_param(#T, &error->params.x);
-        I915_PARAMS_FOR_EACH(FREE);
-#undef FREE
+        cleanup_params(error);
+        cleanup_uc_state(error);
 
         kfree(error);
 }
@@ -1172,11 +1217,13 @@ static void error_record_engine_registers(struct i915_gpu_state *error,
 
         if (INTEL_GEN(dev_priv) >= 6) {
                 ee->rc_psmi = I915_READ(RING_PSMI_CTL(engine->mmio_base));
-                ee->fault_reg = I915_READ(RING_FAULT_REG(engine));
-                if (INTEL_GEN(dev_priv) >= 8)
+                if (INTEL_GEN(dev_priv) >= 8) {
                         gen8_record_semaphore_state(error, engine, ee);
-                else
+                        ee->fault_reg = I915_READ(GEN8_RING_FAULT_REG);
+                } else {
                         gen6_record_semaphore_state(engine, ee);
+                        ee->fault_reg = I915_READ(RING_FAULT_REG(engine));
+                }
         }
 
         if (INTEL_GEN(dev_priv) >= 4) {
@@ -1559,15 +1606,25 @@ static void i915_capture_pinned_buffers(struct drm_i915_private *dev_priv,
         error->pinned_bo = bo;
 }
 
-static void i915_gem_capture_guc_log_buffer(struct drm_i915_private *dev_priv,
-                                            struct i915_gpu_state *error)
+static void capture_uc_state(struct i915_gpu_state *error)
 {
-        /* Capturing log buf contents won't be useful if logging was disabled */
-        if (!dev_priv->guc.log.vma || (i915_modparams.guc_log_level < 0))
+        struct drm_i915_private *i915 = error->i915;
+        struct i915_error_uc *error_uc = &error->uc;
+
+        /* Capturing uC state won't be useful if there is no GuC */
+        if (!error->device_info.has_guc)
                 return;
 
-        error->guc_log = i915_error_object_create(dev_priv,
-                                                  dev_priv->guc.log.vma);
+        error_uc->guc_fw = i915->guc.fw;
+        error_uc->huc_fw = i915->huc.fw;
+
+        /* Non-default firmware paths will be specified by the modparam.
+         * As modparams are generally accesible from the userspace make
+         * explicit copies of the firmware paths.
+         */
+        error_uc->guc_fw.path = kstrdup(i915->guc.fw.path, GFP_ATOMIC);
+        error_uc->huc_fw.path = kstrdup(i915->huc.fw.path, GFP_ATOMIC);
+        error_uc->guc_log = i915_error_object_create(i915, i915->guc.log.vma);
 }
 
 /* Capture all registers which don't fit into another category. */
@@ -1695,6 +1752,14 @@ static __always_inline void dup_param(const char *type, void *x)
                 *(void **)x = kstrdup(*(void **)x, GFP_ATOMIC);
 }
 
+static void capture_params(struct i915_gpu_state *error)
+{
+        error->params = i915_modparams;
+#define DUP(T, x, ...) dup_param(#T, &error->params.x);
+        I915_PARAMS_FOR_EACH(DUP);
+#undef DUP
+}
+
 static int capture(void *data)
 {
         struct i915_gpu_state *error = data;
@@ -1705,10 +1770,8 @@ static int capture(void *data)
                 ktime_to_timeval(ktime_sub(ktime_get(),
                                            error->i915->gt.last_init_time));
 
-        error->params = i915_modparams;
-#define DUP(T, x, ...) dup_param(#T, &error->params.x);
-        I915_PARAMS_FOR_EACH(DUP);
-#undef DUP
+        capture_params(error);
+        capture_uc_state(error);
 
         i915_capture_gen_state(error->i915, error);
         i915_capture_reg_state(error->i915, error);
@@ -1716,7 +1779,6 @@ static int capture(void *data)
         i915_gem_record_rings(error->i915, error);
         i915_capture_active_buffers(error->i915, error);
         i915_capture_pinned_buffers(error->i915, error);
-        i915_gem_capture_guc_log_buffer(error->i915, error);
 
         error->overlay = intel_overlay_capture_error_state(error->i915);
         error->display = intel_display_capture_error_state(error->i915);
diff --git a/drivers/gpu/drm/i915/i915_guc_reg.h b/drivers/gpu/drm/i915/i915_guc_reg.h
index 35cf9918d09a..bc1ae7d8f424 100644
--- a/drivers/gpu/drm/i915/i915_guc_reg.h
+++ b/drivers/gpu/drm/i915/i915_guc_reg.h
@@ -102,13 +102,6 @@
 #define   GUC_ENABLE_MIA_CLOCK_GATING           (1<<15)
 #define   GUC_GEN10_SHIM_WC_ENABLE              (1<<21)
 
-#define GUC_SHIM_CONTROL_VALUE  (GUC_DISABLE_SRAM_INIT_TO_ZEROES        | \
-                                 GUC_ENABLE_READ_CACHE_LOGIC            | \
-                                 GUC_ENABLE_MIA_CACHING                 | \
-                                 GUC_ENABLE_READ_CACHE_FOR_SRAM_DATA    | \
-                                 GUC_ENABLE_READ_CACHE_FOR_WOPCM_DATA   | \
-                                 GUC_ENABLE_MIA_CLOCK_GATING)
-
 #define GUC_SEND_INTERRUPT              _MMIO(0xc4c8)
 #define   GUC_SEND_TRIGGER                (1<<0)
 
diff --git a/drivers/gpu/drm/i915/i915_irq.c b/drivers/gpu/drm/i915/i915_irq.c
index f8205841868b..4fb183ae7a07 100644
--- a/drivers/gpu/drm/i915/i915_irq.c
+++ b/drivers/gpu/drm/i915/i915_irq.c
@@ -1068,6 +1068,9 @@ static void notify_ring(struct intel_engine_cs *engine)
         struct drm_i915_gem_request *rq = NULL;
         struct intel_wait *wait;
 
+        if (!engine->breadcrumbs.irq_armed)
+                return;
+
         atomic_inc(&engine->irq_count);
         set_bit(ENGINE_IRQ_BREADCRUMB, &engine->irq_posted);
 
@@ -1101,7 +1104,8 @@ static void notify_ring(struct intel_engine_cs *engine)
                 if (wakeup)
                         wake_up_process(wait->tsk);
         } else {
-                __intel_engine_disarm_breadcrumbs(engine);
+                if (engine->breadcrumbs.irq_armed)
+                        __intel_engine_disarm_breadcrumbs(engine);
         }
         spin_unlock(&engine->breadcrumbs.irq_lock);
 
@@ -1400,7 +1404,7 @@ gen8_cs_irq_handler(struct intel_engine_cs *engine, u32 iir, int test_shift)
         }
 
         if (tasklet)
-                tasklet_hi_schedule(&execlists->irq_tasklet);
+                tasklet_hi_schedule(&execlists->tasklet);
 }
 
 static irqreturn_t gen8_gt_irq_ack(struct drm_i915_private *dev_priv,
diff --git a/drivers/gpu/drm/i915/i915_oa_bdw.c b/drivers/gpu/drm/i915/i915_oa_bdw.c
index abdf4d0abcce..4abd2e8b5083 100644
--- a/drivers/gpu/drm/i915/i915_oa_bdw.c
+++ b/drivers/gpu/drm/i915/i915_oa_bdw.c
@@ -85,9 +85,9 @@ show_test_oa_id(struct device *kdev, struct device_attribute *attr, char *buf)
 void
 i915_perf_load_test_config_bdw(struct drm_i915_private *dev_priv)
 {
-        strncpy(dev_priv->perf.oa.test_config.uuid,
+        strlcpy(dev_priv->perf.oa.test_config.uuid,
                 "d6de6f55-e526-4f79-a6a6-d7315c09044e",
-                UUID_STRING_LEN);
+                sizeof(dev_priv->perf.oa.test_config.uuid));
         dev_priv->perf.oa.test_config.id = 1;
 
         dev_priv->perf.oa.test_config.mux_regs = mux_config_test_oa;
diff --git a/drivers/gpu/drm/i915/i915_oa_bxt.c b/drivers/gpu/drm/i915/i915_oa_bxt.c
index b69b900de0fe..cb6f304ec16a 100644
--- a/drivers/gpu/drm/i915/i915_oa_bxt.c
+++ b/drivers/gpu/drm/i915/i915_oa_bxt.c
@@ -83,9 +83,9 @@ show_test_oa_id(struct device *kdev, struct device_attribute *attr, char *buf)
 void
 i915_perf_load_test_config_bxt(struct drm_i915_private *dev_priv)
 {
-        strncpy(dev_priv->perf.oa.test_config.uuid,
+        strlcpy(dev_priv->perf.oa.test_config.uuid,
                 "5ee72f5c-092f-421e-8b70-225f7c3e9612",
-                UUID_STRING_LEN);
+                sizeof(dev_priv->perf.oa.test_config.uuid));
         dev_priv->perf.oa.test_config.id = 1;
 
         dev_priv->perf.oa.test_config.mux_regs = mux_config_test_oa;
diff --git a/drivers/gpu/drm/i915/i915_oa_cflgt2.c b/drivers/gpu/drm/i915/i915_oa_cflgt2.c
index 368c87d7ee9a..8641ae30e343 100644
--- a/drivers/gpu/drm/i915/i915_oa_cflgt2.c
+++ b/drivers/gpu/drm/i915/i915_oa_cflgt2.c
@@ -84,9 +84,9 @@ show_test_oa_id(struct device *kdev, struct device_attribute *attr, char *buf)
 void
 i915_perf_load_test_config_cflgt2(struct drm_i915_private *dev_priv)
 {
-        strncpy(dev_priv->perf.oa.test_config.uuid,
+        strlcpy(dev_priv->perf.oa.test_config.uuid,
                 "74fb4902-d3d3-4237-9e90-cbdc68d0a446",
-                UUID_STRING_LEN);
+                sizeof(dev_priv->perf.oa.test_config.uuid));
         dev_priv->perf.oa.test_config.id = 1;
 
         dev_priv->perf.oa.test_config.mux_regs = mux_config_test_oa;
diff --git a/drivers/gpu/drm/i915/i915_oa_cflgt3.c b/drivers/gpu/drm/i915/i915_oa_cflgt3.c
new file mode 100644
index 000000000000..42ff06fe54a3
--- /dev/null
+++ b/drivers/gpu/drm/i915/i915_oa_cflgt3.c
@@ -0,0 +1,109 @@
+/*
+ * Autogenerated file by GPU Top : https://github.com/rib/gputop
+ * DO NOT EDIT manually!
+ *
+ *
+ * Copyright (c) 2015 Intel Corporation
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ *
+ */
+
+#include <linux/sysfs.h>
+
+#include "i915_drv.h"
+#include "i915_oa_cflgt3.h"
+
+static const struct i915_oa_reg b_counter_config_test_oa[] = {
+        { _MMIO(0x2740), 0x00000000 },
+        { _MMIO(0x2744), 0x00800000 },
+        { _MMIO(0x2714), 0xf0800000 },
+        { _MMIO(0x2710), 0x00000000 },
+        { _MMIO(0x2724), 0xf0800000 },
+        { _MMIO(0x2720), 0x00000000 },
+        { _MMIO(0x2770), 0x00000004 },
+        { _MMIO(0x2774), 0x00000000 },
+        { _MMIO(0x2778), 0x00000003 },
+        { _MMIO(0x277c), 0x00000000 },
+        { _MMIO(0x2780), 0x00000007 },
+        { _MMIO(0x2784), 0x00000000 },
+        { _MMIO(0x2788), 0x00100002 },
+        { _MMIO(0x278c), 0x0000fff7 },
+        { _MMIO(0x2790), 0x00100002 },
+        { _MMIO(0x2794), 0x0000ffcf },
+        { _MMIO(0x2798), 0x00100082 },
+        { _MMIO(0x279c), 0x0000ffef },
+        { _MMIO(0x27a0), 0x001000c2 },
+        { _MMIO(0x27a4), 0x0000ffe7 },
+        { _MMIO(0x27a8), 0x00100001 },
+        { _MMIO(0x27ac), 0x0000ffe7 },
+};
+
+static const struct i915_oa_reg flex_eu_config_test_oa[] = {
+};
+
+static const struct i915_oa_reg mux_config_test_oa[] = {
+        { _MMIO(0x9840), 0x00000080 },
+        { _MMIO(0x9888), 0x11810000 },
+        { _MMIO(0x9888), 0x07810013 },
+        { _MMIO(0x9888), 0x1f810000 },
+        { _MMIO(0x9888), 0x1d810000 },
+        { _MMIO(0x9888), 0x1b930040 },
+        { _MMIO(0x9888), 0x07e54000 },
+        { _MMIO(0x9888), 0x1f908000 },
+        { _MMIO(0x9888), 0x11900000 },
+        { _MMIO(0x9888), 0x37900000 },
+        { _MMIO(0x9888), 0x53900000 },
+        { _MMIO(0x9888), 0x45900000 },
+        { _MMIO(0x9888), 0x33900000 },
+};
+
+static ssize_t
+show_test_oa_id(struct device *kdev, struct device_attribute *attr, char *buf)
+{
+        return sprintf(buf, "1\n");
+}
+
+void
+i915_perf_load_test_config_cflgt3(struct drm_i915_private *dev_priv)
+{
+        strncpy(dev_priv->perf.oa.test_config.uuid,
+                "577e8e2c-3fa0-4875-8743-3538d585e3b0",
+                UUID_STRING_LEN);
+        dev_priv->perf.oa.test_config.id = 1;
+
+        dev_priv->perf.oa.test_config.mux_regs = mux_config_test_oa;
+        dev_priv->perf.oa.test_config.mux_regs_len = ARRAY_SIZE(mux_config_test_oa);
+
+        dev_priv->perf.oa.test_config.b_counter_regs = b_counter_config_test_oa;
+        dev_priv->perf.oa.test_config.b_counter_regs_len = ARRAY_SIZE(b_counter_config_test_oa);
+
+        dev_priv->perf.oa.test_config.flex_regs = flex_eu_config_test_oa;
+        dev_priv->perf.oa.test_config.flex_regs_len = ARRAY_SIZE(flex_eu_config_test_oa);
+
+        dev_priv->perf.oa.test_config.sysfs_metric.name = "577e8e2c-3fa0-4875-8743-3538d585e3b0";
+        dev_priv->perf.oa.test_config.sysfs_metric.attrs = dev_priv->perf.oa.test_config.attrs;
+
+        dev_priv->perf.oa.test_config.attrs[0] = &dev_priv->perf.oa.test_config.sysfs_metric_id.attr;
+
+        dev_priv->perf.oa.test_config.sysfs_metric_id.attr.name = "id";
+        dev_priv->perf.oa.test_config.sysfs_metric_id.attr.mode = 0444;
+        dev_priv->perf.oa.test_config.sysfs_metric_id.show = show_test_oa_id;
+}
diff --git a/drivers/gpu/drm/i915/i915_oa_cflgt3.h b/drivers/gpu/drm/i915/i915_oa_cflgt3.h
new file mode 100644
index 000000000000..c13b5aac01b9
--- /dev/null
+++ b/drivers/gpu/drm/i915/i915_oa_cflgt3.h
@@ -0,0 +1,34 @@
+/*
+ * Autogenerated file by GPU Top : https://github.com/rib/gputop
+ * DO NOT EDIT manually!
+ *
+ *
+ * Copyright (c) 2015 Intel Corporation
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ *
+ */
+
+#ifndef __I915_OA_CFLGT3_H__
+#define __I915_OA_CFLGT3_H__
+
+extern void i915_perf_load_test_config_cflgt3(struct drm_i915_private *dev_priv);
+
+#endif
diff --git a/drivers/gpu/drm/i915/i915_oa_chv.c b/drivers/gpu/drm/i915/i915_oa_chv.c
index 322a3f94cd16..556febb2c3c8 100644
--- a/drivers/gpu/drm/i915/i915_oa_chv.c
+++ b/drivers/gpu/drm/i915/i915_oa_chv.c
@@ -84,9 +84,9 @@ show_test_oa_id(struct device *kdev, struct device_attribute *attr, char *buf)
 void
 i915_perf_load_test_config_chv(struct drm_i915_private *dev_priv)
 {
-        strncpy(dev_priv->perf.oa.test_config.uuid,
+        strlcpy(dev_priv->perf.oa.test_config.uuid,
                 "4a534b07-cba3-414d-8d60-874830e883aa",
-                UUID_STRING_LEN);
+                sizeof(dev_priv->perf.oa.test_config.uuid));
         dev_priv->perf.oa.test_config.id = 1;
 
         dev_priv->perf.oa.test_config.mux_regs = mux_config_test_oa;
diff --git a/drivers/gpu/drm/i915/i915_oa_cnl.c b/drivers/gpu/drm/i915/i915_oa_cnl.c
new file mode 100644
index 000000000000..ff0ac3627cc4
--- /dev/null
+++ b/drivers/gpu/drm/i915/i915_oa_cnl.c
@@ -0,0 +1,121 @@
+/*
+ * Autogenerated file by GPU Top : https://github.com/rib/gputop
+ * DO NOT EDIT manually!
+ *
+ *
+ * Copyright (c) 2015 Intel Corporation
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ *
+ */
+
+#include <linux/sysfs.h>
+
+#include "i915_drv.h"
+#include "i915_oa_cnl.h"
+
+static const struct i915_oa_reg b_counter_config_test_oa[] = {
+        { _MMIO(0x2740), 0x00000000 },
+        { _MMIO(0x2710), 0x00000000 },
+        { _MMIO(0x2714), 0xf0800000 },
+        { _MMIO(0x2720), 0x00000000 },
+        { _MMIO(0x2724), 0xf0800000 },
+        { _MMIO(0x2770), 0x00000004 },
+        { _MMIO(0x2774), 0x0000ffff },
+        { _MMIO(0x2778), 0x00000003 },
+        { _MMIO(0x277c), 0x0000ffff },
+        { _MMIO(0x2780), 0x00000007 },
+        { _MMIO(0x2784), 0x0000ffff },
+        { _MMIO(0x2788), 0x00100002 },
+        { _MMIO(0x278c), 0x0000fff7 },
+        { _MMIO(0x2790), 0x00100002 },
+        { _MMIO(0x2794), 0x0000ffcf },
+        { _MMIO(0x2798), 0x00100082 },
+        { _MMIO(0x279c), 0x0000ffef },
+        { _MMIO(0x27a0), 0x001000c2 },
+        { _MMIO(0x27a4), 0x0000ffe7 },
+        { _MMIO(0x27a8), 0x00100001 },
+        { _MMIO(0x27ac), 0x0000ffe7 },
+};
+
+static const struct i915_oa_reg flex_eu_config_test_oa[] = {
+};
+
+static const struct i915_oa_reg mux_config_test_oa[] = {
+        { _MMIO(0xd04), 0x00000200 },
+        { _MMIO(0x9884), 0x00000007 },
+        { _MMIO(0x9888), 0x17060000 },
+        { _MMIO(0x9840), 0x00000000 },
+        { _MMIO(0x9884), 0x00000007 },
+        { _MMIO(0x9888), 0x13034000 },
+        { _MMIO(0x9884), 0x00000007 },
+        { _MMIO(0x9888), 0x07060066 },
+        { _MMIO(0x9884), 0x00000007 },
+        { _MMIO(0x9888), 0x05060000 },
+        { _MMIO(0x9884), 0x00000007 },
+        { _MMIO(0x9888), 0x0f080040 },
+        { _MMIO(0x9884), 0x00000007 },
+        { _MMIO(0x9888), 0x07091000 },
+        { _MMIO(0x9884), 0x00000007 },
+        { _MMIO(0x9888), 0x0f041000 },
+        { _MMIO(0x9884), 0x00000007 },
+        { _MMIO(0x9888), 0x1d004000 },
+        { _MMIO(0x9884), 0x00000007 },
+        { _MMIO(0x9888), 0x35000000 },
+        { _MMIO(0x9884), 0x00000007 },
+        { _MMIO(0x9888), 0x49000000 },
+        { _MMIO(0x9884), 0x00000007 },
+        { _MMIO(0x9888), 0x3d000000 },
+        { _MMIO(0x9884), 0x00000007 },
+        { _MMIO(0x9888), 0x31000000 },
+};
+
+static ssize_t
+show_test_oa_id(struct device *kdev, struct device_attribute *attr, char *buf)
+{
+        return sprintf(buf, "1\n");
+}
+
+void
+i915_perf_load_test_config_cnl(struct drm_i915_private *dev_priv)
+{
+        strncpy(dev_priv->perf.oa.test_config.uuid,
+                "db41edd4-d8e7-4730-ad11-b9a2d6833503",
+                UUID_STRING_LEN);
+        dev_priv->perf.oa.test_config.id = 1;
+
+        dev_priv->perf.oa.test_config.mux_regs = mux_config_test_oa;
+        dev_priv->perf.oa.test_config.mux_regs_len = ARRAY_SIZE(mux_config_test_oa);
+
+        dev_priv->perf.oa.test_config.b_counter_regs = b_counter_config_test_oa;
+        dev_priv->perf.oa.test_config.b_counter_regs_len = ARRAY_SIZE(b_counter_config_test_oa);
+
+        dev_priv->perf.oa.test_config.flex_regs = flex_eu_config_test_oa;
+        dev_priv->perf.oa.test_config.flex_regs_len = ARRAY_SIZE(flex_eu_config_test_oa);
+
+        dev_priv->perf.oa.test_config.sysfs_metric.name = "db41edd4-d8e7-4730-ad11-b9a2d6833503";
+        dev_priv->perf.oa.test_config.sysfs_metric.attrs = dev_priv->perf.oa.test_config.attrs;
+
+        dev_priv->perf.oa.test_config.attrs[0] = &dev_priv->perf.oa.test_config.sysfs_metric_id.attr;
+
+        dev_priv->perf.oa.test_config.sysfs_metric_id.attr.name = "id";
+        dev_priv->perf.oa.test_config.sysfs_metric_id.attr.mode = 0444;
+        dev_priv->perf.oa.test_config.sysfs_metric_id.show = show_test_oa_id;
+}
diff --git a/drivers/gpu/drm/i915/i915_oa_cnl.h b/drivers/gpu/drm/i915/i915_oa_cnl.h
new file mode 100644
index 000000000000..fb918b131105
--- /dev/null
+++ b/drivers/gpu/drm/i915/i915_oa_cnl.h
@@ -0,0 +1,34 @@
+/*
+ * Autogenerated file by GPU Top : https://github.com/rib/gputop
+ * DO NOT EDIT manually!
+ *
+ *
+ * Copyright (c) 2015 Intel Corporation
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ *
+ */
+
+#ifndef __I915_OA_CNL_H__
+#define __I915_OA_CNL_H__
+
+extern void i915_perf_load_test_config_cnl(struct drm_i915_private *dev_priv);
+
+#endif
diff --git a/drivers/gpu/drm/i915/i915_oa_glk.c b/drivers/gpu/drm/i915/i915_oa_glk.c
index 4ee527e4c926..971db587957c 100644
--- a/drivers/gpu/drm/i915/i915_oa_glk.c
+++ b/drivers/gpu/drm/i915/i915_oa_glk.c
@@ -83,9 +83,9 @@ show_test_oa_id(struct device *kdev, struct device_attribute *attr, char *buf)
 void
 i915_perf_load_test_config_glk(struct drm_i915_private *dev_priv)
 {
-        strncpy(dev_priv->perf.oa.test_config.uuid,
+        strlcpy(dev_priv->perf.oa.test_config.uuid,
                 "dd3fd789-e783-4204-8cd0-b671bbccb0cf",
-                UUID_STRING_LEN);
+                sizeof(dev_priv->perf.oa.test_config.uuid));
         dev_priv->perf.oa.test_config.id = 1;
 
         dev_priv->perf.oa.test_config.mux_regs = mux_config_test_oa;
diff --git a/drivers/gpu/drm/i915/i915_oa_hsw.c b/drivers/gpu/drm/i915/i915_oa_hsw.c
index 56b03773bb9d..434a9b96d7ab 100644
--- a/drivers/gpu/drm/i915/i915_oa_hsw.c
+++ b/drivers/gpu/drm/i915/i915_oa_hsw.c
@@ -113,9 +113,9 @@ show_render_basic_id(struct device *kdev, struct device_attribute *attr, char *b
 void
 i915_perf_load_test_config_hsw(struct drm_i915_private *dev_priv)
 {
-        strncpy(dev_priv->perf.oa.test_config.uuid,
+        strlcpy(dev_priv->perf.oa.test_config.uuid,
                 "403d8832-1a27-4aa6-a64e-f5389ce7b212",
-                UUID_STRING_LEN);
+                sizeof(dev_priv->perf.oa.test_config.uuid));
         dev_priv->perf.oa.test_config.id = 1;
 
         dev_priv->perf.oa.test_config.mux_regs = mux_config_render_basic;
diff --git a/drivers/gpu/drm/i915/i915_oa_kblgt2.c b/drivers/gpu/drm/i915/i915_oa_kblgt2.c
index b6e7cc774136..2fa98a40bbc8 100644
--- a/drivers/gpu/drm/i915/i915_oa_kblgt2.c
+++ b/drivers/gpu/drm/i915/i915_oa_kblgt2.c
@@ -84,9 +84,9 @@ show_test_oa_id(struct device *kdev, struct device_attribute *attr, char *buf)
 void
 i915_perf_load_test_config_kblgt2(struct drm_i915_private *dev_priv)
 {
-        strncpy(dev_priv->perf.oa.test_config.uuid,
+        strlcpy(dev_priv->perf.oa.test_config.uuid,
                 "baa3c7e4-52b6-4b85-801e-465a94b746dd",
-                UUID_STRING_LEN);
+                sizeof(dev_priv->perf.oa.test_config.uuid));
         dev_priv->perf.oa.test_config.id = 1;
 
         dev_priv->perf.oa.test_config.mux_regs = mux_config_test_oa;
diff --git a/drivers/gpu/drm/i915/i915_oa_kblgt3.c b/drivers/gpu/drm/i915/i915_oa_kblgt3.c
index 5576afdd9a7e..f3cb6679a1bc 100644
--- a/drivers/gpu/drm/i915/i915_oa_kblgt3.c
+++ b/drivers/gpu/drm/i915/i915_oa_kblgt3.c
@@ -84,9 +84,9 @@ show_test_oa_id(struct device *kdev, struct device_attribute *attr, char *buf)
 void
 i915_perf_load_test_config_kblgt3(struct drm_i915_private *dev_priv)
 {
-        strncpy(dev_priv->perf.oa.test_config.uuid,
+        strlcpy(dev_priv->perf.oa.test_config.uuid,
                 "f1792f32-6db2-4b50-b4b2-557128f1688d",
-                UUID_STRING_LEN);
+                sizeof(dev_priv->perf.oa.test_config.uuid));
         dev_priv->perf.oa.test_config.id = 1;
 
         dev_priv->perf.oa.test_config.mux_regs = mux_config_test_oa;
diff --git a/drivers/gpu/drm/i915/i915_oa_sklgt2.c b/drivers/gpu/drm/i915/i915_oa_sklgt2.c
index 890d55879946..bf8b8cd8a50d 100644
--- a/drivers/gpu/drm/i915/i915_oa_sklgt2.c
+++ b/drivers/gpu/drm/i915/i915_oa_sklgt2.c
@@ -83,9 +83,9 @@ show_test_oa_id(struct device *kdev, struct device_attribute *attr, char *buf)
 void
 i915_perf_load_test_config_sklgt2(struct drm_i915_private *dev_priv)
 {
-        strncpy(dev_priv->perf.oa.test_config.uuid,
+        strlcpy(dev_priv->perf.oa.test_config.uuid,
                 "1651949f-0ac0-4cb1-a06f-dafd74a407d1",
-                UUID_STRING_LEN);
+                sizeof(dev_priv->perf.oa.test_config.uuid));
         dev_priv->perf.oa.test_config.id = 1;
 
         dev_priv->perf.oa.test_config.mux_regs = mux_config_test_oa;
diff --git a/drivers/gpu/drm/i915/i915_oa_sklgt3.c b/drivers/gpu/drm/i915/i915_oa_sklgt3.c
index 85e51addf86a..ae534c7c8135 100644
--- a/drivers/gpu/drm/i915/i915_oa_sklgt3.c
+++ b/drivers/gpu/drm/i915/i915_oa_sklgt3.c
@@ -84,9 +84,9 @@ show_test_oa_id(struct device *kdev, struct device_attribute *attr, char *buf)
 void
 i915_perf_load_test_config_sklgt3(struct drm_i915_private *dev_priv)
 {
-        strncpy(dev_priv->perf.oa.test_config.uuid,
+        strlcpy(dev_priv->perf.oa.test_config.uuid,
                 "2b985803-d3c9-4629-8a4f-634bfecba0e8",
-                UUID_STRING_LEN);
+                sizeof(dev_priv->perf.oa.test_config.uuid));
         dev_priv->perf.oa.test_config.id = 1;
 
         dev_priv->perf.oa.test_config.mux_regs = mux_config_test_oa;
diff --git a/drivers/gpu/drm/i915/i915_oa_sklgt4.c b/drivers/gpu/drm/i915/i915_oa_sklgt4.c
index bce031ee4445..817fba2d82df 100644
--- a/drivers/gpu/drm/i915/i915_oa_sklgt4.c
+++ b/drivers/gpu/drm/i915/i915_oa_sklgt4.c
@@ -84,9 +84,9 @@ show_test_oa_id(struct device *kdev, struct device_attribute *attr, char *buf)
 void
 i915_perf_load_test_config_sklgt4(struct drm_i915_private *dev_priv)
 {
-        strncpy(dev_priv->perf.oa.test_config.uuid,
+        strlcpy(dev_priv->perf.oa.test_config.uuid,
                 "882fa433-1f4a-4a67-a962-c741888fe5f5",
-                UUID_STRING_LEN);
+                sizeof(dev_priv->perf.oa.test_config.uuid));
         dev_priv->perf.oa.test_config.id = 1;
 
         dev_priv->perf.oa.test_config.mux_regs = mux_config_test_oa;
diff --git a/drivers/gpu/drm/i915/i915_perf.c b/drivers/gpu/drm/i915/i915_perf.c
index 59ee808f8fd9..00be015e01df 100644
--- a/drivers/gpu/drm/i915/i915_perf.c
+++ b/drivers/gpu/drm/i915/i915_perf.c
@@ -207,6 +207,8 @@
 #include "i915_oa_kblgt3.h"
 #include "i915_oa_glk.h"
 #include "i915_oa_cflgt2.h"
+#include "i915_oa_cflgt3.h"
+#include "i915_oa_cnl.h"
 
 /* HW requires this to be a power of two, between 128k and 16M, though driver
  * is currently generally designed assuming the largest 16M size is used such
@@ -1851,7 +1853,7 @@ static int gen8_enable_metric_set(struct drm_i915_private *dev_priv,
          * be read back from automatically triggered reports, as part of the
          * RPT_ID field.
          */
-        if (IS_GEN9(dev_priv)) {
+        if (IS_GEN9(dev_priv) || IS_GEN10(dev_priv)) {
                 I915_WRITE(GEN8_OA_DEBUG,
                            _MASKED_BIT_ENABLE(GEN9_OA_DEBUG_DISABLE_CLK_RATIO_REPORTS |
                                               GEN9_OA_DEBUG_INCLUDE_CLK_RATIO));
@@ -1884,6 +1886,16 @@ static void gen8_disable_metric_set(struct drm_i915_private *dev_priv)
 
 }
 
+static void gen10_disable_metric_set(struct drm_i915_private *dev_priv)
+{
+        /* Reset all contexts' slices/subslices configurations. */
+        gen8_configure_all_contexts(dev_priv, NULL, false);
+
+        /* Make sure we disable noa to save power. */
+        I915_WRITE(RPM_CONFIG1,
+                   I915_READ(RPM_CONFIG1) & ~GEN10_GT_NOA_ENABLE);
+}
+
 static void gen7_oa_enable(struct drm_i915_private *dev_priv)
 {
         /*
@@ -2934,6 +2946,10 @@ void i915_perf_register(struct drm_i915_private *dev_priv)
         } else if (IS_COFFEELAKE(dev_priv)) {
                 if (IS_CFL_GT2(dev_priv))
                         i915_perf_load_test_config_cflgt2(dev_priv);
+                if (IS_CFL_GT3(dev_priv))
+                        i915_perf_load_test_config_cflgt3(dev_priv);
+        } else if (IS_CANNONLAKE(dev_priv)) {
+                i915_perf_load_test_config_cnl(dev_priv);
         }
 
         if (dev_priv->perf.oa.test_config.id == 0)
@@ -3019,11 +3035,18 @@ static bool gen8_is_valid_mux_addr(struct drm_i915_private *dev_priv, u32 addr)
                 (addr >= RPM_CONFIG0.reg && addr <= NOA_CONFIG(8).reg);
 }
 
+static bool gen10_is_valid_mux_addr(struct drm_i915_private *dev_priv, u32 addr)
+{
+        return gen8_is_valid_mux_addr(dev_priv, addr) ||
+                (addr >= OA_PERFCNT3_LO.reg && addr <= OA_PERFCNT4_HI.reg);
+}
+
 static bool hsw_is_valid_mux_addr(struct drm_i915_private *dev_priv, u32 addr)
 {
         return gen7_is_valid_mux_addr(dev_priv, addr) ||
                 (addr >= 0x25100 && addr <= 0x2FF90) ||
-                addr == 0x9ec0;
+                (addr >= HSW_MBVID2_NOA0.reg && addr <= HSW_MBVID2_NOA9.reg) ||
+                addr == HSW_MBVID2_MISR0.reg;
 }
 
 static bool chv_is_valid_mux_addr(struct drm_i915_private *dev_priv, u32 addr)
@@ -3419,41 +3442,46 @@ void i915_perf_init(struct drm_i915_private *dev_priv)
                  * worth the complexity to maintain now that BDW+ enable
                  * execlist mode by default.
                  */
-                dev_priv->perf.oa.ops.is_valid_b_counter_reg =
-                        gen7_is_valid_b_counter_addr;
-                dev_priv->perf.oa.ops.is_valid_mux_reg =
-                        gen8_is_valid_mux_addr;
-                dev_priv->perf.oa.ops.is_valid_flex_reg =
-                        gen8_is_valid_flex_addr;
+                dev_priv->perf.oa.oa_formats = gen8_plus_oa_formats;
 
                 dev_priv->perf.oa.ops.init_oa_buffer = gen8_init_oa_buffer;
-                dev_priv->perf.oa.ops.enable_metric_set = gen8_enable_metric_set;
-                dev_priv->perf.oa.ops.disable_metric_set = gen8_disable_metric_set;
                 dev_priv->perf.oa.ops.oa_enable = gen8_oa_enable;
                 dev_priv->perf.oa.ops.oa_disable = gen8_oa_disable;
                 dev_priv->perf.oa.ops.read = gen8_oa_read;
                 dev_priv->perf.oa.ops.oa_hw_tail_read = gen8_oa_hw_tail_read;
 
-                dev_priv->perf.oa.oa_formats = gen8_plus_oa_formats;
-
-                if (IS_GEN8(dev_priv)) {
-                        dev_priv->perf.oa.ctx_oactxctrl_offset = 0x120;
-                        dev_priv->perf.oa.ctx_flexeu0_offset = 0x2ce;
-
-                        dev_priv->perf.oa.timestamp_frequency = 12500000;
+                if (IS_GEN8(dev_priv) || IS_GEN9(dev_priv)) {
+                        dev_priv->perf.oa.ops.is_valid_b_counter_reg =
+                                gen7_is_valid_b_counter_addr;
+                        dev_priv->perf.oa.ops.is_valid_mux_reg =
+                                gen8_is_valid_mux_addr;
+                        dev_priv->perf.oa.ops.is_valid_flex_reg =
+                                gen8_is_valid_flex_addr;
 
-                        dev_priv->perf.oa.gen8_valid_ctx_bit = (1<<25);
                         if (IS_CHERRYVIEW(dev_priv)) {
                                 dev_priv->perf.oa.ops.is_valid_mux_reg =
                                         chv_is_valid_mux_addr;
                         }
-                } else if (IS_GEN9(dev_priv)) {
-                        dev_priv->perf.oa.ctx_oactxctrl_offset = 0x128;
-                        dev_priv->perf.oa.ctx_flexeu0_offset = 0x3de;
 
-                        dev_priv->perf.oa.gen8_valid_ctx_bit = (1<<16);
+                        dev_priv->perf.oa.ops.enable_metric_set = gen8_enable_metric_set;
+                        dev_priv->perf.oa.ops.disable_metric_set = gen8_disable_metric_set;
+
+                        if (IS_GEN8(dev_priv)) {
+                                dev_priv->perf.oa.ctx_oactxctrl_offset = 0x120;
+                                dev_priv->perf.oa.ctx_flexeu0_offset = 0x2ce;
+
+                                dev_priv->perf.oa.gen8_valid_ctx_bit = (1<<25);
+                        } else {
+                                dev_priv->perf.oa.ctx_oactxctrl_offset = 0x128;
+                                dev_priv->perf.oa.ctx_flexeu0_offset = 0x3de;
+
+                                dev_priv->perf.oa.gen8_valid_ctx_bit = (1<<16);
+                        }
 
                         switch (dev_priv->info.platform) {
+                        case INTEL_BROADWELL:
+                                dev_priv->perf.oa.timestamp_frequency = 12500000;
+                                break;
                         case INTEL_BROXTON:
                         case INTEL_GEMINILAKE:
                                 dev_priv->perf.oa.timestamp_frequency = 19200000;
@@ -3464,11 +3492,28 @@ void i915_perf_init(struct drm_i915_private *dev_priv)
                                 dev_priv->perf.oa.timestamp_frequency = 12000000;
                                 break;
                         default:
-                                /* Leave timestamp_frequency to 0 so we can
-                                 * detect unsupported platforms.
-                                 */
                                 break;
                         }
+                } else if (IS_GEN10(dev_priv)) {
+                        dev_priv->perf.oa.ops.is_valid_b_counter_reg =
+                                gen7_is_valid_b_counter_addr;
+                        dev_priv->perf.oa.ops.is_valid_mux_reg =
+                                gen10_is_valid_mux_addr;
+                        dev_priv->perf.oa.ops.is_valid_flex_reg =
+                                gen8_is_valid_flex_addr;
+
+                        dev_priv->perf.oa.ops.enable_metric_set = gen8_enable_metric_set;
+                        dev_priv->perf.oa.ops.disable_metric_set = gen10_disable_metric_set;
+
+                        dev_priv->perf.oa.ctx_oactxctrl_offset = 0x128;
+                        dev_priv->perf.oa.ctx_flexeu0_offset = 0x3de;
+
+                        dev_priv->perf.oa.gen8_valid_ctx_bit = (1<<16);
+
+                        /* Default frequency, although we need to read it from
+                         * the register as it might vary between parts.
+                         */
+                        dev_priv->perf.oa.timestamp_frequency = 12000000;
                 }
         }
 
diff --git a/drivers/gpu/drm/i915/i915_reg.h b/drivers/gpu/drm/i915/i915_reg.h
index 68a58cce6ab1..96c80fa0fcac 100644
--- a/drivers/gpu/drm/i915/i915_reg.h
+++ b/drivers/gpu/drm/i915/i915_reg.h
@@ -355,9 +355,6 @@ static inline bool i915_mmio_reg_valid(i915_reg_t reg)
 #define   ECOCHK_PPGTT_WT_HSW           (0x2<<3)
 #define   ECOCHK_PPGTT_WB_HSW           (0x3<<3)
 
-#define GEN8_CONFIG0                    _MMIO(0xD00)
-#define  GEN9_DEFAULT_FIXES             (1 << 3 | 1 << 2 | 1 << 1)
-
 #define GAC_ECO_BITS                    _MMIO(0x14090)
 #define   ECOBITS_SNB_BIT               (1<<13)
 #define   ECOBITS_PPGTT_CACHE64B        (3<<8)
@@ -382,6 +379,7 @@ static inline bool i915_mmio_reg_valid(i915_reg_t reg)
 #define GEN8_STOLEN_RESERVED_2M         (1 << 7)
 #define GEN8_STOLEN_RESERVED_4M         (2 << 7)
 #define GEN8_STOLEN_RESERVED_8M         (3 << 7)
+#define GEN6_STOLEN_RESERVED_ENABLE     (1 << 0)
 
 /* VGA stuff */
 
@@ -1109,16 +1107,50 @@ static inline bool i915_mmio_reg_valid(i915_reg_t reg)
 #define OA_PERFCNT1_HI      _MMIO(0x91BC)
 #define OA_PERFCNT2_LO      _MMIO(0x91C0)
 #define OA_PERFCNT2_HI      _MMIO(0x91C4)
+#define OA_PERFCNT3_LO      _MMIO(0x91C8)
+#define OA_PERFCNT3_HI      _MMIO(0x91CC)
+#define OA_PERFCNT4_LO      _MMIO(0x91D8)
+#define OA_PERFCNT4_HI      _MMIO(0x91DC)
 
 #define OA_PERFMATRIX_LO    _MMIO(0x91C8)
 #define OA_PERFMATRIX_HI    _MMIO(0x91CC)
 
 /* RPM unit config (Gen8+) */
 #define RPM_CONFIG0         _MMIO(0x0D00)
-#define RPM_CONFIG1         _MMIO(0x0D04)
+#define  GEN9_RPM_CONFIG0_CRYSTAL_CLOCK_FREQ_SHIFT      3
+#define  GEN9_RPM_CONFIG0_CRYSTAL_CLOCK_FREQ_MASK       (1 << GEN9_RPM_CONFIG0_CRYSTAL_CLOCK_FREQ_SHIFT)
+#define  GEN9_RPM_CONFIG0_CRYSTAL_CLOCK_FREQ_19_2_MHZ   0
+#define  GEN9_RPM_CONFIG0_CRYSTAL_CLOCK_FREQ_24_MHZ     1
+#define  GEN10_RPM_CONFIG0_CTC_SHIFT_PARAMETER_SHIFT    1
+#define  GEN10_RPM_CONFIG0_CTC_SHIFT_PARAMETER_MASK     (0x3 << GEN10_RPM_CONFIG0_CTC_SHIFT_PARAMETER_SHIFT)
 
-/* RPC unit config (Gen8+) */
-#define RPM_CONFIG          _MMIO(0x0D08)
+#define RPM_CONFIG1         _MMIO(0x0D04)
+#define  GEN10_GT_NOA_ENABLE  (1 << 9)
+
+/* GPM unit config (Gen9+) */
+#define CTC_MODE                        _MMIO(0xA26C)
+#define  CTC_SOURCE_PARAMETER_MASK 1
+#define  CTC_SOURCE_CRYSTAL_CLOCK       0
+#define  CTC_SOURCE_DIVIDE_LOGIC        1
+#define  CTC_SHIFT_PARAMETER_SHIFT      1
+#define  CTC_SHIFT_PARAMETER_MASK       (0x3 << CTC_SHIFT_PARAMETER_SHIFT)
+
+/* RCP unit config (Gen8+) */
+#define RCP_CONFIG          _MMIO(0x0D08)
+
+/* NOA (HSW) */
+#define HSW_MBVID2_NOA0         _MMIO(0x9E80)
+#define HSW_MBVID2_NOA1         _MMIO(0x9E84)
+#define HSW_MBVID2_NOA2         _MMIO(0x9E88)
+#define HSW_MBVID2_NOA3         _MMIO(0x9E8C)
+#define HSW_MBVID2_NOA4         _MMIO(0x9E90)
+#define HSW_MBVID2_NOA5         _MMIO(0x9E94)
+#define HSW_MBVID2_NOA6         _MMIO(0x9E98)
+#define HSW_MBVID2_NOA7         _MMIO(0x9E9C)
+#define HSW_MBVID2_NOA8         _MMIO(0x9EA0)
+#define HSW_MBVID2_NOA9         _MMIO(0x9EA4)
+
+#define HSW_MBVID2_MISR0        _MMIO(0x9EC0)
 
 /* NOA (Gen8+) */
 #define NOA_CONFIG(i)       _MMIO(0x0D0C + (i) * 4)
@@ -2329,6 +2361,8 @@ enum i915_power_well_id {
 #define   ARB_MODE_SWIZZLE_BDW  (1<<1)
 #define RENDER_HWS_PGA_GEN7     _MMIO(0x04080)
 #define RING_FAULT_REG(engine)  _MMIO(0x4094 + 0x100*(engine)->hw_id)
+#define GEN8_RING_FAULT_REG     _MMIO(0x4094)
+#define   GEN8_RING_FAULT_ENGINE_ID(x)  (((x) >> 12) & 0x7)
 #define   RING_FAULT_GTTSEL_MASK (1<<11)
 #define   RING_FAULT_SRCID(x)   (((x) >> 3) & 0xff)
 #define   RING_FAULT_FAULT_TYPE(x) (((x) >> 1) & 0x3)
@@ -2951,9 +2985,6 @@ enum i915_power_well_id {
 #define ILK_DPFC_CHICKEN        _MMIO(0x43224)
 #define   ILK_DPFC_DISABLE_DUMMY0 (1<<8)
 #define   ILK_DPFC_NUKE_ON_ANY_MODIFICATION     (1<<23)
-#define   GLK_SKIP_SEG_EN               (1<<12)
-#define   GLK_SKIP_SEG_COUNT_MASK       (3<<10)
-#define   GLK_SKIP_SEG_COUNT(x)         ((x)<<10)
 #define ILK_FBC_RT_BASE         _MMIO(0x2128)
 #define   ILK_FBC_RT_VALID      (1<<0)
 #define   SNB_FBC_FRONT_BUFFER  (1<<1)
@@ -3398,6 +3429,7 @@ enum i915_power_well_id {
 #define ELK_STOLEN_RESERVED             _MMIO(MCHBAR_MIRROR_BASE + 0x48)
 #define G4X_STOLEN_RESERVED_ADDR1_MASK  (0xFFFF << 16)
 #define G4X_STOLEN_RESERVED_ADDR2_MASK  (0xFFF << 4)
+#define G4X_STOLEN_RESERVED_ENABLE      (1 << 0)
 
 /* Memory controller frequency in MCHBAR for Haswell (possible SNB+) */
 #define DCLK _MMIO(MCHBAR_MIRROR_BASE_SNB + 0x5e04)
@@ -3819,6 +3851,7 @@ enum {
  * GEN9 clock gating regs
  */
 #define GEN9_CLKGATE_DIS_0              _MMIO(0x46530)
+#define   DARBF_GATING_DIS              (1 << 27)
 #define   PWM2_GATING_DIS               (1 << 14)
 #define   PWM1_GATING_DIS               (1 << 13)
 
@@ -3837,6 +3870,7 @@ enum {
  */
 #define SLICE_UNIT_LEVEL_CLKGATE        _MMIO(0x94d4)
 #define  SARBUNIT_CLKGATE_DIS           (1 << 5)
+#define  RCCUNIT_CLKGATE_DIS            (1 << 7)
 
 /*
  * Display engine regs
@@ -6263,7 +6297,7 @@ enum {
 #define _PLANE_CTL_2_A                          0x70280
 #define _PLANE_CTL_3_A                          0x70380
 #define   PLANE_CTL_ENABLE                      (1 << 31)
-#define   PLANE_CTL_PIPE_GAMMA_ENABLE           (1 << 30)
+#define   PLANE_CTL_PIPE_GAMMA_ENABLE           (1 << 30)   /* Pre-GLK */
 #define   PLANE_CTL_FORMAT_MASK                 (0xf << 24)
 #define   PLANE_CTL_FORMAT_YUV422               (  0 << 24)
 #define   PLANE_CTL_FORMAT_NV12                 (  1 << 24)
@@ -6273,7 +6307,7 @@ enum {
 #define   PLANE_CTL_FORMAT_AYUV                 (  8 << 24)
 #define   PLANE_CTL_FORMAT_INDEXED              ( 12 << 24)
 #define   PLANE_CTL_FORMAT_RGB_565              ( 14 << 24)
-#define   PLANE_CTL_PIPE_CSC_ENABLE             (1 << 23)
+#define   PLANE_CTL_PIPE_CSC_ENABLE             (1 << 23) /* Pre-GLK */
 #define   PLANE_CTL_KEY_ENABLE_MASK             (0x3 << 21)
 #define   PLANE_CTL_KEY_ENABLE_SOURCE           (  1 << 21)
 #define   PLANE_CTL_KEY_ENABLE_DESTINATION      (  2 << 21)
@@ -6286,13 +6320,13 @@ enum {
 #define   PLANE_CTL_YUV422_VYUY                 (  3 << 16)
 #define   PLANE_CTL_DECOMPRESSION_ENABLE        (1 << 15)
 #define   PLANE_CTL_TRICKLE_FEED_DISABLE        (1 << 14)
-#define   PLANE_CTL_PLANE_GAMMA_DISABLE         (1 << 13)
+#define   PLANE_CTL_PLANE_GAMMA_DISABLE         (1 << 13) /* Pre-GLK */
 #define   PLANE_CTL_TILED_MASK                  (0x7 << 10)
 #define   PLANE_CTL_TILED_LINEAR                (  0 << 10)
 #define   PLANE_CTL_TILED_X                     (  1 << 10)
 #define   PLANE_CTL_TILED_Y                     (  4 << 10)
 #define   PLANE_CTL_TILED_YF                    (  5 << 10)
-#define   PLANE_CTL_ALPHA_MASK                  (0x3 << 4)
+#define   PLANE_CTL_ALPHA_MASK                  (0x3 << 4) /* Pre-GLK */
 #define   PLANE_CTL_ALPHA_DISABLE               (  0 << 4)
 #define   PLANE_CTL_ALPHA_SW_PREMULTIPLY        (  2 << 4)
 #define   PLANE_CTL_ALPHA_HW_PREMULTIPLY        (  3 << 4)
@@ -6332,6 +6366,10 @@ enum {
 #define   PLANE_COLOR_PIPE_GAMMA_ENABLE         (1 << 30)
 #define   PLANE_COLOR_PIPE_CSC_ENABLE           (1 << 23)
 #define   PLANE_COLOR_PLANE_GAMMA_DISABLE       (1 << 13)
+#define   PLANE_COLOR_ALPHA_MASK                (0x3 << 4)
+#define   PLANE_COLOR_ALPHA_DISABLE             (0 << 4)
+#define   PLANE_COLOR_ALPHA_SW_PREMULTIPLY      (2 << 4)
+#define   PLANE_COLOR_ALPHA_HW_PREMULTIPLY      (3 << 4)
 #define _PLANE_BUF_CFG_1_A                      0x7027c
 #define _PLANE_BUF_CFG_2_A                      0x7037c
 #define _PLANE_NV12_BUF_CFG_1_A         0x70278
@@ -7774,8 +7812,9 @@ enum {
 #define  FORCEWAKE_ACK_MEDIA_GEN9               _MMIO(0x0D88)
 #define  FORCEWAKE_ACK_RENDER_GEN9              _MMIO(0x0D84)
 #define  FORCEWAKE_ACK_BLITTER_GEN9             _MMIO(0x130044)
-#define   FORCEWAKE_KERNEL                      0x1
-#define   FORCEWAKE_USER                        0x2
+#define   FORCEWAKE_KERNEL                      BIT(0)
+#define   FORCEWAKE_USER                        BIT(1)
+#define   FORCEWAKE_KERNEL_FALLBACK             BIT(15)
 #define  FORCEWAKE_MT_ACK                       _MMIO(0x130040)
 #define  ECOBUS                                 _MMIO(0xa180)
 #define    FORCEWAKE_MT_ENABLE                  (1<<5)
@@ -7905,6 +7944,7 @@ enum {
 #define GEN6_RC1_WAKE_RATE_LIMIT                _MMIO(0xA098)
 #define GEN6_RC6_WAKE_RATE_LIMIT                _MMIO(0xA09C)
 #define GEN6_RC6pp_WAKE_RATE_LIMIT              _MMIO(0xA0A0)
+#define GEN10_MEDIA_WAKE_RATE_LIMIT             _MMIO(0xA0A0)
 #define GEN6_RC_EVALUATION_INTERVAL             _MMIO(0xA0A8)
 #define GEN6_RC_IDLE_HYSTERSIS                  _MMIO(0xA0AC)
 #define GEN6_RC_SLEEP                           _MMIO(0xA0B0)
@@ -8036,11 +8076,18 @@ enum {
 #define   CHV_EU311_PG_ENABLE           (1<<1)
 
 #define GEN9_SLICE_PGCTL_ACK(slice)     _MMIO(0x804c + (slice)*0x4)
+#define GEN10_SLICE_PGCTL_ACK(slice)    _MMIO(0x804c + ((slice) / 3) * 0x34 + \
+                                              ((slice) % 3) * 0x4)
 #define   GEN9_PGCTL_SLICE_ACK          (1 << 0)
 #define   GEN9_PGCTL_SS_ACK(subslice)   (1 << (2 + (subslice)*2))
+#define   GEN10_PGCTL_VALID_SS_MASK(slice) ((slice) == 0 ? 0x7F : 0x1F)
 
 #define GEN9_SS01_EU_PGCTL_ACK(slice)   _MMIO(0x805c + (slice)*0x8)
+#define GEN10_SS01_EU_PGCTL_ACK(slice)  _MMIO(0x805c + ((slice) / 3) * 0x30 + \
+                                              ((slice) % 3) * 0x8)
 #define GEN9_SS23_EU_PGCTL_ACK(slice)   _MMIO(0x8060 + (slice)*0x8)
+#define GEN10_SS23_EU_PGCTL_ACK(slice)  _MMIO(0x8060 + ((slice) / 3) * 0x30 + \
+                                              ((slice) % 3) * 0x8)
 #define   GEN9_PGCTL_SSA_EU08_ACK       (1 << 0)
 #define   GEN9_PGCTL_SSA_EU19_ACK       (1 << 2)
 #define   GEN9_PGCTL_SSA_EU210_ACK      (1 << 4)
@@ -8837,6 +8884,12 @@ enum skl_power_gate {
 #define ILK_TIMESTAMP_HI        _MMIO(0x70070)
 #define IVB_TIMESTAMP_CTR       _MMIO(0x44070)
 
+#define GEN9_TIMESTAMP_OVERRIDE                         _MMIO(0x44074)
+#define  GEN9_TIMESTAMP_OVERRIDE_US_COUNTER_DIVIDER_SHIFT       0
+#define  GEN9_TIMESTAMP_OVERRIDE_US_COUNTER_DIVIDER_MASK        0x3ff
+#define  GEN9_TIMESTAMP_OVERRIDE_US_COUNTER_DENOMINATOR_SHIFT   12
+#define  GEN9_TIMESTAMP_OVERRIDE_US_COUNTER_DENOMINATOR_MASK    (0xf << 12)
+
 #define _PIPE_FRMTMSTMP_A               0x70048
 #define PIPE_FRMTMSTMP(pipe)            \
                         _MMIO_PIPE2(pipe, _PIPE_FRMTMSTMP_A)
diff --git a/drivers/gpu/drm/i915/i915_selftest.h b/drivers/gpu/drm/i915/i915_selftest.h
index 78e1a1b168ff..9766e806dce6 100644
--- a/drivers/gpu/drm/i915/i915_selftest.h
+++ b/drivers/gpu/drm/i915/i915_selftest.h
@@ -99,6 +99,6 @@ __printf(2, 3)
 bool __igt_timeout(unsigned long timeout, const char *fmt, ...);
 
 #define igt_timeout(t, fmt, ...) \
-        __igt_timeout((t), KERN_WARNING pr_fmt(fmt), ##__VA_ARGS__)
+        __igt_timeout((t), KERN_NOTICE pr_fmt(fmt), ##__VA_ARGS__)
 
 #endif /* !__I915_SELFTEST_H__ */
diff --git a/drivers/gpu/drm/i915/i915_utils.h b/drivers/gpu/drm/i915/i915_utils.h
index af3d7cc53fa1..8d07764887ec 100644
--- a/drivers/gpu/drm/i915/i915_utils.h
+++ b/drivers/gpu/drm/i915/i915_utils.h
@@ -83,8 +83,11 @@
         (typeof(ptr))(__v & -BIT(n));                                   \
 })
 
-#define ptr_pack_bits(ptr, bits, n)                                     \
-        ((typeof(ptr))((unsigned long)(ptr) | (bits)))
+#define ptr_pack_bits(ptr, bits, n) ({                                  \
+        unsigned long __bits = (bits);                                  \
+        GEM_BUG_ON(__bits & -BIT(n));                                   \
+        ((typeof(ptr))((unsigned long)(ptr) | __bits));                 \
+})
 
 #define page_mask_bits(ptr) ptr_mask_bits(ptr, PAGE_SHIFT)
 #define page_unmask_bits(ptr) ptr_unmask_bits(ptr, PAGE_SHIFT)
diff --git a/drivers/gpu/drm/i915/i915_vma.c b/drivers/gpu/drm/i915/i915_vma.c
index fbfab2f33023..bf6d8d1eaabe 100644
--- a/drivers/gpu/drm/i915/i915_vma.c
+++ b/drivers/gpu/drm/i915/i915_vma.c
@@ -640,15 +640,17 @@ int __i915_vma_do_pin(struct i915_vma *vma,
                 if (ret)
                         goto err_unpin;
         }
+        GEM_BUG_ON(!drm_mm_node_allocated(&vma->node));
 
         ret = i915_vma_bind(vma, vma->obj->cache_level, flags);
         if (ret)
                 goto err_remove;
 
+        GEM_BUG_ON((vma->flags & I915_VMA_BIND_MASK) == 0);
+
         if ((bound ^ vma->flags) & I915_VMA_GLOBAL_BIND)
                 __i915_vma_set_map_and_fenceable(vma);
 
-        GEM_BUG_ON(!drm_mm_node_allocated(&vma->node));
         GEM_BUG_ON(i915_vma_misplaced(vma, size, alignment, flags));
         return 0;
 
@@ -656,6 +658,7 @@ err_remove:
         if ((bound & I915_VMA_BIND_MASK) == 0) {
                 i915_vma_remove(vma);
                 GEM_BUG_ON(vma->pages);
+                GEM_BUG_ON(vma->flags & I915_VMA_BIND_MASK);
         }
 err_unpin:
         __i915_vma_unpin(vma);
@@ -740,6 +743,7 @@ int i915_vma_unbind(struct i915_vma *vma)
         /* First wait upon any activity as retiring the request may
          * have side-effects such as unpinning or even unbinding this vma.
          */
+        might_sleep();
         active = i915_vma_get_active(vma);
         if (active) {
                 int idx;
diff --git a/drivers/gpu/drm/i915/intel_audio.c b/drivers/gpu/drm/i915/intel_audio.c
index 0ddba16fde1b..f1502a0188eb 100644
--- a/drivers/gpu/drm/i915/intel_audio.c
+++ b/drivers/gpu/drm/i915/intel_audio.c
@@ -102,13 +102,13 @@ static const struct dp_aud_n_m dp_aud_n_m[] = {
 };
 
 static const struct dp_aud_n_m *
-audio_config_dp_get_n_m(struct intel_crtc *intel_crtc, int rate)
+audio_config_dp_get_n_m(const struct intel_crtc_state *crtc_state, int rate)
 {
         int i;
 
         for (i = 0; i < ARRAY_SIZE(dp_aud_n_m); i++) {
                 if (rate == dp_aud_n_m[i].sample_rate &&
-                    intel_crtc->config->port_clock == dp_aud_n_m[i].clock)
+                    crtc_state->port_clock == dp_aud_n_m[i].clock)
                         return &dp_aud_n_m[i];
         }
 
@@ -157,8 +157,10 @@ static const struct {
 };
 
 /* get AUD_CONFIG_PIXEL_CLOCK_HDMI_* value for mode */
-static u32 audio_config_hdmi_pixel_clock(const struct drm_display_mode *adjusted_mode)
+static u32 audio_config_hdmi_pixel_clock(const struct intel_crtc_state *crtc_state)
 {
+        const struct drm_display_mode *adjusted_mode =
+                &crtc_state->base.adjusted_mode;
         int i;
 
         for (i = 0; i < ARRAY_SIZE(hdmi_audio_clock); i++) {
@@ -179,9 +181,11 @@ static u32 audio_config_hdmi_pixel_clock(const struct drm_display_mode *adjusted
         return hdmi_audio_clock[i].config;
 }
 
-static int audio_config_hdmi_get_n(const struct drm_display_mode *adjusted_mode,
+static int audio_config_hdmi_get_n(const struct intel_crtc_state *crtc_state,
                                    int rate)
 {
+        const struct drm_display_mode *adjusted_mode =
+                &crtc_state->base.adjusted_mode;
         int i;
 
         for (i = 0; i < ARRAY_SIZE(hdmi_aud_ncts); i++) {
@@ -220,7 +224,9 @@ static bool intel_eld_uptodate(struct drm_connector *connector,
         return true;
 }
 
-static void g4x_audio_codec_disable(struct intel_encoder *encoder)
+static void g4x_audio_codec_disable(struct intel_encoder *encoder,
+                                    const struct intel_crtc_state *old_crtc_state,
+                                    const struct drm_connector_state *old_conn_state)
 {
         struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
         uint32_t eldv, tmp;
@@ -239,11 +245,12 @@ static void g4x_audio_codec_disable(struct intel_encoder *encoder)
         I915_WRITE(G4X_AUD_CNTL_ST, tmp);
 }
 
-static void g4x_audio_codec_enable(struct drm_connector *connector,
-                                   struct intel_encoder *encoder,
-                                   const struct drm_display_mode *adjusted_mode)
+static void g4x_audio_codec_enable(struct intel_encoder *encoder,
+                                   const struct intel_crtc_state *crtc_state,
+                                   const struct drm_connector_state *conn_state)
 {
-        struct drm_i915_private *dev_priv = to_i915(connector->dev);
+        struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
+        struct drm_connector *connector = conn_state->connector;
         uint8_t *eld = connector->eld;
         uint32_t eldv;
         uint32_t tmp;
@@ -279,16 +286,20 @@ static void g4x_audio_codec_enable(struct drm_connector *connector,
 }
 
 static void
-hsw_dp_audio_config_update(struct intel_crtc *intel_crtc, enum port port,
-                           const struct drm_display_mode *adjusted_mode)
+hsw_dp_audio_config_update(struct intel_encoder *encoder,
+                           const struct intel_crtc_state *crtc_state)
 {
-        struct drm_i915_private *dev_priv = to_i915(intel_crtc->base.dev);
+        struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
         struct i915_audio_component *acomp = dev_priv->audio_component;
-        int rate = acomp ? acomp->aud_sample_rate[port] : 0;
-        const struct dp_aud_n_m *nm = audio_config_dp_get_n_m(intel_crtc, rate);
-        enum pipe pipe = intel_crtc->pipe;
+        struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
+        enum port port = encoder->port;
+        enum pipe pipe = crtc->pipe;
+        const struct dp_aud_n_m *nm;
+        int rate;
         u32 tmp;
 
+        rate = acomp ? acomp->aud_sample_rate[port] : 0;
+        nm = audio_config_dp_get_n_m(crtc_state, rate);
         if (nm)
                 DRM_DEBUG_KMS("using Maud %u, Naud %u\n", nm->m, nm->n);
         else
@@ -323,23 +334,26 @@ hsw_dp_audio_config_update(struct intel_crtc *intel_crtc, enum port port,
 }
 
 static void
-hsw_hdmi_audio_config_update(struct intel_crtc *intel_crtc, enum port port,
-                             const struct drm_display_mode *adjusted_mode)
+hsw_hdmi_audio_config_update(struct intel_encoder *encoder,
+                             const struct intel_crtc_state *crtc_state)
 {
-        struct drm_i915_private *dev_priv = to_i915(intel_crtc->base.dev);
+        struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
         struct i915_audio_component *acomp = dev_priv->audio_component;
-        int rate = acomp ? acomp->aud_sample_rate[port] : 0;
-        enum pipe pipe = intel_crtc->pipe;
-        int n;
+        struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
+        enum port port = encoder->port;
+        enum pipe pipe = crtc->pipe;
+        int n, rate;
         u32 tmp;
 
+        rate = acomp ? acomp->aud_sample_rate[port] : 0;
+
         tmp = I915_READ(HSW_AUD_CFG(pipe));
         tmp &= ~AUD_CONFIG_N_VALUE_INDEX;
         tmp &= ~AUD_CONFIG_PIXEL_CLOCK_HDMI_MASK;
         tmp &= ~AUD_CONFIG_N_PROG_ENABLE;
-        tmp |= audio_config_hdmi_pixel_clock(adjusted_mode);
+        tmp |= audio_config_hdmi_pixel_clock(crtc_state);
 
-        n = audio_config_hdmi_get_n(adjusted_mode, rate);
+        n = audio_config_hdmi_get_n(crtc_state, rate);
         if (n != 0) {
                 DRM_DEBUG_KMS("using N %d\n", n);
 
@@ -363,20 +377,22 @@ hsw_hdmi_audio_config_update(struct intel_crtc *intel_crtc, enum port port,
 }
 
 static void
-hsw_audio_config_update(struct intel_crtc *intel_crtc, enum port port,
-                        const struct drm_display_mode *adjusted_mode)
+hsw_audio_config_update(struct intel_encoder *encoder,
+                        const struct intel_crtc_state *crtc_state)
 {
-        if (intel_crtc_has_dp_encoder(intel_crtc->config))
-                hsw_dp_audio_config_update(intel_crtc, port, adjusted_mode);
+        if (intel_crtc_has_dp_encoder(crtc_state))
+                hsw_dp_audio_config_update(encoder, crtc_state);
         else
-                hsw_hdmi_audio_config_update(intel_crtc, port, adjusted_mode);
+                hsw_hdmi_audio_config_update(encoder, crtc_state);
 }
 
-static void hsw_audio_codec_disable(struct intel_encoder *encoder)
+static void hsw_audio_codec_disable(struct intel_encoder *encoder,
+                                    const struct intel_crtc_state *old_crtc_state,
+                                    const struct drm_connector_state *old_conn_state)
 {
         struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
-        struct intel_crtc *intel_crtc = to_intel_crtc(encoder->base.crtc);
-        enum pipe pipe = intel_crtc->pipe;
+        struct intel_crtc *crtc = to_intel_crtc(old_crtc_state->base.crtc);
+        enum pipe pipe = crtc->pipe;
         uint32_t tmp;
 
         DRM_DEBUG_KMS("Disable audio codec on pipe %c\n", pipe_name(pipe));
@@ -389,7 +405,7 @@ static void hsw_audio_codec_disable(struct intel_encoder *encoder)
         tmp |= AUD_CONFIG_N_PROG_ENABLE;
         tmp &= ~AUD_CONFIG_UPPER_N_MASK;
         tmp &= ~AUD_CONFIG_LOWER_N_MASK;
-        if (intel_crtc_has_dp_encoder(intel_crtc->config))
+        if (intel_crtc_has_dp_encoder(old_crtc_state))
                 tmp |= AUD_CONFIG_N_VALUE_INDEX;
         I915_WRITE(HSW_AUD_CFG(pipe), tmp);
 
@@ -402,14 +418,14 @@ static void hsw_audio_codec_disable(struct intel_encoder *encoder)
         mutex_unlock(&dev_priv->av_mutex);
 }
 
-static void hsw_audio_codec_enable(struct drm_connector *connector,
-                                   struct intel_encoder *intel_encoder,
-                                   const struct drm_display_mode *adjusted_mode)
+static void hsw_audio_codec_enable(struct intel_encoder *encoder,
+                                   const struct intel_crtc_state *crtc_state,
+                                   const struct drm_connector_state *conn_state)
 {
-        struct drm_i915_private *dev_priv = to_i915(connector->dev);
-        struct intel_crtc *intel_crtc = to_intel_crtc(intel_encoder->base.crtc);
-        enum pipe pipe = intel_crtc->pipe;
-        enum port port = intel_encoder->port;
+        struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
+        struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
+        struct drm_connector *connector = conn_state->connector;
+        enum pipe pipe = crtc->pipe;
         const uint8_t *eld = connector->eld;
         uint32_t tmp;
         int len, i;
@@ -448,17 +464,19 @@ static void hsw_audio_codec_enable(struct drm_connector *connector,
         I915_WRITE(HSW_AUD_PIN_ELD_CP_VLD, tmp);
 
         /* Enable timestamps */
-        hsw_audio_config_update(intel_crtc, port, adjusted_mode);
+        hsw_audio_config_update(encoder, crtc_state);
 
         mutex_unlock(&dev_priv->av_mutex);
 }
 
-static void ilk_audio_codec_disable(struct intel_encoder *intel_encoder)
+static void ilk_audio_codec_disable(struct intel_encoder *encoder,
+                                    const struct intel_crtc_state *old_crtc_state,
+                                    const struct drm_connector_state *old_conn_state)
 {
-        struct drm_i915_private *dev_priv = to_i915(intel_encoder->base.dev);
-        struct intel_crtc *intel_crtc = to_intel_crtc(intel_encoder->base.crtc);
-        enum pipe pipe = intel_crtc->pipe;
-        enum port port = intel_encoder->port;
+        struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
+        struct intel_crtc *crtc = to_intel_crtc(old_crtc_state->base.crtc);
+        enum pipe pipe = crtc->pipe;
+        enum port port = encoder->port;
         uint32_t tmp, eldv;
         i915_reg_t aud_config, aud_cntrl_st2;
 
@@ -485,7 +503,7 @@ static void ilk_audio_codec_disable(struct intel_encoder *intel_encoder)
         tmp |= AUD_CONFIG_N_PROG_ENABLE;
         tmp &= ~AUD_CONFIG_UPPER_N_MASK;
         tmp &= ~AUD_CONFIG_LOWER_N_MASK;
-        if (intel_crtc_has_dp_encoder(intel_crtc->config))
+        if (intel_crtc_has_dp_encoder(old_crtc_state))
                 tmp |= AUD_CONFIG_N_VALUE_INDEX;
         I915_WRITE(aud_config, tmp);
 
@@ -497,14 +515,15 @@ static void ilk_audio_codec_disable(struct intel_encoder *intel_encoder)
         I915_WRITE(aud_cntrl_st2, tmp);
 }
 
-static void ilk_audio_codec_enable(struct drm_connector *connector,
-                                   struct intel_encoder *intel_encoder,
-                                   const struct drm_display_mode *adjusted_mode)
+static void ilk_audio_codec_enable(struct intel_encoder *encoder,
+                                   const struct intel_crtc_state *crtc_state,
+                                   const struct drm_connector_state *conn_state)
 {
-        struct drm_i915_private *dev_priv = to_i915(connector->dev);
-        struct intel_crtc *intel_crtc = to_intel_crtc(intel_encoder->base.crtc);
-        enum pipe pipe = intel_crtc->pipe;
-        enum port port = intel_encoder->port;
+        struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
+        struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
+        struct drm_connector *connector = conn_state->connector;
+        enum pipe pipe = crtc->pipe;
+        enum port port = encoder->port;
         uint8_t *eld = connector->eld;
         uint32_t tmp, eldv;
         int len, i;
@@ -568,36 +587,36 @@ static void ilk_audio_codec_enable(struct drm_connector *connector,
         tmp &= ~AUD_CONFIG_N_VALUE_INDEX;
         tmp &= ~AUD_CONFIG_N_PROG_ENABLE;
         tmp &= ~AUD_CONFIG_PIXEL_CLOCK_HDMI_MASK;
-        if (intel_crtc_has_dp_encoder(intel_crtc->config))
+        if (intel_crtc_has_dp_encoder(crtc_state))
                 tmp |= AUD_CONFIG_N_VALUE_INDEX;
         else
-                tmp |= audio_config_hdmi_pixel_clock(adjusted_mode);
+                tmp |= audio_config_hdmi_pixel_clock(crtc_state);
         I915_WRITE(aud_config, tmp);
 }
 
 /**
  * intel_audio_codec_enable - Enable the audio codec for HD audio
- * @intel_encoder: encoder on which to enable audio
+ * @encoder: encoder on which to enable audio
  * @crtc_state: pointer to the current crtc state.
  * @conn_state: pointer to the current connector state.
  *
  * The enable sequences may only be performed after enabling the transcoder and
  * port, and after completed link training.
  */
-void intel_audio_codec_enable(struct intel_encoder *intel_encoder,
+void intel_audio_codec_enable(struct intel_encoder *encoder,
                               const struct intel_crtc_state *crtc_state,
                               const struct drm_connector_state *conn_state)
 {
-        struct drm_encoder *encoder = &intel_encoder->base;
-        const struct drm_display_mode *adjusted_mode = &crtc_state->base.adjusted_mode;
-        struct drm_connector *connector;
-        struct drm_i915_private *dev_priv = to_i915(encoder->dev);
+        struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
         struct i915_audio_component *acomp = dev_priv->audio_component;
-        enum port port = intel_encoder->port;
-        enum pipe pipe = to_intel_crtc(crtc_state->base.crtc)->pipe;
+        struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
+        struct drm_connector *connector = conn_state->connector;
+        const struct drm_display_mode *adjusted_mode =
+                &crtc_state->base.adjusted_mode;
+        enum port port = encoder->port;
+        enum pipe pipe = crtc->pipe;
 
-        connector = conn_state->connector;
-        if (!connector || !connector->eld[0])
+        if (!connector->eld[0])
                 return;
 
         DRM_DEBUG_DRIVER("ELD on [CONNECTOR:%d:%s], [ENCODER:%d:%s]\n",
@@ -609,19 +628,20 @@ void intel_audio_codec_enable(struct intel_encoder *intel_encoder,
         connector->eld[6] = drm_av_sync_delay(connector, adjusted_mode) / 2;
 
         if (dev_priv->display.audio_codec_enable)
-                dev_priv->display.audio_codec_enable(connector, intel_encoder,
-                                                     adjusted_mode);
+                dev_priv->display.audio_codec_enable(encoder,
+                                                     crtc_state,
+                                                     conn_state);
 
         mutex_lock(&dev_priv->av_mutex);
-        intel_encoder->audio_connector = connector;
+        encoder->audio_connector = connector;
 
         /* referred in audio callbacks */
-        dev_priv->av_enc_map[pipe] = intel_encoder;
+        dev_priv->av_enc_map[pipe] = encoder;
         mutex_unlock(&dev_priv->av_mutex);
 
         if (acomp && acomp->audio_ops && acomp->audio_ops->pin_eld_notify) {
                 /* audio drivers expect pipe = -1 to indicate Non-MST cases */
-                if (intel_encoder->type != INTEL_OUTPUT_DP_MST)
+                if (!intel_crtc_has_type(crtc_state, INTEL_OUTPUT_DP_MST))
                         pipe = -1;
                 acomp->audio_ops->pin_eld_notify(acomp->audio_ops->audio_ptr,
                                                  (int) port, (int) pipe);
@@ -629,36 +649,41 @@ void intel_audio_codec_enable(struct intel_encoder *intel_encoder,
 
         intel_lpe_audio_notify(dev_priv, pipe, port, connector->eld,
                                crtc_state->port_clock,
-                               intel_encoder->type == INTEL_OUTPUT_DP);
+                               intel_crtc_has_dp_encoder(crtc_state));
 }
 
 /**
  * intel_audio_codec_disable - Disable the audio codec for HD audio
- * @intel_encoder: encoder on which to disable audio
+ * @encoder: encoder on which to disable audio
+ * @old_crtc_state: pointer to the old crtc state.
+ * @old_conn_state: pointer to the old connector state.
  *
  * The disable sequences must be performed before disabling the transcoder or
  * port.
  */
-void intel_audio_codec_disable(struct intel_encoder *intel_encoder)
+void intel_audio_codec_disable(struct intel_encoder *encoder,
+                               const struct intel_crtc_state *old_crtc_state,
+                               const struct drm_connector_state *old_conn_state)
 {
-        struct drm_encoder *encoder = &intel_encoder->base;
-        struct drm_i915_private *dev_priv = to_i915(encoder->dev);
+        struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
         struct i915_audio_component *acomp = dev_priv->audio_component;
-        enum port port = intel_encoder->port;
-        struct intel_crtc *crtc = to_intel_crtc(encoder->crtc);
+        struct intel_crtc *crtc = to_intel_crtc(old_crtc_state->base.crtc);
+        enum port port = encoder->port;
         enum pipe pipe = crtc->pipe;
 
         if (dev_priv->display.audio_codec_disable)
-                dev_priv->display.audio_codec_disable(intel_encoder);
+                dev_priv->display.audio_codec_disable(encoder,
+                                                      old_crtc_state,
+                                                      old_conn_state);
 
         mutex_lock(&dev_priv->av_mutex);
-        intel_encoder->audio_connector = NULL;
+        encoder->audio_connector = NULL;
         dev_priv->av_enc_map[pipe] = NULL;
         mutex_unlock(&dev_priv->av_mutex);
 
         if (acomp && acomp->audio_ops && acomp->audio_ops->pin_eld_notify) {
                 /* audio drivers expect pipe = -1 to indicate Non-MST cases */
-                if (intel_encoder->type != INTEL_OUTPUT_DP_MST)
+                if (!intel_crtc_has_type(old_crtc_state, INTEL_OUTPUT_DP_MST))
                         pipe = -1;
                 acomp->audio_ops->pin_eld_notify(acomp->audio_ops->audio_ptr,
                                                  (int) port, (int) pipe);
@@ -793,10 +818,9 @@ static int i915_audio_component_sync_audio_rate(struct device *kdev, int port,
                                                 int pipe, int rate)
 {
         struct drm_i915_private *dev_priv = kdev_to_i915(kdev);
-        struct intel_encoder *intel_encoder;
-        struct intel_crtc *crtc;
-        struct drm_display_mode *adjusted_mode;
         struct i915_audio_component *acomp = dev_priv->audio_component;
+        struct intel_encoder *encoder;
+        struct intel_crtc *crtc;
         int err = 0;
 
         if (!HAS_DDI(dev_priv))
@@ -806,23 +830,19 @@ static int i915_audio_component_sync_audio_rate(struct device *kdev, int port,
         mutex_lock(&dev_priv->av_mutex);
 
         /* 1. get the pipe */
-        intel_encoder = get_saved_enc(dev_priv, port, pipe);
-        if (!intel_encoder || !intel_encoder->base.crtc) {
+        encoder = get_saved_enc(dev_priv, port, pipe);
+        if (!encoder || !encoder->base.crtc) {
                 DRM_DEBUG_KMS("Not valid for port %c\n", port_name(port));
                 err = -ENODEV;
                 goto unlock;
         }
 
-        /* pipe passed from the audio driver will be -1 for Non-MST case */
-        crtc = to_intel_crtc(intel_encoder->base.crtc);
-        pipe = crtc->pipe;
-
-        adjusted_mode = &crtc->config->base.adjusted_mode;
+        crtc = to_intel_crtc(encoder->base.crtc);
 
         /* port must be valid now, otherwise the pipe will be invalid */
         acomp->aud_sample_rate[port] = rate;
 
-        hsw_audio_config_update(crtc, port, adjusted_mode);
+        hsw_audio_config_update(encoder, crtc->config);
 
  unlock:
         mutex_unlock(&dev_priv->av_mutex);
diff --git a/drivers/gpu/drm/i915/intel_bios.c b/drivers/gpu/drm/i915/intel_bios.c
index fd23023df7c1..51108ffc28d1 100644
--- a/drivers/gpu/drm/i915/intel_bios.c
+++ b/drivers/gpu/drm/i915/intel_bios.c
@@ -1234,6 +1234,30 @@ static void parse_ddi_port(struct drm_i915_private *dev_priv, enum port port,
                 info->hdmi_level_shift = hdmi_level_shift;
         }
 
+        if (bdb_version >= 204) {
+                int max_tmds_clock;
+
+                switch (child->hdmi_max_data_rate) {
+                default:
+                        MISSING_CASE(child->hdmi_max_data_rate);
+                        /* fall through */
+                case HDMI_MAX_DATA_RATE_PLATFORM:
+                        max_tmds_clock = 0;
+                        break;
+                case HDMI_MAX_DATA_RATE_297:
+                        max_tmds_clock = 297000;
+                        break;
+                case HDMI_MAX_DATA_RATE_165:
+                        max_tmds_clock = 165000;
+                        break;
+                }
+
+                if (max_tmds_clock)
+                        DRM_DEBUG_KMS("VBT HDMI max TMDS clock for port %c: %d kHz\n",
+                                      port_name(port), max_tmds_clock);
+                info->max_tmds_clock = max_tmds_clock;
+        }
+
         /* Parse the I_boost config for SKL and above */
         if (bdb_version >= 196 && child->iboost) {
                 info->dp_boost_level = translate_iboost(child->dp_iboost_level);
diff --git a/drivers/gpu/drm/i915/intel_breadcrumbs.c b/drivers/gpu/drm/i915/intel_breadcrumbs.c
index 5f8b9f1f40f1..5ae2d276f7f3 100644
--- a/drivers/gpu/drm/i915/intel_breadcrumbs.c
+++ b/drivers/gpu/drm/i915/intel_breadcrumbs.c
@@ -123,7 +123,7 @@ static void intel_breadcrumbs_fake_irq(struct timer_list *t)
          */
 
         spin_lock_irq(&b->irq_lock);
-        if (!__intel_breadcrumbs_wakeup(b))
+        if (b->irq_armed && !__intel_breadcrumbs_wakeup(b))
                 __intel_engine_disarm_breadcrumbs(engine);
         spin_unlock_irq(&b->irq_lock);
         if (!b->irq_armed)
@@ -145,6 +145,14 @@ static void intel_breadcrumbs_fake_irq(struct timer_list *t)
 
 static void irq_enable(struct intel_engine_cs *engine)
 {
+        /*
+         * FIXME: Ideally we want this on the API boundary, but for the
+         * sake of testing with mock breadcrumbs (no HW so unable to
+         * enable irqs) we place it deep within the bowels, at the point
+         * of no return.
+         */
+        GEM_BUG_ON(!intel_irqs_enabled(engine->i915));
+
         /* Enabling the IRQ may miss the generation of the interrupt, but
          * we still need to force the barrier before reading the seqno,
          * just in case.
@@ -171,15 +179,37 @@ void __intel_engine_disarm_breadcrumbs(struct intel_engine_cs *engine)
 
         lockdep_assert_held(&b->irq_lock);
         GEM_BUG_ON(b->irq_wait);
+        GEM_BUG_ON(!b->irq_armed);
 
-        if (b->irq_enabled) {
+        GEM_BUG_ON(!b->irq_enabled);
+        if (!--b->irq_enabled)
                 irq_disable(engine);
-                b->irq_enabled = false;
-        }
 
         b->irq_armed = false;
 }
 
+void intel_engine_pin_breadcrumbs_irq(struct intel_engine_cs *engine)
+{
+        struct intel_breadcrumbs *b = &engine->breadcrumbs;
+
+        spin_lock_irq(&b->irq_lock);
+        if (!b->irq_enabled++)
+                irq_enable(engine);
+        GEM_BUG_ON(!b->irq_enabled); /* no overflow! */
+        spin_unlock_irq(&b->irq_lock);
+}
+
+void intel_engine_unpin_breadcrumbs_irq(struct intel_engine_cs *engine)
+{
+        struct intel_breadcrumbs *b = &engine->breadcrumbs;
+
+        spin_lock_irq(&b->irq_lock);
+        GEM_BUG_ON(!b->irq_enabled); /* no underflow! */
+        if (!--b->irq_enabled)
+                irq_disable(engine);
+        spin_unlock_irq(&b->irq_lock);
+}
+
 void intel_engine_disarm_breadcrumbs(struct intel_engine_cs *engine)
 {
         struct intel_breadcrumbs *b = &engine->breadcrumbs;
@@ -197,7 +227,8 @@ void intel_engine_disarm_breadcrumbs(struct intel_engine_cs *engine)
 
         spin_lock(&b->irq_lock);
         first = fetch_and_zero(&b->irq_wait);
-        __intel_engine_disarm_breadcrumbs(engine);
+        if (b->irq_armed)
+                __intel_engine_disarm_breadcrumbs(engine);
         spin_unlock(&b->irq_lock);
 
         rbtree_postorder_for_each_entry_safe(wait, n, &b->waiters, node) {
@@ -241,6 +272,7 @@ static bool __intel_breadcrumbs_enable_irq(struct intel_breadcrumbs *b)
         struct intel_engine_cs *engine =
                 container_of(b, struct intel_engine_cs, breadcrumbs);
         struct drm_i915_private *i915 = engine->i915;
+        bool enabled;
 
         lockdep_assert_held(&b->irq_lock);
         if (b->irq_armed)
@@ -252,7 +284,6 @@ static bool __intel_breadcrumbs_enable_irq(struct intel_breadcrumbs *b)
          * the irq.
          */
         b->irq_armed = true;
-        GEM_BUG_ON(b->irq_enabled);
 
         if (I915_SELFTEST_ONLY(b->mock)) {
                 /* For our mock objects we want to avoid interaction
@@ -273,14 +304,15 @@ static bool __intel_breadcrumbs_enable_irq(struct intel_breadcrumbs *b)
          */
 
         /* No interrupts? Kick the waiter every jiffie! */
-        if (intel_irqs_enabled(i915)) {
-                if (!test_bit(engine->id, &i915->gpu_error.test_irq_rings))
-                        irq_enable(engine);
-                b->irq_enabled = true;
+        enabled = false;
+        if (!b->irq_enabled++ &&
+            !test_bit(engine->id, &i915->gpu_error.test_irq_rings)) {
+                irq_enable(engine);
+                enabled = true;
         }
 
         enable_fake_irq(b);
-        return true;
+        return enabled;
 }
 
 static inline struct intel_wait *to_wait(struct rb_node *node)
diff --git a/drivers/gpu/drm/i915/intel_cdclk.c b/drivers/gpu/drm/i915/intel_cdclk.c
index b2a6d62b71c0..e8884c2ade98 100644
--- a/drivers/gpu/drm/i915/intel_cdclk.c
+++ b/drivers/gpu/drm/i915/intel_cdclk.c
@@ -437,13 +437,45 @@ static int vlv_calc_cdclk(struct drm_i915_private *dev_priv, int min_cdclk)
                 return 200000;
 }
 
+static u8 vlv_calc_voltage_level(struct drm_i915_private *dev_priv, int cdclk)
+{
+        if (IS_VALLEYVIEW(dev_priv)) {
+                if (cdclk >= 320000) /* jump to highest voltage for 400MHz too */
+                        return 2;
+                else if (cdclk >= 266667)
+                        return 1;
+                else
+                        return 0;
+        } else {
+                /*
+                 * Specs are full of misinformation, but testing on actual
+                 * hardware has shown that we just need to write the desired
+                 * CCK divider into the Punit register.
+                 */
+                return DIV_ROUND_CLOSEST(dev_priv->hpll_freq << 1, cdclk) - 1;
+        }
+}
+
 static void vlv_get_cdclk(struct drm_i915_private *dev_priv,
                           struct intel_cdclk_state *cdclk_state)
 {
+        u32 val;
+
         cdclk_state->vco = vlv_get_hpll_vco(dev_priv);
         cdclk_state->cdclk = vlv_get_cck_clock(dev_priv, "cdclk",
                                                CCK_DISPLAY_CLOCK_CONTROL,
                                                cdclk_state->vco);
+
+        mutex_lock(&dev_priv->pcu_lock);
+        val = vlv_punit_read(dev_priv, PUNIT_REG_DSPFREQ);
+        mutex_unlock(&dev_priv->pcu_lock);
+
+        if (IS_VALLEYVIEW(dev_priv))
+                cdclk_state->voltage_level = (val & DSPFREQGUAR_MASK) >>
+                        DSPFREQGUAR_SHIFT;
+        else
+                cdclk_state->voltage_level = (val & DSPFREQGUAR_MASK_CHV) >>
+                        DSPFREQGUAR_SHIFT_CHV;
 }
 
 static void vlv_program_pfi_credits(struct drm_i915_private *dev_priv)
@@ -486,7 +518,19 @@ static void vlv_set_cdclk(struct drm_i915_private *dev_priv,
                           const struct intel_cdclk_state *cdclk_state)
 {
         int cdclk = cdclk_state->cdclk;
-        u32 val, cmd;
+        u32 val, cmd = cdclk_state->voltage_level;
+
+        switch (cdclk) {
+        case 400000:
+        case 333333:
+        case 320000:
+        case 266667:
+        case 200000:
+                break;
+        default:
+                MISSING_CASE(cdclk);
+                return;
+        }
 
         /* There are cases where we can end up here with power domains
          * off and a CDCLK frequency other than the minimum, like when
@@ -496,13 +540,6 @@ static void vlv_set_cdclk(struct drm_i915_private *dev_priv,
          */
         intel_display_power_get(dev_priv, POWER_DOMAIN_PIPE_A);
 
-        if (cdclk >= 320000) /* jump to highest voltage for 400MHz too */
-                cmd = 2;
-        else if (cdclk == 266667)
-                cmd = 1;
-        else
-                cmd = 0;
-
         mutex_lock(&dev_priv->pcu_lock);
         val = vlv_punit_read(dev_priv, PUNIT_REG_DSPFREQ);
         val &= ~DSPFREQGUAR_MASK;
@@ -562,7 +599,7 @@ static void chv_set_cdclk(struct drm_i915_private *dev_priv,
                           const struct intel_cdclk_state *cdclk_state)
 {
         int cdclk = cdclk_state->cdclk;
-        u32 val, cmd;
+        u32 val, cmd = cdclk_state->voltage_level;
 
         switch (cdclk) {
         case 333333:
@@ -583,13 +620,6 @@ static void chv_set_cdclk(struct drm_i915_private *dev_priv,
          */
         intel_display_power_get(dev_priv, POWER_DOMAIN_PIPE_A);
 
-        /*
-         * Specs are full of misinformation, but testing on actual
-         * hardware has shown that we just need to write the desired
-         * CCK divider into the Punit register.
-         */
-        cmd = DIV_ROUND_CLOSEST(dev_priv->hpll_freq << 1, cdclk) - 1;
-
         mutex_lock(&dev_priv->pcu_lock);
         val = vlv_punit_read(dev_priv, PUNIT_REG_DSPFREQ);
         val &= ~DSPFREQGUAR_MASK_CHV;
@@ -621,6 +651,21 @@ static int bdw_calc_cdclk(int min_cdclk)
                 return 337500;
 }
 
+static u8 bdw_calc_voltage_level(int cdclk)
+{
+        switch (cdclk) {
+        default:
+        case 337500:
+                return 2;
+        case 450000:
+                return 0;
+        case 540000:
+                return 1;
+        case 675000:
+                return 3;
+        }
+}
+
 static void bdw_get_cdclk(struct drm_i915_private *dev_priv,
                           struct intel_cdclk_state *cdclk_state)
 {
@@ -639,13 +684,20 @@ static void bdw_get_cdclk(struct drm_i915_private *dev_priv,
                 cdclk_state->cdclk = 337500;
         else
                 cdclk_state->cdclk = 675000;
+
+        /*
+         * Can't read this out :( Let's assume it's
+         * at least what the CDCLK frequency requires.
+         */
+        cdclk_state->voltage_level =
+                bdw_calc_voltage_level(cdclk_state->cdclk);
 }
 
 static void bdw_set_cdclk(struct drm_i915_private *dev_priv,
                           const struct intel_cdclk_state *cdclk_state)
 {
         int cdclk = cdclk_state->cdclk;
-        uint32_t val, data;
+        uint32_t val;
         int ret;
 
         if (WARN((I915_READ(LCPLL_CTL) &
@@ -681,25 +733,21 @@ static void bdw_set_cdclk(struct drm_i915_private *dev_priv,
         val &= ~LCPLL_CLK_FREQ_MASK;
 
         switch (cdclk) {
+        default:
+                MISSING_CASE(cdclk);
+                /* fall through */
+        case 337500:
+                val |= LCPLL_CLK_FREQ_337_5_BDW;
+                break;
         case 450000:
                 val |= LCPLL_CLK_FREQ_450;
-                data = 0;
                 break;
         case 540000:
                 val |= LCPLL_CLK_FREQ_54O_BDW;
-                data = 1;
-                break;
-        case 337500:
-                val |= LCPLL_CLK_FREQ_337_5_BDW;
-                data = 2;
                 break;
         case 675000:
                 val |= LCPLL_CLK_FREQ_675_BDW;
-                data = 3;
                 break;
-        default:
-                WARN(1, "invalid cdclk frequency\n");
-                return;
         }
 
         I915_WRITE(LCPLL_CTL, val);
@@ -713,16 +761,13 @@ static void bdw_set_cdclk(struct drm_i915_private *dev_priv,
                 DRM_ERROR("Switching back to LCPLL failed\n");
 
         mutex_lock(&dev_priv->pcu_lock);
-        sandybridge_pcode_write(dev_priv, HSW_PCODE_DE_WRITE_FREQ_REQ, data);
+        sandybridge_pcode_write(dev_priv, HSW_PCODE_DE_WRITE_FREQ_REQ,
+                                cdclk_state->voltage_level);
         mutex_unlock(&dev_priv->pcu_lock);
 
         I915_WRITE(CDCLK_FREQ, DIV_ROUND_CLOSEST(cdclk, 1000) - 1);
 
         intel_update_cdclk(dev_priv);
-
-        WARN(cdclk != dev_priv->cdclk.hw.cdclk,
-             "cdclk requested %d kHz but got %d kHz\n",
-             cdclk, dev_priv->cdclk.hw.cdclk);
 }
 
 static int skl_calc_cdclk(int min_cdclk, int vco)
@@ -748,6 +793,24 @@ static int skl_calc_cdclk(int min_cdclk, int vco)
         }
 }
 
+static u8 skl_calc_voltage_level(int cdclk)
+{
+        switch (cdclk) {
+        default:
+        case 308571:
+        case 337500:
+                return 0;
+        case 450000:
+        case 432000:
+                return 1;
+        case 540000:
+                return 2;
+        case 617143:
+        case 675000:
+                return 3;
+        }
+}
+
 static void skl_dpll0_update(struct drm_i915_private *dev_priv,
                              struct intel_cdclk_state *cdclk_state)
 {
@@ -798,7 +861,7 @@ static void skl_get_cdclk(struct drm_i915_private *dev_priv,
         cdclk_state->cdclk = cdclk_state->ref;
 
         if (cdclk_state->vco == 0)
-                return;
+                goto out;
 
         cdctl = I915_READ(CDCLK_CTL);
 
@@ -839,6 +902,14 @@ static void skl_get_cdclk(struct drm_i915_private *dev_priv,
                         break;
                 }
         }
+
+ out:
+        /*
+         * Can't read this out :( Let's assume it's
+         * at least what the CDCLK frequency requires.
+         */
+        cdclk_state->voltage_level =
+                skl_calc_voltage_level(cdclk_state->cdclk);
 }
 
 /* convert from kHz to .1 fixpoint MHz with -1MHz offset */
@@ -923,11 +994,9 @@ static void skl_set_cdclk(struct drm_i915_private *dev_priv,
 {
         int cdclk = cdclk_state->cdclk;
         int vco = cdclk_state->vco;
-        u32 freq_select, pcu_ack;
+        u32 freq_select;
         int ret;
 
-        WARN_ON((cdclk == 24000) != (vco == 0));
-
         mutex_lock(&dev_priv->pcu_lock);
         ret = skl_pcode_request(dev_priv, SKL_PCODE_CDCLK_CONTROL,
                                 SKL_CDCLK_PREPARE_FOR_CHANGE,
@@ -942,25 +1011,24 @@ static void skl_set_cdclk(struct drm_i915_private *dev_priv,
 
         /* set CDCLK_CTL */
         switch (cdclk) {
+        default:
+                WARN_ON(cdclk != dev_priv->cdclk.hw.ref);
+                WARN_ON(vco != 0);
+                /* fall through */
+        case 308571:
+        case 337500:
+                freq_select = CDCLK_FREQ_337_308;
+                break;
         case 450000:
         case 432000:
                 freq_select = CDCLK_FREQ_450_432;
-                pcu_ack = 1;
                 break;
         case 540000:
                 freq_select = CDCLK_FREQ_540;
-                pcu_ack = 2;
-                break;
-        case 308571:
-        case 337500:
-        default:
-                freq_select = CDCLK_FREQ_337_308;
-                pcu_ack = 0;
                 break;
         case 617143:
         case 675000:
                 freq_select = CDCLK_FREQ_675_617;
-                pcu_ack = 3;
                 break;
         }
 
@@ -976,7 +1044,8 @@ static void skl_set_cdclk(struct drm_i915_private *dev_priv,
 
         /* inform PCU of the change */
         mutex_lock(&dev_priv->pcu_lock);
-        sandybridge_pcode_write(dev_priv, SKL_PCODE_CDCLK_CONTROL, pcu_ack);
+        sandybridge_pcode_write(dev_priv, SKL_PCODE_CDCLK_CONTROL,
+                                cdclk_state->voltage_level);
         mutex_unlock(&dev_priv->pcu_lock);
 
         intel_update_cdclk(dev_priv);
@@ -995,6 +1064,8 @@ static void skl_sanitize_cdclk(struct drm_i915_private *dev_priv)
                 goto sanitize;
 
         intel_update_cdclk(dev_priv);
+        intel_dump_cdclk_state(&dev_priv->cdclk.hw, "Current CDCLK");
+
         /* Is PLL enabled and locked ? */
         if (dev_priv->cdclk.hw.vco == 0 ||
             dev_priv->cdclk.hw.cdclk == dev_priv->cdclk.hw.ref)
@@ -1055,6 +1126,7 @@ void skl_init_cdclk(struct drm_i915_private *dev_priv)
         if (cdclk_state.vco == 0)
                 cdclk_state.vco = 8100000;
         cdclk_state.cdclk = skl_calc_cdclk(0, cdclk_state.vco);
+        cdclk_state.voltage_level = skl_calc_voltage_level(cdclk_state.cdclk);
 
         skl_set_cdclk(dev_priv, &cdclk_state);
 }
@@ -1072,6 +1144,7 @@ void skl_uninit_cdclk(struct drm_i915_private *dev_priv)
 
         cdclk_state.cdclk = cdclk_state.ref;
         cdclk_state.vco = 0;
+        cdclk_state.voltage_level = skl_calc_voltage_level(cdclk_state.cdclk);
 
         skl_set_cdclk(dev_priv, &cdclk_state);
 }
@@ -1100,6 +1173,11 @@ static int glk_calc_cdclk(int min_cdclk)
                 return 79200;
 }
 
+static u8 bxt_calc_voltage_level(int cdclk)
+{
+        return DIV_ROUND_UP(cdclk, 25000);
+}
+
 static int bxt_de_pll_vco(struct drm_i915_private *dev_priv, int cdclk)
 {
         int ratio;
@@ -1110,6 +1188,7 @@ static int bxt_de_pll_vco(struct drm_i915_private *dev_priv, int cdclk)
         switch (cdclk) {
         default:
                 MISSING_CASE(cdclk);
+                /* fall through */
         case 144000:
         case 288000:
         case 384000:
@@ -1134,6 +1213,7 @@ static int glk_de_pll_vco(struct drm_i915_private *dev_priv, int cdclk)
         switch (cdclk) {
         default:
                 MISSING_CASE(cdclk);
+                /* fall through */
         case  79200:
         case 158400:
         case 316800:
@@ -1174,7 +1254,7 @@ static void bxt_get_cdclk(struct drm_i915_private *dev_priv,
         cdclk_state->cdclk = cdclk_state->ref;
 
         if (cdclk_state->vco == 0)
-                return;
+                goto out;
 
         divider = I915_READ(CDCLK_CTL) & BXT_CDCLK_CD2X_DIV_SEL_MASK;
 
@@ -1198,6 +1278,14 @@ static void bxt_get_cdclk(struct drm_i915_private *dev_priv,
         }
 
         cdclk_state->cdclk = DIV_ROUND_CLOSEST(cdclk_state->vco, div);
+
+ out:
+        /*
+         * Can't read this out :( Let's assume it's
+         * at least what the CDCLK frequency requires.
+         */
+        cdclk_state->voltage_level =
+                bxt_calc_voltage_level(cdclk_state->cdclk);
 }
 
 static void bxt_de_pll_disable(struct drm_i915_private *dev_priv)
@@ -1246,24 +1334,22 @@ static void bxt_set_cdclk(struct drm_i915_private *dev_priv,
 
         /* cdclk = vco / 2 / div{1,1.5,2,4} */
         switch (DIV_ROUND_CLOSEST(vco, cdclk)) {
-        case 8:
-                divider = BXT_CDCLK_CD2X_DIV_SEL_4;
-                break;
-        case 4:
-                divider = BXT_CDCLK_CD2X_DIV_SEL_2;
+        default:
+                WARN_ON(cdclk != dev_priv->cdclk.hw.ref);
+                WARN_ON(vco != 0);
+                /* fall through */
+        case 2:
+                divider = BXT_CDCLK_CD2X_DIV_SEL_1;
                 break;
         case 3:
                 WARN(IS_GEMINILAKE(dev_priv), "Unsupported divider\n");
                 divider = BXT_CDCLK_CD2X_DIV_SEL_1_5;
                 break;
-        case 2:
-                divider = BXT_CDCLK_CD2X_DIV_SEL_1;
+        case 4:
+                divider = BXT_CDCLK_CD2X_DIV_SEL_2;
                 break;
-        default:
-                WARN_ON(cdclk != dev_priv->cdclk.hw.ref);
-                WARN_ON(vco != 0);
-
-                divider = BXT_CDCLK_CD2X_DIV_SEL_1;
+        case 8:
+                divider = BXT_CDCLK_CD2X_DIV_SEL_4;
                 break;
         }
 
@@ -1302,7 +1388,7 @@ static void bxt_set_cdclk(struct drm_i915_private *dev_priv,
 
         mutex_lock(&dev_priv->pcu_lock);
         ret = sandybridge_pcode_write(dev_priv, HSW_PCODE_DE_WRITE_FREQ_REQ,
-                                      DIV_ROUND_UP(cdclk, 25000));
+                                      cdclk_state->voltage_level);
         mutex_unlock(&dev_priv->pcu_lock);
 
         if (ret) {
@@ -1319,6 +1405,7 @@ static void bxt_sanitize_cdclk(struct drm_i915_private *dev_priv)
         u32 cdctl, expected;
 
         intel_update_cdclk(dev_priv);
+        intel_dump_cdclk_state(&dev_priv->cdclk.hw, "Current CDCLK");
 
         if (dev_priv->cdclk.hw.vco == 0 ||
             dev_priv->cdclk.hw.cdclk == dev_priv->cdclk.hw.ref)
@@ -1394,6 +1481,7 @@ void bxt_init_cdclk(struct drm_i915_private *dev_priv)
                 cdclk_state.cdclk = bxt_calc_cdclk(0);
                 cdclk_state.vco = bxt_de_pll_vco(dev_priv, cdclk_state.cdclk);
         }
+        cdclk_state.voltage_level = bxt_calc_voltage_level(cdclk_state.cdclk);
 
         bxt_set_cdclk(dev_priv, &cdclk_state);
 }
@@ -1411,6 +1499,7 @@ void bxt_uninit_cdclk(struct drm_i915_private *dev_priv)
 
         cdclk_state.cdclk = cdclk_state.ref;
         cdclk_state.vco = 0;
+        cdclk_state.voltage_level = bxt_calc_voltage_level(cdclk_state.cdclk);
 
         bxt_set_cdclk(dev_priv, &cdclk_state);
 }
@@ -1425,6 +1514,19 @@ static int cnl_calc_cdclk(int min_cdclk)
                 return 168000;
 }
 
+static u8 cnl_calc_voltage_level(int cdclk)
+{
+        switch (cdclk) {
+        default:
+        case 168000:
+                return 0;
+        case 336000:
+                return 1;
+        case 528000:
+                return 2;
+        }
+}
+
 static void cnl_cdclk_pll_update(struct drm_i915_private *dev_priv,
                                  struct intel_cdclk_state *cdclk_state)
 {
@@ -1458,7 +1560,7 @@ static void cnl_get_cdclk(struct drm_i915_private *dev_priv,
         cdclk_state->cdclk = cdclk_state->ref;
 
         if (cdclk_state->vco == 0)
-                return;
+                goto out;
 
         divider = I915_READ(CDCLK_CTL) & BXT_CDCLK_CD2X_DIV_SEL_MASK;
 
@@ -1475,6 +1577,14 @@ static void cnl_get_cdclk(struct drm_i915_private *dev_priv,
         }
 
         cdclk_state->cdclk = DIV_ROUND_CLOSEST(cdclk_state->vco, div);
+
+ out:
+        /*
+         * Can't read this out :( Let's assume it's
+         * at least what the CDCLK frequency requires.
+         */
+        cdclk_state->voltage_level =
+                cnl_calc_voltage_level(cdclk_state->cdclk);
 }
 
 static void cnl_cdclk_pll_disable(struct drm_i915_private *dev_priv)
@@ -1515,7 +1625,7 @@ static void cnl_set_cdclk(struct drm_i915_private *dev_priv,
 {
         int cdclk = cdclk_state->cdclk;
         int vco = cdclk_state->vco;
-        u32 val, divider, pcu_ack;
+        u32 val, divider;
         int ret;
 
         mutex_lock(&dev_priv->pcu_lock);
@@ -1532,30 +1642,15 @@ static void cnl_set_cdclk(struct drm_i915_private *dev_priv,
 
         /* cdclk = vco / 2 / div{1,2} */
         switch (DIV_ROUND_CLOSEST(vco, cdclk)) {
-        case 4:
-                divider = BXT_CDCLK_CD2X_DIV_SEL_2;
-                break;
-        case 2:
-                divider = BXT_CDCLK_CD2X_DIV_SEL_1;
-                break;
         default:
                 WARN_ON(cdclk != dev_priv->cdclk.hw.ref);
                 WARN_ON(vco != 0);
-
+                /* fall through */
+        case 2:
                 divider = BXT_CDCLK_CD2X_DIV_SEL_1;
                 break;
-        }
-
-        switch (cdclk) {
-        case 528000:
-                pcu_ack = 2;
-                break;
-        case 336000:
-                pcu_ack = 1;
-                break;
-        case 168000:
-        default:
-                pcu_ack = 0;
+        case 4:
+                divider = BXT_CDCLK_CD2X_DIV_SEL_2;
                 break;
         }
 
@@ -1576,10 +1671,17 @@ static void cnl_set_cdclk(struct drm_i915_private *dev_priv,
 
         /* inform PCU of the change */
         mutex_lock(&dev_priv->pcu_lock);
-        sandybridge_pcode_write(dev_priv, SKL_PCODE_CDCLK_CONTROL, pcu_ack);
+        sandybridge_pcode_write(dev_priv, SKL_PCODE_CDCLK_CONTROL,
+                                cdclk_state->voltage_level);
         mutex_unlock(&dev_priv->pcu_lock);
 
         intel_update_cdclk(dev_priv);
+
+        /*
+         * Can't read out the voltage level :(
+         * Let's just assume everything is as expected.
+         */
+        dev_priv->cdclk.hw.voltage_level = cdclk_state->voltage_level;
 }
 
 static int cnl_cdclk_pll_vco(struct drm_i915_private *dev_priv, int cdclk)
@@ -1592,6 +1694,7 @@ static int cnl_cdclk_pll_vco(struct drm_i915_private *dev_priv, int cdclk)
         switch (cdclk) {
         default:
                 MISSING_CASE(cdclk);
+                /* fall through */
         case 168000:
         case 336000:
                 ratio = dev_priv->cdclk.hw.ref == 19200 ? 35 : 28;
@@ -1609,6 +1712,7 @@ static void cnl_sanitize_cdclk(struct drm_i915_private *dev_priv)
         u32 cdctl, expected;
 
         intel_update_cdclk(dev_priv);
+        intel_dump_cdclk_state(&dev_priv->cdclk.hw, "Current CDCLK");
 
         if (dev_priv->cdclk.hw.vco == 0 ||
             dev_priv->cdclk.hw.cdclk == dev_priv->cdclk.hw.ref)
@@ -1668,6 +1772,7 @@ void cnl_init_cdclk(struct drm_i915_private *dev_priv)
 
         cdclk_state.cdclk = cnl_calc_cdclk(0);
         cdclk_state.vco = cnl_cdclk_pll_vco(dev_priv, cdclk_state.cdclk);
+        cdclk_state.voltage_level = cnl_calc_voltage_level(cdclk_state.cdclk);
 
         cnl_set_cdclk(dev_priv, &cdclk_state);
 }
@@ -1685,22 +1790,48 @@ void cnl_uninit_cdclk(struct drm_i915_private *dev_priv)
 
         cdclk_state.cdclk = cdclk_state.ref;
         cdclk_state.vco = 0;
+        cdclk_state.voltage_level = cnl_calc_voltage_level(cdclk_state.cdclk);
 
         cnl_set_cdclk(dev_priv, &cdclk_state);
 }
 
 /**
- * intel_cdclk_state_compare - Determine if two CDCLK states differ
+ * intel_cdclk_needs_modeset - Determine if two CDCLK states require a modeset on all pipes
  * @a: first CDCLK state
  * @b: second CDCLK state
  *
  * Returns:
- * True if the CDCLK states are identical, false if they differ.
+ * True if the CDCLK states require pipes to be off during reprogramming, false if not.
  */
-bool intel_cdclk_state_compare(const struct intel_cdclk_state *a,
+bool intel_cdclk_needs_modeset(const struct intel_cdclk_state *a,
                                const struct intel_cdclk_state *b)
 {
-        return memcmp(a, b, sizeof(*a)) == 0;
+        return a->cdclk != b->cdclk ||
+                a->vco != b->vco ||
+                a->ref != b->ref;
+}
+
+/**
+ * intel_cdclk_changed - Determine if two CDCLK states are different
+ * @a: first CDCLK state
+ * @b: second CDCLK state
+ *
+ * Returns:
+ * True if the CDCLK states don't match, false if they do.
+ */
+bool intel_cdclk_changed(const struct intel_cdclk_state *a,
+                         const struct intel_cdclk_state *b)
+{
+        return intel_cdclk_needs_modeset(a, b) ||
+                a->voltage_level != b->voltage_level;
+}
+
+void intel_dump_cdclk_state(const struct intel_cdclk_state *cdclk_state,
+                            const char *context)
+{
+        DRM_DEBUG_DRIVER("%s %d kHz, VCO %d kHz, ref %d kHz, voltage level %d\n",
+                         context, cdclk_state->cdclk, cdclk_state->vco,
+                         cdclk_state->ref, cdclk_state->voltage_level);
 }
 
 /**
@@ -1714,29 +1845,28 @@ bool intel_cdclk_state_compare(const struct intel_cdclk_state *a,
 void intel_set_cdclk(struct drm_i915_private *dev_priv,
                      const struct intel_cdclk_state *cdclk_state)
 {
-        if (intel_cdclk_state_compare(&dev_priv->cdclk.hw, cdclk_state))
+        if (!intel_cdclk_changed(&dev_priv->cdclk.hw, cdclk_state))
                 return;
 
         if (WARN_ON_ONCE(!dev_priv->display.set_cdclk))
                 return;
 
-        DRM_DEBUG_DRIVER("Changing CDCLK to %d kHz, VCO %d kHz, ref %d kHz\n",
-                         cdclk_state->cdclk, cdclk_state->vco,
-                         cdclk_state->ref);
+        intel_dump_cdclk_state(cdclk_state, "Changing CDCLK to");
 
         dev_priv->display.set_cdclk(dev_priv, cdclk_state);
+
+        if (WARN(intel_cdclk_changed(&dev_priv->cdclk.hw, cdclk_state),
+                 "cdclk state doesn't match!\n")) {
+                intel_dump_cdclk_state(&dev_priv->cdclk.hw, "[hw state]");
+                intel_dump_cdclk_state(cdclk_state, "[sw state]");
+        }
 }
 
 static int intel_pixel_rate_to_cdclk(struct drm_i915_private *dev_priv,
                                      int pixel_rate)
 {
         if (INTEL_GEN(dev_priv) >= 10)
-                /*
-                 * FIXME: Switch to DIV_ROUND_UP(pixel_rate, 2)
-                 * once DDI clock voltage requirements are
-                 * handled correctly.
-                 */
-                return pixel_rate;
+                return DIV_ROUND_UP(pixel_rate, 2);
         else if (IS_GEMINILAKE(dev_priv))
                 /*
                  * FIXME: Avoid using a pixel clock that is more than 99% of the cdclk
@@ -1829,6 +1959,43 @@ static int intel_compute_min_cdclk(struct drm_atomic_state *state)
         return min_cdclk;
 }
 
+/*
+ * Note that this functions assumes that 0 is
+ * the lowest voltage value, and higher values
+ * correspond to increasingly higher voltages.
+ *
+ * Should that relationship no longer hold on
+ * future platforms this code will need to be
+ * adjusted.
+ */
+static u8 cnl_compute_min_voltage_level(struct intel_atomic_state *state)
+{
+        struct drm_i915_private *dev_priv = to_i915(state->base.dev);
+        struct intel_crtc *crtc;
+        struct intel_crtc_state *crtc_state;
+        u8 min_voltage_level;
+        int i;
+        enum pipe pipe;
+
+        memcpy(state->min_voltage_level, dev_priv->min_voltage_level,
+               sizeof(state->min_voltage_level));
+
+        for_each_new_intel_crtc_in_state(state, crtc, crtc_state, i) {
+                if (crtc_state->base.enable)
+                        state->min_voltage_level[i] =
+                                crtc_state->min_voltage_level;
+                else
+                        state->min_voltage_level[i] = 0;
+        }
+
+        min_voltage_level = 0;
+        for_each_pipe(dev_priv, pipe)
+                min_voltage_level = max(state->min_voltage_level[pipe],
+                                        min_voltage_level);
+
+        return min_voltage_level;
+}
+
 static int vlv_modeset_calc_cdclk(struct drm_atomic_state *state)
 {
         struct drm_i915_private *dev_priv = to_i915(state->dev);
@@ -1842,11 +2009,15 @@ static int vlv_modeset_calc_cdclk(struct drm_atomic_state *state)
         cdclk = vlv_calc_cdclk(dev_priv, min_cdclk);
 
         intel_state->cdclk.logical.cdclk = cdclk;
+        intel_state->cdclk.logical.voltage_level =
+                vlv_calc_voltage_level(dev_priv, cdclk);
 
         if (!intel_state->active_crtcs) {
                 cdclk = vlv_calc_cdclk(dev_priv, 0);
 
                 intel_state->cdclk.actual.cdclk = cdclk;
+                intel_state->cdclk.actual.voltage_level =
+                        vlv_calc_voltage_level(dev_priv, cdclk);
         } else {
                 intel_state->cdclk.actual =
                         intel_state->cdclk.logical;
@@ -1871,11 +2042,15 @@ static int bdw_modeset_calc_cdclk(struct drm_atomic_state *state)
         cdclk = bdw_calc_cdclk(min_cdclk);
 
         intel_state->cdclk.logical.cdclk = cdclk;
+        intel_state->cdclk.logical.voltage_level =
+                bdw_calc_voltage_level(cdclk);
 
         if (!intel_state->active_crtcs) {
                 cdclk = bdw_calc_cdclk(0);
 
                 intel_state->cdclk.actual.cdclk = cdclk;
+                intel_state->cdclk.actual.voltage_level =
+                        bdw_calc_voltage_level(cdclk);
         } else {
                 intel_state->cdclk.actual =
                         intel_state->cdclk.logical;
@@ -1906,12 +2081,16 @@ static int skl_modeset_calc_cdclk(struct drm_atomic_state *state)
 
         intel_state->cdclk.logical.vco = vco;
         intel_state->cdclk.logical.cdclk = cdclk;
+        intel_state->cdclk.logical.voltage_level =
+                skl_calc_voltage_level(cdclk);
 
         if (!intel_state->active_crtcs) {
                 cdclk = skl_calc_cdclk(0, vco);
 
                 intel_state->cdclk.actual.vco = vco;
                 intel_state->cdclk.actual.cdclk = cdclk;
+                intel_state->cdclk.actual.voltage_level =
+                        skl_calc_voltage_level(cdclk);
         } else {
                 intel_state->cdclk.actual =
                         intel_state->cdclk.logical;
@@ -1940,6 +2119,8 @@ static int bxt_modeset_calc_cdclk(struct drm_atomic_state *state)
 
         intel_state->cdclk.logical.vco = vco;
         intel_state->cdclk.logical.cdclk = cdclk;
+        intel_state->cdclk.logical.voltage_level =
+                bxt_calc_voltage_level(cdclk);
 
         if (!intel_state->active_crtcs) {
                 if (IS_GEMINILAKE(dev_priv)) {
@@ -1952,6 +2133,8 @@ static int bxt_modeset_calc_cdclk(struct drm_atomic_state *state)
 
                 intel_state->cdclk.actual.vco = vco;
                 intel_state->cdclk.actual.cdclk = cdclk;
+                intel_state->cdclk.actual.voltage_level =
+                        bxt_calc_voltage_level(cdclk);
         } else {
                 intel_state->cdclk.actual =
                         intel_state->cdclk.logical;
@@ -1975,6 +2158,9 @@ static int cnl_modeset_calc_cdclk(struct drm_atomic_state *state)
 
         intel_state->cdclk.logical.vco = vco;
         intel_state->cdclk.logical.cdclk = cdclk;
+        intel_state->cdclk.logical.voltage_level =
+                max(cnl_calc_voltage_level(cdclk),
+                    cnl_compute_min_voltage_level(intel_state));
 
         if (!intel_state->active_crtcs) {
                 cdclk = cnl_calc_cdclk(0);
@@ -1982,6 +2168,8 @@ static int cnl_modeset_calc_cdclk(struct drm_atomic_state *state)
 
                 intel_state->cdclk.actual.vco = vco;
                 intel_state->cdclk.actual.cdclk = cdclk;
+                intel_state->cdclk.actual.voltage_level =
+                        cnl_calc_voltage_level(cdclk);
         } else {
                 intel_state->cdclk.actual =
                         intel_state->cdclk.logical;
@@ -1995,12 +2183,7 @@ static int intel_compute_max_dotclk(struct drm_i915_private *dev_priv)
         int max_cdclk_freq = dev_priv->max_cdclk_freq;
 
         if (INTEL_GEN(dev_priv) >= 10)
-                /*
-                 * FIXME: Allow '2 * max_cdclk_freq'
-                 * once DDI clock voltage requirements are
-                 * handled correctly.
-                 */
-                return max_cdclk_freq;
+                return 2 * max_cdclk_freq;
         else if (IS_GEMINILAKE(dev_priv))
                 /*
                  * FIXME: Limiting to 99% as a temporary workaround. See
@@ -2099,10 +2282,6 @@ void intel_update_cdclk(struct drm_i915_private *dev_priv)
 {
         dev_priv->display.get_cdclk(dev_priv, &dev_priv->cdclk.hw);
 
-        DRM_DEBUG_DRIVER("Current CD clock rate: %d kHz, VCO: %d kHz, ref: %d kHz\n",
-                         dev_priv->cdclk.hw.cdclk, dev_priv->cdclk.hw.vco,
-                         dev_priv->cdclk.hw.ref);
-
         /*
          * 9:0 CMBUS [sic] CDCLK frequency (cdfreq):
          * Programmng [sic] note: bit[9:2] should be programmed to the number
diff --git a/drivers/gpu/drm/i915/intel_color.c b/drivers/gpu/drm/i915/intel_color.c
index b8315bca852b..aa66e952a95d 100644
--- a/drivers/gpu/drm/i915/intel_color.c
+++ b/drivers/gpu/drm/i915/intel_color.c
@@ -370,7 +370,7 @@ static void haswell_load_luts(struct drm_crtc_state *crtc_state)
          */
         if (IS_HASWELL(dev_priv) && intel_crtc_state->ips_enabled &&
             (intel_crtc_state->gamma_mode == GAMMA_MODE_MODE_SPLIT)) {
-                hsw_disable_ips(intel_crtc);
+                hsw_disable_ips(intel_crtc_state);
                 reenable_ips = true;
         }
 
@@ -380,7 +380,7 @@ static void haswell_load_luts(struct drm_crtc_state *crtc_state)
         i9xx_load_luts(crtc_state);
 
         if (reenable_ips)
-                hsw_enable_ips(intel_crtc);
+                hsw_enable_ips(intel_crtc_state);
 }
 
 static void bdw_load_degamma_lut(struct drm_crtc_state *state)
diff --git a/drivers/gpu/drm/i915/intel_crt.c b/drivers/gpu/drm/i915/intel_crt.c
index 437339f5d098..9f31aea51dff 100644
--- a/drivers/gpu/drm/i915/intel_crt.c
+++ b/drivers/gpu/drm/i915/intel_crt.c
@@ -119,6 +119,8 @@ static unsigned int intel_crt_get_flags(struct intel_encoder *encoder)
 static void intel_crt_get_config(struct intel_encoder *encoder,
                                  struct intel_crtc_state *pipe_config)
 {
+        pipe_config->output_types |= BIT(INTEL_OUTPUT_ANALOG);
+
         pipe_config->base.adjusted_mode.flags |= intel_crt_get_flags(encoder);
 
         pipe_config->base.adjusted_mode.crtc_clock = pipe_config->port_clock;
@@ -217,11 +219,9 @@ static void hsw_disable_crt(struct intel_encoder *encoder,
                             const struct intel_crtc_state *old_crtc_state,
                             const struct drm_connector_state *old_conn_state)
 {
-        struct drm_crtc *crtc = old_crtc_state->base.crtc;
-        struct drm_i915_private *dev_priv = to_i915(crtc->dev);
-        struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
+        struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
 
-        WARN_ON(!intel_crtc->config->has_pch_encoder);
+        WARN_ON(!old_crtc_state->has_pch_encoder);
 
         intel_set_pch_fifo_underrun_reporting(dev_priv, PIPE_A, false);
 }
@@ -245,46 +245,42 @@ static void hsw_post_disable_crt(struct intel_encoder *encoder,
 }
 
 static void hsw_pre_pll_enable_crt(struct intel_encoder *encoder,
-                                   const struct intel_crtc_state *pipe_config,
+                                   const struct intel_crtc_state *crtc_state,
                                    const struct drm_connector_state *conn_state)
 {
-        struct drm_crtc *crtc = pipe_config->base.crtc;
-        struct drm_i915_private *dev_priv = to_i915(crtc->dev);
-        struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
+        struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
 
-        WARN_ON(!intel_crtc->config->has_pch_encoder);
+        WARN_ON(!crtc_state->has_pch_encoder);
 
         intel_set_pch_fifo_underrun_reporting(dev_priv, PIPE_A, false);
 }
 
 static void hsw_pre_enable_crt(struct intel_encoder *encoder,
-                               const struct intel_crtc_state *pipe_config,
+                               const struct intel_crtc_state *crtc_state,
                                const struct drm_connector_state *conn_state)
 {
-        struct drm_crtc *crtc = pipe_config->base.crtc;
-        struct drm_i915_private *dev_priv = to_i915(crtc->dev);
-        struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
-        int pipe = intel_crtc->pipe;
+        struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
+        struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
+        enum pipe pipe = crtc->pipe;
 
-        WARN_ON(!intel_crtc->config->has_pch_encoder);
+        WARN_ON(!crtc_state->has_pch_encoder);
 
         intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, false);
 
-        dev_priv->display.fdi_link_train(intel_crtc, pipe_config);
+        dev_priv->display.fdi_link_train(crtc, crtc_state);
 }
 
 static void hsw_enable_crt(struct intel_encoder *encoder,
-                           const struct intel_crtc_state *pipe_config,
+                           const struct intel_crtc_state *crtc_state,
                            const struct drm_connector_state *conn_state)
 {
-        struct drm_crtc *crtc = pipe_config->base.crtc;
-        struct drm_i915_private *dev_priv = to_i915(crtc->dev);
-        struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
-        int pipe = intel_crtc->pipe;
+        struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
+        struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
+        enum pipe pipe = crtc->pipe;
 
-        WARN_ON(!intel_crtc->config->has_pch_encoder);
+        WARN_ON(!crtc_state->has_pch_encoder);
 
-        intel_crt_set_dpms(encoder, pipe_config, DRM_MODE_DPMS_ON);
+        intel_crt_set_dpms(encoder, crtc_state, DRM_MODE_DPMS_ON);
 
         intel_wait_for_vblank(dev_priv, pipe);
         intel_wait_for_vblank(dev_priv, pipe);
@@ -293,10 +289,10 @@ static void hsw_enable_crt(struct intel_encoder *encoder,
 }
 
 static void intel_enable_crt(struct intel_encoder *encoder,
-                             const struct intel_crtc_state *pipe_config,
+                             const struct intel_crtc_state *crtc_state,
                              const struct drm_connector_state *conn_state)
 {
-        intel_crt_set_dpms(encoder, pipe_config, DRM_MODE_DPMS_ON);
+        intel_crt_set_dpms(encoder, crtc_state, DRM_MODE_DPMS_ON);
 }
 
 static enum drm_mode_status
diff --git a/drivers/gpu/drm/i915/intel_csr.c b/drivers/gpu/drm/i915/intel_csr.c
index da9de47562b8..77d8b3d483ca 100644
--- a/drivers/gpu/drm/i915/intel_csr.c
+++ b/drivers/gpu/drm/i915/intel_csr.c
@@ -37,8 +37,8 @@
 #define I915_CSR_GLK "i915/glk_dmc_ver1_04.bin"
 #define GLK_CSR_VERSION_REQUIRED        CSR_VERSION(1, 4)
 
-#define I915_CSR_CNL "i915/cnl_dmc_ver1_04.bin"
-#define CNL_CSR_VERSION_REQUIRED        CSR_VERSION(1, 4)
+#define I915_CSR_CNL "i915/cnl_dmc_ver1_06.bin"
+#define CNL_CSR_VERSION_REQUIRED        CSR_VERSION(1, 6)
 
 #define I915_CSR_KBL "i915/kbl_dmc_ver1_01.bin"
 MODULE_FIRMWARE(I915_CSR_KBL);
@@ -198,6 +198,7 @@ intel_get_stepping_info(struct drm_i915_private *dev_priv)
                 si = bxt_stepping_info;
         } else {
                 size = 0;
+                si = NULL;
         }
 
         if (INTEL_REVID(dev_priv) < size)
diff --git a/drivers/gpu/drm/i915/intel_ddi.c b/drivers/gpu/drm/i915/intel_ddi.c
index 933c18fd4258..eff3b51872eb 100644
--- a/drivers/gpu/drm/i915/intel_ddi.c
+++ b/drivers/gpu/drm/i915/intel_ddi.c
@@ -492,24 +492,6 @@ static const struct cnl_ddi_buf_trans cnl_ddi_translations_edp_1_05V[] = {
         { 0x2, 0x7F, 0x3F, 0x00, 0x00 },        /* 400   400      0.0   */
 };
 
-enum port intel_ddi_get_encoder_port(struct intel_encoder *encoder)
-{
-        switch (encoder->type) {
-        case INTEL_OUTPUT_DP_MST:
-                return enc_to_mst(&encoder->base)->primary->port;
-        case INTEL_OUTPUT_DP:
-        case INTEL_OUTPUT_EDP:
-        case INTEL_OUTPUT_HDMI:
-        case INTEL_OUTPUT_UNKNOWN:
-                return enc_to_dig_port(&encoder->base)->port;
-        case INTEL_OUTPUT_ANALOG:
-                return PORT_E;
-        default:
-                MISSING_CASE(encoder->type);
-                return PORT_A;
-        }
-}
-
 static const struct ddi_buf_trans *
 bdw_get_buf_trans_edp(struct drm_i915_private *dev_priv, int *n_entries)
 {
@@ -811,31 +793,24 @@ static int intel_ddi_hdmi_level(struct drm_i915_private *dev_priv, enum port por
  * values in advance. This function programs the correct values for
  * DP/eDP/FDI use cases.
  */
-static void intel_prepare_dp_ddi_buffers(struct intel_encoder *encoder)
+static void intel_prepare_dp_ddi_buffers(struct intel_encoder *encoder,
+                                         const struct intel_crtc_state *crtc_state)
 {
         struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
         u32 iboost_bit = 0;
         int i, n_entries;
-        enum port port = intel_ddi_get_encoder_port(encoder);
+        enum port port = encoder->port;
         const struct ddi_buf_trans *ddi_translations;
 
-        switch (encoder->type) {
-        case INTEL_OUTPUT_EDP:
+        if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_ANALOG))
+                ddi_translations = intel_ddi_get_buf_trans_fdi(dev_priv,
+                                                               &n_entries);
+        else if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_EDP))
                 ddi_translations = intel_ddi_get_buf_trans_edp(dev_priv, port,
                                                                &n_entries);
-                break;
-        case INTEL_OUTPUT_DP:
+        else
                 ddi_translations = intel_ddi_get_buf_trans_dp(dev_priv, port,
                                                               &n_entries);
-                break;
-        case INTEL_OUTPUT_ANALOG:
-                ddi_translations = intel_ddi_get_buf_trans_fdi(dev_priv,
-                                                               &n_entries);
-                break;
-        default:
-                MISSING_CASE(encoder->type);
-                return;
-        }
 
         /* If we're boosting the current, set bit 31 of trans1 */
         if (IS_GEN9_BC(dev_priv) &&
@@ -861,7 +836,7 @@ static void intel_prepare_hdmi_ddi_buffers(struct intel_encoder *encoder,
         struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
         u32 iboost_bit = 0;
         int n_entries;
-        enum port port = intel_ddi_get_encoder_port(encoder);
+        enum port port = encoder->port;
         const struct ddi_buf_trans *ddi_translations;
 
         ddi_translations = intel_ddi_get_buf_trans_hdmi(dev_priv, &n_entries);
@@ -937,7 +912,7 @@ void hsw_fdi_link_train(struct intel_crtc *crtc,
 
         for_each_encoder_on_crtc(dev, &crtc->base, encoder) {
                 WARN_ON(encoder->type != INTEL_OUTPUT_ANALOG);
-                intel_prepare_dp_ddi_buffers(encoder);
+                intel_prepare_dp_ddi_buffers(encoder, crtc_state);
         }
 
         /* Set the FDI_RX_MISC pwrdn lanes and the 2 workarounds listed at the
@@ -1448,19 +1423,16 @@ static void hsw_ddi_clock_get(struct intel_encoder *encoder,
         ddi_dotclock_get(pipe_config);
 }
 
-static int bxt_calc_pll_link(struct drm_i915_private *dev_priv,
-                             enum intel_dpll_id pll_id)
+static int bxt_calc_pll_link(struct intel_crtc_state *crtc_state)
 {
-        struct intel_shared_dpll *pll;
         struct intel_dpll_hw_state *state;
         struct dpll clock;
 
         /* For DDI ports we always use a shared PLL. */
-        if (WARN_ON(pll_id == DPLL_ID_PRIVATE))
+        if (WARN_ON(!crtc_state->shared_dpll))
                 return 0;
 
-        pll = &dev_priv->shared_dplls[pll_id];
-        state = &pll->state.hw_state;
+        state = &crtc_state->dpll_hw_state;
 
         clock.m1 = 2;
         clock.m2 = (state->pll0 & PORT_PLL_M2_MASK) << 22;
@@ -1474,19 +1446,15 @@ static int bxt_calc_pll_link(struct drm_i915_private *dev_priv,
 }
 
 static void bxt_ddi_clock_get(struct intel_encoder *encoder,
-                                struct intel_crtc_state *pipe_config)
+                              struct intel_crtc_state *pipe_config)
 {
-        struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
-        enum port port = intel_ddi_get_encoder_port(encoder);
-        enum intel_dpll_id pll_id = port;
-
-        pipe_config->port_clock = bxt_calc_pll_link(dev_priv, pll_id);
+        pipe_config->port_clock = bxt_calc_pll_link(pipe_config);
 
         ddi_dotclock_get(pipe_config);
 }
 
-void intel_ddi_clock_get(struct intel_encoder *encoder,
-                         struct intel_crtc_state *pipe_config)
+static void intel_ddi_clock_get(struct intel_encoder *encoder,
+                                struct intel_crtc_state *pipe_config)
 {
         struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
 
@@ -1504,33 +1472,34 @@ void intel_ddi_set_pipe_settings(const struct intel_crtc_state *crtc_state)
 {
         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
-        struct intel_encoder *encoder = intel_ddi_get_crtc_encoder(crtc);
         enum transcoder cpu_transcoder = crtc_state->cpu_transcoder;
-        int type = encoder->type;
-        uint32_t temp;
+        u32 temp;
 
-        if (type == INTEL_OUTPUT_DP || type == INTEL_OUTPUT_EDP || type == INTEL_OUTPUT_DP_MST) {
-                WARN_ON(transcoder_is_dsi(cpu_transcoder));
+        if (!intel_crtc_has_dp_encoder(crtc_state))
+                return;
 
-                temp = TRANS_MSA_SYNC_CLK;
-                switch (crtc_state->pipe_bpp) {
-                case 18:
-                        temp |= TRANS_MSA_6_BPC;
-                        break;
-                case 24:
-                        temp |= TRANS_MSA_8_BPC;
-                        break;
-                case 30:
-                        temp |= TRANS_MSA_10_BPC;
-                        break;
-                case 36:
-                        temp |= TRANS_MSA_12_BPC;
-                        break;
-                default:
-                        BUG();
-                }
-                I915_WRITE(TRANS_MSA_MISC(cpu_transcoder), temp);
+        WARN_ON(transcoder_is_dsi(cpu_transcoder));
+
+        temp = TRANS_MSA_SYNC_CLK;
+        switch (crtc_state->pipe_bpp) {
+        case 18:
+                temp |= TRANS_MSA_6_BPC;
+                break;
+        case 24:
+                temp |= TRANS_MSA_8_BPC;
+                break;
+        case 30:
+                temp |= TRANS_MSA_10_BPC;
+                break;
+        case 36:
+                temp |= TRANS_MSA_12_BPC;
+                break;
+        default:
+                MISSING_CASE(crtc_state->pipe_bpp);
+                break;
         }
+
+        I915_WRITE(TRANS_MSA_MISC(cpu_transcoder), temp);
 }
 
 void intel_ddi_set_vc_payload_alloc(const struct intel_crtc_state *crtc_state,
@@ -1540,6 +1509,7 @@ void intel_ddi_set_vc_payload_alloc(const struct intel_crtc_state *crtc_state,
         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
         enum transcoder cpu_transcoder = crtc_state->cpu_transcoder;
         uint32_t temp;
+
         temp = I915_READ(TRANS_DDI_FUNC_CTL(cpu_transcoder));
         if (state == true)
                 temp |= TRANS_DDI_DP_VC_PAYLOAD_ALLOC;
@@ -1555,8 +1525,7 @@ void intel_ddi_enable_transcoder_func(const struct intel_crtc_state *crtc_state)
         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
         enum pipe pipe = crtc->pipe;
         enum transcoder cpu_transcoder = crtc_state->cpu_transcoder;
-        enum port port = intel_ddi_get_encoder_port(encoder);
-        int type = encoder->type;
+        enum port port = encoder->port;
         uint32_t temp;
 
         /* Enable TRANS_DDI_FUNC_CTL for the pipe to work in HDMI mode */
@@ -1611,7 +1580,7 @@ void intel_ddi_enable_transcoder_func(const struct intel_crtc_state *crtc_state)
                 }
         }
 
-        if (type == INTEL_OUTPUT_HDMI) {
+        if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_HDMI)) {
                 if (crtc_state->has_hdmi_sink)
                         temp |= TRANS_DDI_MODE_SELECT_HDMI;
                 else
@@ -1621,19 +1590,15 @@ void intel_ddi_enable_transcoder_func(const struct intel_crtc_state *crtc_state)
                         temp |= TRANS_DDI_HDMI_SCRAMBLING_MASK;
                 if (crtc_state->hdmi_high_tmds_clock_ratio)
                         temp |= TRANS_DDI_HIGH_TMDS_CHAR_RATE;
-        } else if (type == INTEL_OUTPUT_ANALOG) {
+        } else if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_ANALOG)) {
                 temp |= TRANS_DDI_MODE_SELECT_FDI;
                 temp |= (crtc_state->fdi_lanes - 1) << 1;
-        } else if (type == INTEL_OUTPUT_DP ||
-                   type == INTEL_OUTPUT_EDP) {
-                temp |= TRANS_DDI_MODE_SELECT_DP_SST;
-                temp |= DDI_PORT_WIDTH(crtc_state->lane_count);
-        } else if (type == INTEL_OUTPUT_DP_MST) {
+        } else if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_DP_MST)) {
                 temp |= TRANS_DDI_MODE_SELECT_DP_MST;
                 temp |= DDI_PORT_WIDTH(crtc_state->lane_count);
         } else {
-                WARN(1, "Invalid encoder type %d for pipe %c\n",
-                     encoder->type, pipe_name(pipe));
+                temp |= TRANS_DDI_MODE_SELECT_DP_SST;
+                temp |= DDI_PORT_WIDTH(crtc_state->lane_count);
         }
 
         I915_WRITE(TRANS_DDI_FUNC_CTL(cpu_transcoder), temp);
@@ -1656,7 +1621,7 @@ bool intel_ddi_connector_get_hw_state(struct intel_connector *intel_connector)
         struct drm_i915_private *dev_priv = to_i915(dev);
         struct intel_encoder *encoder = intel_connector->encoder;
         int type = intel_connector->base.connector_type;
-        enum port port = intel_ddi_get_encoder_port(encoder);
+        enum port port = encoder->port;
         enum pipe pipe = 0;
         enum transcoder cpu_transcoder;
         uint32_t tmp;
@@ -1715,9 +1680,9 @@ bool intel_ddi_get_hw_state(struct intel_encoder *encoder,
 {
         struct drm_device *dev = encoder->base.dev;
         struct drm_i915_private *dev_priv = to_i915(dev);
-        enum port port = intel_ddi_get_encoder_port(encoder);
+        enum port port = encoder->port;
+        enum pipe p;
         u32 tmp;
-        int i;
         bool ret;
 
         if (!intel_display_power_get_if_enabled(dev_priv,
@@ -1752,15 +1717,17 @@ bool intel_ddi_get_hw_state(struct intel_encoder *encoder,
                 goto out;
         }
 
-        for (i = TRANSCODER_A; i <= TRANSCODER_C; i++) {
-                tmp = I915_READ(TRANS_DDI_FUNC_CTL(i));
+        for_each_pipe(dev_priv, p) {
+                enum transcoder cpu_transcoder = (enum transcoder) p;
+
+                tmp = I915_READ(TRANS_DDI_FUNC_CTL(cpu_transcoder));
 
                 if ((tmp & TRANS_DDI_PORT_MASK) == TRANS_DDI_SELECT_PORT(port)) {
                         if ((tmp & TRANS_DDI_MODE_SELECT_MASK) ==
                             TRANS_DDI_MODE_SELECT_DP_MST)
                                 goto out;
 
-                        *pipe = i;
+                        *pipe = p;
                         ret = true;
 
                         goto out;
@@ -1800,7 +1767,7 @@ void intel_ddi_enable_pipe_clock(const struct intel_crtc_state *crtc_state)
         struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
         struct intel_encoder *encoder = intel_ddi_get_crtc_encoder(crtc);
-        enum port port = intel_ddi_get_encoder_port(encoder);
+        enum port port = encoder->port;
         enum transcoder cpu_transcoder = crtc_state->cpu_transcoder;
 
         if (cpu_transcoder != TRANSCODER_EDP)
@@ -1836,8 +1803,8 @@ static void skl_ddi_set_iboost(struct intel_encoder *encoder,
                                int level, enum intel_output_type type)
 {
         struct intel_digital_port *intel_dig_port = enc_to_dig_port(&encoder->base);
-        struct drm_i915_private *dev_priv = to_i915(intel_dig_port->base.base.dev);
-        enum port port = intel_dig_port->port;
+        struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
+        enum port port = encoder->port;
         uint8_t iboost;
 
         if (type == INTEL_OUTPUT_HDMI)
@@ -1939,8 +1906,8 @@ static void cnl_ddi_vswing_program(struct intel_encoder *encoder,
                                    int level, enum intel_output_type type)
 {
         struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
-        enum port port = intel_ddi_get_encoder_port(encoder);
         const struct cnl_ddi_buf_trans *ddi_translations;
+        enum port port = encoder->port;
         int n_entries, ln;
         u32 val;
 
@@ -2003,7 +1970,7 @@ static void cnl_ddi_vswing_sequence(struct intel_encoder *encoder,
                                     int level, enum intel_output_type type)
 {
         struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
-        enum port port = intel_ddi_get_encoder_port(encoder);
+        enum port port = encoder->port;
         int width, rate, ln;
         u32 val;
 
@@ -2122,7 +2089,7 @@ static void intel_ddi_clk_select(struct intel_encoder *encoder,
                                  const struct intel_shared_dpll *pll)
 {
         struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
-        enum port port = intel_ddi_get_encoder_port(encoder);
+        enum port port = encoder->port;
         uint32_t val;
 
         if (WARN_ON(!pll))
@@ -2161,7 +2128,7 @@ static void intel_ddi_clk_select(struct intel_encoder *encoder,
 static void intel_ddi_clk_disable(struct intel_encoder *encoder)
 {
         struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
-        enum port port = intel_ddi_get_encoder_port(encoder);
+        enum port port = encoder->port;
 
         if (IS_CANNONLAKE(dev_priv))
                 I915_WRITE(DPCLKA_CFGCR0, I915_READ(DPCLKA_CFGCR0) |
@@ -2179,7 +2146,7 @@ static void intel_ddi_pre_enable_dp(struct intel_encoder *encoder,
 {
         struct intel_dp *intel_dp = enc_to_intel_dp(&encoder->base);
         struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
-        enum port port = intel_ddi_get_encoder_port(encoder);
+        enum port port = encoder->port;
         struct intel_digital_port *dig_port = enc_to_dig_port(&encoder->base);
         bool is_mst = intel_crtc_has_type(crtc_state, INTEL_OUTPUT_DP_MST);
         int level = intel_ddi_dp_level(intel_dp);
@@ -2200,7 +2167,7 @@ static void intel_ddi_pre_enable_dp(struct intel_encoder *encoder,
         else if (IS_GEN9_LP(dev_priv))
                 bxt_ddi_vswing_sequence(encoder, level, encoder->type);
         else
-                intel_prepare_dp_ddi_buffers(encoder);
+                intel_prepare_dp_ddi_buffers(encoder, crtc_state);
 
         intel_ddi_init_dp_buf_reg(encoder);
         if (!is_mst)
@@ -2217,7 +2184,7 @@ static void intel_ddi_pre_enable_hdmi(struct intel_encoder *encoder,
         struct intel_digital_port *intel_dig_port = enc_to_dig_port(&encoder->base);
         struct intel_hdmi *intel_hdmi = &intel_dig_port->hdmi;
         struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
-        enum port port = intel_ddi_get_encoder_port(encoder);
+        enum port port = encoder->port;
         int level = intel_ddi_hdmi_level(dev_priv, port);
         struct intel_digital_port *dig_port = enc_to_dig_port(&encoder->base);
 
@@ -2249,6 +2216,19 @@ static void intel_ddi_pre_enable(struct intel_encoder *encoder,
         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
         enum pipe pipe = crtc->pipe;
 
+        /*
+         * When called from DP MST code:
+         * - conn_state will be NULL
+         * - encoder will be the main encoder (ie. mst->primary)
+         * - the main connector associated with this port
+         *   won't be active or linked to a crtc
+         * - crtc_state will be the state of the first stream to
+         *   be activated on this port, and it may not be the same
+         *   stream that will be deactivated last, but each stream
+         *   should have a state that is identical when it comes to
+         *   the DP link parameteres
+         */
+
         WARN_ON(crtc_state->has_pch_encoder);
 
         intel_set_cpu_fifo_underrun_reporting(dev_priv, pipe, true);
@@ -2262,7 +2242,7 @@ static void intel_ddi_pre_enable(struct intel_encoder *encoder,
 static void intel_disable_ddi_buf(struct intel_encoder *encoder)
 {
         struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
-        enum port port = intel_ddi_get_encoder_port(encoder);
+        enum port port = encoder->port;
         bool wait = false;
         u32 val;
 
@@ -2289,12 +2269,7 @@ static void intel_ddi_post_disable_dp(struct intel_encoder *encoder,
         struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
         struct intel_digital_port *dig_port = enc_to_dig_port(&encoder->base);
         struct intel_dp *intel_dp = &dig_port->dp;
-        /*
-         * old_crtc_state and old_conn_state are NULL when called from
-         * DP_MST. The main connector associated with this port is never
-         * bound to a crtc for MST.
-         */
-        bool is_mst = !old_crtc_state;
+        bool is_mst = intel_crtc_has_type(old_crtc_state, INTEL_OUTPUT_DP_MST);
 
         /*
          * Power down sink before disabling the port, otherwise we end
@@ -2338,12 +2313,19 @@ static void intel_ddi_post_disable(struct intel_encoder *encoder,
                                    const struct drm_connector_state *old_conn_state)
 {
         /*
-         * old_crtc_state and old_conn_state are NULL when called from
-         * DP_MST. The main connector associated with this port is never
-         * bound to a crtc for MST.
+         * When called from DP MST code:
+         * - old_conn_state will be NULL
+         * - encoder will be the main encoder (ie. mst->primary)
+         * - the main connector associated with this port
+         *   won't be active or linked to a crtc
+         * - old_crtc_state will be the state of the last stream to
+         *   be deactivated on this port, and it may not be the same
+         *   stream that was activated last, but each stream
+         *   should have a state that is identical when it comes to
+         *   the DP link parameteres
          */
-        if (old_crtc_state &&
-            intel_crtc_has_type(old_crtc_state, INTEL_OUTPUT_HDMI))
+
+        if (intel_crtc_has_type(old_crtc_state, INTEL_OUTPUT_HDMI))
                 intel_ddi_post_disable_hdmi(encoder,
                                             old_crtc_state, old_conn_state);
         else
@@ -2391,7 +2373,7 @@ static void intel_enable_ddi_dp(struct intel_encoder *encoder,
 {
         struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
         struct intel_dp *intel_dp = enc_to_intel_dp(&encoder->base);
-        enum port port = intel_ddi_get_encoder_port(encoder);
+        enum port port = encoder->port;
 
         if (port == PORT_A && INTEL_GEN(dev_priv) < 9)
                 intel_dp_stop_link_train(intel_dp);
@@ -2410,7 +2392,7 @@ static void intel_enable_ddi_hdmi(struct intel_encoder *encoder,
 {
         struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
         struct intel_digital_port *dig_port = enc_to_dig_port(&encoder->base);
-        enum port port = intel_ddi_get_encoder_port(encoder);
+        enum port port = encoder->port;
 
         intel_hdmi_handle_sink_scrambling(encoder,
                                           conn_state->connector,
@@ -2445,7 +2427,8 @@ static void intel_disable_ddi_dp(struct intel_encoder *encoder,
         struct intel_dp *intel_dp = enc_to_intel_dp(&encoder->base);
 
         if (old_crtc_state->has_audio)
-                intel_audio_codec_disable(encoder);
+                intel_audio_codec_disable(encoder,
+                                          old_crtc_state, old_conn_state);
 
         intel_edp_drrs_disable(intel_dp, old_crtc_state);
         intel_psr_disable(intel_dp, old_crtc_state);
@@ -2457,7 +2440,8 @@ static void intel_disable_ddi_hdmi(struct intel_encoder *encoder,
                                    const struct drm_connector_state *old_conn_state)
 {
         if (old_crtc_state->has_audio)
-                intel_audio_codec_disable(encoder);
+                intel_audio_codec_disable(encoder,
+                                          old_crtc_state, old_conn_state);
 
         intel_hdmi_handle_sink_scrambling(encoder,
                                           old_conn_state->connector,
@@ -2488,7 +2472,7 @@ void intel_ddi_prepare_link_retrain(struct intel_dp *intel_dp)
         struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp);
         struct drm_i915_private *dev_priv =
                 to_i915(intel_dig_port->base.base.dev);
-        enum port port = intel_dig_port->port;
+        enum port port = intel_dig_port->base.port;
         uint32_t val;
         bool wait = false;
 
@@ -2542,11 +2526,18 @@ bool intel_ddi_is_audio_enabled(struct drm_i915_private *dev_priv,
         return false;
 }
 
+void intel_ddi_compute_min_voltage_level(struct drm_i915_private *dev_priv,
+                                         struct intel_crtc_state *crtc_state)
+{
+        if (IS_CANNONLAKE(dev_priv) && crtc_state->port_clock > 594000)
+                crtc_state->min_voltage_level = 2;
+}
+
 void intel_ddi_get_config(struct intel_encoder *encoder,
                           struct intel_crtc_state *pipe_config)
 {
         struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
-        struct intel_crtc *intel_crtc = to_intel_crtc(encoder->base.crtc);
+        struct intel_crtc *intel_crtc = to_intel_crtc(pipe_config->base.crtc);
         enum transcoder cpu_transcoder = pipe_config->cpu_transcoder;
         struct intel_digital_port *intel_dig_port;
         u32 temp, flags = 0;
@@ -2599,12 +2590,23 @@ void intel_ddi_get_config(struct intel_encoder *encoder,
                         pipe_config->hdmi_high_tmds_clock_ratio = true;
                 /* fall through */
         case TRANS_DDI_MODE_SELECT_DVI:
+                pipe_config->output_types |= BIT(INTEL_OUTPUT_HDMI);
                 pipe_config->lane_count = 4;
                 break;
         case TRANS_DDI_MODE_SELECT_FDI:
+                pipe_config->output_types |= BIT(INTEL_OUTPUT_ANALOG);
                 break;
         case TRANS_DDI_MODE_SELECT_DP_SST:
+                if (encoder->type == INTEL_OUTPUT_EDP)
+                        pipe_config->output_types |= BIT(INTEL_OUTPUT_EDP);
+                else
+                        pipe_config->output_types |= BIT(INTEL_OUTPUT_DP);
+                pipe_config->lane_count =
+                        ((temp & DDI_PORT_WIDTH_MASK) >> DDI_PORT_WIDTH_SHIFT) + 1;
+                intel_dp_get_m_n(intel_crtc, pipe_config);
+                break;
         case TRANS_DDI_MODE_SELECT_DP_MST:
+                pipe_config->output_types |= BIT(INTEL_OUTPUT_DP_MST);
                 pipe_config->lane_count =
                         ((temp & DDI_PORT_WIDTH_MASK) >> DDI_PORT_WIDTH_SHIFT) + 1;
                 intel_dp_get_m_n(intel_crtc, pipe_config);
@@ -2641,6 +2643,26 @@ void intel_ddi_get_config(struct intel_encoder *encoder,
         if (IS_GEN9_LP(dev_priv))
                 pipe_config->lane_lat_optim_mask =
                         bxt_ddi_phy_get_lane_lat_optim_mask(encoder);
+
+        intel_ddi_compute_min_voltage_level(dev_priv, pipe_config);
+}
+
+static enum intel_output_type
+intel_ddi_compute_output_type(struct intel_encoder *encoder,
+                              struct intel_crtc_state *crtc_state,
+                              struct drm_connector_state *conn_state)
+{
+        switch (conn_state->connector->connector_type) {
+        case DRM_MODE_CONNECTOR_HDMIA:
+                return INTEL_OUTPUT_HDMI;
+        case DRM_MODE_CONNECTOR_eDP:
+                return INTEL_OUTPUT_EDP;
+        case DRM_MODE_CONNECTOR_DisplayPort:
+                return INTEL_OUTPUT_DP;
+        default:
+                MISSING_CASE(conn_state->connector->connector_type);
+                return INTEL_OUTPUT_UNUSED;
+        }
 }
 
 static bool intel_ddi_compute_config(struct intel_encoder *encoder,
@@ -2648,24 +2670,22 @@ static bool intel_ddi_compute_config(struct intel_encoder *encoder,
                                      struct drm_connector_state *conn_state)
 {
         struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
-        int type = encoder->type;
-        int port = intel_ddi_get_encoder_port(encoder);
+        enum port port = encoder->port;
         int ret;
 
-        WARN(type == INTEL_OUTPUT_UNKNOWN, "compute_config() on unknown output!\n");
-
         if (port == PORT_A)
                 pipe_config->cpu_transcoder = TRANSCODER_EDP;
 
-        if (type == INTEL_OUTPUT_HDMI)
+        if (intel_crtc_has_type(pipe_config, INTEL_OUTPUT_HDMI))
                 ret = intel_hdmi_compute_config(encoder, pipe_config, conn_state);
         else
                 ret = intel_dp_compute_config(encoder, pipe_config, conn_state);
 
         if (IS_GEN9_LP(dev_priv) && ret)
                 pipe_config->lane_lat_optim_mask =
-                        bxt_ddi_phy_calc_lane_lat_optim_mask(encoder,
-                                                             pipe_config->lane_count);
+                        bxt_ddi_phy_calc_lane_lat_optim_mask(pipe_config->lane_count);
+
+        intel_ddi_compute_min_voltage_level(dev_priv, pipe_config);
 
         return ret;
 
@@ -2680,7 +2700,7 @@ static struct intel_connector *
 intel_ddi_init_dp_connector(struct intel_digital_port *intel_dig_port)
 {
         struct intel_connector *connector;
-        enum port port = intel_dig_port->port;
+        enum port port = intel_dig_port->base.port;
 
         connector = intel_connector_alloc();
         if (!connector)
@@ -2699,7 +2719,7 @@ static struct intel_connector *
 intel_ddi_init_hdmi_connector(struct intel_digital_port *intel_dig_port)
 {
         struct intel_connector *connector;
-        enum port port = intel_dig_port->port;
+        enum port port = intel_dig_port->base.port;
 
         connector = intel_connector_alloc();
         if (!connector)
@@ -2711,6 +2731,34 @@ intel_ddi_init_hdmi_connector(struct intel_digital_port *intel_dig_port)
         return connector;
 }
 
+static bool intel_ddi_a_force_4_lanes(struct intel_digital_port *dport)
+{
+        struct drm_i915_private *dev_priv = to_i915(dport->base.base.dev);
+
+        if (dport->base.port != PORT_A)
+                return false;
+
+        if (dport->saved_port_bits & DDI_A_4_LANES)
+                return false;
+
+        /* Broxton/Geminilake: Bspec says that DDI_A_4_LANES is the only
+         *                     supported configuration
+         */
+        if (IS_GEN9_LP(dev_priv))
+                return true;
+
+        /* Cannonlake: Most of SKUs don't support DDI_E, and the only
+         *             one who does also have a full A/E split called
+         *             DDI_F what makes DDI_E useless. However for this
+         *             case let's trust VBT info.
+         */
+        if (IS_CANNONLAKE(dev_priv) &&
+            !intel_bios_is_port_present(dev_priv, PORT_E))
+                return true;
+
+        return false;
+}
+
 void intel_ddi_init(struct drm_i915_private *dev_priv, enum port port)
 {
         struct intel_digital_port *intel_dig_port;
@@ -2777,6 +2825,7 @@ void intel_ddi_init(struct drm_i915_private *dev_priv, enum port port)
         drm_encoder_init(&dev_priv->drm, encoder, &intel_ddi_funcs,
                          DRM_MODE_ENCODER_TMDS, "DDI %c", port_name(port));
 
+        intel_encoder->compute_output_type = intel_ddi_compute_output_type;
         intel_encoder->compute_config = intel_ddi_compute_config;
         intel_encoder->enable = intel_enable_ddi;
         if (IS_GEN9_LP(dev_priv))
@@ -2789,7 +2838,6 @@ void intel_ddi_init(struct drm_i915_private *dev_priv, enum port port)
         intel_encoder->suspend = intel_dp_encoder_suspend;
         intel_encoder->get_power_domains = intel_ddi_get_power_domains;
 
-        intel_dig_port->port = port;
         intel_dig_port->saved_port_bits = I915_READ(DDI_BUF_CTL(port)) &
                                           (DDI_BUF_PORT_REVERSAL |
                                            DDI_A_4_LANES);
@@ -2820,23 +2868,20 @@ void intel_ddi_init(struct drm_i915_private *dev_priv, enum port port)
         }
 
         /*
-         * Bspec says that DDI_A_4_LANES is the only supported configuration
-         * for Broxton.  Yet some BIOS fail to set this bit on port A if eDP
-         * wasn't lit up at boot.  Force this bit on in our internal
-         * configuration so that we use the proper lane count for our
-         * calculations.
+         * Some BIOS might fail to set this bit on port A if eDP
+         * wasn't lit up at boot.  Force this bit set when needed
+         * so we use the proper lane count for our calculations.
          */
-        if (IS_GEN9_LP(dev_priv) && port == PORT_A) {
-                if (!(intel_dig_port->saved_port_bits & DDI_A_4_LANES)) {
-                        DRM_DEBUG_KMS("BXT BIOS forgot to set DDI_A_4_LANES for port A; fixing\n");
-                        intel_dig_port->saved_port_bits |= DDI_A_4_LANES;
-                        max_lanes = 4;
-                }
+        if (intel_ddi_a_force_4_lanes(intel_dig_port)) {
+                DRM_DEBUG_KMS("Forcing DDI_A_4_LANES for port A\n");
+                intel_dig_port->saved_port_bits |= DDI_A_4_LANES;
+                max_lanes = 4;
         }
 
+        intel_dig_port->dp.output_reg = INVALID_MMIO_REG;
         intel_dig_port->max_lanes = max_lanes;
 
-        intel_encoder->type = INTEL_OUTPUT_UNKNOWN;
+        intel_encoder->type = INTEL_OUTPUT_DDI;
         intel_encoder->power_domain = intel_port_to_power_domain(port);
         intel_encoder->port = port;
         intel_encoder->crtc_mask = (1 << 0) | (1 << 1) | (1 << 2);
diff --git a/drivers/gpu/drm/i915/intel_device_info.c b/drivers/gpu/drm/i915/intel_device_info.c
index 875d428ea75f..02f8bf101ccd 100644
--- a/drivers/gpu/drm/i915/intel_device_info.c
+++ b/drivers/gpu/drm/i915/intel_device_info.c
@@ -235,16 +235,6 @@ static void gen9_sseu_info_init(struct drm_i915_private *dev_priv)
 #define IS_SS_DISABLED(ss)      (!(sseu->subslice_mask & BIT(ss)))
                 info->has_pooled_eu = hweight8(sseu->subslice_mask) == 3;
 
-                /*
-                 * There is a HW issue in 2x6 fused down parts that requires
-                 * Pooled EU to be enabled as a WA. The pool configuration
-                 * changes depending upon which subslice is fused down. This
-                 * doesn't affect if the device has all 3 subslices enabled.
-                 */
-                /* WaEnablePooledEuFor2x6:bxt */
-                info->has_pooled_eu |= (hweight8(sseu->subslice_mask) == 2 &&
-                                        IS_BXT_REVID(dev_priv, 0, BXT_REVID_B_LAST));
-
                 sseu->min_eu_in_pool = 0;
                 if (info->has_pooled_eu) {
                         if (IS_SS_DISABLED(2) || IS_SS_DISABLED(0))
@@ -329,6 +319,107 @@ static void broadwell_sseu_info_init(struct drm_i915_private *dev_priv)
         sseu->has_eu_pg = 0;
 }
 
+static u32 read_reference_ts_freq(struct drm_i915_private *dev_priv)
+{
+        u32 ts_override = I915_READ(GEN9_TIMESTAMP_OVERRIDE);
+        u32 base_freq, frac_freq;
+
+        base_freq = ((ts_override & GEN9_TIMESTAMP_OVERRIDE_US_COUNTER_DIVIDER_MASK) >>
+                     GEN9_TIMESTAMP_OVERRIDE_US_COUNTER_DIVIDER_SHIFT) + 1;
+        base_freq *= 1000;
+
+        frac_freq = ((ts_override &
+                      GEN9_TIMESTAMP_OVERRIDE_US_COUNTER_DENOMINATOR_MASK) >>
+                     GEN9_TIMESTAMP_OVERRIDE_US_COUNTER_DENOMINATOR_SHIFT);
+        frac_freq = 1000 / (frac_freq + 1);
+
+        return base_freq + frac_freq;
+}
+
+static u32 read_timestamp_frequency(struct drm_i915_private *dev_priv)
+{
+        u32 f12_5_mhz = 12500;
+        u32 f19_2_mhz = 19200;
+        u32 f24_mhz = 24000;
+
+        if (INTEL_GEN(dev_priv) <= 4) {
+                /* PRMs say:
+                 *
+                 *     "The value in this register increments once every 16
+                 *      hclks." (through the “Clocking Configuration”
+                 *      (“CLKCFG”) MCHBAR register)
+                 */
+                return dev_priv->rawclk_freq / 16;
+        } else if (INTEL_GEN(dev_priv) <= 8) {
+                /* PRMs say:
+                 *
+                 *     "The PCU TSC counts 10ns increments; this timestamp
+                 *      reflects bits 38:3 of the TSC (i.e. 80ns granularity,
+                 *      rolling over every 1.5 hours).
+                 */
+                return f12_5_mhz;
+        } else if (INTEL_GEN(dev_priv) <= 9) {
+                u32 ctc_reg = I915_READ(CTC_MODE);
+                u32 freq = 0;
+
+                if ((ctc_reg & CTC_SOURCE_PARAMETER_MASK) == CTC_SOURCE_DIVIDE_LOGIC) {
+                        freq = read_reference_ts_freq(dev_priv);
+                } else {
+                        freq = IS_GEN9_LP(dev_priv) ? f19_2_mhz : f24_mhz;
+
+                        /* Now figure out how the command stream's timestamp
+                         * register increments from this frequency (it might
+                         * increment only every few clock cycle).
+                         */
+                        freq >>= 3 - ((ctc_reg & CTC_SHIFT_PARAMETER_MASK) >>
+                                      CTC_SHIFT_PARAMETER_SHIFT);
+                }
+
+                return freq;
+        } else if (INTEL_GEN(dev_priv) <= 10) {
+                u32 ctc_reg = I915_READ(CTC_MODE);
+                u32 freq = 0;
+                u32 rpm_config_reg = 0;
+
+                /* First figure out the reference frequency. There are 2 ways
+                 * we can compute the frequency, either through the
+                 * TIMESTAMP_OVERRIDE register or through RPM_CONFIG. CTC_MODE
+                 * tells us which one we should use.
+                 */
+                if ((ctc_reg & CTC_SOURCE_PARAMETER_MASK) == CTC_SOURCE_DIVIDE_LOGIC) {
+                        freq = read_reference_ts_freq(dev_priv);
+                } else {
+                        u32 crystal_clock;
+
+                        rpm_config_reg = I915_READ(RPM_CONFIG0);
+                        crystal_clock = (rpm_config_reg &
+                                         GEN9_RPM_CONFIG0_CRYSTAL_CLOCK_FREQ_MASK) >>
+                                GEN9_RPM_CONFIG0_CRYSTAL_CLOCK_FREQ_SHIFT;
+                        switch (crystal_clock) {
+                        case GEN9_RPM_CONFIG0_CRYSTAL_CLOCK_FREQ_19_2_MHZ:
+                                freq = f19_2_mhz;
+                                break;
+                        case GEN9_RPM_CONFIG0_CRYSTAL_CLOCK_FREQ_24_MHZ:
+                                freq = f24_mhz;
+                                break;
+                        }
+                }
+
+                /* Now figure out how the command stream's timestamp register
+                 * increments from this frequency (it might increment only
+                 * every few clock cycle).
+                 */
+                freq >>= 3 - ((rpm_config_reg &
+                               GEN10_RPM_CONFIG0_CTC_SHIFT_PARAMETER_MASK) >>
+                              GEN10_RPM_CONFIG0_CTC_SHIFT_PARAMETER_SHIFT);
+
+                return freq;
+        }
+
+        DRM_ERROR("Unknown gen, unable to compute command stream timestamp frequency\n");
+        return 0;
+}
+
 /*
  * Determine various intel_device_info fields at runtime.
  *
@@ -347,7 +438,10 @@ void intel_device_info_runtime_init(struct drm_i915_private *dev_priv)
         struct intel_device_info *info = mkwrite_device_info(dev_priv);
         enum pipe pipe;
 
-        if (INTEL_GEN(dev_priv) >= 9) {
+        if (INTEL_GEN(dev_priv) >= 10) {
+                for_each_pipe(dev_priv, pipe)
+                        info->num_scalers[pipe] = 2;
+        } else if (INTEL_GEN(dev_priv) == 9) {
                 info->num_scalers[PIPE_A] = 2;
                 info->num_scalers[PIPE_B] = 2;
                 info->num_scalers[PIPE_C] = 1;
@@ -447,6 +541,9 @@ void intel_device_info_runtime_init(struct drm_i915_private *dev_priv)
         else if (INTEL_GEN(dev_priv) >= 10)
                 gen10_sseu_info_init(dev_priv);
 
+        /* Initialize command stream timestamp frequency */
+        info->cs_timestamp_frequency_khz = read_timestamp_frequency(dev_priv);
+
         DRM_DEBUG_DRIVER("slice mask: %04x\n", info->sseu.slice_mask);
         DRM_DEBUG_DRIVER("slice total: %u\n", hweight8(info->sseu.slice_mask));
         DRM_DEBUG_DRIVER("subslice total: %u\n",
@@ -462,4 +559,6 @@ void intel_device_info_runtime_init(struct drm_i915_private *dev_priv)
                          info->sseu.has_subslice_pg ? "y" : "n");
         DRM_DEBUG_DRIVER("has EU power gating: %s\n",
                          info->sseu.has_eu_pg ? "y" : "n");
+        DRM_DEBUG_DRIVER("CS timestamp frequency: %u kHz\n",
+                         info->cs_timestamp_frequency_khz);
 }
diff --git a/drivers/gpu/drm/i915/intel_display.c b/drivers/gpu/drm/i915/intel_display.c
index e0fffd883b54..2007c69468b9 100644
--- a/drivers/gpu/drm/i915/intel_display.c
+++ b/drivers/gpu/drm/i915/intel_display.c
@@ -219,10 +219,8 @@ intel_fdi_link_freq(struct drm_i915_private *dev_priv,
 {
         if (HAS_DDI(dev_priv))
                 return pipe_config->port_clock; /* SPLL */
-        else if (IS_GEN5(dev_priv))
-                return ((I915_READ(FDI_PLL_BIOS_0) & FDI_PLL_FB_CLOCK_MASK) + 2) * 10000;
         else
-                return 270000;
+                return dev_priv->fdi_pll_freq;
 }
 
 static const struct intel_limit intel_limits_i8xx_dac = {
@@ -1703,7 +1701,7 @@ void vlv_wait_port_ready(struct drm_i915_private *dev_priv,
         u32 port_mask;
         i915_reg_t dpll_reg;
 
-        switch (dport->port) {
+        switch (dport->base.port) {
         case PORT_B:
                 port_mask = DPLL_PORTB_READY_MASK;
                 dpll_reg = DPLL(0);
@@ -1725,7 +1723,8 @@ void vlv_wait_port_ready(struct drm_i915_private *dev_priv,
                                     dpll_reg, port_mask, expected_mask,
                                     1000))
                 WARN(1, "timed out waiting for port %c ready: got 0x%x, expected 0x%x\n",
-                     port_name(dport->port), I915_READ(dpll_reg) & port_mask, expected_mask);
+                     port_name(dport->base.port),
+                     I915_READ(dpll_reg) & port_mask, expected_mask);
 }
 
 static void ironlake_enable_pch_transcoder(struct drm_i915_private *dev_priv,
@@ -1873,8 +1872,6 @@ enum pipe intel_crtc_pch_transcoder(struct intel_crtc *crtc)
 {
         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
 
-        WARN_ON(!crtc->config->has_pch_encoder);
-
         if (HAS_PCH_LPT(dev_priv))
                 return PIPE_A;
         else
@@ -3433,20 +3430,11 @@ static u32 skl_plane_ctl_format(uint32_t pixel_format)
         case DRM_FORMAT_RGB565:
                 return PLANE_CTL_FORMAT_RGB_565;
         case DRM_FORMAT_XBGR8888:
+        case DRM_FORMAT_ABGR8888:
                 return PLANE_CTL_FORMAT_XRGB_8888 | PLANE_CTL_ORDER_RGBX;
         case DRM_FORMAT_XRGB8888:
-                return PLANE_CTL_FORMAT_XRGB_8888;
-        /*
-         * XXX: For ARBG/ABGR formats we default to expecting scanout buffers
-         * to be already pre-multiplied. We need to add a knob (or a different
-         * DRM_FORMAT) for user-space to configure that.
-         */
-        case DRM_FORMAT_ABGR8888:
-                return PLANE_CTL_FORMAT_XRGB_8888 | PLANE_CTL_ORDER_RGBX |
-                        PLANE_CTL_ALPHA_SW_PREMULTIPLY;
         case DRM_FORMAT_ARGB8888:
-                return PLANE_CTL_FORMAT_XRGB_8888 |
-                        PLANE_CTL_ALPHA_SW_PREMULTIPLY;
+                return PLANE_CTL_FORMAT_XRGB_8888;
         case DRM_FORMAT_XRGB2101010:
                 return PLANE_CTL_FORMAT_XRGB_2101010;
         case DRM_FORMAT_XBGR2101010:
@@ -3466,6 +3454,33 @@ static u32 skl_plane_ctl_format(uint32_t pixel_format)
         return 0;
 }
 
+/*
+ * XXX: For ARBG/ABGR formats we default to expecting scanout buffers
+ * to be already pre-multiplied. We need to add a knob (or a different
+ * DRM_FORMAT) for user-space to configure that.
+ */
+static u32 skl_plane_ctl_alpha(uint32_t pixel_format)
+{
+        switch (pixel_format) {
+        case DRM_FORMAT_ABGR8888:
+        case DRM_FORMAT_ARGB8888:
+                return PLANE_CTL_ALPHA_SW_PREMULTIPLY;
+        default:
+                return PLANE_CTL_ALPHA_DISABLE;
+        }
+}
+
+static u32 glk_plane_color_ctl_alpha(uint32_t pixel_format)
+{
+        switch (pixel_format) {
+        case DRM_FORMAT_ABGR8888:
+        case DRM_FORMAT_ARGB8888:
+                return PLANE_COLOR_ALPHA_SW_PREMULTIPLY;
+        default:
+                return PLANE_COLOR_ALPHA_DISABLE;
+        }
+}
+
 static u32 skl_plane_ctl_tiling(uint64_t fb_modifier)
 {
         switch (fb_modifier) {
@@ -3522,7 +3537,8 @@ u32 skl_plane_ctl(const struct intel_crtc_state *crtc_state,
 
         plane_ctl = PLANE_CTL_ENABLE;
 
-        if (!IS_GEMINILAKE(dev_priv) && !IS_CANNONLAKE(dev_priv)) {
+        if (INTEL_GEN(dev_priv) < 10 && !IS_GEMINILAKE(dev_priv)) {
+                plane_ctl |= skl_plane_ctl_alpha(fb->format->format);
                 plane_ctl |=
                         PLANE_CTL_PIPE_GAMMA_ENABLE |
                         PLANE_CTL_PIPE_CSC_ENABLE |
@@ -3541,6 +3557,20 @@ u32 skl_plane_ctl(const struct intel_crtc_state *crtc_state,
         return plane_ctl;
 }
 
+u32 glk_plane_color_ctl(const struct intel_crtc_state *crtc_state,
+                        const struct intel_plane_state *plane_state)
+{
+        const struct drm_framebuffer *fb = plane_state->base.fb;
+        u32 plane_color_ctl = 0;
+
+        plane_color_ctl |= PLANE_COLOR_PIPE_GAMMA_ENABLE;
+        plane_color_ctl |= PLANE_COLOR_PIPE_CSC_ENABLE;
+        plane_color_ctl |= PLANE_COLOR_PLANE_GAMMA_DISABLE;
+        plane_color_ctl |= glk_plane_color_ctl_alpha(fb->format->format);
+
+        return plane_color_ctl;
+}
+
 static int
 __intel_display_resume(struct drm_device *dev,
                        struct drm_atomic_state *state,
@@ -4483,7 +4513,7 @@ intel_trans_dp_port_sel(struct intel_crtc *crtc)
         for_each_encoder_on_crtc(dev, &crtc->base, encoder) {
                 if (encoder->type == INTEL_OUTPUT_DP ||
                     encoder->type == INTEL_OUTPUT_EDP)
-                        return enc_to_dig_port(&encoder->base)->port;
+                        return encoder->port;
         }
 
         return -1;
@@ -4834,8 +4864,9 @@ static void ironlake_pfit_enable(struct intel_crtc *crtc)
         }
 }
 
-void hsw_enable_ips(struct intel_crtc *crtc)
+void hsw_enable_ips(const struct intel_crtc_state *crtc_state)
 {
+        struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
         struct drm_device *dev = crtc->base.dev;
         struct drm_i915_private *dev_priv = to_i915(dev);
 
@@ -4873,12 +4904,13 @@ void hsw_enable_ips(struct intel_crtc *crtc)
         }
 }
 
-void hsw_disable_ips(struct intel_crtc *crtc)
+void hsw_disable_ips(const struct intel_crtc_state *crtc_state)
 {
+        struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
         struct drm_device *dev = crtc->base.dev;
         struct drm_i915_private *dev_priv = to_i915(dev);
 
-        if (!crtc->config->ips_enabled)
+        if (!crtc_state->ips_enabled)
                 return;
 
         assert_plane_enabled(dev_priv, crtc->plane);
@@ -4926,7 +4958,8 @@ static void intel_crtc_dpms_overlay_disable(struct intel_crtc *intel_crtc)
  * completely hide the primary plane.
  */
 static void
-intel_post_enable_primary(struct drm_crtc *crtc)
+intel_post_enable_primary(struct drm_crtc *crtc,
+                          const struct intel_crtc_state *new_crtc_state)
 {
         struct drm_device *dev = crtc->dev;
         struct drm_i915_private *dev_priv = to_i915(dev);
@@ -4939,7 +4972,7 @@ intel_post_enable_primary(struct drm_crtc *crtc)
          * when going from primary only to sprite only and vice
          * versa.
          */
-        hsw_enable_ips(intel_crtc);
+        hsw_enable_ips(new_crtc_state);
 
         /*
          * Gen2 reports pipe underruns whenever all planes are disabled.
@@ -4958,7 +4991,8 @@ intel_post_enable_primary(struct drm_crtc *crtc)
 
 /* FIXME move all this to pre_plane_update() with proper state tracking */
 static void
-intel_pre_disable_primary(struct drm_crtc *crtc)
+intel_pre_disable_primary(struct drm_crtc *crtc,
+                          const struct intel_crtc_state *old_crtc_state)
 {
         struct drm_device *dev = crtc->dev;
         struct drm_i915_private *dev_priv = to_i915(dev);
@@ -4980,7 +5014,7 @@ intel_pre_disable_primary(struct drm_crtc *crtc)
          * when going from primary only to sprite only and vice
          * versa.
          */
-        hsw_disable_ips(intel_crtc);
+        hsw_disable_ips(old_crtc_state);
 }
 
 /* FIXME get rid of this and use pre_plane_update */
@@ -4992,7 +5026,7 @@ intel_pre_disable_primary_noatomic(struct drm_crtc *crtc)
         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
         int pipe = intel_crtc->pipe;
 
-        intel_pre_disable_primary(crtc);
+        intel_pre_disable_primary(crtc, to_intel_crtc_state(crtc->state));
 
         /*
          * Vblank time updates from the shadow to live plane control register
@@ -5036,7 +5070,7 @@ static void intel_post_plane_update(struct intel_crtc_state *old_crtc_state)
                 if (primary_state->base.visible &&
                     (needs_modeset(&pipe_config->base) ||
                      !old_primary_state->base.visible))
-                        intel_post_enable_primary(&crtc->base);
+                        intel_post_enable_primary(&crtc->base, pipe_config);
         }
 }
 
@@ -5065,7 +5099,7 @@ static void intel_pre_plane_update(struct intel_crtc_state *old_crtc_state,
 
                 if (old_primary_state->base.visible &&
                     (modeset || !primary_state->base.visible))
-                        intel_pre_disable_primary(&crtc->base);
+                        intel_pre_disable_primary(&crtc->base, old_crtc_state);
         }
 
         /*
@@ -5939,6 +5973,7 @@ static void intel_crtc_disable_noatomic(struct drm_crtc *crtc,
 
         dev_priv->active_crtcs &= ~(1 << intel_crtc->pipe);
         dev_priv->min_cdclk[intel_crtc->pipe] = 0;
+        dev_priv->min_voltage_level[intel_crtc->pipe] = 0;
 }
 
 /*
@@ -7633,7 +7668,7 @@ static void ironlake_init_pch_refclk(struct drm_i915_private *dev_priv)
                         break;
                 case INTEL_OUTPUT_EDP:
                         has_panel = true;
-                        if (enc_to_dig_port(&encoder->base)->port == PORT_A)
+                        if (encoder->port == PORT_A)
                                 has_cpu_edp = true;
                         break;
                 default:
@@ -8426,7 +8461,7 @@ skylake_get_initial_plane_config(struct intel_crtc *crtc,
 {
         struct drm_device *dev = crtc->base.dev;
         struct drm_i915_private *dev_priv = to_i915(dev);
-        u32 val, base, offset, stride_mult, tiling;
+        u32 val, base, offset, stride_mult, tiling, alpha;
         int pipe = crtc->pipe;
         int fourcc, pixel_format;
         unsigned int aligned_height;
@@ -8448,9 +8483,16 @@ skylake_get_initial_plane_config(struct intel_crtc *crtc,
                 goto error;
 
         pixel_format = val & PLANE_CTL_FORMAT_MASK;
+
+        if (INTEL_GEN(dev_priv) >= 10 || IS_GEMINILAKE(dev_priv)) {
+                alpha = I915_READ(PLANE_COLOR_CTL(pipe, 0));
+                alpha &= PLANE_COLOR_ALPHA_MASK;
+        } else {
+                alpha = val & PLANE_CTL_ALPHA_MASK;
+        }
+
         fourcc = skl_format_to_fourcc(pixel_format,
-                                      val & PLANE_CTL_ORDER_RGBX,
-                                      val & PLANE_CTL_ALPHA_MASK);
+                                      val & PLANE_CTL_ORDER_RGBX, alpha);
         fb->format = drm_format_info(fourcc);
 
         tiling = val & PLANE_CTL_TILED_MASK;
@@ -8857,7 +8899,9 @@ static void hsw_restore_lcpll(struct drm_i915_private *dev_priv)
         }
 
         intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
+
         intel_update_cdclk(dev_priv);
+        intel_dump_cdclk_state(&dev_priv->cdclk.hw, "Current CDCLK");
 }
 
 /*
@@ -9231,10 +9275,6 @@ static bool haswell_get_pipe_config(struct intel_crtc *crtc,
                         ironlake_get_pfit_config(crtc, pipe_config);
         }
 
-        if (IS_HASWELL(dev_priv))
-                pipe_config->ips_enabled = hsw_crtc_supports_ips(crtc) &&
-                        (I915_READ(IPS_CTL) & IPS_ENABLE);
-
         if (pipe_config->cpu_transcoder != TRANSCODER_EDP &&
             !transcoder_is_dsi(pipe_config->cpu_transcoder)) {
                 pipe_config->pixel_multiplier =
@@ -10578,7 +10618,7 @@ static const char * const output_type_str[] = {
         OUTPUT_TYPE(DP),
         OUTPUT_TYPE(EDP),
         OUTPUT_TYPE(DSI),
-        OUTPUT_TYPE(UNKNOWN),
+        OUTPUT_TYPE(DDI),
         OUTPUT_TYPE(DP_MST),
 };
 
@@ -10749,13 +10789,13 @@ static bool check_digital_port_conflicts(struct drm_atomic_state *state)
 
                 switch (encoder->type) {
                         unsigned int port_mask;
-                case INTEL_OUTPUT_UNKNOWN:
+                case INTEL_OUTPUT_DDI:
                         if (WARN_ON(!HAS_DDI(to_i915(dev))))
                                 break;
                 case INTEL_OUTPUT_DP:
                 case INTEL_OUTPUT_HDMI:
                 case INTEL_OUTPUT_EDP:
-                        port_mask = 1 << enc_to_dig_port(&encoder->base)->port;
+                        port_mask = 1 << encoder->port;
 
                         /* the same port mustn't appear more than once */
                         if (used_ports & port_mask)
@@ -10765,7 +10805,7 @@ static bool check_digital_port_conflicts(struct drm_atomic_state *state)
                         break;
                 case INTEL_OUTPUT_DP_MST:
                         used_mst_ports |=
-                                1 << enc_to_mst(&encoder->base)->primary->port;
+                                1 << encoder->port;
                         break;
                 default:
                         break;
@@ -10882,7 +10922,12 @@ intel_modeset_pipe_config(struct drm_crtc *crtc,
                  * Determine output_types before calling the .compute_config()
                  * hooks so that the hooks can use this information safely.
                  */
-                pipe_config->output_types |= 1 << encoder->type;
+                if (encoder->compute_output_type)
+                        pipe_config->output_types |=
+                                BIT(encoder->compute_output_type(encoder, pipe_config,
+                                                                 connector_state));
+                else
+                        pipe_config->output_types |= BIT(encoder->type);
         }
 
 encoder_retry:
@@ -11071,6 +11116,9 @@ intel_pipe_config_compare(struct drm_i915_private *dev_priv,
                           bool adjust)
 {
         bool ret = true;
+        bool fixup_inherited = adjust &&
+                (current_config->base.mode.private_flags & I915_MODE_FLAG_INHERITED) &&
+                !(pipe_config->base.mode.private_flags & I915_MODE_FLAG_INHERITED);
 
 #define PIPE_CONF_CHECK_X(name) \
         if (current_config->name != pipe_config->name) { \
@@ -11090,6 +11138,31 @@ intel_pipe_config_compare(struct drm_i915_private *dev_priv,
                 ret = false; \
         }
 
+#define PIPE_CONF_CHECK_BOOL(name)      \
+        if (current_config->name != pipe_config->name) { \
+                pipe_config_err(adjust, __stringify(name), \
+                          "(expected %s, found %s)\n", \
+                          yesno(current_config->name), \
+                          yesno(pipe_config->name)); \
+                ret = false; \
+        }
+
+/*
+ * Checks state where we only read out the enabling, but not the entire
+ * state itself (like full infoframes or ELD for audio). These states
+ * require a full modeset on bootup to fix up.
+ */
+#define PIPE_CONF_CHECK_BOOL_INCOMPLETE(name) \
+        if (!fixup_inherited || (!current_config->name && !pipe_config->name)) { \
+                PIPE_CONF_CHECK_BOOL(name); \
+        } else { \
+                pipe_config_err(adjust, __stringify(name), \
+                          "unable to verify whether state matches exactly, forcing modeset (expected %s, found %s)\n", \
+                          yesno(current_config->name), \
+                          yesno(pipe_config->name)); \
+                ret = false; \
+        }
+
 #define PIPE_CONF_CHECK_P(name) \
         if (current_config->name != pipe_config->name) { \
                 pipe_config_err(adjust, __stringify(name), \
@@ -11175,7 +11248,7 @@ intel_pipe_config_compare(struct drm_i915_private *dev_priv,
 
         PIPE_CONF_CHECK_I(cpu_transcoder);
 
-        PIPE_CONF_CHECK_I(has_pch_encoder);
+        PIPE_CONF_CHECK_BOOL(has_pch_encoder);
         PIPE_CONF_CHECK_I(fdi_lanes);
         PIPE_CONF_CHECK_M_N(fdi_m_n);
 
@@ -11207,17 +11280,17 @@ intel_pipe_config_compare(struct drm_i915_private *dev_priv,
         PIPE_CONF_CHECK_I(base.adjusted_mode.crtc_vsync_end);
 
         PIPE_CONF_CHECK_I(pixel_multiplier);
-        PIPE_CONF_CHECK_I(has_hdmi_sink);
+        PIPE_CONF_CHECK_BOOL(has_hdmi_sink);
         if ((INTEL_GEN(dev_priv) < 8 && !IS_HASWELL(dev_priv)) ||
             IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
-                PIPE_CONF_CHECK_I(limited_color_range);
+                PIPE_CONF_CHECK_BOOL(limited_color_range);
 
-        PIPE_CONF_CHECK_I(hdmi_scrambling);
-        PIPE_CONF_CHECK_I(hdmi_high_tmds_clock_ratio);
-        PIPE_CONF_CHECK_I(has_infoframe);
-        PIPE_CONF_CHECK_I(ycbcr420);
+        PIPE_CONF_CHECK_BOOL(hdmi_scrambling);
+        PIPE_CONF_CHECK_BOOL(hdmi_high_tmds_clock_ratio);
+        PIPE_CONF_CHECK_BOOL_INCOMPLETE(has_infoframe);
+        PIPE_CONF_CHECK_BOOL(ycbcr420);
 
-        PIPE_CONF_CHECK_I(has_audio);
+        PIPE_CONF_CHECK_BOOL_INCOMPLETE(has_audio);
 
         PIPE_CONF_CHECK_FLAGS(base.adjusted_mode.flags,
                               DRM_MODE_FLAG_INTERLACE);
@@ -11243,7 +11316,7 @@ intel_pipe_config_compare(struct drm_i915_private *dev_priv,
                 PIPE_CONF_CHECK_I(pipe_src_w);
                 PIPE_CONF_CHECK_I(pipe_src_h);
 
-                PIPE_CONF_CHECK_I(pch_pfit.enabled);
+                PIPE_CONF_CHECK_BOOL(pch_pfit.enabled);
                 if (current_config->pch_pfit.enabled) {
                         PIPE_CONF_CHECK_X(pch_pfit.pos);
                         PIPE_CONF_CHECK_X(pch_pfit.size);
@@ -11253,11 +11326,7 @@ intel_pipe_config_compare(struct drm_i915_private *dev_priv,
                 PIPE_CONF_CHECK_CLOCK_FUZZY(pixel_rate);
         }
 
-        /* BDW+ don't expose a synchronous way to read the state */
-        if (IS_HASWELL(dev_priv))
-                PIPE_CONF_CHECK_I(ips_enabled);
-
-        PIPE_CONF_CHECK_I(double_wide);
+        PIPE_CONF_CHECK_BOOL(double_wide);
 
         PIPE_CONF_CHECK_P(shared_dpll);
         PIPE_CONF_CHECK_X(dpll_hw_state.dpll);
@@ -11291,8 +11360,12 @@ intel_pipe_config_compare(struct drm_i915_private *dev_priv,
         PIPE_CONF_CHECK_CLOCK_FUZZY(base.adjusted_mode.crtc_clock);
         PIPE_CONF_CHECK_CLOCK_FUZZY(port_clock);
 
+        PIPE_CONF_CHECK_I(min_voltage_level);
+
 #undef PIPE_CONF_CHECK_X
 #undef PIPE_CONF_CHECK_I
+#undef PIPE_CONF_CHECK_BOOL
+#undef PIPE_CONF_CHECK_BOOL_INCOMPLETE
 #undef PIPE_CONF_CHECK_P
 #undef PIPE_CONF_CHECK_FLAGS
 #undef PIPE_CONF_CHECK_CLOCK_FUZZY
@@ -11559,10 +11632,8 @@ verify_crtc_state(struct drm_crtc *crtc,
                                 "Encoder connected to wrong pipe %c\n",
                                 pipe_name(pipe));
 
-                if (active) {
-                        pipe_config->output_types |= 1 << encoder->type;
+                if (active)
                         encoder->get_config(encoder, pipe_config);
-                }
         }
 
         intel_crtc_compute_pixel_rate(pipe_config);
@@ -11933,16 +12004,16 @@ static int intel_modeset_checks(struct drm_atomic_state *state)
                  * holding all the crtc locks, even if we don't end up
                  * touching the hardware
                  */
-                if (!intel_cdclk_state_compare(&dev_priv->cdclk.logical,
-                                               &intel_state->cdclk.logical)) {
+                if (intel_cdclk_changed(&dev_priv->cdclk.logical,
+                                        &intel_state->cdclk.logical)) {
                         ret = intel_lock_all_pipes(state);
                         if (ret < 0)
                                 return ret;
                 }
 
                 /* All pipes must be switched off while we change the cdclk. */
-                if (!intel_cdclk_state_compare(&dev_priv->cdclk.actual,
-                                               &intel_state->cdclk.actual)) {
+                if (intel_cdclk_needs_modeset(&dev_priv->cdclk.actual,
+                                              &intel_state->cdclk.actual)) {
                         ret = intel_modeset_all_pipes(state);
                         if (ret < 0)
                                 return ret;
@@ -11951,6 +12022,9 @@ static int intel_modeset_checks(struct drm_atomic_state *state)
                 DRM_DEBUG_KMS("New cdclk calculated to be logical %u kHz, actual %u kHz\n",
                               intel_state->cdclk.logical.cdclk,
                               intel_state->cdclk.actual.cdclk);
+                DRM_DEBUG_KMS("New voltage level calculated to be logical %u, actual %u\n",
+                              intel_state->cdclk.logical.voltage_level,
+                              intel_state->cdclk.actual.voltage_level);
         } else {
                 to_intel_atomic_state(state)->cdclk.logical = dev_priv->cdclk.logical;
         }
@@ -12519,6 +12593,9 @@ static int intel_atomic_commit(struct drm_device *dev,
         if (intel_state->modeset) {
                 memcpy(dev_priv->min_cdclk, intel_state->min_cdclk,
                        sizeof(intel_state->min_cdclk));
+                memcpy(dev_priv->min_voltage_level,
+                       intel_state->min_voltage_level,
+                       sizeof(intel_state->min_voltage_level));
                 dev_priv->active_crtcs = intel_state->active_crtcs;
                 dev_priv->cdclk.logical = intel_state->cdclk.logical;
                 dev_priv->cdclk.actual = intel_state->cdclk.actual;
@@ -12756,7 +12833,7 @@ skl_max_scale(struct intel_crtc *intel_crtc, struct intel_crtc_state *crtc_state
         crtc_clock = crtc_state->base.adjusted_mode.crtc_clock;
         max_dotclk = to_intel_atomic_state(crtc_state->base.state)->cdclk.logical.cdclk;
 
-        if (IS_GEMINILAKE(dev_priv))
+        if (IS_GEMINILAKE(dev_priv) || INTEL_GEN(dev_priv) >= 10)
                 max_dotclk *= 2;
 
         if (WARN_ON_ONCE(!crtc_clock || max_dotclk < crtc_clock))
@@ -12820,6 +12897,9 @@ intel_check_primary_plane(struct intel_plane *plane,
                 state->ctl = i9xx_plane_ctl(crtc_state, state);
         }
 
+        if (INTEL_GEN(dev_priv) >= 10 || IS_GEMINILAKE(dev_priv))
+                state->color_ctl = glk_plane_color_ctl(crtc_state, state);
+
         return 0;
 }
 
@@ -12864,6 +12944,7 @@ out:
 static void intel_finish_crtc_commit(struct drm_crtc *crtc,
                                      struct drm_crtc_state *old_crtc_state)
 {
+        struct drm_i915_private *dev_priv = to_i915(crtc->dev);
         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
         struct intel_atomic_state *old_intel_state =
                 to_intel_atomic_state(old_crtc_state->state);
@@ -12871,6 +12952,20 @@ static void intel_finish_crtc_commit(struct drm_crtc *crtc,
                 intel_atomic_get_new_crtc_state(old_intel_state, intel_crtc);
 
         intel_pipe_update_end(new_crtc_state);
+
+        if (new_crtc_state->update_pipe &&
+            !needs_modeset(&new_crtc_state->base) &&
+            old_crtc_state->mode.private_flags & I915_MODE_FLAG_INHERITED) {
+                if (!IS_GEN2(dev_priv))
+                        intel_set_cpu_fifo_underrun_reporting(dev_priv, intel_crtc->pipe, true);
+
+                if (new_crtc_state->has_pch_encoder) {
+                        enum pipe pch_transcoder =
+                                intel_crtc_pch_transcoder(intel_crtc);
+
+                        intel_set_pch_fifo_underrun_reporting(dev_priv, pch_transcoder, true);
+                }
+        }
 }
 
 /**
@@ -14352,6 +14447,7 @@ void intel_modeset_init_hw(struct drm_device *dev)
         struct drm_i915_private *dev_priv = to_i915(dev);
 
         intel_update_cdclk(dev_priv);
+        intel_dump_cdclk_state(&dev_priv->cdclk.hw, "Current CDCLK");
         dev_priv->cdclk.logical = dev_priv->cdclk.actual = dev_priv->cdclk.hw;
 }
 
@@ -14431,6 +14527,8 @@ retry:
 
                 cs->wm.need_postvbl_update = true;
                 dev_priv->display.optimize_watermarks(intel_state, cs);
+
+                to_intel_crtc_state(crtc->state)->wm = cs->wm;
         }
 
 put_state:
@@ -14440,6 +14538,22 @@ fail:
         drm_modeset_acquire_fini(&ctx);
 }
 
+static void intel_update_fdi_pll_freq(struct drm_i915_private *dev_priv)
+{
+        if (IS_GEN5(dev_priv)) {
+                u32 fdi_pll_clk =
+                        I915_READ(FDI_PLL_BIOS_0) & FDI_PLL_FB_CLOCK_MASK;
+
+                dev_priv->fdi_pll_freq = (fdi_pll_clk + 2) * 10000;
+        } else if (IS_GEN6(dev_priv) || IS_IVYBRIDGE(dev_priv)) {
+                dev_priv->fdi_pll_freq = 270000;
+        } else {
+                return;
+        }
+
+        DRM_DEBUG_DRIVER("FDI PLL freq=%d\n", dev_priv->fdi_pll_freq);
+}
+
 int intel_modeset_init(struct drm_device *dev)
 {
         struct drm_i915_private *dev_priv = to_i915(dev);
@@ -14527,6 +14641,7 @@ int intel_modeset_init(struct drm_device *dev)
         }
 
         intel_shared_dpll_init(dev);
+        intel_update_fdi_pll_freq(dev_priv);
 
         intel_update_czclk(dev_priv);
         intel_modeset_init_hw(dev);
@@ -14716,7 +14831,7 @@ static void intel_sanitize_crtc(struct intel_crtc *crtc,
         enum transcoder cpu_transcoder = crtc->config->cpu_transcoder;
 
         /* Clear any frame start delays used for debugging left by the BIOS */
-        if (!transcoder_is_dsi(cpu_transcoder)) {
+        if (crtc->active && !transcoder_is_dsi(cpu_transcoder)) {
                 i915_reg_t reg = PIPECONF(cpu_transcoder);
 
                 I915_WRITE(reg,
@@ -14949,7 +15064,6 @@ static void intel_modeset_readout_hw_state(struct drm_device *dev)
                         crtc_state = to_intel_crtc_state(crtc->base.state);
 
                         encoder->base.crtc = &crtc->base;
-                        crtc_state->output_types |= 1 << encoder->type;
                         encoder->get_config(encoder, crtc_state);
                 } else {
                         encoder->base.crtc = NULL;
@@ -15028,6 +15142,8 @@ static void intel_modeset_readout_hw_state(struct drm_device *dev)
                 }
 
                 dev_priv->min_cdclk[crtc->pipe] = min_cdclk;
+                dev_priv->min_voltage_level[crtc->pipe] =
+                        crtc_state->min_voltage_level;
 
                 intel_pipe_config_sanity_check(dev_priv, crtc_state);
         }
@@ -15051,6 +15167,23 @@ get_encoder_power_domains(struct drm_i915_private *dev_priv)
         }
 }
 
+static void intel_early_display_was(struct drm_i915_private *dev_priv)
+{
+        /* Display WA #1185 WaDisableDARBFClkGating:cnl,glk */
+        if (IS_CANNONLAKE(dev_priv) || IS_GEMINILAKE(dev_priv))
+                I915_WRITE(GEN9_CLKGATE_DIS_0, I915_READ(GEN9_CLKGATE_DIS_0) |
+                           DARBF_GATING_DIS);
+
+        if (IS_HASWELL(dev_priv)) {
+                /*
+                 * WaRsPkgCStateDisplayPMReq:hsw
+                 * System hang if this isn't done before disabling all planes!
+                 */
+                I915_WRITE(CHICKEN_PAR1_1,
+                           I915_READ(CHICKEN_PAR1_1) | FORCE_ARB_IDLE_PLANES);
+        }
+}
+
 /* Scan out the current hw modeset state,
  * and sanitizes it to the current state
  */
@@ -15064,15 +15197,7 @@ intel_modeset_setup_hw_state(struct drm_device *dev,
         struct intel_encoder *encoder;
         int i;
 
-        if (IS_HASWELL(dev_priv)) {
-                /*
-                 * WaRsPkgCStateDisplayPMReq:hsw
-                 * System hang if this isn't done before disabling all planes!
-                 */
-                I915_WRITE(CHICKEN_PAR1_1,
-                           I915_READ(CHICKEN_PAR1_1) | FORCE_ARB_IDLE_PLANES);
-        }
-
+        intel_early_display_was(dev_priv);
         intel_modeset_readout_hw_state(dev);
 
         /* HW state is read out, now we need to sanitize this mess. */
@@ -15164,17 +15289,6 @@ void intel_display_resume(struct drm_device *dev)
                 drm_atomic_state_put(state);
 }
 
-void intel_modeset_gem_init(struct drm_device *dev)
-{
-        struct drm_i915_private *dev_priv = to_i915(dev);
-
-        intel_init_gt_powersave(dev_priv);
-
-        intel_init_clock_gating(dev_priv);
-
-        intel_setup_overlay(dev_priv);
-}
-
 int intel_connector_register(struct drm_connector *connector)
 {
         struct intel_connector *intel_connector = to_intel_connector(connector);
diff --git a/drivers/gpu/drm/i915/intel_dp.c b/drivers/gpu/drm/i915/intel_dp.c
index 65260fb35d2a..bbf2256ba574 100644
--- a/drivers/gpu/drm/i915/intel_dp.c
+++ b/drivers/gpu/drm/i915/intel_dp.c
@@ -129,11 +129,13 @@ static struct intel_dp *intel_attached_dp(struct drm_connector *connector)
         return enc_to_intel_dp(&intel_attached_encoder(connector)->base);
 }
 
-static void intel_dp_link_down(struct intel_dp *intel_dp);
+static void intel_dp_link_down(struct intel_encoder *encoder,
+                               const struct intel_crtc_state *old_crtc_state);
 static bool edp_panel_vdd_on(struct intel_dp *intel_dp);
 static void edp_panel_vdd_off(struct intel_dp *intel_dp, bool sync);
-static void vlv_init_panel_power_sequencer(struct intel_dp *intel_dp);
-static void vlv_steal_power_sequencer(struct drm_device *dev,
+static void vlv_init_panel_power_sequencer(struct intel_encoder *encoder,
+                                           const struct intel_crtc_state *crtc_state);
+static void vlv_steal_power_sequencer(struct drm_i915_private *dev_priv,
                                       enum pipe pipe);
 static void intel_dp_unset_edid(struct intel_dp *intel_dp);
 
@@ -221,7 +223,7 @@ intel_dp_set_source_rates(struct intel_dp *intel_dp)
 {
         struct intel_digital_port *dig_port = dp_to_dig_port(intel_dp);
         struct drm_i915_private *dev_priv = to_i915(dig_port->base.base.dev);
-        enum port port = dig_port->port;
+        enum port port = dig_port->base.port;
         const int *source_rates;
         int size;
         u32 voltage;
@@ -427,24 +429,19 @@ static void intel_dp_unpack_aux(uint32_t src, uint8_t *dst, int dst_bytes)
 }
 
 static void
-intel_dp_init_panel_power_sequencer(struct drm_device *dev,
-                                    struct intel_dp *intel_dp);
+intel_dp_init_panel_power_sequencer(struct intel_dp *intel_dp);
 static void
-intel_dp_init_panel_power_sequencer_registers(struct drm_device *dev,
-                                              struct intel_dp *intel_dp,
+intel_dp_init_panel_power_sequencer_registers(struct intel_dp *intel_dp,
                                               bool force_disable_vdd);
 static void
-intel_dp_pps_init(struct drm_device *dev, struct intel_dp *intel_dp);
+intel_dp_pps_init(struct intel_dp *intel_dp);
 
 static void pps_lock(struct intel_dp *intel_dp)
 {
-        struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp);
-        struct intel_encoder *encoder = &intel_dig_port->base;
-        struct drm_device *dev = encoder->base.dev;
-        struct drm_i915_private *dev_priv = to_i915(dev);
+        struct drm_i915_private *dev_priv = to_i915(intel_dp_to_dev(intel_dp));
 
         /*
-         * See vlv_power_sequencer_reset() why we need
+         * See intel_power_sequencer_reset() why we need
          * a power domain reference here.
          */
         intel_display_power_get(dev_priv, intel_dp->aux_power_domain);
@@ -454,10 +451,7 @@ static void pps_lock(struct intel_dp *intel_dp)
 
 static void pps_unlock(struct intel_dp *intel_dp)
 {
-        struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp);
-        struct intel_encoder *encoder = &intel_dig_port->base;
-        struct drm_device *dev = encoder->base.dev;
-        struct drm_i915_private *dev_priv = to_i915(dev);
+        struct drm_i915_private *dev_priv = to_i915(intel_dp_to_dev(intel_dp));
 
         mutex_unlock(&dev_priv->pps_mutex);
 
@@ -467,8 +461,8 @@ static void pps_unlock(struct intel_dp *intel_dp)
 static void
 vlv_power_sequencer_kick(struct intel_dp *intel_dp)
 {
+        struct drm_i915_private *dev_priv = to_i915(intel_dp_to_dev(intel_dp));
         struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp);
-        struct drm_i915_private *dev_priv = to_i915(intel_dig_port->base.base.dev);
         enum pipe pipe = intel_dp->pps_pipe;
         bool pll_enabled, release_cl_override = false;
         enum dpio_phy phy = DPIO_PHY(pipe);
@@ -477,11 +471,11 @@ vlv_power_sequencer_kick(struct intel_dp *intel_dp)
 
         if (WARN(I915_READ(intel_dp->output_reg) & DP_PORT_EN,
                  "skipping pipe %c power seqeuncer kick due to port %c being active\n",
-                 pipe_name(pipe), port_name(intel_dig_port->port)))
+                 pipe_name(pipe), port_name(intel_dig_port->base.port)))
                 return;
 
         DRM_DEBUG_KMS("kicking pipe %c power sequencer for port %c\n",
-                      pipe_name(pipe), port_name(intel_dig_port->port));
+                      pipe_name(pipe), port_name(intel_dig_port->base.port));
 
         /* Preserve the BIOS-computed detected bit. This is
          * supposed to be read-only.
@@ -578,9 +572,8 @@ static enum pipe vlv_find_free_pps(struct drm_i915_private *dev_priv)
 static enum pipe
 vlv_power_sequencer_pipe(struct intel_dp *intel_dp)
 {
+        struct drm_i915_private *dev_priv = to_i915(intel_dp_to_dev(intel_dp));
         struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp);
-        struct drm_device *dev = intel_dig_port->base.base.dev;
-        struct drm_i915_private *dev_priv = to_i915(dev);
         enum pipe pipe;
 
         lockdep_assert_held(&dev_priv->pps_mutex);
@@ -603,16 +596,16 @@ vlv_power_sequencer_pipe(struct intel_dp *intel_dp)
         if (WARN_ON(pipe == INVALID_PIPE))
                 pipe = PIPE_A;
 
-        vlv_steal_power_sequencer(dev, pipe);
+        vlv_steal_power_sequencer(dev_priv, pipe);
         intel_dp->pps_pipe = pipe;
 
         DRM_DEBUG_KMS("picked pipe %c power sequencer for port %c\n",
                       pipe_name(intel_dp->pps_pipe),
-                      port_name(intel_dig_port->port));
+                      port_name(intel_dig_port->base.port));
 
         /* init power sequencer on this pipe and port */
-        intel_dp_init_panel_power_sequencer(dev, intel_dp);
-        intel_dp_init_panel_power_sequencer_registers(dev, intel_dp, true);
+        intel_dp_init_panel_power_sequencer(intel_dp);
+        intel_dp_init_panel_power_sequencer_registers(intel_dp, true);
 
         /*
          * Even vdd force doesn't work until we've made
@@ -626,9 +619,7 @@ vlv_power_sequencer_pipe(struct intel_dp *intel_dp)
 static int
 bxt_power_sequencer_idx(struct intel_dp *intel_dp)
 {
-        struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp);
-        struct drm_device *dev = intel_dig_port->base.base.dev;
-        struct drm_i915_private *dev_priv = to_i915(dev);
+        struct drm_i915_private *dev_priv = to_i915(intel_dp_to_dev(intel_dp));
 
         lockdep_assert_held(&dev_priv->pps_mutex);
 
@@ -649,7 +640,7 @@ bxt_power_sequencer_idx(struct intel_dp *intel_dp)
          * Only the HW needs to be reprogrammed, the SW state is fixed and
          * has been setup during connector init.
          */
-        intel_dp_init_panel_power_sequencer_registers(dev, intel_dp, false);
+        intel_dp_init_panel_power_sequencer_registers(intel_dp, false);
 
         return 0;
 }
@@ -701,10 +692,9 @@ vlv_initial_pps_pipe(struct drm_i915_private *dev_priv,
 static void
 vlv_initial_power_sequencer_setup(struct intel_dp *intel_dp)
 {
+        struct drm_i915_private *dev_priv = to_i915(intel_dp_to_dev(intel_dp));
         struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp);
-        struct drm_device *dev = intel_dig_port->base.base.dev;
-        struct drm_i915_private *dev_priv = to_i915(dev);
-        enum port port = intel_dig_port->port;
+        enum port port = intel_dig_port->base.port;
 
         lockdep_assert_held(&dev_priv->pps_mutex);
 
@@ -731,13 +721,12 @@ vlv_initial_power_sequencer_setup(struct intel_dp *intel_dp)
         DRM_DEBUG_KMS("initial power sequencer for port %c: pipe %c\n",
                       port_name(port), pipe_name(intel_dp->pps_pipe));
 
-        intel_dp_init_panel_power_sequencer(dev, intel_dp);
-        intel_dp_init_panel_power_sequencer_registers(dev, intel_dp, false);
+        intel_dp_init_panel_power_sequencer(intel_dp);
+        intel_dp_init_panel_power_sequencer_registers(intel_dp, false);
 }
 
 void intel_power_sequencer_reset(struct drm_i915_private *dev_priv)
 {
-        struct drm_device *dev = &dev_priv->drm;
         struct intel_encoder *encoder;
 
         if (WARN_ON(!IS_VALLEYVIEW(dev_priv) && !IS_CHERRYVIEW(dev_priv) &&
@@ -754,15 +743,20 @@ void intel_power_sequencer_reset(struct drm_i915_private *dev_priv)
          * should use them always.
          */
 
-        for_each_intel_encoder(dev, encoder) {
+        for_each_intel_encoder(&dev_priv->drm, encoder) {
                 struct intel_dp *intel_dp;
 
                 if (encoder->type != INTEL_OUTPUT_DP &&
-                    encoder->type != INTEL_OUTPUT_EDP)
+                    encoder->type != INTEL_OUTPUT_EDP &&
+                    encoder->type != INTEL_OUTPUT_DDI)
                         continue;
 
                 intel_dp = enc_to_intel_dp(&encoder->base);
 
+                /* Skip pure DVI/HDMI DDI encoders */
+                if (!i915_mmio_reg_valid(intel_dp->output_reg))
+                        continue;
+
                 WARN_ON(intel_dp->active_pipe != INVALID_PIPE);
 
                 if (encoder->type != INTEL_OUTPUT_EDP)
@@ -783,10 +777,10 @@ struct pps_registers {
         i915_reg_t pp_div;
 };
 
-static void intel_pps_get_registers(struct drm_i915_private *dev_priv,
-                                    struct intel_dp *intel_dp,
+static void intel_pps_get_registers(struct intel_dp *intel_dp,
                                     struct pps_registers *regs)
 {
+        struct drm_i915_private *dev_priv = to_i915(intel_dp_to_dev(intel_dp));
         int pps_idx = 0;
 
         memset(regs, 0, sizeof(*regs));
@@ -809,8 +803,7 @@ _pp_ctrl_reg(struct intel_dp *intel_dp)
 {
         struct pps_registers regs;
 
-        intel_pps_get_registers(to_i915(intel_dp_to_dev(intel_dp)), intel_dp,
-                                &regs);
+        intel_pps_get_registers(intel_dp, &regs);
 
         return regs.pp_ctrl;
 }
@@ -820,8 +813,7 @@ _pp_stat_reg(struct intel_dp *intel_dp)
 {
         struct pps_registers regs;
 
-        intel_pps_get_registers(to_i915(intel_dp_to_dev(intel_dp)), intel_dp,
-                                &regs);
+        intel_pps_get_registers(intel_dp, &regs);
 
         return regs.pp_stat;
 }
@@ -833,8 +825,7 @@ static int edp_notify_handler(struct notifier_block *this, unsigned long code,
 {
         struct intel_dp *intel_dp = container_of(this, typeof(* intel_dp),
                                                  edp_notifier);
-        struct drm_device *dev = intel_dp_to_dev(intel_dp);
-        struct drm_i915_private *dev_priv = to_i915(dev);
+        struct drm_i915_private *dev_priv = to_i915(intel_dp_to_dev(intel_dp));
 
         if (!intel_dp_is_edp(intel_dp) || code != SYS_RESTART)
                 return 0;
@@ -864,8 +855,7 @@ static int edp_notify_handler(struct notifier_block *this, unsigned long code,
 
 static bool edp_have_panel_power(struct intel_dp *intel_dp)
 {
-        struct drm_device *dev = intel_dp_to_dev(intel_dp);
-        struct drm_i915_private *dev_priv = to_i915(dev);
+        struct drm_i915_private *dev_priv = to_i915(intel_dp_to_dev(intel_dp));
 
         lockdep_assert_held(&dev_priv->pps_mutex);
 
@@ -878,8 +868,7 @@ static bool edp_have_panel_power(struct intel_dp *intel_dp)
 
 static bool edp_have_panel_vdd(struct intel_dp *intel_dp)
 {
-        struct drm_device *dev = intel_dp_to_dev(intel_dp);
-        struct drm_i915_private *dev_priv = to_i915(dev);
+        struct drm_i915_private *dev_priv = to_i915(intel_dp_to_dev(intel_dp));
 
         lockdep_assert_held(&dev_priv->pps_mutex);
 
@@ -893,8 +882,7 @@ static bool edp_have_panel_vdd(struct intel_dp *intel_dp)
 static void
 intel_dp_check_edp(struct intel_dp *intel_dp)
 {
-        struct drm_device *dev = intel_dp_to_dev(intel_dp);
-        struct drm_i915_private *dev_priv = to_i915(dev);
+        struct drm_i915_private *dev_priv = to_i915(intel_dp_to_dev(intel_dp));
 
         if (!intel_dp_is_edp(intel_dp))
                 return;
@@ -910,9 +898,7 @@ intel_dp_check_edp(struct intel_dp *intel_dp)
 static uint32_t
 intel_dp_aux_wait_done(struct intel_dp *intel_dp, bool has_aux_irq)
 {
-        struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp);
-        struct drm_device *dev = intel_dig_port->base.base.dev;
-        struct drm_i915_private *dev_priv = to_i915(dev);
+        struct drm_i915_private *dev_priv = to_i915(intel_dp_to_dev(intel_dp));
         i915_reg_t ch_ctl = intel_dp->aux_ch_ctl_reg;
         uint32_t status;
         bool done;
@@ -959,7 +945,7 @@ static uint32_t ilk_get_aux_clock_divider(struct intel_dp *intel_dp, int index)
          * like to run at 2MHz.  So, take the cdclk or PCH rawclk value and
          * divide by 2000 and use that
          */
-        if (intel_dig_port->port == PORT_A)
+        if (intel_dig_port->base.port == PORT_A)
                 return DIV_ROUND_CLOSEST(dev_priv->cdclk.hw.cdclk, 2000);
         else
                 return DIV_ROUND_CLOSEST(dev_priv->rawclk_freq, 2000);
@@ -970,7 +956,7 @@ static uint32_t hsw_get_aux_clock_divider(struct intel_dp *intel_dp, int index)
         struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp);
         struct drm_i915_private *dev_priv = to_i915(intel_dig_port->base.base.dev);
 
-        if (intel_dig_port->port != PORT_A && HAS_PCH_LPT_H(dev_priv)) {
+        if (intel_dig_port->base.port != PORT_A && HAS_PCH_LPT_H(dev_priv)) {
                 /* Workaround for non-ULT HSW */
                 switch (index) {
                 case 0: return 63;
@@ -1440,7 +1426,7 @@ static void intel_aux_reg_init(struct intel_dp *intel_dp)
 {
         struct drm_i915_private *dev_priv = to_i915(intel_dp_to_dev(intel_dp));
         enum port port = intel_aux_port(dev_priv,
-                                        dp_to_dig_port(intel_dp)->port);
+                                        dp_to_dig_port(intel_dp)->base.port);
         int i;
 
         intel_dp->aux_ch_ctl_reg = intel_aux_ctl_reg(dev_priv, port);
@@ -1458,7 +1444,7 @@ static void
 intel_dp_aux_init(struct intel_dp *intel_dp)
 {
         struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp);
-        enum port port = intel_dig_port->port;
+        enum port port = intel_dig_port->base.port;
 
         intel_aux_reg_init(intel_dp);
         drm_dp_aux_init(&intel_dp->aux);
@@ -1479,8 +1465,7 @@ static void
 intel_dp_set_clock(struct intel_encoder *encoder,
                    struct intel_crtc_state *pipe_config)
 {
-        struct drm_device *dev = encoder->base.dev;
-        struct drm_i915_private *dev_priv = to_i915(dev);
+        struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
         const struct dp_link_dpll *divisor = NULL;
         int i, count = 0;
 
@@ -1628,7 +1613,7 @@ intel_dp_compute_config(struct intel_encoder *encoder,
         struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
         struct drm_display_mode *adjusted_mode = &pipe_config->base.adjusted_mode;
         struct intel_dp *intel_dp = enc_to_intel_dp(&encoder->base);
-        enum port port = dp_to_dig_port(intel_dp)->port;
+        enum port port = encoder->port;
         struct intel_crtc *intel_crtc = to_intel_crtc(pipe_config->base.crtc);
         struct intel_connector *intel_connector = intel_dp->attached_connector;
         struct intel_digital_connector_state *intel_conn_state =
@@ -1849,11 +1834,10 @@ void intel_dp_set_link_params(struct intel_dp *intel_dp,
 static void intel_dp_prepare(struct intel_encoder *encoder,
                              const struct intel_crtc_state *pipe_config)
 {
-        struct drm_device *dev = encoder->base.dev;
-        struct drm_i915_private *dev_priv = to_i915(dev);
+        struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
         struct intel_dp *intel_dp = enc_to_intel_dp(&encoder->base);
-        enum port port = dp_to_dig_port(intel_dp)->port;
-        struct intel_crtc *crtc = to_intel_crtc(encoder->base.crtc);
+        enum port port = encoder->port;
+        struct intel_crtc *crtc = to_intel_crtc(pipe_config->base.crtc);
         const struct drm_display_mode *adjusted_mode = &pipe_config->base.adjusted_mode;
 
         intel_dp_set_link_params(intel_dp, pipe_config->port_clock,
@@ -1940,20 +1924,18 @@ static void intel_dp_prepare(struct intel_encoder *encoder,
 #define IDLE_CYCLE_MASK         (PP_ON | PP_SEQUENCE_MASK | PP_CYCLE_DELAY_ACTIVE | PP_SEQUENCE_STATE_MASK)
 #define IDLE_CYCLE_VALUE        (0     | PP_SEQUENCE_NONE | 0                     | PP_SEQUENCE_STATE_OFF_IDLE)
 
-static void intel_pps_verify_state(struct drm_i915_private *dev_priv,
-                                   struct intel_dp *intel_dp);
+static void intel_pps_verify_state(struct intel_dp *intel_dp);
 
 static void wait_panel_status(struct intel_dp *intel_dp,
                                        u32 mask,
                                        u32 value)
 {
-        struct drm_device *dev = intel_dp_to_dev(intel_dp);
-        struct drm_i915_private *dev_priv = to_i915(dev);
+        struct drm_i915_private *dev_priv = to_i915(intel_dp_to_dev(intel_dp));
         i915_reg_t pp_stat_reg, pp_ctrl_reg;
 
         lockdep_assert_held(&dev_priv->pps_mutex);
 
-        intel_pps_verify_state(dev_priv, intel_dp);
+        intel_pps_verify_state(intel_dp);
 
         pp_stat_reg = _pp_stat_reg(intel_dp);
         pp_ctrl_reg = _pp_ctrl_reg(intel_dp);
@@ -2024,8 +2006,7 @@ static void edp_wait_backlight_off(struct intel_dp *intel_dp)
 
 static  u32 ironlake_get_pp_control(struct intel_dp *intel_dp)
 {
-        struct drm_device *dev = intel_dp_to_dev(intel_dp);
-        struct drm_i915_private *dev_priv = to_i915(dev);
+        struct drm_i915_private *dev_priv = to_i915(intel_dp_to_dev(intel_dp));
         u32 control;
 
         lockdep_assert_held(&dev_priv->pps_mutex);
@@ -2046,9 +2027,8 @@ static  u32 ironlake_get_pp_control(struct intel_dp *intel_dp)
  */
 static bool edp_panel_vdd_on(struct intel_dp *intel_dp)
 {
-        struct drm_device *dev = intel_dp_to_dev(intel_dp);
+        struct drm_i915_private *dev_priv = to_i915(intel_dp_to_dev(intel_dp));
         struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp);
-        struct drm_i915_private *dev_priv = to_i915(dev);
         u32 pp;
         i915_reg_t pp_stat_reg, pp_ctrl_reg;
         bool need_to_disable = !intel_dp->want_panel_vdd;
@@ -2067,7 +2047,7 @@ static bool edp_panel_vdd_on(struct intel_dp *intel_dp)
         intel_display_power_get(dev_priv, intel_dp->aux_power_domain);
 
         DRM_DEBUG_KMS("Turning eDP port %c VDD on\n",
-                      port_name(intel_dig_port->port));
+                      port_name(intel_dig_port->base.port));
 
         if (!edp_have_panel_power(intel_dp))
                 wait_panel_power_cycle(intel_dp);
@@ -2087,7 +2067,7 @@ static bool edp_panel_vdd_on(struct intel_dp *intel_dp)
          */
         if (!edp_have_panel_power(intel_dp)) {
                 DRM_DEBUG_KMS("eDP port %c panel power wasn't enabled\n",
-                              port_name(intel_dig_port->port));
+                              port_name(intel_dig_port->base.port));
                 msleep(intel_dp->panel_power_up_delay);
         }
 
@@ -2113,13 +2093,12 @@ void intel_edp_panel_vdd_on(struct intel_dp *intel_dp)
         pps_unlock(intel_dp);
 
         I915_STATE_WARN(!vdd, "eDP port %c VDD already requested on\n",
-             port_name(dp_to_dig_port(intel_dp)->port));
+             port_name(dp_to_dig_port(intel_dp)->base.port));
 }
 
 static void edp_panel_vdd_off_sync(struct intel_dp *intel_dp)
 {
-        struct drm_device *dev = intel_dp_to_dev(intel_dp);
-        struct drm_i915_private *dev_priv = to_i915(dev);
+        struct drm_i915_private *dev_priv = to_i915(intel_dp_to_dev(intel_dp));
         struct intel_digital_port *intel_dig_port =
                 dp_to_dig_port(intel_dp);
         u32 pp;
@@ -2133,7 +2112,7 @@ static void edp_panel_vdd_off_sync(struct intel_dp *intel_dp)
                 return;
 
         DRM_DEBUG_KMS("Turning eDP port %c VDD off\n",
-                      port_name(intel_dig_port->port));
+                      port_name(intel_dig_port->base.port));
 
         pp = ironlake_get_pp_control(intel_dp);
         pp &= ~EDP_FORCE_VDD;
@@ -2193,7 +2172,7 @@ static void edp_panel_vdd_off(struct intel_dp *intel_dp, bool sync)
                 return;
 
         I915_STATE_WARN(!intel_dp->want_panel_vdd, "eDP port %c VDD not forced on",
-             port_name(dp_to_dig_port(intel_dp)->port));
+             port_name(dp_to_dig_port(intel_dp)->base.port));
 
         intel_dp->want_panel_vdd = false;
 
@@ -2205,8 +2184,7 @@ static void edp_panel_vdd_off(struct intel_dp *intel_dp, bool sync)
 
 static void edp_panel_on(struct intel_dp *intel_dp)
 {
-        struct drm_device *dev = intel_dp_to_dev(intel_dp);
-        struct drm_i915_private *dev_priv = to_i915(dev);
+        struct drm_i915_private *dev_priv = to_i915(intel_dp_to_dev(intel_dp));
         u32 pp;
         i915_reg_t pp_ctrl_reg;
 
@@ -2216,11 +2194,11 @@ static void edp_panel_on(struct intel_dp *intel_dp)
                 return;
 
         DRM_DEBUG_KMS("Turn eDP port %c panel power on\n",
-                      port_name(dp_to_dig_port(intel_dp)->port));
+                      port_name(dp_to_dig_port(intel_dp)->base.port));
 
         if (WARN(edp_have_panel_power(intel_dp),
                  "eDP port %c panel power already on\n",
-                 port_name(dp_to_dig_port(intel_dp)->port)))
+                 port_name(dp_to_dig_port(intel_dp)->base.port)))
                 return;
 
         wait_panel_power_cycle(intel_dp);
@@ -2264,8 +2242,7 @@ void intel_edp_panel_on(struct intel_dp *intel_dp)
 
 static void edp_panel_off(struct intel_dp *intel_dp)
 {
-        struct drm_device *dev = intel_dp_to_dev(intel_dp);
-        struct drm_i915_private *dev_priv = to_i915(dev);
+        struct drm_i915_private *dev_priv = to_i915(intel_dp_to_dev(intel_dp));
         u32 pp;
         i915_reg_t pp_ctrl_reg;
 
@@ -2275,10 +2252,10 @@ static void edp_panel_off(struct intel_dp *intel_dp)
                 return;
 
         DRM_DEBUG_KMS("Turn eDP port %c panel power off\n",
-                      port_name(dp_to_dig_port(intel_dp)->port));
+                      port_name(dp_to_dig_port(intel_dp)->base.port));
 
         WARN(!intel_dp->want_panel_vdd, "Need eDP port %c VDD to turn off panel\n",
-             port_name(dp_to_dig_port(intel_dp)->port));
+             port_name(dp_to_dig_port(intel_dp)->base.port));
 
         pp = ironlake_get_pp_control(intel_dp);
         /* We need to switch off panel power _and_ force vdd, for otherwise some
@@ -2313,9 +2290,7 @@ void intel_edp_panel_off(struct intel_dp *intel_dp)
 /* Enable backlight in the panel power control. */
 static void _intel_edp_backlight_on(struct intel_dp *intel_dp)
 {
-        struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp);
-        struct drm_device *dev = intel_dig_port->base.base.dev;
-        struct drm_i915_private *dev_priv = to_i915(dev);
+        struct drm_i915_private *dev_priv = to_i915(intel_dp_to_dev(intel_dp));
         u32 pp;
         i915_reg_t pp_ctrl_reg;
 
@@ -2358,8 +2333,7 @@ void intel_edp_backlight_on(const struct intel_crtc_state *crtc_state,
 /* Disable backlight in the panel power control. */
 static void _intel_edp_backlight_off(struct intel_dp *intel_dp)
 {
-        struct drm_device *dev = intel_dp_to_dev(intel_dp);
-        struct drm_i915_private *dev_priv = to_i915(dev);
+        struct drm_i915_private *dev_priv = to_i915(intel_dp_to_dev(intel_dp));
         u32 pp;
         i915_reg_t pp_ctrl_reg;
 
@@ -2430,7 +2404,7 @@ static void assert_dp_port(struct intel_dp *intel_dp, bool state)
 
         I915_STATE_WARN(cur_state != state,
                         "DP port %c state assertion failure (expected %s, current %s)\n",
-                        port_name(dig_port->port),
+                        port_name(dig_port->base.port),
                         onoff(state), onoff(cur_state));
 }
 #define assert_dp_port_disabled(d) assert_dp_port((d), false)
@@ -2486,10 +2460,10 @@ static void ironlake_edp_pll_on(struct intel_dp *intel_dp,
         udelay(200);
 }
 
-static void ironlake_edp_pll_off(struct intel_dp *intel_dp)
+static void ironlake_edp_pll_off(struct intel_dp *intel_dp,
+                                 const struct intel_crtc_state *old_crtc_state)
 {
-        struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp);
-        struct intel_crtc *crtc = to_intel_crtc(intel_dig_port->base.base.crtc);
+        struct intel_crtc *crtc = to_intel_crtc(old_crtc_state->base.crtc);
         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
 
         assert_pipe_disabled(dev_priv, crtc->pipe);
@@ -2505,6 +2479,21 @@ static void ironlake_edp_pll_off(struct intel_dp *intel_dp)
         udelay(200);
 }
 
+static bool downstream_hpd_needs_d0(struct intel_dp *intel_dp)
+{
+        /*
+         * DPCD 1.2+ should support BRANCH_DEVICE_CTRL, and thus
+         * be capable of signalling downstream hpd with a long pulse.
+         * Whether or not that means D3 is safe to use is not clear,
+         * but let's assume so until proven otherwise.
+         *
+         * FIXME should really check all downstream ports...
+         */
+        return intel_dp->dpcd[DP_DPCD_REV] == 0x11 &&
+                intel_dp->dpcd[DP_DOWNSTREAMPORT_PRESENT] & DP_DWN_STRM_PORT_PRESENT &&
+                intel_dp->downstream_ports[0] & DP_DS_PORT_HPD;
+}
+
 /* If the sink supports it, try to set the power state appropriately */
 void intel_dp_sink_dpms(struct intel_dp *intel_dp, int mode)
 {
@@ -2515,6 +2504,9 @@ void intel_dp_sink_dpms(struct intel_dp *intel_dp, int mode)
                 return;
 
         if (mode != DRM_MODE_DPMS_ON) {
+                if (downstream_hpd_needs_d0(intel_dp))
+                        return;
+
                 ret = drm_dp_dpcd_writeb(&intel_dp->aux, DP_SET_POWER,
                                          DP_SET_POWER_D3);
         } else {
@@ -2544,10 +2536,9 @@ void intel_dp_sink_dpms(struct intel_dp *intel_dp, int mode)
 static bool intel_dp_get_hw_state(struct intel_encoder *encoder,
                                   enum pipe *pipe)
 {
+        struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
         struct intel_dp *intel_dp = enc_to_intel_dp(&encoder->base);
-        enum port port = dp_to_dig_port(intel_dp)->port;
-        struct drm_device *dev = encoder->base.dev;
-        struct drm_i915_private *dev_priv = to_i915(dev);
+        enum port port = encoder->port;
         u32 tmp;
         bool ret;
 
@@ -2596,12 +2587,16 @@ out:
 static void intel_dp_get_config(struct intel_encoder *encoder,
                                 struct intel_crtc_state *pipe_config)
 {
+        struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
         struct intel_dp *intel_dp = enc_to_intel_dp(&encoder->base);
         u32 tmp, flags = 0;
-        struct drm_device *dev = encoder->base.dev;
-        struct drm_i915_private *dev_priv = to_i915(dev);
-        enum port port = dp_to_dig_port(intel_dp)->port;
-        struct intel_crtc *crtc = to_intel_crtc(encoder->base.crtc);
+        enum port port = encoder->port;
+        struct intel_crtc *crtc = to_intel_crtc(pipe_config->base.crtc);
+
+        if (encoder->type == INTEL_OUTPUT_EDP)
+                pipe_config->output_types |= BIT(INTEL_OUTPUT_EDP);
+        else
+                pipe_config->output_types |= BIT(INTEL_OUTPUT_DP);
 
         tmp = I915_READ(intel_dp->output_reg);
 
@@ -2680,7 +2675,8 @@ static void intel_disable_dp(struct intel_encoder *encoder,
         struct intel_dp *intel_dp = enc_to_intel_dp(&encoder->base);
 
         if (old_crtc_state->has_audio)
-                intel_audio_codec_disable(encoder);
+                intel_audio_codec_disable(encoder,
+                                          old_crtc_state, old_conn_state);
 
         /* Make sure the panel is off before trying to change the mode. But also
          * ensure that we have vdd while we switch off the panel. */
@@ -2694,12 +2690,10 @@ static void g4x_disable_dp(struct intel_encoder *encoder,
                            const struct intel_crtc_state *old_crtc_state,
                            const struct drm_connector_state *old_conn_state)
 {
-        struct intel_dp *intel_dp = enc_to_intel_dp(&encoder->base);
-
         intel_disable_dp(encoder, old_crtc_state, old_conn_state);
 
         /* disable the port before the pipe on g4x */
-        intel_dp_link_down(intel_dp);
+        intel_dp_link_down(encoder, old_crtc_state);
 }
 
 static void ilk_disable_dp(struct intel_encoder *encoder,
@@ -2725,38 +2719,34 @@ static void ilk_post_disable_dp(struct intel_encoder *encoder,
                                 const struct drm_connector_state *old_conn_state)
 {
         struct intel_dp *intel_dp = enc_to_intel_dp(&encoder->base);
-        enum port port = dp_to_dig_port(intel_dp)->port;
+        enum port port = encoder->port;
 
-        intel_dp_link_down(intel_dp);
+        intel_dp_link_down(encoder, old_crtc_state);
 
         /* Only ilk+ has port A */
         if (port == PORT_A)
-                ironlake_edp_pll_off(intel_dp);
+                ironlake_edp_pll_off(intel_dp, old_crtc_state);
 }
 
 static void vlv_post_disable_dp(struct intel_encoder *encoder,
                                 const struct intel_crtc_state *old_crtc_state,
                                 const struct drm_connector_state *old_conn_state)
 {
-        struct intel_dp *intel_dp = enc_to_intel_dp(&encoder->base);
-
-        intel_dp_link_down(intel_dp);
+        intel_dp_link_down(encoder, old_crtc_state);
 }
 
 static void chv_post_disable_dp(struct intel_encoder *encoder,
                                 const struct intel_crtc_state *old_crtc_state,
                                 const struct drm_connector_state *old_conn_state)
 {
-        struct intel_dp *intel_dp = enc_to_intel_dp(&encoder->base);
-        struct drm_device *dev = encoder->base.dev;
-        struct drm_i915_private *dev_priv = to_i915(dev);
+        struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
 
-        intel_dp_link_down(intel_dp);
+        intel_dp_link_down(encoder, old_crtc_state);
 
         mutex_lock(&dev_priv->sb_lock);
 
         /* Assert data lane reset */
-        chv_data_lane_soft_reset(encoder, true);
+        chv_data_lane_soft_reset(encoder, old_crtc_state, true);
 
         mutex_unlock(&dev_priv->sb_lock);
 }
@@ -2766,10 +2756,9 @@ _intel_dp_set_link_train(struct intel_dp *intel_dp,
                          uint32_t *DP,
                          uint8_t dp_train_pat)
 {
+        struct drm_i915_private *dev_priv = to_i915(intel_dp_to_dev(intel_dp));
         struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp);
-        struct drm_device *dev = intel_dig_port->base.base.dev;
-        struct drm_i915_private *dev_priv = to_i915(dev);
-        enum port port = intel_dig_port->port;
+        enum port port = intel_dig_port->base.port;
 
         if (dp_train_pat & DP_TRAINING_PATTERN_MASK)
                 DRM_DEBUG_KMS("Using DP training pattern TPS%d\n",
@@ -2852,8 +2841,7 @@ _intel_dp_set_link_train(struct intel_dp *intel_dp,
 static void intel_dp_enable_port(struct intel_dp *intel_dp,
                                  const struct intel_crtc_state *old_crtc_state)
 {
-        struct drm_device *dev = intel_dp_to_dev(intel_dp);
-        struct drm_i915_private *dev_priv = to_i915(dev);
+        struct drm_i915_private *dev_priv = to_i915(intel_dp_to_dev(intel_dp));
 
         /* enable with pattern 1 (as per spec) */
 
@@ -2877,10 +2865,9 @@ static void intel_enable_dp(struct intel_encoder *encoder,
                             const struct intel_crtc_state *pipe_config,
                             const struct drm_connector_state *conn_state)
 {
+        struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
         struct intel_dp *intel_dp = enc_to_intel_dp(&encoder->base);
-        struct drm_device *dev = encoder->base.dev;
-        struct drm_i915_private *dev_priv = to_i915(dev);
-        struct intel_crtc *crtc = to_intel_crtc(encoder->base.crtc);
+        struct intel_crtc *crtc = to_intel_crtc(pipe_config->base.crtc);
         uint32_t dp_reg = I915_READ(intel_dp->output_reg);
         enum pipe pipe = crtc->pipe;
 
@@ -2890,7 +2877,7 @@ static void intel_enable_dp(struct intel_encoder *encoder,
         pps_lock(intel_dp);
 
         if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
-                vlv_init_panel_power_sequencer(intel_dp);
+                vlv_init_panel_power_sequencer(encoder, pipe_config);
 
         intel_dp_enable_port(intel_dp, pipe_config);
 
@@ -2944,7 +2931,7 @@ static void g4x_pre_enable_dp(struct intel_encoder *encoder,
                               const struct drm_connector_state *conn_state)
 {
         struct intel_dp *intel_dp = enc_to_intel_dp(&encoder->base);
-        enum port port = dp_to_dig_port(intel_dp)->port;
+        enum port port = encoder->port;
 
         intel_dp_prepare(encoder, pipe_config);
 
@@ -2977,22 +2964,21 @@ static void vlv_detach_power_sequencer(struct intel_dp *intel_dp)
          * from a port.
          */
         DRM_DEBUG_KMS("detaching pipe %c power sequencer from port %c\n",
-                      pipe_name(pipe), port_name(intel_dig_port->port));
+                      pipe_name(pipe), port_name(intel_dig_port->base.port));
         I915_WRITE(pp_on_reg, 0);
         POSTING_READ(pp_on_reg);
 
         intel_dp->pps_pipe = INVALID_PIPE;
 }
 
-static void vlv_steal_power_sequencer(struct drm_device *dev,
+static void vlv_steal_power_sequencer(struct drm_i915_private *dev_priv,
                                       enum pipe pipe)
 {
-        struct drm_i915_private *dev_priv = to_i915(dev);
         struct intel_encoder *encoder;
 
         lockdep_assert_held(&dev_priv->pps_mutex);
 
-        for_each_intel_encoder(dev, encoder) {
+        for_each_intel_encoder(&dev_priv->drm, encoder) {
                 struct intel_dp *intel_dp;
                 enum port port;
 
@@ -3001,7 +2987,7 @@ static void vlv_steal_power_sequencer(struct drm_device *dev,
                         continue;
 
                 intel_dp = enc_to_intel_dp(&encoder->base);
-                port = dp_to_dig_port(intel_dp)->port;
+                port = dp_to_dig_port(intel_dp)->base.port;
 
                 WARN(intel_dp->active_pipe == pipe,
                      "stealing pipe %c power sequencer from active (e)DP port %c\n",
@@ -3018,13 +3004,12 @@ static void vlv_steal_power_sequencer(struct drm_device *dev,
         }
 }
 
-static void vlv_init_panel_power_sequencer(struct intel_dp *intel_dp)
+static void vlv_init_panel_power_sequencer(struct intel_encoder *encoder,
+                                           const struct intel_crtc_state *crtc_state)
 {
-        struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp);
-        struct intel_encoder *encoder = &intel_dig_port->base;
-        struct drm_device *dev = encoder->base.dev;
-        struct drm_i915_private *dev_priv = to_i915(dev);
-        struct intel_crtc *crtc = to_intel_crtc(encoder->base.crtc);
+        struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
+        struct intel_dp *intel_dp = enc_to_intel_dp(&encoder->base);
+        struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
 
         lockdep_assert_held(&dev_priv->pps_mutex);
 
@@ -3044,7 +3029,7 @@ static void vlv_init_panel_power_sequencer(struct intel_dp *intel_dp)
          * We may be stealing the power
          * sequencer from another port.
          */
-        vlv_steal_power_sequencer(dev, crtc->pipe);
+        vlv_steal_power_sequencer(dev_priv, crtc->pipe);
 
         intel_dp->active_pipe = crtc->pipe;
 
@@ -3055,18 +3040,18 @@ static void vlv_init_panel_power_sequencer(struct intel_dp *intel_dp)
         intel_dp->pps_pipe = crtc->pipe;
 
         DRM_DEBUG_KMS("initializing pipe %c power sequencer for port %c\n",
-                      pipe_name(intel_dp->pps_pipe), port_name(intel_dig_port->port));
+                      pipe_name(intel_dp->pps_pipe), port_name(encoder->port));
 
         /* init power sequencer on this pipe and port */
-        intel_dp_init_panel_power_sequencer(dev, intel_dp);
-        intel_dp_init_panel_power_sequencer_registers(dev, intel_dp, true);
+        intel_dp_init_panel_power_sequencer(intel_dp);
+        intel_dp_init_panel_power_sequencer_registers(intel_dp, true);
 }
 
 static void vlv_pre_enable_dp(struct intel_encoder *encoder,
                               const struct intel_crtc_state *pipe_config,
                               const struct drm_connector_state *conn_state)
 {
-        vlv_phy_pre_encoder_enable(encoder);
+        vlv_phy_pre_encoder_enable(encoder, pipe_config);
 
         intel_enable_dp(encoder, pipe_config, conn_state);
 }
@@ -3077,14 +3062,14 @@ static void vlv_dp_pre_pll_enable(struct intel_encoder *encoder,
 {
         intel_dp_prepare(encoder, pipe_config);
 
-        vlv_phy_pre_pll_enable(encoder);
+        vlv_phy_pre_pll_enable(encoder, pipe_config);
 }
 
 static void chv_pre_enable_dp(struct intel_encoder *encoder,
                               const struct intel_crtc_state *pipe_config,
                               const struct drm_connector_state *conn_state)
 {
-        chv_phy_pre_encoder_enable(encoder);
+        chv_phy_pre_encoder_enable(encoder, pipe_config);
 
         intel_enable_dp(encoder, pipe_config, conn_state);
 
@@ -3098,14 +3083,14 @@ static void chv_dp_pre_pll_enable(struct intel_encoder *encoder,
 {
         intel_dp_prepare(encoder, pipe_config);
 
-        chv_phy_pre_pll_enable(encoder);
+        chv_phy_pre_pll_enable(encoder, pipe_config);
 }
 
 static void chv_dp_post_pll_disable(struct intel_encoder *encoder,
-                                    const struct intel_crtc_state *pipe_config,
-                                    const struct drm_connector_state *conn_state)
+                                    const struct intel_crtc_state *old_crtc_state,
+                                    const struct drm_connector_state *old_conn_state)
 {
-        chv_phy_post_pll_disable(encoder);
+        chv_phy_post_pll_disable(encoder, old_crtc_state);
 }
 
 /*
@@ -3153,7 +3138,7 @@ uint8_t
 intel_dp_voltage_max(struct intel_dp *intel_dp)
 {
         struct drm_i915_private *dev_priv = to_i915(intel_dp_to_dev(intel_dp));
-        enum port port = dp_to_dig_port(intel_dp)->port;
+        enum port port = dp_to_dig_port(intel_dp)->base.port;
 
         if (INTEL_GEN(dev_priv) >= 9) {
                 struct intel_encoder *encoder = &dp_to_dig_port(intel_dp)->base;
@@ -3172,7 +3157,7 @@ uint8_t
 intel_dp_pre_emphasis_max(struct intel_dp *intel_dp, uint8_t voltage_swing)
 {
         struct drm_i915_private *dev_priv = to_i915(intel_dp_to_dev(intel_dp));
-        enum port port = dp_to_dig_port(intel_dp)->port;
+        enum port port = dp_to_dig_port(intel_dp)->base.port;
 
         if (INTEL_GEN(dev_priv) >= 9) {
                 switch (voltage_swing & DP_TRAIN_VOLTAGE_SWING_MASK) {
@@ -3505,10 +3490,9 @@ gen7_edp_signal_levels(uint8_t train_set)
 void
 intel_dp_set_signal_levels(struct intel_dp *intel_dp)
 {
+        struct drm_i915_private *dev_priv = to_i915(intel_dp_to_dev(intel_dp));
         struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp);
-        enum port port = intel_dig_port->port;
-        struct drm_device *dev = intel_dig_port->base.base.dev;
-        struct drm_i915_private *dev_priv = to_i915(dev);
+        enum port port = intel_dig_port->base.port;
         uint32_t signal_levels, mask = 0;
         uint8_t train_set = intel_dp->train_set[0];
 
@@ -3563,10 +3547,9 @@ intel_dp_program_link_training_pattern(struct intel_dp *intel_dp,
 
 void intel_dp_set_idle_link_train(struct intel_dp *intel_dp)
 {
+        struct drm_i915_private *dev_priv = to_i915(intel_dp_to_dev(intel_dp));
         struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp);
-        struct drm_device *dev = intel_dig_port->base.base.dev;
-        struct drm_i915_private *dev_priv = to_i915(dev);
-        enum port port = intel_dig_port->port;
+        enum port port = intel_dig_port->base.port;
         uint32_t val;
 
         if (!HAS_DDI(dev_priv))
@@ -3595,13 +3578,13 @@ void intel_dp_set_idle_link_train(struct intel_dp *intel_dp)
 }
 
 static void
-intel_dp_link_down(struct intel_dp *intel_dp)
+intel_dp_link_down(struct intel_encoder *encoder,
+                   const struct intel_crtc_state *old_crtc_state)
 {
-        struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp);
-        struct intel_crtc *crtc = to_intel_crtc(intel_dig_port->base.base.crtc);
-        enum port port = intel_dig_port->port;
-        struct drm_device *dev = intel_dig_port->base.base.dev;
-        struct drm_i915_private *dev_priv = to_i915(dev);
+        struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
+        struct intel_dp *intel_dp = enc_to_intel_dp(&encoder->base);
+        struct intel_crtc *crtc = to_intel_crtc(old_crtc_state->base.crtc);
+        enum port port = encoder->port;
         uint32_t DP = intel_dp->DP;
 
         if (WARN_ON(HAS_DDI(dev_priv)))
@@ -3747,11 +3730,11 @@ intel_edp_init_dpcd(struct intel_dp *intel_dp)
         if (drm_dp_dpcd_read(&intel_dp->aux, DP_EDP_DPCD_REV,
                              intel_dp->edp_dpcd, sizeof(intel_dp->edp_dpcd)) ==
                              sizeof(intel_dp->edp_dpcd))
-                DRM_DEBUG_KMS("EDP DPCD : %*ph\n", (int) sizeof(intel_dp->edp_dpcd),
+                DRM_DEBUG_KMS("eDP DPCD: %*ph\n", (int) sizeof(intel_dp->edp_dpcd),
                               intel_dp->edp_dpcd);
 
-        /* Intermediate frequency support */
-        if (intel_dp->edp_dpcd[0] >= 0x03) { /* eDp v1.4 or higher */
+        /* Read the eDP 1.4+ supported link rates. */
+        if (intel_dp->edp_dpcd[0] >= DP_EDP_14) {
                 __le16 sink_rates[DP_MAX_SUPPORTED_RATES];
                 int i;
 
@@ -3775,6 +3758,10 @@ intel_edp_init_dpcd(struct intel_dp *intel_dp)
                 intel_dp->num_sink_rates = i;
         }
 
+        /*
+         * Use DP_LINK_RATE_SET if DP_SUPPORTED_LINK_RATES are available,
+         * default to DP_MAX_LINK_RATE and DP_LINK_BW_SET otherwise.
+         */
         if (intel_dp->num_sink_rates)
                 intel_dp->use_rate_select = true;
         else
@@ -3874,11 +3861,12 @@ intel_dp_configure_mst(struct intel_dp *intel_dp)
                                         intel_dp->is_mst);
 }
 
-static int intel_dp_sink_crc_stop(struct intel_dp *intel_dp)
+static int intel_dp_sink_crc_stop(struct intel_dp *intel_dp,
+                                  struct intel_crtc_state *crtc_state, bool disable_wa)
 {
         struct intel_digital_port *dig_port = dp_to_dig_port(intel_dp);
         struct drm_i915_private *dev_priv = to_i915(dig_port->base.base.dev);
-        struct intel_crtc *intel_crtc = to_intel_crtc(dig_port->base.base.crtc);
+        struct intel_crtc *intel_crtc = to_intel_crtc(crtc_state->base.crtc);
         u8 buf;
         int ret = 0;
         int count = 0;
@@ -3914,15 +3902,17 @@ static int intel_dp_sink_crc_stop(struct intel_dp *intel_dp)
         }
 
  out:
-        hsw_enable_ips(intel_crtc);
+        if (disable_wa)
+                hsw_enable_ips(crtc_state);
         return ret;
 }
 
-static int intel_dp_sink_crc_start(struct intel_dp *intel_dp)
+static int intel_dp_sink_crc_start(struct intel_dp *intel_dp,
+                                   struct intel_crtc_state *crtc_state)
 {
         struct intel_digital_port *dig_port = dp_to_dig_port(intel_dp);
         struct drm_i915_private *dev_priv = to_i915(dig_port->base.base.dev);
-        struct intel_crtc *intel_crtc = to_intel_crtc(dig_port->base.base.crtc);
+        struct intel_crtc *intel_crtc = to_intel_crtc(crtc_state->base.crtc);
         u8 buf;
         int ret;
 
@@ -3936,16 +3926,16 @@ static int intel_dp_sink_crc_start(struct intel_dp *intel_dp)
                 return -EIO;
 
         if (buf & DP_TEST_SINK_START) {
-                ret = intel_dp_sink_crc_stop(intel_dp);
+                ret = intel_dp_sink_crc_stop(intel_dp, crtc_state, false);
                 if (ret)
                         return ret;
         }
 
-        hsw_disable_ips(intel_crtc);
+        hsw_disable_ips(crtc_state);
 
         if (drm_dp_dpcd_writeb(&intel_dp->aux, DP_TEST_SINK,
                                buf | DP_TEST_SINK_START) < 0) {
-                hsw_enable_ips(intel_crtc);
+                hsw_enable_ips(crtc_state);
                 return -EIO;
         }
 
@@ -3953,16 +3943,16 @@ static int intel_dp_sink_crc_start(struct intel_dp *intel_dp)
         return 0;
 }
 
-int intel_dp_sink_crc(struct intel_dp *intel_dp, u8 *crc)
+int intel_dp_sink_crc(struct intel_dp *intel_dp, struct intel_crtc_state *crtc_state, u8 *crc)
 {
         struct intel_digital_port *dig_port = dp_to_dig_port(intel_dp);
         struct drm_i915_private *dev_priv = to_i915(dig_port->base.base.dev);
-        struct intel_crtc *intel_crtc = to_intel_crtc(dig_port->base.base.crtc);
+        struct intel_crtc *intel_crtc = to_intel_crtc(crtc_state->base.crtc);
         u8 buf;
         int count, ret;
         int attempts = 6;
 
-        ret = intel_dp_sink_crc_start(intel_dp);
+        ret = intel_dp_sink_crc_start(intel_dp, crtc_state);
         if (ret)
                 return ret;
 
@@ -3990,7 +3980,7 @@ int intel_dp_sink_crc(struct intel_dp *intel_dp, u8 *crc)
         }
 
 stop:
-        intel_dp_sink_crc_stop(intel_dp);
+        intel_dp_sink_crc_stop(intel_dp, crtc_state, true);
         return ret;
 }
 
@@ -4285,11 +4275,11 @@ intel_dp_retrain_link(struct intel_dp *intel_dp)
 static void
 intel_dp_check_link_status(struct intel_dp *intel_dp)
 {
+        struct drm_i915_private *dev_priv = to_i915(intel_dp_to_dev(intel_dp));
         struct intel_encoder *intel_encoder = &dp_to_dig_port(intel_dp)->base;
-        struct drm_device *dev = intel_dp_to_dev(intel_dp);
         u8 link_status[DP_LINK_STATUS_SIZE];
 
-        WARN_ON(!drm_modeset_is_locked(&dev->mode_config.connection_mutex));
+        WARN_ON(!drm_modeset_is_locked(&dev_priv->drm.mode_config.connection_mutex));
 
         if (!intel_dp_get_link_status(intel_dp, link_status)) {
                 DRM_ERROR("Failed to get link status\n");
@@ -4335,8 +4325,7 @@ intel_dp_check_link_status(struct intel_dp *intel_dp)
 static bool
 intel_dp_short_pulse(struct intel_dp *intel_dp)
 {
-        struct drm_device *dev = intel_dp_to_dev(intel_dp);
-        struct intel_encoder *intel_encoder = &dp_to_dig_port(intel_dp)->base;
+        struct drm_i915_private *dev_priv = to_i915(intel_dp_to_dev(intel_dp));
         u8 sink_irq_vector = 0;
         u8 old_sink_count = intel_dp->sink_count;
         bool ret;
@@ -4375,13 +4364,13 @@ intel_dp_short_pulse(struct intel_dp *intel_dp)
                         DRM_DEBUG_DRIVER("CP or sink specific irq unhandled\n");
         }
 
-        drm_modeset_lock(&dev->mode_config.connection_mutex, NULL);
+        drm_modeset_lock(&dev_priv->drm.mode_config.connection_mutex, NULL);
         intel_dp_check_link_status(intel_dp);
-        drm_modeset_unlock(&dev->mode_config.connection_mutex);
+        drm_modeset_unlock(&dev_priv->drm.mode_config.connection_mutex);
         if (intel_dp->compliance.test_type == DP_TEST_LINK_TRAINING) {
                 DRM_DEBUG_KMS("Link Training Compliance Test requested\n");
                 /* Send a Hotplug Uevent to userspace to start modeset */
-                drm_kms_helper_hotplug_event(intel_encoder->base.dev);
+                drm_kms_helper_hotplug_event(&dev_priv->drm);
         }
 
         return true;
@@ -4445,8 +4434,7 @@ intel_dp_detect_dpcd(struct intel_dp *intel_dp)
 static enum drm_connector_status
 edp_detect(struct intel_dp *intel_dp)
 {
-        struct drm_device *dev = intel_dp_to_dev(intel_dp);
-        struct drm_i915_private *dev_priv = to_i915(dev);
+        struct drm_i915_private *dev_priv = to_i915(intel_dp_to_dev(intel_dp));
         enum drm_connector_status status;
 
         status = intel_panel_detect(dev_priv);
@@ -4461,7 +4449,7 @@ static bool ibx_digital_port_connected(struct drm_i915_private *dev_priv,
 {
         u32 bit;
 
-        switch (port->port) {
+        switch (port->base.port) {
         case PORT_B:
                 bit = SDE_PORTB_HOTPLUG;
                 break;
@@ -4472,7 +4460,7 @@ static bool ibx_digital_port_connected(struct drm_i915_private *dev_priv,
                 bit = SDE_PORTD_HOTPLUG;
                 break;
         default:
-                MISSING_CASE(port->port);
+                MISSING_CASE(port->base.port);
                 return false;
         }
 
@@ -4484,7 +4472,7 @@ static bool cpt_digital_port_connected(struct drm_i915_private *dev_priv,
 {
         u32 bit;
 
-        switch (port->port) {
+        switch (port->base.port) {
         case PORT_B:
                 bit = SDE_PORTB_HOTPLUG_CPT;
                 break;
@@ -4495,7 +4483,7 @@ static bool cpt_digital_port_connected(struct drm_i915_private *dev_priv,
                 bit = SDE_PORTD_HOTPLUG_CPT;
                 break;
         default:
-                MISSING_CASE(port->port);
+                MISSING_CASE(port->base.port);
                 return false;
         }
 
@@ -4507,7 +4495,7 @@ static bool spt_digital_port_connected(struct drm_i915_private *dev_priv,
 {
         u32 bit;
 
-        switch (port->port) {
+        switch (port->base.port) {
         case PORT_A:
                 bit = SDE_PORTA_HOTPLUG_SPT;
                 break;
@@ -4526,7 +4514,7 @@ static bool g4x_digital_port_connected(struct drm_i915_private *dev_priv,
 {
         u32 bit;
 
-        switch (port->port) {
+        switch (port->base.port) {
         case PORT_B:
                 bit = PORTB_HOTPLUG_LIVE_STATUS_G4X;
                 break;
@@ -4537,7 +4525,7 @@ static bool g4x_digital_port_connected(struct drm_i915_private *dev_priv,
                 bit = PORTD_HOTPLUG_LIVE_STATUS_G4X;
                 break;
         default:
-                MISSING_CASE(port->port);
+                MISSING_CASE(port->base.port);
                 return false;
         }
 
@@ -4549,7 +4537,7 @@ static bool gm45_digital_port_connected(struct drm_i915_private *dev_priv,
 {
         u32 bit;
 
-        switch (port->port) {
+        switch (port->base.port) {
         case PORT_B:
                 bit = PORTB_HOTPLUG_LIVE_STATUS_GM45;
                 break;
@@ -4560,7 +4548,7 @@ static bool gm45_digital_port_connected(struct drm_i915_private *dev_priv,
                 bit = PORTD_HOTPLUG_LIVE_STATUS_GM45;
                 break;
         default:
-                MISSING_CASE(port->port);
+                MISSING_CASE(port->base.port);
                 return false;
         }
 
@@ -4570,7 +4558,7 @@ static bool gm45_digital_port_connected(struct drm_i915_private *dev_priv,
 static bool ilk_digital_port_connected(struct drm_i915_private *dev_priv,
                                        struct intel_digital_port *port)
 {
-        if (port->port == PORT_A)
+        if (port->base.port == PORT_A)
                 return I915_READ(DEISR) & DE_DP_A_HOTPLUG;
         else
                 return ibx_digital_port_connected(dev_priv, port);
@@ -4579,7 +4567,7 @@ static bool ilk_digital_port_connected(struct drm_i915_private *dev_priv,
 static bool snb_digital_port_connected(struct drm_i915_private *dev_priv,
                                        struct intel_digital_port *port)
 {
-        if (port->port == PORT_A)
+        if (port->base.port == PORT_A)
                 return I915_READ(DEISR) & DE_DP_A_HOTPLUG;
         else
                 return cpt_digital_port_connected(dev_priv, port);
@@ -4588,7 +4576,7 @@ static bool snb_digital_port_connected(struct drm_i915_private *dev_priv,
 static bool ivb_digital_port_connected(struct drm_i915_private *dev_priv,
                                        struct intel_digital_port *port)
 {
-        if (port->port == PORT_A)
+        if (port->base.port == PORT_A)
                 return I915_READ(DEISR) & DE_DP_A_HOTPLUG_IVB;
         else
                 return cpt_digital_port_connected(dev_priv, port);
@@ -4597,7 +4585,7 @@ static bool ivb_digital_port_connected(struct drm_i915_private *dev_priv,
 static bool bdw_digital_port_connected(struct drm_i915_private *dev_priv,
                                        struct intel_digital_port *port)
 {
-        if (port->port == PORT_A)
+        if (port->base.port == PORT_A)
                 return I915_READ(GEN8_DE_PORT_ISR) & GEN8_PORT_DP_A_HOTPLUG;
         else
                 return cpt_digital_port_connected(dev_priv, port);
@@ -4702,24 +4690,21 @@ intel_dp_unset_edid(struct intel_dp *intel_dp)
 }
 
 static int
-intel_dp_long_pulse(struct intel_connector *intel_connector)
+intel_dp_long_pulse(struct intel_connector *connector)
 {
-        struct drm_connector *connector = &intel_connector->base;
-        struct intel_dp *intel_dp = intel_attached_dp(connector);
-        struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp);
-        struct intel_encoder *intel_encoder = &intel_dig_port->base;
-        struct drm_device *dev = connector->dev;
+        struct drm_i915_private *dev_priv = to_i915(connector->base.dev);
+        struct intel_dp *intel_dp = intel_attached_dp(&connector->base);
         enum drm_connector_status status;
         u8 sink_irq_vector = 0;
 
-        WARN_ON(!drm_modeset_is_locked(&connector->dev->mode_config.connection_mutex));
+        WARN_ON(!drm_modeset_is_locked(&dev_priv->drm.mode_config.connection_mutex));
 
-        intel_display_power_get(to_i915(dev), intel_dp->aux_power_domain);
+        intel_display_power_get(dev_priv, intel_dp->aux_power_domain);
 
         /* Can't disconnect eDP, but you can close the lid... */
         if (intel_dp_is_edp(intel_dp))
                 status = edp_detect(intel_dp);
-        else if (intel_digital_port_connected(to_i915(dev),
+        else if (intel_digital_port_connected(dev_priv,
                                               dp_to_dig_port(intel_dp)))
                 status = intel_dp_detect_dpcd(intel_dp);
         else
@@ -4740,9 +4725,6 @@ intel_dp_long_pulse(struct intel_connector *intel_connector)
                 goto out;
         }
 
-        if (intel_encoder->type != INTEL_OUTPUT_EDP)
-                intel_encoder->type = INTEL_OUTPUT_DP;
-
         if (intel_dp->reset_link_params) {
                 /* Initial max link lane count */
                 intel_dp->max_link_lane_count = intel_dp_max_common_lane_count(intel_dp);
@@ -4793,7 +4775,7 @@ intel_dp_long_pulse(struct intel_connector *intel_connector)
         intel_dp->aux.i2c_defer_count = 0;
 
         intel_dp_set_edid(intel_dp);
-        if (intel_dp_is_edp(intel_dp) || intel_connector->detect_edid)
+        if (intel_dp_is_edp(intel_dp) || connector->detect_edid)
                 status = connector_status_connected;
         intel_dp->detect_done = true;
 
@@ -4816,7 +4798,7 @@ out:
         if (status != connector_status_connected && !intel_dp->is_mst)
                 intel_dp_unset_edid(intel_dp);
 
-        intel_display_power_put(to_i915(dev), intel_dp->aux_power_domain);
+        intel_display_power_put(dev_priv, intel_dp->aux_power_domain);
         return status;
 }
 
@@ -4859,9 +4841,6 @@ intel_dp_force(struct drm_connector *connector)
         intel_dp_set_edid(intel_dp);
 
         intel_display_power_put(dev_priv, intel_dp->aux_power_domain);
-
-        if (intel_encoder->type != INTEL_OUTPUT_EDP)
-                intel_encoder->type = INTEL_OUTPUT_DP;
 }
 
 static int intel_dp_get_modes(struct drm_connector *connector)
@@ -4986,9 +4965,7 @@ void intel_dp_encoder_suspend(struct intel_encoder *intel_encoder)
 
 static void intel_edp_panel_vdd_sanitize(struct intel_dp *intel_dp)
 {
-        struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp);
-        struct drm_device *dev = intel_dig_port->base.base.dev;
-        struct drm_i915_private *dev_priv = to_i915(dev);
+        struct drm_i915_private *dev_priv = to_i915(intel_dp_to_dev(intel_dp));
 
         lockdep_assert_held(&dev_priv->pps_mutex);
 
@@ -5041,7 +5018,7 @@ void intel_dp_encoder_reset(struct drm_encoder *encoder)
 
         if (intel_dp_is_edp(intel_dp)) {
                 /* Reinit the power sequencer, in case BIOS did something with it. */
-                intel_dp_pps_init(encoder->dev, intel_dp);
+                intel_dp_pps_init(intel_dp);
                 intel_edp_panel_vdd_sanitize(intel_dp);
         }
 
@@ -5076,14 +5053,9 @@ enum irqreturn
 intel_dp_hpd_pulse(struct intel_digital_port *intel_dig_port, bool long_hpd)
 {
         struct intel_dp *intel_dp = &intel_dig_port->dp;
-        struct drm_device *dev = intel_dig_port->base.base.dev;
-        struct drm_i915_private *dev_priv = to_i915(dev);
+        struct drm_i915_private *dev_priv = to_i915(intel_dp_to_dev(intel_dp));
         enum irqreturn ret = IRQ_NONE;
 
-        if (intel_dig_port->base.type != INTEL_OUTPUT_EDP &&
-            intel_dig_port->base.type != INTEL_OUTPUT_HDMI)
-                intel_dig_port->base.type = INTEL_OUTPUT_DP;
-
         if (long_hpd && intel_dig_port->base.type == INTEL_OUTPUT_EDP) {
                 /*
                  * vdd off can generate a long pulse on eDP which
@@ -5092,12 +5064,12 @@ intel_dp_hpd_pulse(struct intel_digital_port *intel_dig_port, bool long_hpd)
                  * "vdd off -> long hpd -> vdd on -> detect -> vdd off -> ..."
                  */
                 DRM_DEBUG_KMS("ignoring long hpd on eDP port %c\n",
-                              port_name(intel_dig_port->port));
+                              port_name(intel_dig_port->base.port));
                 return IRQ_HANDLED;
         }
 
         DRM_DEBUG_KMS("got hpd irq on port %c - %s\n",
-                      port_name(intel_dig_port->port),
+                      port_name(intel_dig_port->base.port),
                       long_hpd ? "long" : "short");
 
         if (long_hpd) {
@@ -5185,13 +5157,13 @@ static void intel_dp_init_panel_power_timestamps(struct intel_dp *intel_dp)
 }
 
 static void
-intel_pps_readout_hw_state(struct drm_i915_private *dev_priv,
-                           struct intel_dp *intel_dp, struct edp_power_seq *seq)
+intel_pps_readout_hw_state(struct intel_dp *intel_dp, struct edp_power_seq *seq)
 {
+        struct drm_i915_private *dev_priv = to_i915(intel_dp_to_dev(intel_dp));
         u32 pp_on, pp_off, pp_div = 0, pp_ctl = 0;
         struct pps_registers regs;
 
-        intel_pps_get_registers(dev_priv, intel_dp, &regs);
+        intel_pps_get_registers(intel_dp, &regs);
 
         /* Workaround: Need to write PP_CONTROL with the unlock key as
          * the very first thing. */
@@ -5235,13 +5207,12 @@ intel_pps_dump_state(const char *state_name, const struct edp_power_seq *seq)
 }
 
 static void
-intel_pps_verify_state(struct drm_i915_private *dev_priv,
-                       struct intel_dp *intel_dp)
+intel_pps_verify_state(struct intel_dp *intel_dp)
 {
         struct edp_power_seq hw;
         struct edp_power_seq *sw = &intel_dp->pps_delays;
 
-        intel_pps_readout_hw_state(dev_priv, intel_dp, &hw);
+        intel_pps_readout_hw_state(intel_dp, &hw);
 
         if (hw.t1_t3 != sw->t1_t3 || hw.t8 != sw->t8 || hw.t9 != sw->t9 ||
             hw.t10 != sw->t10 || hw.t11_t12 != sw->t11_t12) {
@@ -5252,10 +5223,9 @@ intel_pps_verify_state(struct drm_i915_private *dev_priv,
 }
 
 static void
-intel_dp_init_panel_power_sequencer(struct drm_device *dev,
-                                    struct intel_dp *intel_dp)
+intel_dp_init_panel_power_sequencer(struct intel_dp *intel_dp)
 {
-        struct drm_i915_private *dev_priv = to_i915(dev);
+        struct drm_i915_private *dev_priv = to_i915(intel_dp_to_dev(intel_dp));
         struct edp_power_seq cur, vbt, spec,
                 *final = &intel_dp->pps_delays;
 
@@ -5265,7 +5235,7 @@ intel_dp_init_panel_power_sequencer(struct drm_device *dev,
         if (final->t11_t12 != 0)
                 return;
 
-        intel_pps_readout_hw_state(dev_priv, intel_dp, &cur);
+        intel_pps_readout_hw_state(intel_dp, &cur);
 
         intel_pps_dump_state("cur", &cur);
 
@@ -5339,20 +5309,19 @@ intel_dp_init_panel_power_sequencer(struct drm_device *dev,
 }
 
 static void
-intel_dp_init_panel_power_sequencer_registers(struct drm_device *dev,
-                                              struct intel_dp *intel_dp,
+intel_dp_init_panel_power_sequencer_registers(struct intel_dp *intel_dp,
                                               bool force_disable_vdd)
 {
-        struct drm_i915_private *dev_priv = to_i915(dev);
+        struct drm_i915_private *dev_priv = to_i915(intel_dp_to_dev(intel_dp));
         u32 pp_on, pp_off, pp_div, port_sel = 0;
         int div = dev_priv->rawclk_freq / 1000;
         struct pps_registers regs;
-        enum port port = dp_to_dig_port(intel_dp)->port;
+        enum port port = dp_to_dig_port(intel_dp)->base.port;
         const struct edp_power_seq *seq = &intel_dp->pps_delays;
 
         lockdep_assert_held(&dev_priv->pps_mutex);
 
-        intel_pps_get_registers(dev_priv, intel_dp, &regs);
+        intel_pps_get_registers(intel_dp, &regs);
 
         /*
          * On some VLV machines the BIOS can leave the VDD
@@ -5424,16 +5393,15 @@ intel_dp_init_panel_power_sequencer_registers(struct drm_device *dev,
                       I915_READ(regs.pp_div));
 }
 
-static void intel_dp_pps_init(struct drm_device *dev,
-                              struct intel_dp *intel_dp)
+static void intel_dp_pps_init(struct intel_dp *intel_dp)
 {
-        struct drm_i915_private *dev_priv = to_i915(dev);
+        struct drm_i915_private *dev_priv = to_i915(intel_dp_to_dev(intel_dp));
 
         if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) {
                 vlv_initial_power_sequencer_setup(intel_dp);
         } else {
-                intel_dp_init_panel_power_sequencer(dev, intel_dp);
-                intel_dp_init_panel_power_sequencer_registers(dev, intel_dp, false);
+                intel_dp_init_panel_power_sequencer(intel_dp);
+                intel_dp_init_panel_power_sequencer_registers(intel_dp, false);
         }
 }
 
@@ -5472,7 +5440,6 @@ static void intel_dp_set_drrs_state(struct drm_i915_private *dev_priv,
 
         dig_port = dp_to_dig_port(intel_dp);
         encoder = &dig_port->base;
-        intel_crtc = to_intel_crtc(encoder->base.crtc);
 
         if (!intel_crtc) {
                 DRM_DEBUG_KMS("DRRS: intel_crtc not initialized\n");
@@ -5545,8 +5512,7 @@ static void intel_dp_set_drrs_state(struct drm_i915_private *dev_priv,
 void intel_edp_drrs_enable(struct intel_dp *intel_dp,
                            const struct intel_crtc_state *crtc_state)
 {
-        struct drm_device *dev = intel_dp_to_dev(intel_dp);
-        struct drm_i915_private *dev_priv = to_i915(dev);
+        struct drm_i915_private *dev_priv = to_i915(intel_dp_to_dev(intel_dp));
 
         if (!crtc_state->has_drrs) {
                 DRM_DEBUG_KMS("Panel doesn't support DRRS\n");
@@ -5581,8 +5547,7 @@ unlock:
 void intel_edp_drrs_disable(struct intel_dp *intel_dp,
                             const struct intel_crtc_state *old_crtc_state)
 {
-        struct drm_device *dev = intel_dp_to_dev(intel_dp);
-        struct drm_i915_private *dev_priv = to_i915(dev);
+        struct drm_i915_private *dev_priv = to_i915(intel_dp_to_dev(intel_dp));
 
         if (!old_crtc_state->has_drrs)
                 return;
@@ -5765,7 +5730,7 @@ void intel_edp_drrs_flush(struct drm_i915_private *dev_priv,
 
 /**
  * intel_dp_drrs_init - Init basic DRRS work and mutex.
- * @intel_connector: eDP connector
+ * @connector: eDP connector
  * @fixed_mode: preferred mode of panel
  *
  * This function is  called only once at driver load to initialize basic
@@ -5777,12 +5742,10 @@ void intel_edp_drrs_flush(struct drm_i915_private *dev_priv,
  * from VBT setting).
  */
 static struct drm_display_mode *
-intel_dp_drrs_init(struct intel_connector *intel_connector,
-                struct drm_display_mode *fixed_mode)
+intel_dp_drrs_init(struct intel_connector *connector,
+                   struct drm_display_mode *fixed_mode)
 {
-        struct drm_connector *connector = &intel_connector->base;
-        struct drm_device *dev = connector->dev;
-        struct drm_i915_private *dev_priv = to_i915(dev);
+        struct drm_i915_private *dev_priv = to_i915(connector->base.dev);
         struct drm_display_mode *downclock_mode = NULL;
 
         INIT_DELAYED_WORK(&dev_priv->drrs.work, intel_edp_drrs_downclock_work);
@@ -5798,8 +5761,8 @@ intel_dp_drrs_init(struct intel_connector *intel_connector,
                 return NULL;
         }
 
-        downclock_mode = intel_find_panel_downclock
-                                        (dev_priv, fixed_mode, connector);
+        downclock_mode = intel_find_panel_downclock(dev_priv, fixed_mode,
+                                                    &connector->base);
 
         if (!downclock_mode) {
                 DRM_DEBUG_KMS("Downclock mode is not found. DRRS not supported\n");
@@ -5816,11 +5779,9 @@ intel_dp_drrs_init(struct intel_connector *intel_connector,
 static bool intel_edp_init_connector(struct intel_dp *intel_dp,
                                      struct intel_connector *intel_connector)
 {
-        struct drm_connector *connector = &intel_connector->base;
-        struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp);
-        struct intel_encoder *intel_encoder = &intel_dig_port->base;
-        struct drm_device *dev = intel_encoder->base.dev;
+        struct drm_device *dev = intel_dp_to_dev(intel_dp);
         struct drm_i915_private *dev_priv = to_i915(dev);
+        struct drm_connector *connector = &intel_connector->base;
         struct drm_display_mode *fixed_mode = NULL;
         struct drm_display_mode *alt_fixed_mode = NULL;
         struct drm_display_mode *downclock_mode = NULL;
@@ -5838,7 +5799,7 @@ static bool intel_edp_init_connector(struct intel_dp *intel_dp,
          * eDP and LVDS bail out early in this case to prevent interfering
          * with an already powered-on LVDS power sequencer.
          */
-        if (intel_get_lvds_encoder(dev)) {
+        if (intel_get_lvds_encoder(&dev_priv->drm)) {
                 WARN_ON(!(HAS_PCH_IBX(dev_priv) || HAS_PCH_CPT(dev_priv)));
                 DRM_INFO("LVDS was detected, not registering eDP\n");
 
@@ -5848,7 +5809,7 @@ static bool intel_edp_init_connector(struct intel_dp *intel_dp,
         pps_lock(intel_dp);
 
         intel_dp_init_panel_power_timestamps(intel_dp);
-        intel_dp_pps_init(dev, intel_dp);
+        intel_dp_pps_init(intel_dp);
         intel_edp_panel_vdd_sanitize(intel_dp);
 
         pps_unlock(intel_dp);
@@ -5948,9 +5909,9 @@ intel_dp_init_connector_port_info(struct intel_digital_port *intel_dig_port)
         struct intel_encoder *encoder = &intel_dig_port->base;
         struct intel_dp *intel_dp = &intel_dig_port->dp;
 
-        encoder->hpd_pin = intel_hpd_pin(intel_dig_port->port);
+        encoder->hpd_pin = intel_hpd_pin(encoder->port);
 
-        switch (intel_dig_port->port) {
+        switch (encoder->port) {
         case PORT_A:
                 intel_dp->aux_power_domain = POWER_DOMAIN_AUX_A;
                 break;
@@ -5968,7 +5929,7 @@ intel_dp_init_connector_port_info(struct intel_digital_port *intel_dig_port)
                 intel_dp->aux_power_domain = POWER_DOMAIN_AUX_D;
                 break;
         default:
-                MISSING_CASE(intel_dig_port->port);
+                MISSING_CASE(encoder->port);
         }
 }
 
@@ -6004,7 +5965,7 @@ intel_dp_init_connector(struct intel_digital_port *intel_dig_port,
         struct intel_encoder *intel_encoder = &intel_dig_port->base;
         struct drm_device *dev = intel_encoder->base.dev;
         struct drm_i915_private *dev_priv = to_i915(dev);
-        enum port port = intel_dig_port->port;
+        enum port port = intel_encoder->port;
         int type;
 
         /* Initialize the work for modeset in case of link train failure */
@@ -6174,7 +6135,6 @@ bool intel_dp_init(struct drm_i915_private *dev_priv,
                 intel_encoder->disable = g4x_disable_dp;
         }
 
-        intel_dig_port->port = port;
         intel_dig_port->dp.output_reg = output_reg;
         intel_dig_port->max_lanes = 4;
 
diff --git a/drivers/gpu/drm/i915/intel_dp_mst.c b/drivers/gpu/drm/i915/intel_dp_mst.c
index 772521440a9f..c3de0918ee13 100644
--- a/drivers/gpu/drm/i915/intel_dp_mst.c
+++ b/drivers/gpu/drm/i915/intel_dp_mst.c
@@ -34,6 +34,7 @@ static bool intel_dp_mst_compute_config(struct intel_encoder *encoder,
                                         struct intel_crtc_state *pipe_config,
                                         struct drm_connector_state *conn_state)
 {
+        struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
         struct intel_dp_mst_encoder *intel_mst = enc_to_mst(&encoder->base);
         struct intel_digital_port *intel_dig_port = intel_mst->primary;
         struct intel_dp *intel_dp = &intel_dig_port->dp;
@@ -87,6 +88,12 @@ static bool intel_dp_mst_compute_config(struct intel_encoder *encoder,
 
         pipe_config->dp_m_n.tu = slots;
 
+        if (IS_GEN9_LP(dev_priv))
+                pipe_config->lane_lat_optim_mask =
+                        bxt_ddi_phy_calc_lane_lat_optim_mask(pipe_config->lane_count);
+
+        intel_ddi_compute_min_voltage_level(dev_priv, pipe_config);
+
         return true;
 }
 
@@ -142,7 +149,8 @@ static void intel_mst_disable_dp(struct intel_encoder *encoder,
                 DRM_ERROR("failed to update payload %d\n", ret);
         }
         if (old_crtc_state->has_audio)
-                intel_audio_codec_disable(encoder);
+                intel_audio_codec_disable(encoder,
+                                          old_crtc_state, old_conn_state);
 }
 
 static void intel_mst_post_disable_dp(struct intel_encoder *encoder,
@@ -172,13 +180,27 @@ static void intel_mst_post_disable_dp(struct intel_encoder *encoder,
         intel_dp->active_mst_links--;
 
         intel_mst->connector = NULL;
-        if (intel_dp->active_mst_links == 0) {
+        if (intel_dp->active_mst_links == 0)
                 intel_dig_port->base.post_disable(&intel_dig_port->base,
-                                                  NULL, NULL);
-        }
+                                                  old_crtc_state, NULL);
+
         DRM_DEBUG_KMS("active links %d\n", intel_dp->active_mst_links);
 }
 
+static void intel_mst_pre_pll_enable_dp(struct intel_encoder *encoder,
+                                        const struct intel_crtc_state *pipe_config,
+                                        const struct drm_connector_state *conn_state)
+{
+        struct intel_dp_mst_encoder *intel_mst = enc_to_mst(&encoder->base);
+        struct intel_digital_port *intel_dig_port = intel_mst->primary;
+        struct intel_dp *intel_dp = &intel_dig_port->dp;
+
+        if (intel_dp->active_mst_links == 0 &&
+            intel_dig_port->base.pre_pll_enable)
+                intel_dig_port->base.pre_pll_enable(&intel_dig_port->base,
+                                                    pipe_config, NULL);
+}
+
 static void intel_mst_pre_enable_dp(struct intel_encoder *encoder,
                                     const struct intel_crtc_state *pipe_config,
                                     const struct drm_connector_state *conn_state)
@@ -187,7 +209,7 @@ static void intel_mst_pre_enable_dp(struct intel_encoder *encoder,
         struct intel_digital_port *intel_dig_port = intel_mst->primary;
         struct intel_dp *intel_dp = &intel_dig_port->dp;
         struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
-        enum port port = intel_dig_port->port;
+        enum port port = intel_dig_port->base.port;
         struct intel_connector *connector =
                 to_intel_connector(conn_state->connector);
         int ret;
@@ -231,7 +253,7 @@ static void intel_mst_enable_dp(struct intel_encoder *encoder,
         struct intel_digital_port *intel_dig_port = intel_mst->primary;
         struct intel_dp *intel_dp = &intel_dig_port->dp;
         struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
-        enum port port = intel_dig_port->port;
+        enum port port = intel_dig_port->base.port;
         int ret;
 
         DRM_DEBUG_KMS("active links %d\n", intel_dp->active_mst_links);
@@ -265,48 +287,8 @@ static void intel_dp_mst_enc_get_config(struct intel_encoder *encoder,
 {
         struct intel_dp_mst_encoder *intel_mst = enc_to_mst(&encoder->base);
         struct intel_digital_port *intel_dig_port = intel_mst->primary;
-        struct intel_crtc *crtc = to_intel_crtc(pipe_config->base.crtc);
-        struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
-        enum transcoder cpu_transcoder = pipe_config->cpu_transcoder;
-        u32 temp, flags = 0;
-
-        pipe_config->has_audio =
-                intel_ddi_is_audio_enabled(dev_priv, crtc);
-
-        temp = I915_READ(TRANS_DDI_FUNC_CTL(cpu_transcoder));
-        if (temp & TRANS_DDI_PHSYNC)
-                flags |= DRM_MODE_FLAG_PHSYNC;
-        else
-                flags |= DRM_MODE_FLAG_NHSYNC;
-        if (temp & TRANS_DDI_PVSYNC)
-                flags |= DRM_MODE_FLAG_PVSYNC;
-        else
-                flags |= DRM_MODE_FLAG_NVSYNC;
-
-        switch (temp & TRANS_DDI_BPC_MASK) {
-        case TRANS_DDI_BPC_6:
-                pipe_config->pipe_bpp = 18;
-                break;
-        case TRANS_DDI_BPC_8:
-                pipe_config->pipe_bpp = 24;
-                break;
-        case TRANS_DDI_BPC_10:
-                pipe_config->pipe_bpp = 30;
-                break;
-        case TRANS_DDI_BPC_12:
-                pipe_config->pipe_bpp = 36;
-                break;
-        default:
-                break;
-        }
-        pipe_config->base.adjusted_mode.flags |= flags;
-
-        pipe_config->lane_count =
-                ((temp & DDI_PORT_WIDTH_MASK) >> DDI_PORT_WIDTH_SHIFT) + 1;
-
-        intel_dp_get_m_n(crtc, pipe_config);
 
-        intel_ddi_clock_get(&intel_dig_port->base, pipe_config);
+        intel_ddi_get_config(&intel_dig_port->base, pipe_config);
 }
 
 static int intel_dp_mst_get_ddc_modes(struct drm_connector *connector)
@@ -570,13 +552,14 @@ intel_dp_create_fake_mst_encoder(struct intel_digital_port *intel_dig_port, enum
 
         intel_encoder->type = INTEL_OUTPUT_DP_MST;
         intel_encoder->power_domain = intel_dig_port->base.power_domain;
-        intel_encoder->port = intel_dig_port->port;
+        intel_encoder->port = intel_dig_port->base.port;
         intel_encoder->crtc_mask = 0x7;
         intel_encoder->cloneable = 0;
 
         intel_encoder->compute_config = intel_dp_mst_compute_config;
         intel_encoder->disable = intel_mst_disable_dp;
         intel_encoder->post_disable = intel_mst_post_disable_dp;
+        intel_encoder->pre_pll_enable = intel_mst_pre_pll_enable_dp;
         intel_encoder->pre_enable = intel_mst_pre_enable_dp;
         intel_encoder->enable = intel_mst_enable_dp;
         intel_encoder->get_hw_state = intel_dp_mst_enc_get_hw_state;
diff --git a/drivers/gpu/drm/i915/intel_dpio_phy.c b/drivers/gpu/drm/i915/intel_dpio_phy.c
index de38d014ed39..76473e9836c6 100644
--- a/drivers/gpu/drm/i915/intel_dpio_phy.c
+++ b/drivers/gpu/drm/i915/intel_dpio_phy.c
@@ -466,21 +466,21 @@ void bxt_ddi_phy_init(struct drm_i915_private *dev_priv, enum dpio_phy phy)
 
         lockdep_assert_held(&dev_priv->power_domains.lock);
 
-        if (rcomp_phy != -1) {
+        was_enabled = true;
+        if (rcomp_phy != -1)
                 was_enabled = bxt_ddi_phy_is_enabled(dev_priv, rcomp_phy);
 
-                /*
-                 * We need to copy the GRC calibration value from rcomp_phy,
-                 * so make sure it's powered up.
-                 */
-                if (!was_enabled)
-                        _bxt_ddi_phy_init(dev_priv, rcomp_phy);
-        }
+        /*
+         * We need to copy the GRC calibration value from rcomp_phy,
+         * so make sure it's powered up.
+         */
+        if (!was_enabled)
+                _bxt_ddi_phy_init(dev_priv, rcomp_phy);
 
         _bxt_ddi_phy_init(dev_priv, phy);
 
-        if (rcomp_phy != -1 && !was_enabled)
-                bxt_ddi_phy_uninit(dev_priv, phy_info->rcomp_phy);
+        if (!was_enabled)
+                bxt_ddi_phy_uninit(dev_priv, rcomp_phy);
 }
 
 static bool __printf(6, 7)
@@ -567,8 +567,7 @@ bool bxt_ddi_phy_verify_state(struct drm_i915_private *dev_priv,
 }
 
 uint8_t
-bxt_ddi_phy_calc_lane_lat_optim_mask(struct intel_encoder *encoder,
-                                     uint8_t lane_count)
+bxt_ddi_phy_calc_lane_lat_optim_mask(uint8_t lane_count)
 {
         switch (lane_count) {
         case 1:
@@ -587,9 +586,8 @@ bxt_ddi_phy_calc_lane_lat_optim_mask(struct intel_encoder *encoder,
 void bxt_ddi_phy_set_lane_optim_mask(struct intel_encoder *encoder,
                                      uint8_t lane_lat_optim_mask)
 {
-        struct intel_digital_port *dport = enc_to_dig_port(&encoder->base);
-        struct drm_i915_private *dev_priv = to_i915(dport->base.base.dev);
-        enum port port = dport->port;
+        struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
+        enum port port = encoder->port;
         enum dpio_phy phy;
         enum dpio_channel ch;
         int lane;
@@ -614,9 +612,8 @@ void bxt_ddi_phy_set_lane_optim_mask(struct intel_encoder *encoder,
 uint8_t
 bxt_ddi_phy_get_lane_lat_optim_mask(struct intel_encoder *encoder)
 {
-        struct intel_digital_port *dport = enc_to_dig_port(&encoder->base);
-        struct drm_i915_private *dev_priv = to_i915(dport->base.base.dev);
-        enum port port = dport->port;
+        struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
+        enum port port = encoder->port;
         enum dpio_phy phy;
         enum dpio_channel ch;
         int lane;
@@ -642,7 +639,7 @@ void chv_set_phy_signal_level(struct intel_encoder *encoder,
 {
         struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
         struct intel_digital_port *dport = enc_to_dig_port(&encoder->base);
-        struct intel_crtc *intel_crtc = to_intel_crtc(dport->base.base.crtc);
+        struct intel_crtc *intel_crtc = to_intel_crtc(encoder->base.crtc);
         enum dpio_channel ch = vlv_dport_to_channel(dport);
         enum pipe pipe = intel_crtc->pipe;
         u32 val;
@@ -734,11 +731,12 @@ void chv_set_phy_signal_level(struct intel_encoder *encoder,
 }
 
 void chv_data_lane_soft_reset(struct intel_encoder *encoder,
+                              const struct intel_crtc_state *crtc_state,
                               bool reset)
 {
         struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
         enum dpio_channel ch = vlv_dport_to_channel(enc_to_dig_port(&encoder->base));
-        struct intel_crtc *crtc = to_intel_crtc(encoder->base.crtc);
+        struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
         enum pipe pipe = crtc->pipe;
         uint32_t val;
 
@@ -777,17 +775,16 @@ void chv_data_lane_soft_reset(struct intel_encoder *encoder,
         }
 }
 
-void chv_phy_pre_pll_enable(struct intel_encoder *encoder)
+void chv_phy_pre_pll_enable(struct intel_encoder *encoder,
+                            const struct intel_crtc_state *crtc_state)
 {
         struct intel_digital_port *dport = enc_to_dig_port(&encoder->base);
-        struct drm_device *dev = encoder->base.dev;
-        struct drm_i915_private *dev_priv = to_i915(dev);
-        struct intel_crtc *intel_crtc =
-                to_intel_crtc(encoder->base.crtc);
+        struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
+        struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
         enum dpio_channel ch = vlv_dport_to_channel(dport);
-        enum pipe pipe = intel_crtc->pipe;
+        enum pipe pipe = crtc->pipe;
         unsigned int lane_mask =
-                intel_dp_unused_lane_mask(intel_crtc->config->lane_count);
+                intel_dp_unused_lane_mask(crtc_state->lane_count);
         u32 val;
 
         /*
@@ -803,7 +800,7 @@ void chv_phy_pre_pll_enable(struct intel_encoder *encoder)
         mutex_lock(&dev_priv->sb_lock);
 
         /* Assert data lane reset */
-        chv_data_lane_soft_reset(encoder, true);
+        chv_data_lane_soft_reset(encoder, crtc_state, true);
 
         /* program left/right clock distribution */
         if (pipe != PIPE_B) {
@@ -833,7 +830,7 @@ void chv_phy_pre_pll_enable(struct intel_encoder *encoder)
                 val |= CHV_PCS_USEDCLKCHANNEL;
         vlv_dpio_write(dev_priv, pipe, VLV_PCS01_DW8(ch), val);
 
-        if (intel_crtc->config->lane_count > 2) {
+        if (crtc_state->lane_count > 2) {
                 val = vlv_dpio_read(dev_priv, pipe, VLV_PCS23_DW8(ch));
                 val |= CHV_PCS_USEDCLKCHANNEL_OVRRIDE;
                 if (pipe != PIPE_B)
@@ -858,16 +855,15 @@ void chv_phy_pre_pll_enable(struct intel_encoder *encoder)
         mutex_unlock(&dev_priv->sb_lock);
 }
 
-void chv_phy_pre_encoder_enable(struct intel_encoder *encoder)
+void chv_phy_pre_encoder_enable(struct intel_encoder *encoder,
+                                const struct intel_crtc_state *crtc_state)
 {
         struct intel_dp *intel_dp = enc_to_intel_dp(&encoder->base);
         struct intel_digital_port *dport = dp_to_dig_port(intel_dp);
-        struct drm_device *dev = encoder->base.dev;
-        struct drm_i915_private *dev_priv = to_i915(dev);
-        struct intel_crtc *intel_crtc =
-                to_intel_crtc(encoder->base.crtc);
+        struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
+        struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
         enum dpio_channel ch = vlv_dport_to_channel(dport);
-        int pipe = intel_crtc->pipe;
+        enum pipe pipe = crtc->pipe;
         int data, i, stagger;
         u32 val;
 
@@ -878,16 +874,16 @@ void chv_phy_pre_encoder_enable(struct intel_encoder *encoder)
         val &= ~DPIO_LANEDESKEW_STRAP_OVRD;
         vlv_dpio_write(dev_priv, pipe, VLV_PCS01_DW11(ch), val);
 
-        if (intel_crtc->config->lane_count > 2) {
+        if (crtc_state->lane_count > 2) {
                 val = vlv_dpio_read(dev_priv, pipe, VLV_PCS23_DW11(ch));
                 val &= ~DPIO_LANEDESKEW_STRAP_OVRD;
                 vlv_dpio_write(dev_priv, pipe, VLV_PCS23_DW11(ch), val);
         }
 
         /* Program Tx lane latency optimal setting*/
-        for (i = 0; i < intel_crtc->config->lane_count; i++) {
+        for (i = 0; i < crtc_state->lane_count; i++) {
                 /* Set the upar bit */
-                if (intel_crtc->config->lane_count == 1)
+                if (crtc_state->lane_count == 1)
                         data = 0x0;
                 else
                         data = (i == 1) ? 0x0 : 0x1;
@@ -896,13 +892,13 @@ void chv_phy_pre_encoder_enable(struct intel_encoder *encoder)
         }
 
         /* Data lane stagger programming */
-        if (intel_crtc->config->port_clock > 270000)
+        if (crtc_state->port_clock > 270000)
                 stagger = 0x18;
-        else if (intel_crtc->config->port_clock > 135000)
+        else if (crtc_state->port_clock > 135000)
                 stagger = 0xd;
-        else if (intel_crtc->config->port_clock > 67500)
+        else if (crtc_state->port_clock > 67500)
                 stagger = 0x7;
-        else if (intel_crtc->config->port_clock > 33750)
+        else if (crtc_state->port_clock > 33750)
                 stagger = 0x4;
         else
                 stagger = 0x2;
@@ -911,7 +907,7 @@ void chv_phy_pre_encoder_enable(struct intel_encoder *encoder)
         val |= DPIO_TX2_STAGGER_MASK(0x1f);
         vlv_dpio_write(dev_priv, pipe, VLV_PCS01_DW11(ch), val);
 
-        if (intel_crtc->config->lane_count > 2) {
+        if (crtc_state->lane_count > 2) {
                 val = vlv_dpio_read(dev_priv, pipe, VLV_PCS23_DW11(ch));
                 val |= DPIO_TX2_STAGGER_MASK(0x1f);
                 vlv_dpio_write(dev_priv, pipe, VLV_PCS23_DW11(ch), val);
@@ -924,7 +920,7 @@ void chv_phy_pre_encoder_enable(struct intel_encoder *encoder)
                        DPIO_TX1_STAGGER_MULT(6) |
                        DPIO_TX2_STAGGER_MULT(0));
 
-        if (intel_crtc->config->lane_count > 2) {
+        if (crtc_state->lane_count > 2) {
                 vlv_dpio_write(dev_priv, pipe, VLV_PCS23_DW12(ch),
                                DPIO_LANESTAGGER_STRAP(stagger) |
                                DPIO_LANESTAGGER_STRAP_OVRD |
@@ -934,7 +930,7 @@ void chv_phy_pre_encoder_enable(struct intel_encoder *encoder)
         }
 
         /* Deassert data lane reset */
-        chv_data_lane_soft_reset(encoder, false);
+        chv_data_lane_soft_reset(encoder, crtc_state, false);
 
         mutex_unlock(&dev_priv->sb_lock);
 }
@@ -950,10 +946,11 @@ void chv_phy_release_cl2_override(struct intel_encoder *encoder)
         }
 }
 
-void chv_phy_post_pll_disable(struct intel_encoder *encoder)
+void chv_phy_post_pll_disable(struct intel_encoder *encoder,
+                              const struct intel_crtc_state *old_crtc_state)
 {
         struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
-        enum pipe pipe = to_intel_crtc(encoder->base.crtc)->pipe;
+        enum pipe pipe = to_intel_crtc(old_crtc_state->base.crtc)->pipe;
         u32 val;
 
         mutex_lock(&dev_priv->sb_lock);
@@ -991,7 +988,7 @@ void vlv_set_phy_signal_level(struct intel_encoder *encoder,
         struct intel_crtc *intel_crtc = to_intel_crtc(encoder->base.crtc);
         struct intel_digital_port *dport = enc_to_dig_port(&encoder->base);
         enum dpio_channel port = vlv_dport_to_channel(dport);
-        int pipe = intel_crtc->pipe;
+        enum pipe pipe = intel_crtc->pipe;
 
         mutex_lock(&dev_priv->sb_lock);
         vlv_dpio_write(dev_priv, pipe, VLV_TX_DW5(port), 0x00000000);
@@ -1009,15 +1006,14 @@ void vlv_set_phy_signal_level(struct intel_encoder *encoder,
         mutex_unlock(&dev_priv->sb_lock);
 }
 
-void vlv_phy_pre_pll_enable(struct intel_encoder *encoder)
+void vlv_phy_pre_pll_enable(struct intel_encoder *encoder,
+                            const struct intel_crtc_state *crtc_state)
 {
         struct intel_digital_port *dport = enc_to_dig_port(&encoder->base);
-        struct drm_device *dev = encoder->base.dev;
-        struct drm_i915_private *dev_priv = to_i915(dev);
-        struct intel_crtc *intel_crtc =
-                to_intel_crtc(encoder->base.crtc);
+        struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
+        struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
         enum dpio_channel port = vlv_dport_to_channel(dport);
-        int pipe = intel_crtc->pipe;
+        enum pipe pipe = crtc->pipe;
 
         /* Program Tx lane resets to default */
         mutex_lock(&dev_priv->sb_lock);
@@ -1037,15 +1033,15 @@ void vlv_phy_pre_pll_enable(struct intel_encoder *encoder)
         mutex_unlock(&dev_priv->sb_lock);
 }
 
-void vlv_phy_pre_encoder_enable(struct intel_encoder *encoder)
+void vlv_phy_pre_encoder_enable(struct intel_encoder *encoder,
+                                const struct intel_crtc_state *crtc_state)
 {
         struct intel_dp *intel_dp = enc_to_intel_dp(&encoder->base);
         struct intel_digital_port *dport = dp_to_dig_port(intel_dp);
-        struct drm_device *dev = encoder->base.dev;
-        struct drm_i915_private *dev_priv = to_i915(dev);
-        struct intel_crtc *intel_crtc = to_intel_crtc(encoder->base.crtc);
+        struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
+        struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
         enum dpio_channel port = vlv_dport_to_channel(dport);
-        int pipe = intel_crtc->pipe;
+        enum pipe pipe = crtc->pipe;
         u32 val;
 
         mutex_lock(&dev_priv->sb_lock);
@@ -1067,14 +1063,14 @@ void vlv_phy_pre_encoder_enable(struct intel_encoder *encoder)
         mutex_unlock(&dev_priv->sb_lock);
 }
 
-void vlv_phy_reset_lanes(struct intel_encoder *encoder)
+void vlv_phy_reset_lanes(struct intel_encoder *encoder,
+                         const struct intel_crtc_state *old_crtc_state)
 {
         struct intel_digital_port *dport = enc_to_dig_port(&encoder->base);
         struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
-        struct intel_crtc *intel_crtc =
-                to_intel_crtc(encoder->base.crtc);
+        struct intel_crtc *crtc = to_intel_crtc(old_crtc_state->base.crtc);
         enum dpio_channel port = vlv_dport_to_channel(dport);
-        int pipe = intel_crtc->pipe;
+        enum pipe pipe = crtc->pipe;
 
         mutex_lock(&dev_priv->sb_lock);
         vlv_dpio_write(dev_priv, pipe, VLV_PCS_DW0(port), 0x00000000);
diff --git a/drivers/gpu/drm/i915/intel_dpll_mgr.c b/drivers/gpu/drm/i915/intel_dpll_mgr.c
index df808a94c511..51c5ae4e9116 100644
--- a/drivers/gpu/drm/i915/intel_dpll_mgr.c
+++ b/drivers/gpu/drm/i915/intel_dpll_mgr.c
@@ -813,15 +813,11 @@ hsw_get_dpll(struct intel_crtc *crtc, struct intel_crtc_state *crtc_state,
         memset(&crtc_state->dpll_hw_state, 0,
                sizeof(crtc_state->dpll_hw_state));
 
-        if (encoder->type == INTEL_OUTPUT_HDMI) {
+        if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_HDMI)) {
                 pll = hsw_ddi_hdmi_get_dpll(clock, crtc, crtc_state);
-
-        } else if (encoder->type == INTEL_OUTPUT_DP ||
-                   encoder->type == INTEL_OUTPUT_DP_MST ||
-                   encoder->type == INTEL_OUTPUT_EDP) {
+        } else if (intel_crtc_has_dp_encoder(crtc_state)) {
                 pll = hsw_ddi_dp_get_dpll(encoder, clock);
-
-        } else if (encoder->type == INTEL_OUTPUT_ANALOG) {
+        } else if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_ANALOG)) {
                 if (WARN_ON(crtc_state->port_clock / 2 != 135000))
                         return NULL;
 
@@ -1369,15 +1365,13 @@ skl_get_dpll(struct intel_crtc *crtc, struct intel_crtc_state *crtc_state,
 
         memset(&dpll_hw_state, 0, sizeof(dpll_hw_state));
 
-        if (encoder->type == INTEL_OUTPUT_HDMI) {
+        if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_HDMI)) {
                 bret = skl_ddi_hdmi_pll_dividers(crtc, crtc_state, clock);
                 if (!bret) {
                         DRM_DEBUG_KMS("Could not get HDMI pll dividers.\n");
                         return NULL;
                 }
-        } else if (encoder->type == INTEL_OUTPUT_DP ||
-                   encoder->type == INTEL_OUTPUT_DP_MST ||
-                   encoder->type == INTEL_OUTPUT_EDP) {
+        } else if (intel_crtc_has_dp_encoder(crtc_state)) {
                 bret = skl_ddi_dp_set_dpll_hw_state(clock, &dpll_hw_state);
                 if (!bret) {
                         DRM_DEBUG_KMS("Could not set DP dpll HW state.\n");
@@ -1388,7 +1382,7 @@ skl_get_dpll(struct intel_crtc *crtc, struct intel_crtc_state *crtc_state,
                 return NULL;
         }
 
-        if (encoder->type == INTEL_OUTPUT_EDP)
+        if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_EDP))
                 pll = intel_find_shared_dpll(crtc, crtc_state,
                                              DPLL_ID_SKL_DPLL0,
                                              DPLL_ID_SKL_DPLL0);
@@ -1808,18 +1802,15 @@ bxt_get_dpll(struct intel_crtc *crtc,
 {
         struct intel_dpll_hw_state dpll_hw_state = { };
         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
-        struct intel_digital_port *intel_dig_port;
         struct intel_shared_dpll *pll;
         int i, clock = crtc_state->port_clock;
 
-        if (encoder->type == INTEL_OUTPUT_HDMI &&
+        if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_HDMI) &&
             !bxt_ddi_hdmi_set_dpll_hw_state(crtc, crtc_state, clock,
                                             &dpll_hw_state))
                 return NULL;
 
-        if ((encoder->type == INTEL_OUTPUT_DP ||
-             encoder->type == INTEL_OUTPUT_EDP ||
-             encoder->type == INTEL_OUTPUT_DP_MST) &&
+        if (intel_crtc_has_dp_encoder(crtc_state) &&
             !bxt_ddi_dp_set_dpll_hw_state(clock, &dpll_hw_state))
                 return NULL;
 
@@ -1828,15 +1819,8 @@ bxt_get_dpll(struct intel_crtc *crtc,
 
         crtc_state->dpll_hw_state = dpll_hw_state;
 
-        if (encoder->type == INTEL_OUTPUT_DP_MST) {
-                struct intel_dp_mst_encoder *intel_mst = enc_to_mst(&encoder->base);
-
-                intel_dig_port = intel_mst->primary;
-        } else
-                intel_dig_port = enc_to_dig_port(&encoder->base);
-
         /* 1:1 mapping between ports and PLLs */
-        i = (enum intel_dpll_id) intel_dig_port->port;
+        i = (enum intel_dpll_id) encoder->port;
         pll = intel_get_shared_dpll_by_id(dev_priv, i);
 
         DRM_DEBUG_KMS("[CRTC:%d:%s] using pre-allocated %s\n",
@@ -2008,8 +1992,8 @@ static void cnl_ddi_pll_enable(struct drm_i915_private *dev_priv,
          * requirement, follow the Display Voltage Frequency Switching
          * Sequence Before Frequency Change
          *
-         * FIXME: (DVFS) is used to adjust the display voltage to match the
-         * display clock frequencies
+         * Note: DVFS is actually handled via the cdclk code paths,
+         * hence we do nothing here.
          */
 
         /* 6. Enable DPLL in DPLL_ENABLE. */
@@ -2030,8 +2014,8 @@ static void cnl_ddi_pll_enable(struct drm_i915_private *dev_priv,
          * requirement, follow the Display Voltage Frequency Switching
          * Sequence After Frequency Change
          *
-         * FIXME: (DVFS) is used to adjust the display voltage to match the
-         * display clock frequencies
+         * Note: DVFS is actually handled via the cdclk code paths,
+         * hence we do nothing here.
          */
 
         /*
@@ -2055,8 +2039,8 @@ static void cnl_ddi_pll_disable(struct drm_i915_private *dev_priv,
          * requirement, follow the Display Voltage Frequency Switching
          * Sequence Before Frequency Change
          *
-         * FIXME: (DVFS) is used to adjust the display voltage to match the
-         * display clock frequencies
+         * Note: DVFS is actually handled via the cdclk code paths,
+         * hence we do nothing here.
          */
 
         /* 3. Disable DPLL through DPLL_ENABLE. */
@@ -2077,8 +2061,8 @@ static void cnl_ddi_pll_disable(struct drm_i915_private *dev_priv,
          * requirement, follow the Display Voltage Frequency Switching
          * Sequence After Frequency Change
          *
-         * FIXME: (DVFS) is used to adjust the display voltage to match the
-         * display clock frequencies
+         * Note: DVFS is actually handled via the cdclk code paths,
+         * hence we do nothing here.
          */
 
         /* 6. Disable DPLL power in DPLL_ENABLE. */
@@ -2126,10 +2110,8 @@ out:
         return ret;
 }
 
-static void cnl_wrpll_get_multipliers(unsigned int bestdiv,
-                                      unsigned int *pdiv,
-                                      unsigned int *qdiv,
-                                      unsigned int *kdiv)
+static void cnl_wrpll_get_multipliers(int bestdiv, int *pdiv,
+                                      int *qdiv, int *kdiv)
 {
         /* even dividers */
         if (bestdiv % 2 == 0) {
@@ -2167,10 +2149,12 @@ static void cnl_wrpll_get_multipliers(unsigned int bestdiv,
         }
 }
 
-static void cnl_wrpll_params_populate(struct skl_wrpll_params *params, uint32_t dco_freq,
-                                      uint32_t ref_freq, uint32_t pdiv, uint32_t qdiv,
-                                      uint32_t kdiv)
+static void cnl_wrpll_params_populate(struct skl_wrpll_params *params,
+                                      u32 dco_freq, u32 ref_freq,
+                                      int pdiv, int qdiv, int kdiv)
 {
+        u32 dco;
+
         switch (kdiv) {
         case 1:
                 params->kdiv = 1;
@@ -2202,39 +2186,35 @@ static void cnl_wrpll_params_populate(struct skl_wrpll_params *params, uint32_t
                 WARN(1, "Incorrect PDiv\n");
         }
 
-        if (kdiv != 2)
-                qdiv = 1;
+        WARN_ON(kdiv != 2 && qdiv != 1);
 
         params->qdiv_ratio = qdiv;
         params->qdiv_mode = (qdiv == 1) ? 0 : 1;
 
-        params->dco_integer = div_u64(dco_freq, ref_freq);
-        params->dco_fraction = div_u64((div_u64((uint64_t)dco_freq<<15, (uint64_t)ref_freq) -
-                                        ((uint64_t)params->dco_integer<<15)) * 0x8000, 0x8000);
+        dco = div_u64((u64)dco_freq << 15, ref_freq);
+
+        params->dco_integer = dco >> 15;
+        params->dco_fraction = dco & 0x7fff;
 }
 
 static bool
-cnl_ddi_calculate_wrpll(int clock /* in Hz */,
+cnl_ddi_calculate_wrpll(int clock,
                         struct drm_i915_private *dev_priv,
                         struct skl_wrpll_params *wrpll_params)
 {
-        uint64_t afe_clock = clock * 5 / KHz(1); /* clocks in kHz */
-        unsigned int dco_min = 7998 * KHz(1);
-        unsigned int dco_max = 10000 * KHz(1);
-        unsigned int dco_mid = (dco_min + dco_max) / 2;
-
+        u32 afe_clock = clock * 5;
+        u32 dco_min = 7998000;
+        u32 dco_max = 10000000;
+        u32 dco_mid = (dco_min + dco_max) / 2;
         static const int dividers[] = {  2,  4,  6,  8, 10, 12,  14,  16,
                                          18, 20, 24, 28, 30, 32,  36,  40,
                                          42, 44, 48, 50, 52, 54,  56,  60,
                                          64, 66, 68, 70, 72, 76,  78,  80,
                                          84, 88, 90, 92, 96, 98, 100, 102,
                                           3,  5,  7,  9, 15, 21 };
-        unsigned int d, dco;
-        unsigned int dco_centrality = 0;
-        unsigned int best_dco_centrality = 999999;
-        unsigned int best_div = 0;
-        unsigned int best_dco = 0;
-        unsigned int pdiv = 0, qdiv = 0, kdiv = 0;
+        u32 dco, best_dco = 0, dco_centrality = 0;
+        u32 best_dco_centrality = U32_MAX; /* Spec meaning of 999999 MHz */
+        int d, best_div = 0, pdiv = 0, qdiv = 0, kdiv = 0;
 
         for (d = 0; d < ARRAY_SIZE(dividers); d++) {
                 dco = afe_clock * dividers[d];
@@ -2271,7 +2251,7 @@ static bool cnl_ddi_hdmi_pll_dividers(struct intel_crtc *crtc,
 
         cfgcr0 = DPLL_CFGCR0_HDMI_MODE;
 
-        if (!cnl_ddi_calculate_wrpll(clock * 1000, dev_priv, &wrpll_params))
+        if (!cnl_ddi_calculate_wrpll(clock, dev_priv, &wrpll_params))
                 return false;
 
         cfgcr0 |= DPLL_CFGCR0_DCO_FRACTION(wrpll_params.dco_fraction) |
@@ -2281,7 +2261,6 @@ static bool cnl_ddi_hdmi_pll_dividers(struct intel_crtc *crtc,
                 DPLL_CFGCR1_QDIV_MODE(wrpll_params.qdiv_mode) |
                 DPLL_CFGCR1_KDIV(wrpll_params.kdiv) |
                 DPLL_CFGCR1_PDIV(wrpll_params.pdiv) |
-                wrpll_params.central_freq |
                 DPLL_CFGCR1_CENTRAL_FREQ;
 
         memset(&crtc_state->dpll_hw_state, 0,
@@ -2345,15 +2324,13 @@ cnl_get_dpll(struct intel_crtc *crtc, struct intel_crtc_state *crtc_state,
 
         memset(&dpll_hw_state, 0, sizeof(dpll_hw_state));
 
-        if (encoder->type == INTEL_OUTPUT_HDMI) {
+        if (intel_crtc_has_type(crtc_state, INTEL_OUTPUT_HDMI)) {
                 bret = cnl_ddi_hdmi_pll_dividers(crtc, crtc_state, clock);
                 if (!bret) {
                         DRM_DEBUG_KMS("Could not get HDMI pll dividers.\n");
                         return NULL;
                 }
-        } else if (encoder->type == INTEL_OUTPUT_DP ||
-                   encoder->type == INTEL_OUTPUT_DP_MST ||
-                   encoder->type == INTEL_OUTPUT_EDP) {
+        } else if (intel_crtc_has_dp_encoder(crtc_state)) {
                 bret = cnl_ddi_dp_set_dpll_hw_state(clock, &dpll_hw_state);
                 if (!bret) {
                         DRM_DEBUG_KMS("Could not set DP dpll HW state.\n");
@@ -2361,8 +2338,8 @@ cnl_get_dpll(struct intel_crtc *crtc, struct intel_crtc_state *crtc_state,
                 }
                 crtc_state->dpll_hw_state = dpll_hw_state;
         } else {
-                DRM_DEBUG_KMS("Skip DPLL setup for encoder %d\n",
-                              encoder->type);
+                DRM_DEBUG_KMS("Skip DPLL setup for output_types 0x%x\n",
+                              crtc_state->output_types);
                 return NULL;
         }
 
diff --git a/drivers/gpu/drm/i915/intel_drv.h b/drivers/gpu/drm/i915/intel_drv.h
index 6c7f8bca574e..bf8b057f72a6 100644
--- a/drivers/gpu/drm/i915/intel_drv.h
+++ b/drivers/gpu/drm/i915/intel_drv.h
@@ -47,14 +47,11 @@
  * contexts. Note that it's important that we check the condition again after
  * having timed out, since the timeout could be due to preemption or similar and
  * we've never had a chance to check the condition before the timeout.
- *
- * TODO: When modesetting has fully transitioned to atomic, the below
- * drm_can_sleep() can be removed and in_atomic()/!in_atomic() asserts
- * added.
  */
 #define _wait_for(COND, US, W) ({ \
         unsigned long timeout__ = jiffies + usecs_to_jiffies(US) + 1;   \
         int ret__;                                                      \
+        might_sleep();                                                  \
         for (;;) {                                                      \
                 bool expired__ = time_after(jiffies, timeout__);        \
                 if (COND) {                                             \
@@ -65,11 +62,7 @@
                         ret__ = -ETIMEDOUT;                             \
                         break;                                          \
                 }                                                       \
-                if ((W) && drm_can_sleep()) {                           \
-                        usleep_range((W), (W)*2);                       \
-                } else {                                                \
-                        cpu_relax();                                    \
-                }                                                       \
+                usleep_range((W), (W) * 2);                             \
         }                                                               \
         ret__;                                                          \
 })
@@ -173,7 +166,7 @@ enum intel_output_type {
         INTEL_OUTPUT_DP = 7,
         INTEL_OUTPUT_EDP = 8,
         INTEL_OUTPUT_DSI = 9,
-        INTEL_OUTPUT_UNKNOWN = 10,
+        INTEL_OUTPUT_DDI = 10,
         INTEL_OUTPUT_DP_MST = 11,
 };
 
@@ -216,6 +209,9 @@ struct intel_encoder {
         enum port port;
         unsigned int cloneable;
         void (*hot_plug)(struct intel_encoder *);
+        enum intel_output_type (*compute_output_type)(struct intel_encoder *,
+                                                      struct intel_crtc_state *,
+                                                      struct drm_connector_state *);
         bool (*compute_config)(struct intel_encoder *,
                                struct intel_crtc_state *,
                                struct drm_connector_state *);
@@ -386,6 +382,8 @@ struct intel_atomic_state {
         unsigned int active_crtcs;
         /* minimum acceptable cdclk for each pipe */
         int min_cdclk[I915_MAX_PIPES];
+        /* minimum acceptable voltage level for each pipe */
+        u8 min_voltage_level[I915_MAX_PIPES];
 
         struct intel_shared_dpll_state shared_dpll[I915_NUM_PLLS];
 
@@ -420,6 +418,9 @@ struct intel_plane_state {
         /* plane control register */
         u32 ctl;
 
+        /* plane color control register */
+        u32 color_ctl;
+
         /*
          * scaler_id
          *    = -1 : not using a scaler
@@ -738,6 +739,9 @@ struct intel_crtc_state {
          */
         uint8_t lane_lat_optim_mask;
 
+        /* minimum acceptable voltage level */
+        u8 min_voltage_level;
+
         /* Panel fitter controls for gen2-gen4 + VLV */
         struct {
                 u32 control;
@@ -1048,7 +1052,6 @@ struct intel_lspcon {
 
 struct intel_digital_port {
         struct intel_encoder base;
-        enum port port;
         u32 saved_port_bits;
         struct intel_dp dp;
         struct intel_hdmi hdmi;
@@ -1080,7 +1083,7 @@ struct intel_dp_mst_encoder {
 static inline enum dpio_channel
 vlv_dport_to_channel(struct intel_digital_port *dport)
 {
-        switch (dport->port) {
+        switch (dport->base.port) {
         case PORT_B:
         case PORT_D:
                 return DPIO_CH0;
@@ -1094,7 +1097,7 @@ vlv_dport_to_channel(struct intel_digital_port *dport)
 static inline enum dpio_phy
 vlv_dport_to_phy(struct intel_digital_port *dport)
 {
-        switch (dport->port) {
+        switch (dport->base.port) {
         case PORT_B:
         case PORT_C:
                 return DPIO_PHY0;
@@ -1147,7 +1150,7 @@ enc_to_dig_port(struct drm_encoder *encoder)
         struct intel_encoder *intel_encoder = to_intel_encoder(encoder);
 
         switch (intel_encoder->type) {
-        case INTEL_OUTPUT_UNKNOWN:
+        case INTEL_OUTPUT_DDI:
                 WARN_ON(!HAS_DDI(to_i915(encoder->dev)));
         case INTEL_OUTPUT_DP:
         case INTEL_OUTPUT_EDP:
@@ -1271,7 +1274,6 @@ void intel_ddi_fdi_post_disable(struct intel_encoder *intel_encoder,
 void hsw_fdi_link_train(struct intel_crtc *crtc,
                         const struct intel_crtc_state *crtc_state);
 void intel_ddi_init(struct drm_i915_private *dev_priv, enum port port);
-enum port intel_ddi_get_encoder_port(struct intel_encoder *intel_encoder);
 bool intel_ddi_get_hw_state(struct intel_encoder *encoder, enum pipe *pipe);
 void intel_ddi_enable_transcoder_func(const struct intel_crtc_state *crtc_state);
 void intel_ddi_disable_transcoder_func(struct drm_i915_private *dev_priv,
@@ -1288,10 +1290,10 @@ bool intel_ddi_is_audio_enabled(struct drm_i915_private *dev_priv,
 void intel_ddi_get_config(struct intel_encoder *encoder,
                           struct intel_crtc_state *pipe_config);
 
-void intel_ddi_clock_get(struct intel_encoder *encoder,
-                         struct intel_crtc_state *pipe_config);
 void intel_ddi_set_vc_payload_alloc(const struct intel_crtc_state *crtc_state,
                                     bool state);
+void intel_ddi_compute_min_voltage_level(struct drm_i915_private *dev_priv,
+                                         struct intel_crtc_state *crtc_state);
 u32 bxt_signal_levels(struct intel_dp *intel_dp);
 uint32_t ddi_signal_levels(struct intel_dp *intel_dp);
 u8 intel_ddi_dp_voltage_max(struct intel_encoder *encoder);
@@ -1304,7 +1306,9 @@ void intel_init_audio_hooks(struct drm_i915_private *dev_priv);
 void intel_audio_codec_enable(struct intel_encoder *encoder,
                               const struct intel_crtc_state *crtc_state,
                               const struct drm_connector_state *conn_state);
-void intel_audio_codec_disable(struct intel_encoder *encoder);
+void intel_audio_codec_disable(struct intel_encoder *encoder,
+                               const struct intel_crtc_state *old_crtc_state,
+                               const struct drm_connector_state *old_conn_state);
 void i915_audio_component_init(struct drm_i915_private *dev_priv);
 void i915_audio_component_cleanup(struct drm_i915_private *dev_priv);
 void intel_audio_init(struct drm_i915_private *dev_priv);
@@ -1322,10 +1326,14 @@ void intel_init_cdclk_hooks(struct drm_i915_private *dev_priv);
 void intel_update_max_cdclk(struct drm_i915_private *dev_priv);
 void intel_update_cdclk(struct drm_i915_private *dev_priv);
 void intel_update_rawclk(struct drm_i915_private *dev_priv);
-bool intel_cdclk_state_compare(const struct intel_cdclk_state *a,
+bool intel_cdclk_needs_modeset(const struct intel_cdclk_state *a,
                                const struct intel_cdclk_state *b);
+bool intel_cdclk_changed(const struct intel_cdclk_state *a,
+                         const struct intel_cdclk_state *b);
 void intel_set_cdclk(struct drm_i915_private *dev_priv,
                      const struct intel_cdclk_state *cdclk_state);
+void intel_dump_cdclk_state(const struct intel_cdclk_state *cdclk_state,
+                            const char *context);
 
 /* intel_display.c */
 void i830_enable_pipe(struct drm_i915_private *dev_priv, enum pipe pipe);
@@ -1477,8 +1485,8 @@ bool bxt_find_best_dpll(struct intel_crtc_state *crtc_state, int target_clock,
 int chv_calc_dpll_params(int refclk, struct dpll *pll_clock);
 
 bool intel_crtc_active(struct intel_crtc *crtc);
-void hsw_enable_ips(struct intel_crtc *crtc);
-void hsw_disable_ips(struct intel_crtc *crtc);
+void hsw_enable_ips(const struct intel_crtc_state *crtc_state);
+void hsw_disable_ips(const struct intel_crtc_state *crtc_state);
 enum intel_display_power_domain intel_port_to_power_domain(enum port port);
 void intel_mode_from_pipe_config(struct drm_display_mode *mode,
                                  struct intel_crtc_state *pipe_config);
@@ -1491,6 +1499,8 @@ static inline u32 intel_plane_ggtt_offset(const struct intel_plane_state *state)
         return i915_ggtt_offset(state->vma);
 }
 
+u32 glk_plane_color_ctl(const struct intel_crtc_state *crtc_state,
+                        const struct intel_plane_state *plane_state);
 u32 skl_plane_ctl(const struct intel_crtc_state *crtc_state,
                   const struct intel_plane_state *plane_state);
 u32 skl_plane_stride(const struct drm_framebuffer *fb, int plane,
@@ -1521,7 +1531,8 @@ void intel_dp_sink_dpms(struct intel_dp *intel_dp, int mode);
 void intel_dp_encoder_reset(struct drm_encoder *encoder);
 void intel_dp_encoder_suspend(struct intel_encoder *intel_encoder);
 void intel_dp_encoder_destroy(struct drm_encoder *encoder);
-int intel_dp_sink_crc(struct intel_dp *intel_dp, u8 *crc);
+int intel_dp_sink_crc(struct intel_dp *intel_dp,
+                      struct intel_crtc_state *crtc_state, u8 *crc);
 bool intel_dp_compute_config(struct intel_encoder *encoder,
                              struct intel_crtc_state *pipe_config,
                              struct drm_connector_state *conn_state);
@@ -1868,7 +1879,6 @@ void intel_init_gt_powersave(struct drm_i915_private *dev_priv);
 void intel_cleanup_gt_powersave(struct drm_i915_private *dev_priv);
 void intel_sanitize_gt_powersave(struct drm_i915_private *dev_priv);
 void intel_enable_gt_powersave(struct drm_i915_private *dev_priv);
-void intel_autoenable_gt_powersave(struct drm_i915_private *dev_priv);
 void intel_disable_gt_powersave(struct drm_i915_private *dev_priv);
 void intel_suspend_gt_powersave(struct drm_i915_private *dev_priv);
 void gen6_rps_busy(struct drm_i915_private *dev_priv);
diff --git a/drivers/gpu/drm/i915/intel_dsi.c b/drivers/gpu/drm/i915/intel_dsi.c
index 83f15848098a..f09474b0c4d3 100644
--- a/drivers/gpu/drm/i915/intel_dsi.c
+++ b/drivers/gpu/drm/i915/intel_dsi.c
@@ -662,11 +662,11 @@ static void vlv_dsi_clear_device_ready(struct intel_encoder *encoder)
         }
 }
 
-static void intel_dsi_port_enable(struct intel_encoder *encoder)
+static void intel_dsi_port_enable(struct intel_encoder *encoder,
+                                  const struct intel_crtc_state *crtc_state)
 {
-        struct drm_device *dev = encoder->base.dev;
-        struct drm_i915_private *dev_priv = to_i915(dev);
-        struct intel_crtc *intel_crtc = to_intel_crtc(encoder->base.crtc);
+        struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
+        struct intel_crtc *crtc = to_intel_crtc(crtc_state->base.crtc);
         struct intel_dsi *intel_dsi = enc_to_intel_dsi(&encoder->base);
         enum port port;
 
@@ -705,7 +705,7 @@ static void intel_dsi_port_enable(struct intel_encoder *encoder)
                         if (IS_BROXTON(dev_priv))
                                 temp |= LANE_CONFIGURATION_DUAL_LINK_A;
                         else
-                                temp |= intel_crtc->pipe ?
+                                temp |= crtc->pipe ?
                                         LANE_CONFIGURATION_DUAL_LINK_B :
                                         LANE_CONFIGURATION_DUAL_LINK_A;
                 }
@@ -875,7 +875,7 @@ static void intel_dsi_pre_enable(struct intel_encoder *encoder,
 
                 intel_dsi_vbt_exec_sequence(intel_dsi, MIPI_SEQ_DISPLAY_ON);
 
-                intel_dsi_port_enable(encoder);
+                intel_dsi_port_enable(encoder, pipe_config);
         }
 
         intel_panel_enable_backlight(pipe_config, conn_state);
@@ -1082,7 +1082,7 @@ static void bxt_dsi_get_pipe_config(struct intel_encoder *encoder,
         struct drm_display_mode *adjusted_mode =
                                         &pipe_config->base.adjusted_mode;
         struct drm_display_mode *adjusted_mode_sw;
-        struct intel_crtc *intel_crtc;
+        struct intel_crtc *crtc = to_intel_crtc(pipe_config->base.crtc);
         struct intel_dsi *intel_dsi = enc_to_intel_dsi(&encoder->base);
         unsigned int lane_count = intel_dsi->lane_count;
         unsigned int bpp, fmt;
@@ -1093,8 +1093,7 @@ static void bxt_dsi_get_pipe_config(struct intel_encoder *encoder,
                                 crtc_hblank_start_sw, crtc_hblank_end_sw;
 
         /* FIXME: hw readout should not depend on SW state */
-        intel_crtc = to_intel_crtc(encoder->base.crtc);
-        adjusted_mode_sw = &intel_crtc->config->base.adjusted_mode;
+        adjusted_mode_sw = &crtc->config->base.adjusted_mode;
 
         /*
          * Atleast one port is active as encoder->get_config called only if
@@ -1243,6 +1242,8 @@ static void intel_dsi_get_config(struct intel_encoder *encoder,
         u32 pclk;
         DRM_DEBUG_KMS("\n");
 
+        pipe_config->output_types |= BIT(INTEL_OUTPUT_DSI);
+
         if (IS_GEN9_LP(dev_priv))
                 bxt_dsi_get_pipe_config(encoder, pipe_config);
 
diff --git a/drivers/gpu/drm/i915/intel_dvo.c b/drivers/gpu/drm/i915/intel_dvo.c
index 53c9b763f4ce..754baa00bea9 100644
--- a/drivers/gpu/drm/i915/intel_dvo.c
+++ b/drivers/gpu/drm/i915/intel_dvo.c
@@ -159,6 +159,8 @@ static void intel_dvo_get_config(struct intel_encoder *encoder,
         struct intel_dvo *intel_dvo = enc_to_dvo(encoder);
         u32 tmp, flags = 0;
 
+        pipe_config->output_types |= BIT(INTEL_OUTPUT_DVO);
+
         tmp = I915_READ(intel_dvo->dev.dvo_reg);
         if (tmp & DVO_HSYNC_ACTIVE_HIGH)
                 flags |= DRM_MODE_FLAG_PHSYNC;
diff --git a/drivers/gpu/drm/i915/intel_engine_cs.c b/drivers/gpu/drm/i915/intel_engine_cs.c
index ab5bf4e2e28e..9897c7f78c51 100644
--- a/drivers/gpu/drm/i915/intel_engine_cs.c
+++ b/drivers/gpu/drm/i915/intel_engine_cs.c
@@ -50,6 +50,8 @@ struct engine_class_info {
         const char *name;
         int (*init_legacy)(struct intel_engine_cs *engine);
         int (*init_execlists)(struct intel_engine_cs *engine);
+
+        u8 uabi_class;
 };
 
 static const struct engine_class_info intel_engine_classes[] = {
@@ -57,21 +59,25 @@ static const struct engine_class_info intel_engine_classes[] = {
                 .name = "rcs",
                 .init_execlists = logical_render_ring_init,
                 .init_legacy = intel_init_render_ring_buffer,
+                .uabi_class = I915_ENGINE_CLASS_RENDER,
         },
         [COPY_ENGINE_CLASS] = {
                 .name = "bcs",
                 .init_execlists = logical_xcs_ring_init,
                 .init_legacy = intel_init_blt_ring_buffer,
+                .uabi_class = I915_ENGINE_CLASS_COPY,
         },
         [VIDEO_DECODE_CLASS] = {
                 .name = "vcs",
                 .init_execlists = logical_xcs_ring_init,
                 .init_legacy = intel_init_bsd_ring_buffer,
+                .uabi_class = I915_ENGINE_CLASS_VIDEO,
         },
         [VIDEO_ENHANCEMENT_CLASS] = {
                 .name = "vecs",
                 .init_execlists = logical_xcs_ring_init,
                 .init_legacy = intel_init_vebox_ring_buffer,
+                .uabi_class = I915_ENGINE_CLASS_VIDEO_ENHANCE,
         },
 };
 
@@ -213,13 +219,15 @@ intel_engine_setup(struct drm_i915_private *dev_priv,
         WARN_ON(snprintf(engine->name, sizeof(engine->name), "%s%u",
                          class_info->name, info->instance) >=
                 sizeof(engine->name));
-        engine->uabi_id = info->uabi_id;
         engine->hw_id = engine->guc_id = info->hw_id;
         engine->mmio_base = info->mmio_base;
         engine->irq_shift = info->irq_shift;
         engine->class = info->class;
         engine->instance = info->instance;
 
+        engine->uabi_id = info->uabi_id;
+        engine->uabi_class = class_info->uabi_class;
+
         engine->context_size = __intel_engine_context_size(dev_priv,
                                                            engine->class);
         if (WARN_ON(engine->context_size > BIT(20)))
@@ -281,6 +289,8 @@ int intel_engines_init_mmio(struct drm_i915_private *dev_priv)
 
         device_info->num_rings = hweight32(mask);
 
+        i915_check_and_clear_faults(dev_priv);
+
         return 0;
 
 cleanup:
@@ -620,7 +630,7 @@ int intel_engine_init_common(struct intel_engine_cs *engine)
          * Similarly the preempt context must always be available so that
          * we can interrupt the engine at any time.
          */
-        if (INTEL_INFO(engine->i915)->has_logical_ring_preemption) {
+        if (HAS_LOGICAL_RING_PREEMPTION(engine->i915)) {
                 ring = engine->context_pin(engine,
                                            engine->i915->preempt_context);
                 if (IS_ERR(ring)) {
@@ -633,25 +643,19 @@ int intel_engine_init_common(struct intel_engine_cs *engine)
         if (ret)
                 goto err_unpin_preempt;
 
-        ret = i915_gem_render_state_init(engine);
-        if (ret)
-                goto err_breadcrumbs;
-
         if (HWS_NEEDS_PHYSICAL(engine->i915))
                 ret = init_phys_status_page(engine);
         else
                 ret = init_status_page(engine);
         if (ret)
-                goto err_rs_fini;
+                goto err_breadcrumbs;
 
         return 0;
 
-err_rs_fini:
-        i915_gem_render_state_fini(engine);
 err_breadcrumbs:
         intel_engine_fini_breadcrumbs(engine);
 err_unpin_preempt:
-        if (INTEL_INFO(engine->i915)->has_logical_ring_preemption)
+        if (HAS_LOGICAL_RING_PREEMPTION(engine->i915))
                 engine->context_unpin(engine, engine->i915->preempt_context);
 err_unpin_kernel:
         engine->context_unpin(engine, engine->i915->kernel_context);
@@ -674,12 +678,14 @@ void intel_engine_cleanup_common(struct intel_engine_cs *engine)
         else
                 cleanup_status_page(engine);
 
-        i915_gem_render_state_fini(engine);
         intel_engine_fini_breadcrumbs(engine);
         intel_engine_cleanup_cmd_parser(engine);
         i915_gem_batch_pool_fini(&engine->batch_pool);
 
-        if (INTEL_INFO(engine->i915)->has_logical_ring_preemption)
+        if (engine->default_state)
+                i915_gem_object_put(engine->default_state);
+
+        if (HAS_LOGICAL_RING_PREEMPTION(engine->i915))
                 engine->context_unpin(engine, engine->i915->preempt_context);
         engine->context_unpin(engine, engine->i915->kernel_context);
 }
@@ -1014,22 +1020,6 @@ static int gen9_init_workarounds(struct intel_engine_cs *engine)
                 WA_SET_BIT_MASKED(HALF_SLICE_CHICKEN3,
                                   GEN9_DISABLE_OCL_OOB_SUPPRESS_LOGIC);
 
-        /* WaDisableDgMirrorFixInHalfSliceChicken5:bxt */
-        if (IS_BXT_REVID(dev_priv, 0, BXT_REVID_A1))
-                WA_CLR_BIT_MASKED(GEN9_HALF_SLICE_CHICKEN5,
-                                  GEN9_DG_MIRROR_FIX_ENABLE);
-
-        /* WaSetDisablePixMaskCammingAndRhwoInCommonSliceChicken:bxt */
-        if (IS_BXT_REVID(dev_priv, 0, BXT_REVID_A1)) {
-                WA_SET_BIT_MASKED(GEN7_COMMON_SLICE_CHICKEN1,
-                                  GEN9_RHWO_OPTIMIZATION_DISABLE);
-                /*
-                 * WA also requires GEN9_SLICE_COMMON_ECO_CHICKEN0[14:14] to be set
-                 * but we do that in per ctx batchbuffer as there is an issue
-                 * with this register not getting restored on ctx restore
-                 */
-        }
-
         /* WaEnableYV12BugFixInHalfSliceChicken7:skl,bxt,kbl,glk,cfl */
         /* WaEnableSamplerGPGPUPreemptionSupport:skl,bxt,kbl,cfl */
         WA_SET_BIT_MASKED(GEN9_HALF_SLICE_CHICKEN7,
@@ -1045,11 +1035,6 @@ static int gen9_init_workarounds(struct intel_engine_cs *engine)
         WA_CLR_BIT_MASKED(GEN9_HALF_SLICE_CHICKEN5,
                           GEN9_CCS_TLB_PREFETCH_ENABLE);
 
-        /* WaDisableMaskBasedCammingInRCC:bxt */
-        if (IS_BXT_REVID(dev_priv, 0, BXT_REVID_A1))
-                WA_SET_BIT_MASKED(SLICE_ECO_CHICKEN0,
-                                  PIXEL_MASK_CAMMING_DISABLE);
-
         /* WaForceContextSaveRestoreNonCoherent:skl,bxt,kbl,cfl */
         WA_SET_BIT_MASKED(HDC_CHICKEN0,
                           HDC_FORCE_CONTEXT_SAVE_RESTORE_NON_COHERENT |
@@ -1079,8 +1064,7 @@ static int gen9_init_workarounds(struct intel_engine_cs *engine)
         /* WaDisableSamplerPowerBypassForSOPingPong:skl,bxt,kbl,cfl */
         if (IS_SKYLAKE(dev_priv) ||
             IS_KABYLAKE(dev_priv) ||
-            IS_COFFEELAKE(dev_priv) ||
-            IS_BXT_REVID(dev_priv, 0, BXT_REVID_B0))
+            IS_COFFEELAKE(dev_priv))
                 WA_SET_BIT_MASKED(HALF_SLICE_CHICKEN3,
                                   GEN8_SAMPLER_POWER_BYPASS_DIS);
 
@@ -1204,72 +1188,35 @@ static int skl_init_workarounds(struct intel_engine_cs *engine)
 static int bxt_init_workarounds(struct intel_engine_cs *engine)
 {
         struct drm_i915_private *dev_priv = engine->i915;
+        u32 val;
         int ret;
 
         ret = gen9_init_workarounds(engine);
         if (ret)
                 return ret;
 
-        /* WaStoreMultiplePTEenable:bxt */
-        /* This is a requirement according to Hardware specification */
-        if (IS_BXT_REVID(dev_priv, 0, BXT_REVID_A1))
-                I915_WRITE(TILECTL, I915_READ(TILECTL) | TILECTL_TLBPF);
-
-        /* WaSetClckGatingDisableMedia:bxt */
-        if (IS_BXT_REVID(dev_priv, 0, BXT_REVID_A1)) {
-                I915_WRITE(GEN7_MISCCPCTL, (I915_READ(GEN7_MISCCPCTL) &
-                                            ~GEN8_DOP_CLOCK_GATE_MEDIA_ENABLE));
-        }
-
         /* WaDisableThreadStallDopClockGating:bxt */
         WA_SET_BIT_MASKED(GEN8_ROW_CHICKEN,
                           STALL_DOP_GATING_DISABLE);
 
         /* WaDisablePooledEuLoadBalancingFix:bxt */
-        if (IS_BXT_REVID(dev_priv, BXT_REVID_B0, REVID_FOREVER)) {
-                I915_WRITE(FF_SLICE_CS_CHICKEN2,
-                           _MASKED_BIT_ENABLE(GEN9_POOLED_EU_LOAD_BALANCING_FIX_DISABLE));
-        }
-
-        /* WaDisableSbeCacheDispatchPortSharing:bxt */
-        if (IS_BXT_REVID(dev_priv, 0, BXT_REVID_B0)) {
-                WA_SET_BIT_MASKED(
-                        GEN7_HALF_SLICE_CHICKEN1,
-                        GEN7_SBE_SS_CACHE_DISPATCH_PORT_SHARING_DISABLE);
-        }
-
-        /* WaDisableObjectLevelPreemptionForTrifanOrPolygon:bxt */
-        /* WaDisableObjectLevelPreemptionForInstancedDraw:bxt */
-        /* WaDisableObjectLevelPreemtionForInstanceId:bxt */
-        /* WaDisableLSQCROPERFforOCL:bxt */
-        if (IS_BXT_REVID(dev_priv, 0, BXT_REVID_A1)) {
-                ret = wa_ring_whitelist_reg(engine, GEN9_CS_DEBUG_MODE1);
-                if (ret)
-                        return ret;
-
-                ret = wa_ring_whitelist_reg(engine, GEN8_L3SQCREG4);
-                if (ret)
-                        return ret;
-        }
+        I915_WRITE(FF_SLICE_CS_CHICKEN2,
+                   _MASKED_BIT_ENABLE(GEN9_POOLED_EU_LOAD_BALANCING_FIX_DISABLE));
 
         /* WaProgramL3SqcReg1DefaultForPerf:bxt */
-        if (IS_BXT_REVID(dev_priv, BXT_REVID_B0, REVID_FOREVER)) {
-                u32 val = I915_READ(GEN8_L3SQCREG1);
-                val &= ~L3_PRIO_CREDITS_MASK;
-                val |= L3_GENERAL_PRIO_CREDITS(62) | L3_HIGH_PRIO_CREDITS(2);
-                I915_WRITE(GEN8_L3SQCREG1, val);
-        }
+        val = I915_READ(GEN8_L3SQCREG1);
+        val &= ~L3_PRIO_CREDITS_MASK;
+        val |= L3_GENERAL_PRIO_CREDITS(62) | L3_HIGH_PRIO_CREDITS(2);
+        I915_WRITE(GEN8_L3SQCREG1, val);
 
         /* WaToEnableHwFixForPushConstHWBug:bxt */
-        if (IS_BXT_REVID(dev_priv, BXT_REVID_C0, REVID_FOREVER))
-                WA_SET_BIT_MASKED(COMMON_SLICE_CHICKEN2,
-                                  GEN8_SBE_DISABLE_REPLAY_BUF_OPTIMIZATION);
+        WA_SET_BIT_MASKED(COMMON_SLICE_CHICKEN2,
+                          GEN8_SBE_DISABLE_REPLAY_BUF_OPTIMIZATION);
 
         /* WaInPlaceDecompressionHang:bxt */
-        if (IS_BXT_REVID(dev_priv, BXT_REVID_C0, REVID_FOREVER))
-                I915_WRITE(GEN9_GAMT_ECO_REG_RW_IA,
-                           (I915_READ(GEN9_GAMT_ECO_REG_RW_IA) |
-                            GAMT_ECO_ENABLE_IN_PLACE_DECOMPRESS));
+        I915_WRITE(GEN9_GAMT_ECO_REG_RW_IA,
+                   (I915_READ(GEN9_GAMT_ECO_REG_RW_IA) |
+                    GAMT_ECO_ENABLE_IN_PLACE_DECOMPRESS));
 
         return 0;
 }
@@ -1585,6 +1532,34 @@ bool intel_engines_are_idle(struct drm_i915_private *dev_priv)
         return true;
 }
 
+/**
+ * intel_engine_has_kernel_context:
+ * @engine: the engine
+ *
+ * Returns true if the last context to be executed on this engine, or has been
+ * executed if the engine is already idle, is the kernel context
+ * (#i915.kernel_context).
+ */
+bool intel_engine_has_kernel_context(const struct intel_engine_cs *engine)
+{
+        const struct i915_gem_context * const kernel_context =
+                engine->i915->kernel_context;
+        struct drm_i915_gem_request *rq;
+
+        lockdep_assert_held(&engine->i915->drm.struct_mutex);
+
+        /*
+         * Check the last context seen by the engine. If active, it will be
+         * the last request that remains in the timeline. When idle, it is
+         * the last executed context as tracked by retirement.
+         */
+        rq = __i915_gem_active_peek(&engine->timeline->last_request);
+        if (rq)
+                return rq->ctx == kernel_context;
+        else
+                return engine->last_retired_context == kernel_context;
+}
+
 void intel_engines_reset_default_submission(struct drm_i915_private *i915)
 {
         struct intel_engine_cs *engine;
@@ -1594,19 +1569,63 @@ void intel_engines_reset_default_submission(struct drm_i915_private *i915)
                 engine->set_default_submission(engine);
 }
 
-void intel_engines_mark_idle(struct drm_i915_private *i915)
+/**
+ * intel_engines_park: called when the GT is transitioning from busy->idle
+ * @i915: the i915 device
+ *
+ * The GT is now idle and about to go to sleep (maybe never to wake again?).
+ * Time for us to tidy and put away our toys (release resources back to the
+ * system).
+ */
+void intel_engines_park(struct drm_i915_private *i915)
 {
         struct intel_engine_cs *engine;
         enum intel_engine_id id;
 
         for_each_engine(engine, i915, id) {
+                /* Flush the residual irq tasklets first. */
                 intel_engine_disarm_breadcrumbs(engine);
+                tasklet_kill(&engine->execlists.tasklet);
+
+                /*
+                 * We are committed now to parking the engines, make sure there
+                 * will be no more interrupts arriving later and the engines
+                 * are truly idle.
+                 */
+                if (wait_for(intel_engine_is_idle(engine), 10)) {
+                        struct drm_printer p = drm_debug_printer(__func__);
+
+                        dev_err(i915->drm.dev,
+                                "%s is not idle before parking\n",
+                                engine->name);
+                        intel_engine_dump(engine, &p);
+                }
+
+                if (engine->park)
+                        engine->park(engine);
+
                 i915_gem_batch_pool_fini(&engine->batch_pool);
-                tasklet_kill(&engine->execlists.irq_tasklet);
                 engine->execlists.no_priolist = false;
         }
 }
 
+/**
+ * intel_engines_unpark: called when the GT is transitioning from idle->busy
+ * @i915: the i915 device
+ *
+ * The GT was idle and now about to fire up with some new user requests.
+ */
+void intel_engines_unpark(struct drm_i915_private *i915)
+{
+        struct intel_engine_cs *engine;
+        enum intel_engine_id id;
+
+        for_each_engine(engine, i915, id) {
+                if (engine->unpark)
+                        engine->unpark(engine);
+        }
+}
+
 bool intel_engine_can_store_dword(struct intel_engine_cs *engine)
 {
         switch (INTEL_GEN(engine->i915)) {
@@ -1622,6 +1641,20 @@ bool intel_engine_can_store_dword(struct intel_engine_cs *engine)
         }
 }
 
+unsigned int intel_engines_has_context_isolation(struct drm_i915_private *i915)
+{
+        struct intel_engine_cs *engine;
+        enum intel_engine_id id;
+        unsigned int which;
+
+        which = 0;
+        for_each_engine(engine, i915, id)
+                if (engine->default_state)
+                        which |= BIT(engine->uabi_class);
+
+        return which;
+}
+
 static void print_request(struct drm_printer *m,
                           struct drm_i915_gem_request *rq,
                           const char *prefix)
@@ -1688,9 +1721,14 @@ void intel_engine_dump(struct intel_engine_cs *engine, struct drm_printer *m)
         drm_printf(m, "\tRING_TAIL:  0x%08x [0x%08x]\n",
                    I915_READ(RING_TAIL(engine->mmio_base)) & TAIL_ADDR,
                    rq ? rq->ring->tail : 0);
-        drm_printf(m, "\tRING_CTL:   0x%08x [%s]\n",
+        drm_printf(m, "\tRING_CTL:   0x%08x%s\n",
                    I915_READ(RING_CTL(engine->mmio_base)),
-                   I915_READ(RING_CTL(engine->mmio_base)) & (RING_WAIT | RING_WAIT_SEMAPHORE) ? "waiting" : "");
+                   I915_READ(RING_CTL(engine->mmio_base)) & (RING_WAIT | RING_WAIT_SEMAPHORE) ? " [waiting]" : "");
+        if (INTEL_GEN(engine->i915) > 2) {
+                drm_printf(m, "\tRING_MODE:  0x%08x%s\n",
+                           I915_READ(RING_MI_MODE(engine->mmio_base)),
+                           I915_READ(RING_MI_MODE(engine->mmio_base)) & (MODE_IDLE) ? " [idle]" : "");
+        }
 
         rcu_read_unlock();
 
@@ -1781,6 +1819,7 @@ void intel_engine_dump(struct intel_engine_cs *engine, struct drm_printer *m)
         }
         spin_unlock_irq(&b->rb_lock);
 
+        drm_printf(m, "Idle? %s\n", yesno(intel_engine_is_idle(engine)));
         drm_printf(m, "\n");
 }
 
diff --git a/drivers/gpu/drm/i915/intel_guc.c b/drivers/gpu/drm/i915/intel_guc.c
index 10037c0fdf95..823d0c2e9ad2 100644
--- a/drivers/gpu/drm/i915/intel_guc.c
+++ b/drivers/gpu/drm/i915/intel_guc.c
@@ -23,6 +23,7 @@
  */
 
 #include "intel_guc.h"
+#include "intel_guc_submission.h"
 #include "i915_drv.h"
 
 static void gen8_guc_raise_irq(struct intel_guc *guc)
@@ -268,7 +269,6 @@ int intel_guc_auth_huc(struct intel_guc *guc, u32 rsa_offset)
 int intel_guc_suspend(struct drm_i915_private *dev_priv)
 {
         struct intel_guc *guc = &dev_priv->guc;
-        struct i915_gem_context *ctx;
         u32 data[3];
 
         if (guc->fw.load_status != INTEL_UC_FIRMWARE_SUCCESS)
@@ -276,14 +276,33 @@ int intel_guc_suspend(struct drm_i915_private *dev_priv)
 
         gen9_disable_guc_interrupts(dev_priv);
 
-        ctx = dev_priv->kernel_context;
-
         data[0] = INTEL_GUC_ACTION_ENTER_S_STATE;
         /* any value greater than GUC_POWER_D0 */
         data[1] = GUC_POWER_D1;
-        /* first page is shared data with GuC */
-        data[2] = guc_ggtt_offset(ctx->engine[RCS].state) +
-                  LRC_GUCSHR_PN * PAGE_SIZE;
+        data[2] = guc_ggtt_offset(guc->shared_data);
+
+        return intel_guc_send(guc, data, ARRAY_SIZE(data));
+}
+
+/**
+ * intel_guc_reset_engine() - ask GuC to reset an engine
+ * @guc:        intel_guc structure
+ * @engine:     engine to be reset
+ */
+int intel_guc_reset_engine(struct intel_guc *guc,
+                           struct intel_engine_cs *engine)
+{
+        u32 data[7];
+
+        GEM_BUG_ON(!guc->execbuf_client);
+
+        data[0] = INTEL_GUC_ACTION_REQUEST_ENGINE_RESET;
+        data[1] = engine->guc_id;
+        data[2] = 0;
+        data[3] = 0;
+        data[4] = 0;
+        data[5] = guc->execbuf_client->stage_id;
+        data[6] = guc_ggtt_offset(guc->shared_data);
 
         return intel_guc_send(guc, data, ARRAY_SIZE(data));
 }
@@ -295,7 +314,6 @@ int intel_guc_suspend(struct drm_i915_private *dev_priv)
 int intel_guc_resume(struct drm_i915_private *dev_priv)
 {
         struct intel_guc *guc = &dev_priv->guc;
-        struct i915_gem_context *ctx;
         u32 data[3];
 
         if (guc->fw.load_status != INTEL_UC_FIRMWARE_SUCCESS)
@@ -304,13 +322,9 @@ int intel_guc_resume(struct drm_i915_private *dev_priv)
         if (i915_modparams.guc_log_level >= 0)
                 gen9_enable_guc_interrupts(dev_priv);
 
-        ctx = dev_priv->kernel_context;
-
         data[0] = INTEL_GUC_ACTION_EXIT_S_STATE;
         data[1] = GUC_POWER_D0;
-        /* first page is shared data with GuC */
-        data[2] = guc_ggtt_offset(ctx->engine[RCS].state) +
-                  LRC_GUCSHR_PN * PAGE_SIZE;
+        data[2] = guc_ggtt_offset(guc->shared_data);
 
         return intel_guc_send(guc, data, ARRAY_SIZE(data));
 }
diff --git a/drivers/gpu/drm/i915/intel_guc.h b/drivers/gpu/drm/i915/intel_guc.h
index 418450b1ae27..75c4cfefdaff 100644
--- a/drivers/gpu/drm/i915/intel_guc.h
+++ b/drivers/gpu/drm/i915/intel_guc.h
@@ -34,9 +34,14 @@
 #include "i915_guc_reg.h"
 #include "i915_vma.h"
 
+struct guc_preempt_work {
+        struct work_struct work;
+        struct intel_engine_cs *engine;
+};
+
 /*
  * Top level structure of GuC. It handles firmware loading and manages client
- * pool and doorbells. intel_guc owns a i915_guc_client to replace the legacy
+ * pool and doorbells. intel_guc owns a intel_guc_client to replace the legacy
  * ExecList submission.
  */
 struct intel_guc {
@@ -54,8 +59,14 @@ struct intel_guc {
         struct i915_vma *stage_desc_pool;
         void *stage_desc_pool_vaddr;
         struct ida stage_ids;
+        struct i915_vma *shared_data;
+        void *shared_data_vaddr;
+
+        struct intel_guc_client *execbuf_client;
+        struct intel_guc_client *preempt_client;
 
-        struct i915_guc_client *execbuf_client;
+        struct guc_preempt_work preempt_work[I915_NUM_ENGINES];
+        struct workqueue_struct *preempt_wq;
 
         DECLARE_BITMAP(doorbell_bitmap, GUC_NUM_DOORBELLS);
         /* Cyclic counter mod pagesize  */
diff --git a/drivers/gpu/drm/i915/intel_guc_ct.c b/drivers/gpu/drm/i915/intel_guc_ct.c
index c4cbec140101..24ad55752396 100644
--- a/drivers/gpu/drm/i915/intel_guc_ct.c
+++ b/drivers/gpu/drm/i915/intel_guc_ct.c
@@ -198,6 +198,7 @@ static int ctch_open(struct intel_guc *guc,
                 err = ctch_init(guc, ctch);
                 if (unlikely(err))
                         goto err_out;
+                GEM_BUG_ON(!ctch->vma);
         }
 
         /* vma should be already allocated and map'ed */
diff --git a/drivers/gpu/drm/i915/intel_guc_fw.c b/drivers/gpu/drm/i915/intel_guc_fw.c
index ef67a36354c5..69ba01599575 100644
--- a/drivers/gpu/drm/i915/intel_guc_fw.c
+++ b/drivers/gpu/drm/i915/intel_guc_fw.c
@@ -97,23 +97,50 @@ int intel_guc_fw_select(struct intel_guc *guc)
         return 0;
 }
 
-/*
- * Read the GuC status register (GUC_STATUS) and store it in the
- * specified location; then return a boolean indicating whether
- * the value matches either of two values representing completion
- * of the GuC boot process.
- *
- * This is used for polling the GuC status in a wait_for()
- * loop below.
- */
-static inline bool guc_ucode_response(struct drm_i915_private *dev_priv,
-                                      u32 *status)
+static void guc_prepare_xfer(struct intel_guc *guc)
 {
-        u32 val = I915_READ(GUC_STATUS);
-        u32 uk_val = val & GS_UKERNEL_MASK;
-        *status = val;
-        return (uk_val == GS_UKERNEL_READY ||
-                ((val & GS_MIA_CORE_STATE) && uk_val == GS_UKERNEL_LAPIC_DONE));
+        struct drm_i915_private *dev_priv = guc_to_i915(guc);
+
+        /* Must program this register before loading the ucode with DMA */
+        I915_WRITE(GUC_SHIM_CONTROL, GUC_DISABLE_SRAM_INIT_TO_ZEROES |
+                                     GUC_ENABLE_READ_CACHE_LOGIC |
+                                     GUC_ENABLE_MIA_CACHING |
+                                     GUC_ENABLE_READ_CACHE_FOR_SRAM_DATA |
+                                     GUC_ENABLE_READ_CACHE_FOR_WOPCM_DATA |
+                                     GUC_ENABLE_MIA_CLOCK_GATING);
+
+        if (IS_GEN9_LP(dev_priv))
+                I915_WRITE(GEN9LP_GT_PM_CONFIG, GT_DOORBELL_ENABLE);
+        else
+                I915_WRITE(GEN9_GT_PM_CONFIG, GT_DOORBELL_ENABLE);
+
+        if (IS_GEN9(dev_priv)) {
+                /* DOP Clock Gating Enable for GuC clocks */
+                I915_WRITE(GEN7_MISCCPCTL, (GEN8_DOP_CLOCK_GATE_GUC_ENABLE |
+                                            I915_READ(GEN7_MISCCPCTL)));
+
+                /* allows for 5us (in 10ns units) before GT can go to RC6 */
+                I915_WRITE(GUC_ARAT_C6DIS, 0x1FF);
+        }
+}
+
+/* Copy RSA signature from the fw image to HW for verification */
+static int guc_xfer_rsa(struct intel_guc *guc, struct i915_vma *vma)
+{
+        struct drm_i915_private *dev_priv = guc_to_i915(guc);
+        struct intel_uc_fw *guc_fw = &guc->fw;
+        struct sg_table *sg = vma->pages;
+        u32 rsa[UOS_RSA_SCRATCH_MAX_COUNT];
+        int i;
+
+        if (sg_pcopy_to_buffer(sg->sgl, sg->nents, rsa, sizeof(rsa),
+                               guc_fw->rsa_offset) != sizeof(rsa))
+                return -EINVAL;
+
+        for (i = 0; i < UOS_RSA_SCRATCH_MAX_COUNT; i++)
+                I915_WRITE(UOS_RSA_SCRATCH(i), rsa[i]);
+
+        return 0;
 }
 
 /*
@@ -122,29 +149,19 @@ static inline bool guc_ucode_response(struct drm_i915_private *dev_priv,
  * Architecturally, the DMA engine is bidirectional, and can potentially even
  * transfer between GTT locations. This functionality is left out of the API
  * for now as there is no need for it.
- *
- * Note that GuC needs the CSS header plus uKernel code to be copied by the
- * DMA engine in one operation, whereas the RSA signature is loaded via MMIO.
  */
-static int guc_ucode_xfer_dma(struct drm_i915_private *dev_priv,
-                              struct i915_vma *vma)
+static int guc_xfer_ucode(struct intel_guc *guc, struct i915_vma *vma)
 {
-        struct intel_uc_fw *guc_fw = &dev_priv->guc.fw;
+        struct drm_i915_private *dev_priv = guc_to_i915(guc);
+        struct intel_uc_fw *guc_fw = &guc->fw;
         unsigned long offset;
-        struct sg_table *sg = vma->pages;
-        u32 status, rsa[UOS_RSA_SCRATCH_MAX_COUNT];
-        int i, ret = 0;
-
-        /* where RSA signature starts */
-        offset = guc_fw->rsa_offset;
-
-        /* Copy RSA signature from the fw image to HW for verification */
-        sg_pcopy_to_buffer(sg->sgl, sg->nents, rsa, sizeof(rsa), offset);
-        for (i = 0; i < UOS_RSA_SCRATCH_MAX_COUNT; i++)
-                I915_WRITE(UOS_RSA_SCRATCH(i), rsa[i]);
+        u32 status;
+        int ret;
 
-        /* The header plus uCode will be copied to WOPCM via DMA, excluding any
-         * other components */
+        /*
+         * The header plus uCode will be copied to WOPCM via DMA, excluding any
+         * other components
+         */
         I915_WRITE(DMA_COPY_SIZE, guc_fw->header_size + guc_fw->ucode_size);
 
         /* Set the source address for the new blob */
@@ -162,33 +179,62 @@ static int guc_ucode_xfer_dma(struct drm_i915_private *dev_priv,
         /* Finally start the DMA */
         I915_WRITE(DMA_CTRL, _MASKED_BIT_ENABLE(UOS_MOVE | START_DMA));
 
+        /* Wait for DMA to finish */
+        ret = __intel_wait_for_register_fw(dev_priv, DMA_CTRL, START_DMA, 0,
+                                           2, 100, &status);
+        DRM_DEBUG_DRIVER("GuC DMA status %#x\n", status);
+
+        return ret;
+}
+
+/*
+ * Read the GuC status register (GUC_STATUS) and store it in the
+ * specified location; then return a boolean indicating whether
+ * the value matches either of two values representing completion
+ * of the GuC boot process.
+ *
+ * This is used for polling the GuC status in a wait_for()
+ * loop below.
+ */
+static inline bool guc_ready(struct intel_guc *guc, u32 *status)
+{
+        struct drm_i915_private *dev_priv = guc_to_i915(guc);
+        u32 val = I915_READ(GUC_STATUS);
+        u32 uk_val = val & GS_UKERNEL_MASK;
+
+        *status = val;
+        return (uk_val == GS_UKERNEL_READY) ||
+                ((val & GS_MIA_CORE_STATE) && (uk_val == GS_UKERNEL_LAPIC_DONE));
+}
+
+static int guc_wait_ucode(struct intel_guc *guc)
+{
+        u32 status;
+        int ret;
+
         /*
-         * Wait for the DMA to complete & the GuC to start up.
+         * Wait for the GuC to start up.
          * NB: Docs recommend not using the interrupt for completion.
          * Measurements indicate this should take no more than 20ms, so a
          * timeout here indicates that the GuC has failed and is unusable.
          * (Higher levels of the driver will attempt to fall back to
          * execlist mode if this happens.)
          */
-        ret = wait_for(guc_ucode_response(dev_priv, &status), 100);
-
-        DRM_DEBUG_DRIVER("DMA status 0x%x, GuC status 0x%x\n",
-                        I915_READ(DMA_CTRL), status);
+        ret = wait_for(guc_ready(guc, &status), 100);
+        DRM_DEBUG_DRIVER("GuC status %#x\n", status);
 
         if ((status & GS_BOOTROM_MASK) == GS_BOOTROM_RSA_FAILED) {
                 DRM_ERROR("GuC firmware signature verification failed\n");
                 ret = -ENOEXEC;
         }
 
-        DRM_DEBUG_DRIVER("returning %d\n", ret);
-
         return ret;
 }
 
 /*
  * Load the GuC firmware blob into the MinuteIA.
  */
-static int guc_ucode_xfer(struct intel_uc_fw *guc_fw, struct i915_vma *vma)
+static int guc_fw_xfer(struct intel_uc_fw *guc_fw, struct i915_vma *vma)
 {
         struct intel_guc *guc = container_of(guc_fw, struct intel_guc, fw);
         struct drm_i915_private *dev_priv = guc_to_i915(guc);
@@ -198,34 +244,24 @@ static int guc_ucode_xfer(struct intel_uc_fw *guc_fw, struct i915_vma *vma)
 
         intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
 
-        /* Enable MIA caching. GuC clock gating is disabled. */
-        I915_WRITE(GUC_SHIM_CONTROL, GUC_SHIM_CONTROL_VALUE);
+        guc_prepare_xfer(guc);
 
-        /* WaDisableMinuteIaClockGating:bxt */
-        if (IS_BXT_REVID(dev_priv, 0, BXT_REVID_A1)) {
-                I915_WRITE(GUC_SHIM_CONTROL, (I915_READ(GUC_SHIM_CONTROL) &
-                                              ~GUC_ENABLE_MIA_CLOCK_GATING));
-        }
-
-        /* WaC6DisallowByGfxPause:bxt */
-        if (IS_BXT_REVID(dev_priv, 0, BXT_REVID_B0))
-                I915_WRITE(GEN6_GFXPAUSE, 0x30FFF);
-
-        if (IS_GEN9_LP(dev_priv))
-                I915_WRITE(GEN9LP_GT_PM_CONFIG, GT_DOORBELL_ENABLE);
-        else
-                I915_WRITE(GEN9_GT_PM_CONFIG, GT_DOORBELL_ENABLE);
-
-        if (IS_GEN9(dev_priv)) {
-                /* DOP Clock Gating Enable for GuC clocks */
-                I915_WRITE(GEN7_MISCCPCTL, (GEN8_DOP_CLOCK_GATE_GUC_ENABLE |
-                                            I915_READ(GEN7_MISCCPCTL)));
+        /*
+         * Note that GuC needs the CSS header plus uKernel code to be copied
+         * by the DMA engine in one operation, whereas the RSA signature is
+         * loaded via MMIO.
+         */
+        ret = guc_xfer_rsa(guc, vma);
+        if (ret)
+                DRM_WARN("GuC firmware signature xfer error %d\n", ret);
 
-                /* allows for 5us (in 10ns units) before GT can go to RC6 */
-                I915_WRITE(GUC_ARAT_C6DIS, 0x1FF);
-        }
+        ret = guc_xfer_ucode(guc, vma);
+        if (ret)
+                DRM_WARN("GuC firmware code xfer error %d\n", ret);
 
-        ret = guc_ucode_xfer_dma(dev_priv, vma);
+        ret = guc_wait_ucode(guc);
+        if (ret)
+                DRM_ERROR("GuC firmware xfer error %d\n", ret);
 
         intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
 
@@ -247,5 +283,5 @@ static int guc_ucode_xfer(struct intel_uc_fw *guc_fw, struct i915_vma *vma)
  */
 int intel_guc_fw_upload(struct intel_guc *guc)
 {
-        return intel_uc_fw_upload(&guc->fw, guc_ucode_xfer);
+        return intel_uc_fw_upload(&guc->fw, guc_fw_xfer);
 }
diff --git a/drivers/gpu/drm/i915/intel_guc_fwif.h b/drivers/gpu/drm/i915/intel_guc_fwif.h
index 80c507435458..6a10aa6f04d3 100644
--- a/drivers/gpu/drm/i915/intel_guc_fwif.h
+++ b/drivers/gpu/drm/i915/intel_guc_fwif.h
@@ -544,9 +544,37 @@ union guc_log_control {
         u32 value;
 } __packed;
 
+struct guc_ctx_report {
+        u32 report_return_status;
+        u32 reserved1[64];
+        u32 affected_count;
+        u32 reserved2[2];
+} __packed;
+
+/* GuC Shared Context Data Struct */
+struct guc_shared_ctx_data {
+        u32 addr_of_last_preempted_data_low;
+        u32 addr_of_last_preempted_data_high;
+        u32 addr_of_last_preempted_data_high_tmp;
+        u32 padding;
+        u32 is_mapped_to_proxy;
+        u32 proxy_ctx_id;
+        u32 engine_reset_ctx_id;
+        u32 media_reset_count;
+        u32 reserved1[8];
+        u32 uk_last_ctx_switch_reason;
+        u32 was_reset;
+        u32 lrca_gpu_addr;
+        u64 execlist_ctx;
+        u32 reserved2[66];
+        struct guc_ctx_report preempt_ctx_report[GUC_MAX_ENGINES_NUM];
+} __packed;
+
 /* This Action will be programmed in C180 - SOFT_SCRATCH_O_REG */
 enum intel_guc_action {
         INTEL_GUC_ACTION_DEFAULT = 0x0,
+        INTEL_GUC_ACTION_REQUEST_PREEMPTION = 0x2,
+        INTEL_GUC_ACTION_REQUEST_ENGINE_RESET = 0x3,
         INTEL_GUC_ACTION_SAMPLE_FORCEWAKE = 0x6,
         INTEL_GUC_ACTION_ALLOCATE_DOORBELL = 0x10,
         INTEL_GUC_ACTION_DEALLOCATE_DOORBELL = 0x20,
@@ -562,6 +590,18 @@ enum intel_guc_action {
         INTEL_GUC_ACTION_LIMIT
 };
 
+enum intel_guc_preempt_options {
+        INTEL_GUC_PREEMPT_OPTION_DROP_WORK_Q = 0x4,
+        INTEL_GUC_PREEMPT_OPTION_DROP_SUBMIT_Q = 0x8,
+};
+
+enum intel_guc_report_status {
+        INTEL_GUC_REPORT_STATUS_UNKNOWN = 0x0,
+        INTEL_GUC_REPORT_STATUS_ACKED = 0x1,
+        INTEL_GUC_REPORT_STATUS_ERROR = 0x2,
+        INTEL_GUC_REPORT_STATUS_COMPLETE = 0x4,
+};
+
 /*
  * The GuC sends its response to a command by overwriting the
  * command in SS0. The response is distinguishable from a command
diff --git a/drivers/gpu/drm/i915/i915_guc_submission.c b/drivers/gpu/drm/i915/intel_guc_submission.c
index f84c267728fd..cbf5a96f5806 100644
--- a/drivers/gpu/drm/i915/i915_guc_submission.c
+++ b/drivers/gpu/drm/i915/intel_guc_submission.c
@@ -25,23 +25,24 @@
 #include <linux/circ_buf.h>
 #include <trace/events/dma_fence.h>
 
-#include "i915_guc_submission.h"
+#include "intel_guc_submission.h"
 #include "i915_drv.h"
 
 /**
  * DOC: GuC-based command submission
  *
  * GuC client:
- * A i915_guc_client refers to a submission path through GuC. Currently, there
- * is only one of these (the execbuf_client) and this one is charged with all
- * submissions to the GuC. This struct is the owner of a doorbell, a process
- * descriptor and a workqueue (all of them inside a single gem object that
- * contains all required pages for these elements).
+ * A intel_guc_client refers to a submission path through GuC. Currently, there
+ * are two clients. One of them (the execbuf_client) is charged with all
+ * submissions to the GuC, the other one (preempt_client) is responsible for
+ * preempting the execbuf_client. This struct is the owner of a doorbell, a
+ * process descriptor and a workqueue (all of them inside a single gem object
+ * that contains all required pages for these elements).
  *
  * GuC stage descriptor:
  * During initialization, the driver allocates a static pool of 1024 such
  * descriptors, and shares them with the GuC.
- * Currently, there exists a 1:1 mapping between a i915_guc_client and a
+ * Currently, there exists a 1:1 mapping between a intel_guc_client and a
  * guc_stage_desc (via the client's stage_id), so effectively only one
  * gets used. This stage descriptor lets the GuC know about the doorbell,
  * workqueue and process descriptor. Theoretically, it also lets the GuC
@@ -70,7 +71,7 @@
  * WQ_TYPE_INORDER is needed to support legacy submission via GuC, which
  * represents in-order queue. The kernel driver packs ring tail pointer and an
  * ELSP context descriptor dword into Work Item.
- * See guc_wq_item_append()
+ * See guc_add_request()
  *
  * ADS:
  * The Additional Data Struct (ADS) has pointers for different buffers used by
@@ -81,12 +82,13 @@
  *
  */
 
-static inline bool is_high_priority(struct i915_guc_client* client)
+static inline bool is_high_priority(struct intel_guc_client *client)
 {
-        return client->priority <= GUC_CLIENT_PRIORITY_HIGH;
+        return (client->priority == GUC_CLIENT_PRIORITY_KMD_HIGH ||
+                client->priority == GUC_CLIENT_PRIORITY_HIGH);
 }
 
-static int __reserve_doorbell(struct i915_guc_client *client)
+static int __reserve_doorbell(struct intel_guc_client *client)
 {
         unsigned long offset;
         unsigned long end;
@@ -100,7 +102,7 @@ static int __reserve_doorbell(struct i915_guc_client *client)
          * priority contexts, the second half for high-priority ones.
          */
         offset = 0;
-        end = GUC_NUM_DOORBELLS/2;
+        end = GUC_NUM_DOORBELLS / 2;
         if (is_high_priority(client)) {
                 offset = end;
                 end += offset;
@@ -118,7 +120,7 @@ static int __reserve_doorbell(struct i915_guc_client *client)
         return 0;
 }
 
-static void __unreserve_doorbell(struct i915_guc_client *client)
+static void __unreserve_doorbell(struct intel_guc_client *client)
 {
         GEM_BUG_ON(client->doorbell_id == GUC_DOORBELL_INVALID);
 
@@ -150,7 +152,7 @@ static int __guc_deallocate_doorbell(struct intel_guc *guc, u32 stage_id)
         return intel_guc_send(guc, action, ARRAY_SIZE(action));
 }
 
-static struct guc_stage_desc *__get_stage_desc(struct i915_guc_client *client)
+static struct guc_stage_desc *__get_stage_desc(struct intel_guc_client *client)
 {
         struct guc_stage_desc *base = client->guc->stage_desc_pool_vaddr;
 
@@ -164,7 +166,7 @@ static struct guc_stage_desc *__get_stage_desc(struct i915_guc_client *client)
  * client object which contains the page being used for the doorbell
  */
 
-static void __update_doorbell_desc(struct i915_guc_client *client, u16 new_id)
+static void __update_doorbell_desc(struct intel_guc_client *client, u16 new_id)
 {
         struct guc_stage_desc *desc;
 
@@ -173,12 +175,12 @@ static void __update_doorbell_desc(struct i915_guc_client *client, u16 new_id)
         desc->db_id = new_id;
 }
 
-static struct guc_doorbell_info *__get_doorbell(struct i915_guc_client *client)
+static struct guc_doorbell_info *__get_doorbell(struct intel_guc_client *client)
 {
         return client->vaddr + client->doorbell_offset;
 }
 
-static bool has_doorbell(struct i915_guc_client *client)
+static bool has_doorbell(struct intel_guc_client *client)
 {
         if (client->doorbell_id == GUC_DOORBELL_INVALID)
                 return false;
@@ -186,7 +188,7 @@ static bool has_doorbell(struct i915_guc_client *client)
         return test_bit(client->doorbell_id, client->guc->doorbell_bitmap);
 }
 
-static int __create_doorbell(struct i915_guc_client *client)
+static int __create_doorbell(struct intel_guc_client *client)
 {
         struct guc_doorbell_info *doorbell;
         int err;
@@ -196,13 +198,16 @@ static int __create_doorbell(struct i915_guc_client *client)
         doorbell->cookie = 0;
 
         err = __guc_allocate_doorbell(client->guc, client->stage_id);
-        if (err)
+        if (err) {
                 doorbell->db_status = GUC_DOORBELL_DISABLED;
+                DRM_ERROR("Couldn't create client %u doorbell: %d\n",
+                          client->stage_id, err);
+        }
 
         return err;
 }
 
-static int __destroy_doorbell(struct i915_guc_client *client)
+static int __destroy_doorbell(struct intel_guc_client *client)
 {
         struct drm_i915_private *dev_priv = guc_to_i915(client->guc);
         struct guc_doorbell_info *doorbell;
@@ -216,14 +221,15 @@ static int __destroy_doorbell(struct i915_guc_client *client)
 
         /* Doorbell release flow requires that we wait for GEN8_DRB_VALID bit
          * to go to zero after updating db_status before we call the GuC to
-         * release the doorbell */
+         * release the doorbell
+         */
         if (wait_for_us(!(I915_READ(GEN8_DRBREGL(db_id)) & GEN8_DRB_VALID), 10))
                 WARN_ONCE(true, "Doorbell never became invalid after disable\n");
 
         return __guc_deallocate_doorbell(client->guc, client->stage_id);
 }
 
-static int create_doorbell(struct i915_guc_client *client)
+static int create_doorbell(struct intel_guc_client *client)
 {
         int ret;
 
@@ -245,7 +251,7 @@ err:
         return ret;
 }
 
-static int destroy_doorbell(struct i915_guc_client *client)
+static int destroy_doorbell(struct intel_guc_client *client)
 {
         int err;
 
@@ -265,7 +271,7 @@ static int destroy_doorbell(struct i915_guc_client *client)
         return 0;
 }
 
-static unsigned long __select_cacheline(struct intel_guc* guc)
+static unsigned long __select_cacheline(struct intel_guc *guc)
 {
         unsigned long offset;
 
@@ -276,12 +282,12 @@ static unsigned long __select_cacheline(struct intel_guc* guc)
         guc->db_cacheline += cache_line_size();
 
         DRM_DEBUG_DRIVER("reserved cacheline 0x%lx, next 0x%x, linesize %u\n",
-                        offset, guc->db_cacheline, cache_line_size());
+                         offset, guc->db_cacheline, cache_line_size());
         return offset;
 }
 
 static inline struct guc_process_desc *
-__get_process_desc(struct i915_guc_client *client)
+__get_process_desc(struct intel_guc_client *client)
 {
         return client->vaddr + client->proc_desc_offset;
 }
@@ -290,7 +296,7 @@ __get_process_desc(struct i915_guc_client *client)
  * Initialise the process descriptor shared with the GuC firmware.
  */
 static void guc_proc_desc_init(struct intel_guc *guc,
-                               struct i915_guc_client *client)
+                               struct intel_guc_client *client)
 {
         struct guc_process_desc *desc;
 
@@ -311,6 +317,37 @@ static void guc_proc_desc_init(struct intel_guc *guc,
         desc->priority = client->priority;
 }
 
+static int guc_stage_desc_pool_create(struct intel_guc *guc)
+{
+        struct i915_vma *vma;
+        void *vaddr;
+
+        vma = intel_guc_allocate_vma(guc,
+                                     PAGE_ALIGN(sizeof(struct guc_stage_desc) *
+                                     GUC_MAX_STAGE_DESCRIPTORS));
+        if (IS_ERR(vma))
+                return PTR_ERR(vma);
+
+        vaddr = i915_gem_object_pin_map(vma->obj, I915_MAP_WB);
+        if (IS_ERR(vaddr)) {
+                i915_vma_unpin_and_release(&vma);
+                return PTR_ERR(vaddr);
+        }
+
+        guc->stage_desc_pool = vma;
+        guc->stage_desc_pool_vaddr = vaddr;
+        ida_init(&guc->stage_ids);
+
+        return 0;
+}
+
+static void guc_stage_desc_pool_destroy(struct intel_guc *guc)
+{
+        ida_destroy(&guc->stage_ids);
+        i915_gem_object_unpin_map(guc->stage_desc_pool->obj);
+        i915_vma_unpin_and_release(&guc->stage_desc_pool);
+}
+
 /*
  * Initialise/clear the stage descriptor shared with the GuC firmware.
  *
@@ -319,7 +356,7 @@ static void guc_proc_desc_init(struct intel_guc *guc,
  * write queue, etc).
  */
 static void guc_stage_desc_init(struct intel_guc *guc,
-                                struct i915_guc_client *client)
+                                struct intel_guc_client *client)
 {
         struct drm_i915_private *dev_priv = guc_to_i915(guc);
         struct intel_engine_cs *engine;
@@ -331,7 +368,10 @@ static void guc_stage_desc_init(struct intel_guc *guc,
         desc = __get_stage_desc(client);
         memset(desc, 0, sizeof(*desc));
 
-        desc->attribute = GUC_STAGE_DESC_ATTR_ACTIVE | GUC_STAGE_DESC_ATTR_KERNEL;
+        desc->attribute = GUC_STAGE_DESC_ATTR_ACTIVE |
+                          GUC_STAGE_DESC_ATTR_KERNEL;
+        if (is_high_priority(client))
+                desc->attribute |= GUC_STAGE_DESC_ATTR_PREEMPT;
         desc->stage_id = client->stage_id;
         desc->priority = client->priority;
         desc->db_id = client->doorbell_id;
@@ -356,7 +396,7 @@ static void guc_stage_desc_init(struct intel_guc *guc,
                  * submission or, in other words, not using a direct submission
                  * model) the KMD's LRCA is not used for any work submission.
                  * Instead, the GuC uses the LRCA of the user mode context (see
-                 * guc_wq_item_append below).
+                 * guc_add_request below).
                  */
                 lrc->context_desc = lower_32_bits(ce->lrc_desc);
 
@@ -365,7 +405,8 @@ static void guc_stage_desc_init(struct intel_guc *guc,
                         guc_ggtt_offset(ce->state) + LRC_STATE_PN * PAGE_SIZE;
 
                 /* XXX: In direct submission, the GuC wants the HW context id
-                 * here. In proxy submission, it wants the stage id */
+                 * here. In proxy submission, it wants the stage id
+                 */
                 lrc->context_id = (client->stage_id << GUC_ELC_CTXID_OFFSET) |
                                 (guc_engine_id << GUC_ELC_ENGINE_OFFSET);
 
@@ -378,7 +419,7 @@ static void guc_stage_desc_init(struct intel_guc *guc,
         }
 
         DRM_DEBUG_DRIVER("Host engines 0x%x => GuC engines used 0x%x\n",
-                        client->engines, desc->engines_used);
+                         client->engines, desc->engines_used);
         WARN_ON(desc->engines_used == 0);
 
         /*
@@ -398,7 +439,7 @@ static void guc_stage_desc_init(struct intel_guc *guc,
 }
 
 static void guc_stage_desc_fini(struct intel_guc *guc,
-                                struct i915_guc_client *client)
+                                struct intel_guc_client *client)
 {
         struct guc_stage_desc *desc;
 
@@ -406,24 +447,47 @@ static void guc_stage_desc_fini(struct intel_guc *guc,
         memset(desc, 0, sizeof(*desc));
 }
 
+static int guc_shared_data_create(struct intel_guc *guc)
+{
+        struct i915_vma *vma;
+        void *vaddr;
+
+        vma = intel_guc_allocate_vma(guc, PAGE_SIZE);
+        if (IS_ERR(vma))
+                return PTR_ERR(vma);
+
+        vaddr = i915_gem_object_pin_map(vma->obj, I915_MAP_WB);
+        if (IS_ERR(vaddr)) {
+                i915_vma_unpin_and_release(&vma);
+                return PTR_ERR(vaddr);
+        }
+
+        guc->shared_data = vma;
+        guc->shared_data_vaddr = vaddr;
+
+        return 0;
+}
+
+static void guc_shared_data_destroy(struct intel_guc *guc)
+{
+        i915_gem_object_unpin_map(guc->shared_data->obj);
+        i915_vma_unpin_and_release(&guc->shared_data);
+}
+
 /* Construct a Work Item and append it to the GuC's Work Queue */
-static void guc_wq_item_append(struct i915_guc_client *client,
-                               struct drm_i915_gem_request *rq)
+static void guc_wq_item_append(struct intel_guc_client *client,
+                               u32 target_engine, u32 context_desc,
+                               u32 ring_tail, u32 fence_id)
 {
         /* wqi_len is in DWords, and does not include the one-word header */
         const size_t wqi_size = sizeof(struct guc_wq_item);
         const u32 wqi_len = wqi_size / sizeof(u32) - 1;
-        struct intel_engine_cs *engine = rq->engine;
-        struct i915_gem_context *ctx = rq->ctx;
         struct guc_process_desc *desc = __get_process_desc(client);
         struct guc_wq_item *wqi;
-        u32 ring_tail, wq_off;
+        u32 wq_off;
 
         lockdep_assert_held(&client->wq_lock);
 
-        ring_tail = intel_ring_set_tail(rq->ring, rq->tail) / sizeof(u64);
-        GEM_BUG_ON(ring_tail > WQ_RING_TAIL_MAX);
-
         /* For now workqueue item is 4 DWs; workqueue buffer is 2 pages. So we
          * should not have the case where structure wqi is across page, neither
          * wrapped to the beginning. This simplifies the implementation below.
@@ -445,19 +509,18 @@ static void guc_wq_item_append(struct i915_guc_client *client,
         /* Now fill in the 4-word work queue item */
         wqi->header = WQ_TYPE_INORDER |
                       (wqi_len << WQ_LEN_SHIFT) |
-                      (engine->guc_id << WQ_TARGET_SHIFT) |
+                      (target_engine << WQ_TARGET_SHIFT) |
                       WQ_NO_WCFLUSH_WAIT;
-
-        wqi->context_desc = lower_32_bits(intel_lr_context_descriptor(ctx, engine));
-
+        wqi->context_desc = context_desc;
         wqi->submit_element_info = ring_tail << WQ_RING_TAIL_SHIFT;
-        wqi->fence_id = rq->global_seqno;
+        GEM_BUG_ON(ring_tail > WQ_RING_TAIL_MAX);
+        wqi->fence_id = fence_id;
 
-        /* Postincrement WQ tail for next time. */
+        /* Make the update visible to GuC */
         WRITE_ONCE(desc->tail, (wq_off + wqi_size) & (GUC_WQ_SIZE - 1));
 }
 
-static void guc_reset_wq(struct i915_guc_client *client)
+static void guc_reset_wq(struct intel_guc_client *client)
 {
         struct guc_process_desc *desc = __get_process_desc(client);
 
@@ -465,7 +528,7 @@ static void guc_reset_wq(struct i915_guc_client *client)
         desc->tail = 0;
 }
 
-static void guc_ring_doorbell(struct i915_guc_client *client)
+static void guc_ring_doorbell(struct intel_guc_client *client)
 {
         struct guc_doorbell_info *db;
         u32 cookie;
@@ -475,29 +538,166 @@ static void guc_ring_doorbell(struct i915_guc_client *client)
         /* pointer of current doorbell cacheline */
         db = __get_doorbell(client);
 
-        /* we're not expecting the doorbell cookie to change behind our back */
+        /*
+         * We're not expecting the doorbell cookie to change behind our back,
+         * we also need to treat 0 as a reserved value.
+         */
         cookie = READ_ONCE(db->cookie);
-        WARN_ON_ONCE(xchg(&db->cookie, cookie + 1) != cookie);
+        WARN_ON_ONCE(xchg(&db->cookie, cookie + 1 ?: cookie + 2) != cookie);
 
         /* XXX: doorbell was lost and need to acquire it again */
         GEM_BUG_ON(db->db_status != GUC_DOORBELL_ENABLED);
 }
 
+static void guc_add_request(struct intel_guc *guc,
+                            struct drm_i915_gem_request *rq)
+{
+        struct intel_guc_client *client = guc->execbuf_client;
+        struct intel_engine_cs *engine = rq->engine;
+        u32 ctx_desc = lower_32_bits(intel_lr_context_descriptor(rq->ctx,
+                                                                 engine));
+        u32 ring_tail = intel_ring_set_tail(rq->ring, rq->tail) / sizeof(u64);
+
+        spin_lock(&client->wq_lock);
+
+        guc_wq_item_append(client, engine->guc_id, ctx_desc,
+                           ring_tail, rq->global_seqno);
+        guc_ring_doorbell(client);
+
+        client->submissions[engine->id] += 1;
+
+        spin_unlock(&client->wq_lock);
+}
+
+/*
+ * When we're doing submissions using regular execlists backend, writing to
+ * ELSP from CPU side is enough to make sure that writes to ringbuffer pages
+ * pinned in mappable aperture portion of GGTT are visible to command streamer.
+ * Writes done by GuC on our behalf are not guaranteeing such ordering,
+ * therefore, to ensure the flush, we're issuing a POSTING READ.
+ */
+static void flush_ggtt_writes(struct i915_vma *vma)
+{
+        struct drm_i915_private *dev_priv = to_i915(vma->obj->base.dev);
+
+        if (i915_vma_is_map_and_fenceable(vma))
+                POSTING_READ_FW(GUC_STATUS);
+}
+
+#define GUC_PREEMPT_FINISHED 0x1
+#define GUC_PREEMPT_BREADCRUMB_DWORDS 0x8
+static void inject_preempt_context(struct work_struct *work)
+{
+        struct guc_preempt_work *preempt_work =
+                container_of(work, typeof(*preempt_work), work);
+        struct intel_engine_cs *engine = preempt_work->engine;
+        struct intel_guc *guc = container_of(preempt_work, typeof(*guc),
+                                             preempt_work[engine->id]);
+        struct intel_guc_client *client = guc->preempt_client;
+        struct guc_stage_desc *stage_desc = __get_stage_desc(client);
+        struct intel_ring *ring = client->owner->engine[engine->id].ring;
+        u32 ctx_desc = lower_32_bits(intel_lr_context_descriptor(client->owner,
+                                                                 engine));
+        u32 *cs = ring->vaddr + ring->tail;
+        u32 data[7];
+
+        if (engine->id == RCS) {
+                cs = gen8_emit_ggtt_write_rcs(cs, GUC_PREEMPT_FINISHED,
+                                intel_hws_preempt_done_address(engine));
+        } else {
+                cs = gen8_emit_ggtt_write(cs, GUC_PREEMPT_FINISHED,
+                                intel_hws_preempt_done_address(engine));
+                *cs++ = MI_NOOP;
+                *cs++ = MI_NOOP;
+        }
+        *cs++ = MI_USER_INTERRUPT;
+        *cs++ = MI_NOOP;
+
+        GEM_BUG_ON(!IS_ALIGNED(ring->size,
+                               GUC_PREEMPT_BREADCRUMB_DWORDS * sizeof(u32)));
+        GEM_BUG_ON((void *)cs - (ring->vaddr + ring->tail) !=
+                   GUC_PREEMPT_BREADCRUMB_DWORDS * sizeof(u32));
+
+        ring->tail += GUC_PREEMPT_BREADCRUMB_DWORDS * sizeof(u32);
+        ring->tail &= (ring->size - 1);
+
+        flush_ggtt_writes(ring->vma);
+
+        spin_lock_irq(&client->wq_lock);
+        guc_wq_item_append(client, engine->guc_id, ctx_desc,
+                           ring->tail / sizeof(u64), 0);
+        spin_unlock_irq(&client->wq_lock);
+
+        /*
+         * If GuC firmware performs an engine reset while that engine had
+         * a preemption pending, it will set the terminated attribute bit
+         * on our preemption stage descriptor. GuC firmware retains all
+         * pending work items for a high-priority GuC client, unlike the
+         * normal-priority GuC client where work items are dropped. It
+         * wants to make sure the preempt-to-idle work doesn't run when
+         * scheduling resumes, and uses this bit to inform its scheduler
+         * and presumably us as well. Our job is to clear it for the next
+         * preemption after reset, otherwise that and future preemptions
+         * will never complete. We'll just clear it every time.
+         */
+        stage_desc->attribute &= ~GUC_STAGE_DESC_ATTR_TERMINATED;
+
+        data[0] = INTEL_GUC_ACTION_REQUEST_PREEMPTION;
+        data[1] = client->stage_id;
+        data[2] = INTEL_GUC_PREEMPT_OPTION_DROP_WORK_Q |
+                  INTEL_GUC_PREEMPT_OPTION_DROP_SUBMIT_Q;
+        data[3] = engine->guc_id;
+        data[4] = guc->execbuf_client->priority;
+        data[5] = guc->execbuf_client->stage_id;
+        data[6] = guc_ggtt_offset(guc->shared_data);
+
+        if (WARN_ON(intel_guc_send(guc, data, ARRAY_SIZE(data)))) {
+                execlists_clear_active(&engine->execlists,
+                                       EXECLISTS_ACTIVE_PREEMPT);
+                tasklet_schedule(&engine->execlists.tasklet);
+        }
+}
+
+/*
+ * We're using user interrupt and HWSP value to mark that preemption has
+ * finished and GPU is idle. Normally, we could unwind and continue similar to
+ * execlists submission path. Unfortunately, with GuC we also need to wait for
+ * it to finish its own postprocessing, before attempting to submit. Otherwise
+ * GuC may silently ignore our submissions, and thus we risk losing request at
+ * best, executing out-of-order and causing kernel panic at worst.
+ */
+#define GUC_PREEMPT_POSTPROCESS_DELAY_MS 10
+static void wait_for_guc_preempt_report(struct intel_engine_cs *engine)
+{
+        struct intel_guc *guc = &engine->i915->guc;
+        struct guc_shared_ctx_data *data = guc->shared_data_vaddr;
+        struct guc_ctx_report *report =
+                &data->preempt_ctx_report[engine->guc_id];
+
+        WARN_ON(wait_for_atomic(report->report_return_status ==
+                                INTEL_GUC_REPORT_STATUS_COMPLETE,
+                                GUC_PREEMPT_POSTPROCESS_DELAY_MS));
+        /*
+         * GuC is expecting that we're also going to clear the affected context
+         * counter, let's also reset the return status to not depend on GuC
+         * resetting it after recieving another preempt action
+         */
+        report->affected_count = 0;
+        report->report_return_status = INTEL_GUC_REPORT_STATUS_UNKNOWN;
+}
+
 /**
- * i915_guc_submit() - Submit commands through GuC
+ * guc_submit() - Submit commands through GuC
  * @engine: engine associated with the commands
  *
  * The only error here arises if the doorbell hardware isn't functioning
  * as expected, which really shouln't happen.
  */
-static void i915_guc_submit(struct intel_engine_cs *engine)
+static void guc_submit(struct intel_engine_cs *engine)
 {
-        struct drm_i915_private *dev_priv = engine->i915;
-        struct intel_guc *guc = &dev_priv->guc;
-        struct i915_guc_client *client = guc->execbuf_client;
+        struct intel_guc *guc = &engine->i915->guc;
         struct intel_engine_execlists * const execlists = &engine->execlists;
         struct execlist_port *port = execlists->port;
-        const unsigned int engine_id = engine->id;
         unsigned int n;
 
         for (n = 0; n < execlists_num_ports(execlists); n++) {
@@ -508,44 +708,13 @@ static void i915_guc_submit(struct intel_engine_cs *engine)
                 if (rq && count == 0) {
                         port_set(&port[n], port_pack(rq, ++count));
 
-                        if (i915_vma_is_map_and_fenceable(rq->ring->vma))
-                                POSTING_READ_FW(GUC_STATUS);
-
-                        spin_lock(&client->wq_lock);
-
-                        guc_wq_item_append(client, rq);
-                        guc_ring_doorbell(client);
+                        flush_ggtt_writes(rq->ring->vma);
 
-                        client->submissions[engine_id] += 1;
-
-                        spin_unlock(&client->wq_lock);
+                        guc_add_request(guc, rq);
                 }
         }
 }
 
-static void nested_enable_signaling(struct drm_i915_gem_request *rq)
-{
-        /* If we use dma_fence_enable_sw_signaling() directly, lockdep
-         * detects an ordering issue between the fence lockclass and the
-         * global_timeline. This circular dependency can only occur via 2
-         * different fences (but same fence lockclass), so we use the nesting
-         * annotation here to prevent the warn, equivalent to the nesting
-         * inside i915_gem_request_submit() for when we also enable the
-         * signaler.
-         */
-
-        if (test_and_set_bit(DMA_FENCE_FLAG_ENABLE_SIGNAL_BIT,
-                             &rq->fence.flags))
-                return;
-
-        GEM_BUG_ON(test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &rq->fence.flags));
-        trace_dma_fence_enable_signal(&rq->fence);
-
-        spin_lock_nested(&rq->lock, SINGLE_DEPTH_NESTING);
-        intel_engine_enable_signaling(rq, true);
-        spin_unlock(&rq->lock);
-}
-
 static void port_assign(struct execlist_port *port,
                         struct drm_i915_gem_request *rq)
 {
@@ -555,10 +724,9 @@ static void port_assign(struct execlist_port *port,
                 i915_gem_request_put(port_request(port));
 
         port_set(port, port_pack(i915_gem_request_get(rq), port_count(port)));
-        nested_enable_signaling(rq);
 }
 
-static void i915_guc_dequeue(struct intel_engine_cs *engine)
+static void guc_dequeue(struct intel_engine_cs *engine)
 {
         struct intel_engine_execlists * const execlists = &engine->execlists;
         struct execlist_port *port = execlists->port;
@@ -568,13 +736,32 @@ static void i915_guc_dequeue(struct intel_engine_cs *engine)
         bool submit = false;
         struct rb_node *rb;
 
-        if (port_isset(port))
-                port++;
-
         spin_lock_irq(&engine->timeline->lock);
         rb = execlists->first;
         GEM_BUG_ON(rb_first(&execlists->queue) != rb);
-        while (rb) {
+
+        if (!rb)
+                goto unlock;
+
+        if (HAS_LOGICAL_RING_PREEMPTION(engine->i915) && port_isset(port)) {
+                struct guc_preempt_work *preempt_work =
+                        &engine->i915->guc.preempt_work[engine->id];
+
+                if (rb_entry(rb, struct i915_priolist, node)->priority >
+                    max(port_request(port)->priotree.priority, 0)) {
+                        execlists_set_active(execlists,
+                                             EXECLISTS_ACTIVE_PREEMPT);
+                        queue_work(engine->i915->guc.preempt_wq,
+                                   &preempt_work->work);
+                        goto unlock;
+                } else if (port_isset(last_port)) {
+                        goto unlock;
+                }
+
+                port++;
+        }
+
+        do {
                 struct i915_priolist *p = rb_entry(rb, typeof(*p), node);
                 struct drm_i915_gem_request *rq, *rn;
 
@@ -592,10 +779,10 @@ static void i915_guc_dequeue(struct intel_engine_cs *engine)
                         }
 
                         INIT_LIST_HEAD(&rq->priotree.link);
-                        rq->priotree.priority = INT_MAX;
 
                         __i915_gem_request_submit(rq);
-                        trace_i915_gem_request_in(rq, port_index(port, execlists));
+                        trace_i915_gem_request_in(rq,
+                                                  port_index(port, execlists));
                         last = rq;
                         submit = true;
                 }
@@ -605,24 +792,23 @@ static void i915_guc_dequeue(struct intel_engine_cs *engine)
                 INIT_LIST_HEAD(&p->requests);
                 if (p->priority != I915_PRIORITY_NORMAL)
                         kmem_cache_free(engine->i915->priorities, p);
-        }
+        } while (rb);
 done:
         execlists->first = rb;
         if (submit) {
                 port_assign(port, last);
                 execlists_set_active(execlists, EXECLISTS_ACTIVE_USER);
-                i915_guc_submit(engine);
+                guc_submit(engine);
         }
+unlock:
         spin_unlock_irq(&engine->timeline->lock);
 }
 
-static void i915_guc_irq_handler(unsigned long data)
+static void guc_submission_tasklet(unsigned long data)
 {
         struct intel_engine_cs * const engine = (struct intel_engine_cs *)data;
         struct intel_engine_execlists * const execlists = &engine->execlists;
         struct execlist_port *port = execlists->port;
-        const struct execlist_port * const last_port =
-                &execlists->port[execlists->port_mask];
         struct drm_i915_gem_request *rq;
 
         rq = port_request(&port[0]);
@@ -637,14 +823,26 @@ static void i915_guc_irq_handler(unsigned long data)
         if (!rq)
                 execlists_clear_active(execlists, EXECLISTS_ACTIVE_USER);
 
-        if (!port_isset(last_port))
-                i915_guc_dequeue(engine);
+        if (execlists_is_active(execlists, EXECLISTS_ACTIVE_PREEMPT) &&
+            intel_read_status_page(engine, I915_GEM_HWS_PREEMPT_INDEX) ==
+            GUC_PREEMPT_FINISHED) {
+                execlists_cancel_port_requests(&engine->execlists);
+                execlists_unwind_incomplete_requests(execlists);
+
+                wait_for_guc_preempt_report(engine);
+
+                execlists_clear_active(execlists, EXECLISTS_ACTIVE_PREEMPT);
+                intel_write_status_page(engine, I915_GEM_HWS_PREEMPT_INDEX, 0);
+        }
+
+        if (!execlists_is_active(execlists, EXECLISTS_ACTIVE_PREEMPT))
+                guc_dequeue(engine);
 }
 
 /*
  * Everything below here is concerned with setup & teardown, and is
  * therefore not part of the somewhat time-critical batch-submission
- * path of i915_guc_submit() above.
+ * path of guc_submit() above.
  */
 
 /* Check that a doorbell register is in the expected state */
@@ -673,7 +871,7 @@ static bool doorbell_ok(struct intel_guc *guc, u16 db_id)
  * reloaded the GuC FW) we can use this function to tell the GuC to reassign the
  * doorbell to the rightful owner.
  */
-static int __reset_doorbell(struct i915_guc_client* client, u16 db_id)
+static int __reset_doorbell(struct intel_guc_client *client, u16 db_id)
 {
         int err;
 
@@ -694,7 +892,7 @@ static int __reset_doorbell(struct i915_guc_client* client, u16 db_id)
  */
 static int guc_init_doorbell_hw(struct intel_guc *guc)
 {
-        struct i915_guc_client *client = guc->execbuf_client;
+        struct intel_guc_client *client = guc->execbuf_client;
         bool recreate_first_client = false;
         u16 db_id;
         int ret;
@@ -717,7 +915,8 @@ static int guc_init_doorbell_hw(struct intel_guc *guc)
         if (recreate_first_client) {
                 ret = __reserve_doorbell(client);
                 if (unlikely(ret)) {
-                        DRM_ERROR("Couldn't re-reserve first client db: %d\n", ret);
+                        DRM_ERROR("Couldn't re-reserve first client db: %d\n",
+                                  ret);
                         return ret;
                 }
 
@@ -725,15 +924,16 @@ static int guc_init_doorbell_hw(struct intel_guc *guc)
         }
 
         /* Now for every client (and not only execbuf_client) make sure their
-         * doorbells are known by the GuC */
-        //for (client = client_list; client != NULL; client = client->next)
-        {
-                ret = __create_doorbell(client);
-                if (ret) {
-                        DRM_ERROR("Couldn't recreate client %u doorbell: %d\n",
-                                client->stage_id, ret);
-                        return ret;
-                }
+         * doorbells are known by the GuC
+         */
+        ret = __create_doorbell(guc->execbuf_client);
+        if (ret)
+                return ret;
+
+        ret = __create_doorbell(guc->preempt_client);
+        if (ret) {
+                __destroy_doorbell(guc->execbuf_client);
+                return ret;
         }
 
         /* Read back & verify all (used & unused) doorbell registers */
@@ -744,25 +944,25 @@ static int guc_init_doorbell_hw(struct intel_guc *guc)
 }
 
 /**
- * guc_client_alloc() - Allocate an i915_guc_client
+ * guc_client_alloc() - Allocate an intel_guc_client
  * @dev_priv:   driver private data structure
  * @engines:    The set of engines to enable for this client
  * @priority:   four levels priority _CRITICAL, _HIGH, _NORMAL and _LOW
- *              The kernel client to replace ExecList submission is created with
- *              NORMAL priority. Priority of a client for scheduler can be HIGH,
- *              while a preemption context can use CRITICAL.
+ *              The kernel client to replace ExecList submission is created with
+ *              NORMAL priority. Priority of a client for scheduler can be HIGH,
+ *              while a preemption context can use CRITICAL.
  * @ctx:        the context that owns the client (we use the default render
- *              context)
+ *              context)
  *
- * Return:      An i915_guc_client object if success, else NULL.
+ * Return:      An intel_guc_client object if success, else NULL.
  */
-static struct i915_guc_client *
+static struct intel_guc_client *
 guc_client_alloc(struct drm_i915_private *dev_priv,
                  u32 engines,
                  u32 priority,
                  struct i915_gem_context *ctx)
 {
-        struct i915_guc_client *client;
+        struct intel_guc_client *client;
         struct intel_guc *guc = &dev_priv->guc;
         struct i915_vma *vma;
         void *vaddr;
@@ -780,7 +980,7 @@ guc_client_alloc(struct drm_i915_private *dev_priv,
         spin_lock_init(&client->wq_lock);
 
         ret = ida_simple_get(&guc->stage_ids, 0, GUC_MAX_STAGE_DESCRIPTORS,
-                                GFP_KERNEL);
+                             GFP_KERNEL);
         if (ret < 0)
                 goto err_client;
 
@@ -840,7 +1040,7 @@ err_client:
         return ERR_PTR(ret);
 }
 
-static void guc_client_free(struct i915_guc_client *client)
+static void guc_client_free(struct intel_guc_client *client)
 {
         /*
          * XXX: wait for any outstanding submissions before freeing memory.
@@ -849,7 +1049,8 @@ static void guc_client_free(struct i915_guc_client *client)
 
         /* FIXME: in many cases, by the time we get here the GuC has been
          * reset, so we cannot destroy the doorbell properly. Ignore the
-         * error message for now */
+         * error message for now
+         */
         destroy_doorbell(client);
         guc_stage_desc_fini(client->guc, client);
         i915_gem_object_unpin_map(client->vma->obj);
@@ -858,6 +1059,50 @@ static void guc_client_free(struct i915_guc_client *client)
         kfree(client);
 }
 
+static int guc_clients_create(struct intel_guc *guc)
+{
+        struct drm_i915_private *dev_priv = guc_to_i915(guc);
+        struct intel_guc_client *client;
+
+        GEM_BUG_ON(guc->execbuf_client);
+        GEM_BUG_ON(guc->preempt_client);
+
+        client = guc_client_alloc(dev_priv,
+                                  INTEL_INFO(dev_priv)->ring_mask,
+                                  GUC_CLIENT_PRIORITY_KMD_NORMAL,
+                                  dev_priv->kernel_context);
+        if (IS_ERR(client)) {
+                DRM_ERROR("Failed to create GuC client for submission!\n");
+                return PTR_ERR(client);
+        }
+        guc->execbuf_client = client;
+
+        client = guc_client_alloc(dev_priv,
+                                  INTEL_INFO(dev_priv)->ring_mask,
+                                  GUC_CLIENT_PRIORITY_KMD_HIGH,
+                                  dev_priv->preempt_context);
+        if (IS_ERR(client)) {
+                DRM_ERROR("Failed to create GuC client for preemption!\n");
+                guc_client_free(guc->execbuf_client);
+                guc->execbuf_client = NULL;
+                return PTR_ERR(client);
+        }
+        guc->preempt_client = client;
+
+        return 0;
+}
+
+static void guc_clients_destroy(struct intel_guc *guc)
+{
+        struct intel_guc_client *client;
+
+        client = fetch_and_zero(&guc->execbuf_client);
+        guc_client_free(client);
+
+        client = fetch_and_zero(&guc->preempt_client);
+        guc_client_free(client);
+}
+
 static void guc_policy_init(struct guc_policy *policy)
 {
         policy->execution_quantum = POLICY_DEFAULT_EXECUTION_QUANTUM_US;
@@ -941,7 +1186,8 @@ static int guc_ads_create(struct intel_guc *guc)
          * because our GuC shared data is there.
          */
         blob->ads.golden_context_lrca =
-                guc_ggtt_offset(dev_priv->kernel_context->engine[RCS].state) + skipped_offset;
+                guc_ggtt_offset(dev_priv->kernel_context->engine[RCS].state) +
+                skipped_offset;
 
         /*
          * The GuC expects us to exclude the portion of the context image that
@@ -950,7 +1196,8 @@ static int guc_ads_create(struct intel_guc *guc)
          * dwords). Weird guc is weird.
          */
         for_each_engine(engine, dev_priv, id)
-                blob->ads.eng_state_size[engine->guc_id] = engine->context_size - skipped_size;
+                blob->ads.eng_state_size[engine->guc_id] =
+                        engine->context_size - skipped_size;
 
         base = guc_ggtt_offset(vma);
         blob->ads.scheduler_policies = base + ptr_offset(blob, policies);
@@ -967,66 +1214,110 @@ static void guc_ads_destroy(struct intel_guc *guc)
         i915_vma_unpin_and_release(&guc->ads_vma);
 }
 
+static int guc_preempt_work_create(struct intel_guc *guc)
+{
+        struct drm_i915_private *dev_priv = guc_to_i915(guc);
+        struct intel_engine_cs *engine;
+        enum intel_engine_id id;
+
+        /*
+         * Even though both sending GuC action, and adding a new workitem to
+         * GuC workqueue are serialized (each with its own locking), since
+         * we're using mutliple engines, it's possible that we're going to
+         * issue a preempt request with two (or more - each for different
+         * engine) workitems in GuC queue. In this situation, GuC may submit
+         * all of them, which will make us very confused.
+         * Our preemption contexts may even already be complete - before we
+         * even had the chance to sent the preempt action to GuC!. Rather
+         * than introducing yet another lock, we can just use ordered workqueue
+         * to make sure we're always sending a single preemption request with a
+         * single workitem.
+         */
+        guc->preempt_wq = alloc_ordered_workqueue("i915-guc_preempt",
+                                                  WQ_HIGHPRI);
+        if (!guc->preempt_wq)
+                return -ENOMEM;
+
+        for_each_engine(engine, dev_priv, id) {
+                guc->preempt_work[id].engine = engine;
+                INIT_WORK(&guc->preempt_work[id].work, inject_preempt_context);
+        }
+
+        return 0;
+}
+
+static void guc_preempt_work_destroy(struct intel_guc *guc)
+{
+        struct drm_i915_private *dev_priv = guc_to_i915(guc);
+        struct intel_engine_cs *engine;
+        enum intel_engine_id id;
+
+        for_each_engine(engine, dev_priv, id)
+                cancel_work_sync(&guc->preempt_work[id].work);
+
+        destroy_workqueue(guc->preempt_wq);
+        guc->preempt_wq = NULL;
+}
+
 /*
  * Set up the memory resources to be shared with the GuC (via the GGTT)
  * at firmware loading time.
  */
-int i915_guc_submission_init(struct drm_i915_private *dev_priv)
+int intel_guc_submission_init(struct intel_guc *guc)
 {
-        struct intel_guc *guc = &dev_priv->guc;
-        struct i915_vma *vma;
-        void *vaddr;
         int ret;
 
         if (guc->stage_desc_pool)
                 return 0;
 
-        vma = intel_guc_allocate_vma(guc,
-                                PAGE_ALIGN(sizeof(struct guc_stage_desc) *
-                                        GUC_MAX_STAGE_DESCRIPTORS));
-        if (IS_ERR(vma))
-                return PTR_ERR(vma);
-
-        guc->stage_desc_pool = vma;
-
-        vaddr = i915_gem_object_pin_map(guc->stage_desc_pool->obj, I915_MAP_WB);
-        if (IS_ERR(vaddr)) {
-                ret = PTR_ERR(vaddr);
-                goto err_vma;
-        }
+        ret = guc_stage_desc_pool_create(guc);
+        if (ret)
+                return ret;
+        /*
+         * Keep static analysers happy, let them know that we allocated the
+         * vma after testing that it didn't exist earlier.
+         */
+        GEM_BUG_ON(!guc->stage_desc_pool);
 
-        guc->stage_desc_pool_vaddr = vaddr;
+        ret = guc_shared_data_create(guc);
+        if (ret)
+                goto err_stage_desc_pool;
+        GEM_BUG_ON(!guc->shared_data);
 
         ret = intel_guc_log_create(guc);
         if (ret < 0)
-                goto err_vaddr;
+                goto err_shared_data;
 
-        ret = guc_ads_create(guc);
-        if (ret < 0)
+        ret = guc_preempt_work_create(guc);
+        if (ret)
                 goto err_log;
+        GEM_BUG_ON(!guc->preempt_wq);
 
-        ida_init(&guc->stage_ids);
+        ret = guc_ads_create(guc);
+        if (ret < 0)
+                goto err_wq;
+        GEM_BUG_ON(!guc->ads_vma);
 
         return 0;
 
+err_wq:
+        guc_preempt_work_destroy(guc);
 err_log:
         intel_guc_log_destroy(guc);
-err_vaddr:
-        i915_gem_object_unpin_map(guc->stage_desc_pool->obj);
-err_vma:
-        i915_vma_unpin_and_release(&guc->stage_desc_pool);
+err_shared_data:
+        guc_shared_data_destroy(guc);
+err_stage_desc_pool:
+        guc_stage_desc_pool_destroy(guc);
         return ret;
 }
 
-void i915_guc_submission_fini(struct drm_i915_private *dev_priv)
+void intel_guc_submission_fini(struct intel_guc *guc)
 {
-        struct intel_guc *guc = &dev_priv->guc;
-
-        ida_destroy(&guc->stage_ids);
         guc_ads_destroy(guc);
+        guc_preempt_work_destroy(guc);
         intel_guc_log_destroy(guc);
-        i915_gem_object_unpin_map(guc->stage_desc_pool->obj);
-        i915_vma_unpin_and_release(&guc->stage_desc_pool);
+        guc_shared_data_destroy(guc);
+        guc_stage_desc_pool_destroy(guc);
 }
 
 static void guc_interrupts_capture(struct drm_i915_private *dev_priv)
@@ -1036,7 +1327,9 @@ static void guc_interrupts_capture(struct drm_i915_private *dev_priv)
         enum intel_engine_id id;
         int irqs;
 
-        /* tell all command streamers to forward interrupts (but not vblank) to GuC */
+        /* tell all command streamers to forward interrupts (but not vblank)
+         * to GuC
+         */
         irqs = _MASKED_BIT_ENABLE(GFX_INTERRUPT_STEERING);
         for_each_engine(engine, dev_priv, id)
                 I915_WRITE(RING_MODE_GEN7(engine), irqs);
@@ -1097,10 +1390,19 @@ static void guc_interrupts_release(struct drm_i915_private *dev_priv)
         rps->pm_intrmsk_mbz &= ~ARAT_EXPIRED_INTRMSK;
 }
 
-int i915_guc_submission_enable(struct drm_i915_private *dev_priv)
+static void guc_submission_park(struct intel_engine_cs *engine)
 {
-        struct intel_guc *guc = &dev_priv->guc;
-        struct i915_guc_client *client = guc->execbuf_client;
+        intel_engine_unpin_breadcrumbs_irq(engine);
+}
+
+static void guc_submission_unpark(struct intel_engine_cs *engine)
+{
+        intel_engine_pin_breadcrumbs_irq(engine);
+}
+
+int intel_guc_submission_enable(struct intel_guc *guc)
+{
+        struct drm_i915_private *dev_priv = guc_to_i915(guc);
         struct intel_engine_cs *engine;
         enum intel_engine_id id;
         int err;
@@ -1118,61 +1420,62 @@ int i915_guc_submission_enable(struct drm_i915_private *dev_priv)
                      sizeof(struct guc_wq_item) *
                      I915_NUM_ENGINES > GUC_WQ_SIZE);
 
-        if (!client) {
-                client = guc_client_alloc(dev_priv,
-                                          INTEL_INFO(dev_priv)->ring_mask,
-                                          GUC_CLIENT_PRIORITY_KMD_NORMAL,
-                                          dev_priv->kernel_context);
-                if (IS_ERR(client)) {
-                        DRM_ERROR("Failed to create GuC client for execbuf!\n");
-                        return PTR_ERR(client);
-                }
-
-                guc->execbuf_client = client;
+        /*
+         * We're being called on both module initialization and on reset,
+         * until this flow is changed, we're using regular client presence to
+         * determine which case are we in, and whether we should allocate new
+         * clients or just reset their workqueues.
+         */
+        if (!guc->execbuf_client) {
+                err = guc_clients_create(guc);
+                if (err)
+                        return err;
+        } else {
+                guc_reset_wq(guc->execbuf_client);
+                guc_reset_wq(guc->preempt_client);
         }
 
         err = intel_guc_sample_forcewake(guc);
         if (err)
-                goto err_execbuf_client;
-
-        guc_reset_wq(client);
+                goto err_free_clients;
 
         err = guc_init_doorbell_hw(guc);
         if (err)
-                goto err_execbuf_client;
+                goto err_free_clients;
 
         /* Take over from manual control of ELSP (execlists) */
         guc_interrupts_capture(dev_priv);
 
         for_each_engine(engine, dev_priv, id) {
-                struct intel_engine_execlists * const execlists = &engine->execlists;
-                /* The tasklet was initialised by execlists, and may be in
-                 * a state of flux (across a reset) and so we just want to
-                 * take over the callback without changing any other state
-                 * in the tasklet.
-                 */
-                execlists->irq_tasklet.func = i915_guc_irq_handler;
-                clear_bit(ENGINE_IRQ_EXECLIST, &engine->irq_posted);
-                tasklet_schedule(&execlists->irq_tasklet);
+                struct intel_engine_execlists * const execlists =
+                        &engine->execlists;
+
+                execlists->tasklet.func = guc_submission_tasklet;
+                engine->park = guc_submission_park;
+                engine->unpark = guc_submission_unpark;
         }
 
         return 0;
 
-err_execbuf_client:
-        guc_client_free(guc->execbuf_client);
-        guc->execbuf_client = NULL;
+err_free_clients:
+        guc_clients_destroy(guc);
         return err;
 }
 
-void i915_guc_submission_disable(struct drm_i915_private *dev_priv)
+void intel_guc_submission_disable(struct intel_guc *guc)
 {
-        struct intel_guc *guc = &dev_priv->guc;
+        struct drm_i915_private *dev_priv = guc_to_i915(guc);
+
+        GEM_BUG_ON(dev_priv->gt.awake); /* GT should be parked first */
 
         guc_interrupts_release(dev_priv);
 
         /* Revert back to manual ELSP submission */
         intel_engines_reset_default_submission(dev_priv);
 
-        guc_client_free(guc->execbuf_client);
-        guc->execbuf_client = NULL;
+        guc_clients_destroy(guc);
 }
+
+#if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
+#include "selftests/intel_guc.c"
+#endif
diff --git a/drivers/gpu/drm/i915/i915_guc_submission.h b/drivers/gpu/drm/i915/intel_guc_submission.h
index cb4353b59059..e901192ee469 100644
--- a/drivers/gpu/drm/i915/i915_guc_submission.h
+++ b/drivers/gpu/drm/i915/intel_guc_submission.h
@@ -52,7 +52,7 @@ struct drm_i915_private;
  * queue (a circular array of work items), again described in the process
  * descriptor. Work queue pages are mapped momentarily as required.
  */
-struct i915_guc_client {
+struct intel_guc_client {
         struct i915_vma *vma;
         void *vaddr;
         struct i915_gem_context *owner;
@@ -67,14 +67,15 @@ struct i915_guc_client {
         u16 doorbell_id;
         unsigned long doorbell_offset;
 
+        /* Protects GuC client's WQ access */
         spinlock_t wq_lock;
         /* Per-engine counts of GuC submissions */
         u64 submissions[I915_NUM_ENGINES];
 };
 
-int i915_guc_submission_init(struct drm_i915_private *dev_priv);
-int i915_guc_submission_enable(struct drm_i915_private *dev_priv);
-void i915_guc_submission_disable(struct drm_i915_private *dev_priv);
-void i915_guc_submission_fini(struct drm_i915_private *dev_priv);
+int intel_guc_submission_init(struct intel_guc *guc);
+int intel_guc_submission_enable(struct intel_guc *guc);
+void intel_guc_submission_disable(struct intel_guc *guc);
+void intel_guc_submission_fini(struct intel_guc *guc);
 
 #endif
diff --git a/drivers/gpu/drm/i915/intel_hdmi.c b/drivers/gpu/drm/i915/intel_hdmi.c
index e039702c1907..a40f35af225c 100644
--- a/drivers/gpu/drm/i915/intel_hdmi.c
+++ b/drivers/gpu/drm/i915/intel_hdmi.c
@@ -186,7 +186,7 @@ static bool g4x_infoframe_enabled(struct drm_encoder *encoder,
         if ((val & VIDEO_DIP_ENABLE) == 0)
                 return false;
 
-        if ((val & VIDEO_DIP_PORT_MASK) != VIDEO_DIP_PORT(intel_dig_port->port))
+        if ((val & VIDEO_DIP_PORT_MASK) != VIDEO_DIP_PORT(intel_dig_port->base.port))
                 return false;
 
         return val & (VIDEO_DIP_ENABLE_AVI |
@@ -245,7 +245,7 @@ static bool ibx_infoframe_enabled(struct drm_encoder *encoder,
         if ((val & VIDEO_DIP_ENABLE) == 0)
                 return false;
 
-        if ((val & VIDEO_DIP_PORT_MASK) != VIDEO_DIP_PORT(intel_dig_port->port))
+        if ((val & VIDEO_DIP_PORT_MASK) != VIDEO_DIP_PORT(intel_dig_port->base.port))
                 return false;
 
         return val & (VIDEO_DIP_ENABLE_AVI |
@@ -362,7 +362,7 @@ static bool vlv_infoframe_enabled(struct drm_encoder *encoder,
         if ((val & VIDEO_DIP_ENABLE) == 0)
                 return false;
 
-        if ((val & VIDEO_DIP_PORT_MASK) != VIDEO_DIP_PORT(intel_dig_port->port))
+        if ((val & VIDEO_DIP_PORT_MASK) != VIDEO_DIP_PORT(intel_dig_port->base.port))
                 return false;
 
         return val & (VIDEO_DIP_ENABLE_AVI |
@@ -538,7 +538,7 @@ static void g4x_set_infoframes(struct drm_encoder *encoder,
         struct intel_hdmi *intel_hdmi = &intel_dig_port->hdmi;
         i915_reg_t reg = VIDEO_DIP_CTL;
         u32 val = I915_READ(reg);
-        u32 port = VIDEO_DIP_PORT(intel_dig_port->port);
+        u32 port = VIDEO_DIP_PORT(intel_dig_port->base.port);
 
         assert_hdmi_port_disabled(intel_hdmi);
 
@@ -689,7 +689,7 @@ static void ibx_set_infoframes(struct drm_encoder *encoder,
         struct intel_hdmi *intel_hdmi = &intel_dig_port->hdmi;
         i915_reg_t reg = TVIDEO_DIP_CTL(intel_crtc->pipe);
         u32 val = I915_READ(reg);
-        u32 port = VIDEO_DIP_PORT(intel_dig_port->port);
+        u32 port = VIDEO_DIP_PORT(intel_dig_port->base.port);
 
         assert_hdmi_port_disabled(intel_hdmi);
 
@@ -785,7 +785,7 @@ static void vlv_set_infoframes(struct drm_encoder *encoder,
         struct intel_hdmi *intel_hdmi = enc_to_intel_hdmi(encoder);
         i915_reg_t reg = VLV_TVIDEO_DIP_CTL(intel_crtc->pipe);
         u32 val = I915_READ(reg);
-        u32 port = VIDEO_DIP_PORT(intel_dig_port->port);
+        u32 port = VIDEO_DIP_PORT(intel_dig_port->base.port);
 
         assert_hdmi_port_disabled(intel_hdmi);
 
@@ -960,6 +960,8 @@ static void intel_hdmi_get_config(struct intel_encoder *encoder,
         u32 tmp, flags = 0;
         int dotclock;
 
+        pipe_config->output_types |= BIT(INTEL_OUTPUT_HDMI);
+
         tmp = I915_READ(intel_hdmi->hdmi_reg);
 
         if (tmp & SDVO_HSYNC_ACTIVE_HIGH)
@@ -1207,7 +1209,8 @@ static void g4x_disable_hdmi(struct intel_encoder *encoder,
                              const struct drm_connector_state *old_conn_state)
 {
         if (old_crtc_state->has_audio)
-                intel_audio_codec_disable(encoder);
+                intel_audio_codec_disable(encoder,
+                                          old_crtc_state, old_conn_state);
 
         intel_disable_hdmi(encoder, old_crtc_state, old_conn_state);
 }
@@ -1217,7 +1220,8 @@ static void pch_disable_hdmi(struct intel_encoder *encoder,
                              const struct drm_connector_state *old_conn_state)
 {
         if (old_crtc_state->has_audio)
-                intel_audio_codec_disable(encoder);
+                intel_audio_codec_disable(encoder,
+                                          old_crtc_state, old_conn_state);
 }
 
 static void pch_post_disable_hdmi(struct intel_encoder *encoder,
@@ -1227,24 +1231,34 @@ static void pch_post_disable_hdmi(struct intel_encoder *encoder,
         intel_disable_hdmi(encoder, old_crtc_state, old_conn_state);
 }
 
-static int intel_hdmi_source_max_tmds_clock(struct drm_i915_private *dev_priv)
+static int intel_hdmi_source_max_tmds_clock(struct intel_encoder *encoder)
 {
-        if (IS_G4X(dev_priv))
-                return 165000;
-        else if (IS_GEMINILAKE(dev_priv))
-                return 594000;
-        else if (IS_HASWELL(dev_priv) || INTEL_INFO(dev_priv)->gen >= 8)
-                return 300000;
+        struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
+        const struct ddi_vbt_port_info *info =
+                &dev_priv->vbt.ddi_port_info[encoder->port];
+        int max_tmds_clock;
+
+        if (INTEL_GEN(dev_priv) >= 10 || IS_GEMINILAKE(dev_priv))
+                max_tmds_clock = 594000;
+        else if (INTEL_GEN(dev_priv) >= 8 || IS_HASWELL(dev_priv))
+                max_tmds_clock = 300000;
+        else if (INTEL_GEN(dev_priv) >= 5)
+                max_tmds_clock = 225000;
         else
-                return 225000;
+                max_tmds_clock = 165000;
+
+        if (info->max_tmds_clock)
+                max_tmds_clock = min(max_tmds_clock, info->max_tmds_clock);
+
+        return max_tmds_clock;
 }
 
 static int hdmi_port_clock_limit(struct intel_hdmi *hdmi,
                                  bool respect_downstream_limits,
                                  bool force_dvi)
 {
-        struct drm_device *dev = intel_hdmi_to_dev(hdmi);
-        int max_tmds_clock = intel_hdmi_source_max_tmds_clock(to_i915(dev));
+        struct intel_encoder *encoder = &hdmi_to_dig_port(hdmi)->base;
+        int max_tmds_clock = intel_hdmi_source_max_tmds_clock(encoder);
 
         if (respect_downstream_limits) {
                 struct intel_connector *connector = hdmi->attached_connector;
@@ -1339,6 +1353,12 @@ static bool hdmi_12bpc_possible(const struct intel_crtc_state *crtc_state)
         if (HAS_GMCH_DISPLAY(dev_priv))
                 return false;
 
+        if (crtc_state->pipe_bpp <= 8*3)
+                return false;
+
+        if (!crtc_state->has_hdmi_sink)
+                return false;
+
         /*
          * HDMI 12bpc affects the clocks, so it's only possible
          * when not cloning with other encoder types.
@@ -1464,9 +1484,8 @@ bool intel_hdmi_compute_config(struct intel_encoder *encoder,
          * outputs. We also need to check that the higher clock still fits
          * within limits.
          */
-        if (pipe_config->pipe_bpp > 8*3 && pipe_config->has_hdmi_sink && !force_dvi &&
-            hdmi_port_clock_valid(intel_hdmi, clock_12bpc, true, force_dvi) == MODE_OK &&
-            hdmi_12bpc_possible(pipe_config)) {
+        if (hdmi_12bpc_possible(pipe_config) &&
+            hdmi_port_clock_valid(intel_hdmi, clock_12bpc, true, force_dvi) == MODE_OK) {
                 DRM_DEBUG_KMS("picking bpc to 12 for HDMI output\n");
                 desired_bpp = 12*3;
 
@@ -1495,7 +1514,8 @@ bool intel_hdmi_compute_config(struct intel_encoder *encoder,
 
         pipe_config->lane_count = 4;
 
-        if (scdc->scrambling.supported && IS_GEMINILAKE(dev_priv)) {
+        if (scdc->scrambling.supported && (INTEL_GEN(dev_priv) >= 10 ||
+                                           IS_GEMINILAKE(dev_priv))) {
                 if (scdc->scrambling.low_rates)
                         pipe_config->hdmi_scrambling = true;
 
@@ -1529,7 +1549,7 @@ intel_hdmi_dp_dual_mode_detect(struct drm_connector *connector, bool has_edid)
 {
         struct drm_i915_private *dev_priv = to_i915(connector->dev);
         struct intel_hdmi *hdmi = intel_attached_hdmi(connector);
-        enum port port = hdmi_to_dig_port(hdmi)->port;
+        enum port port = hdmi_to_dig_port(hdmi)->base.port;
         struct i2c_adapter *adapter =
                 intel_gmbus_get_adapter(dev_priv, hdmi->ddc_bus);
         enum drm_dp_dual_mode_type type = drm_dp_dual_mode_detect(adapter);
@@ -1613,12 +1633,9 @@ intel_hdmi_detect(struct drm_connector *connector, bool force)
 
         intel_hdmi_unset_edid(connector);
 
-        if (intel_hdmi_set_edid(connector)) {
-                struct intel_hdmi *intel_hdmi = intel_attached_hdmi(connector);
-
-                hdmi_to_dig_port(intel_hdmi)->base.type = INTEL_OUTPUT_HDMI;
+        if (intel_hdmi_set_edid(connector))
                 status = connector_status_connected;
-        } else
+        else
                 status = connector_status_disconnected;
 
         intel_display_power_put(dev_priv, POWER_DOMAIN_GMBUS);
@@ -1629,8 +1646,6 @@ intel_hdmi_detect(struct drm_connector *connector, bool force)
 static void
 intel_hdmi_force(struct drm_connector *connector)
 {
-        struct intel_hdmi *intel_hdmi = intel_attached_hdmi(connector);
-
         DRM_DEBUG_KMS("[CONNECTOR:%d:%s]\n",
                       connector->base.id, connector->name);
 
@@ -1640,7 +1655,6 @@ intel_hdmi_force(struct drm_connector *connector)
                 return;
 
         intel_hdmi_set_edid(connector);
-        hdmi_to_dig_port(intel_hdmi)->base.type = INTEL_OUTPUT_HDMI;
 }
 
 static int intel_hdmi_get_modes(struct drm_connector *connector)
@@ -1673,10 +1687,9 @@ static void vlv_hdmi_pre_enable(struct intel_encoder *encoder,
                                 const struct drm_connector_state *conn_state)
 {
         struct intel_digital_port *dport = enc_to_dig_port(&encoder->base);
-        struct drm_device *dev = encoder->base.dev;
-        struct drm_i915_private *dev_priv = to_i915(dev);
+        struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
 
-        vlv_phy_pre_encoder_enable(encoder);
+        vlv_phy_pre_encoder_enable(encoder, pipe_config);
 
         /* HDMI 1.0V-2dB */
         vlv_set_phy_signal_level(encoder, 0x2b245f5f, 0x00002000, 0x5578b83a,
@@ -1697,7 +1710,7 @@ static void vlv_hdmi_pre_pll_enable(struct intel_encoder *encoder,
 {
         intel_hdmi_prepare(encoder, pipe_config);
 
-        vlv_phy_pre_pll_enable(encoder);
+        vlv_phy_pre_pll_enable(encoder, pipe_config);
 }
 
 static void chv_hdmi_pre_pll_enable(struct intel_encoder *encoder,
@@ -1706,14 +1719,14 @@ static void chv_hdmi_pre_pll_enable(struct intel_encoder *encoder,
 {
         intel_hdmi_prepare(encoder, pipe_config);
 
-        chv_phy_pre_pll_enable(encoder);
+        chv_phy_pre_pll_enable(encoder, pipe_config);
 }
 
 static void chv_hdmi_post_pll_disable(struct intel_encoder *encoder,
                                       const struct intel_crtc_state *old_crtc_state,
                                       const struct drm_connector_state *old_conn_state)
 {
-        chv_phy_post_pll_disable(encoder);
+        chv_phy_post_pll_disable(encoder, old_crtc_state);
 }
 
 static void vlv_hdmi_post_disable(struct intel_encoder *encoder,
@@ -1721,7 +1734,7 @@ static void vlv_hdmi_post_disable(struct intel_encoder *encoder,
                                   const struct drm_connector_state *old_conn_state)
 {
         /* Reset lanes to avoid HDMI flicker (VLV w/a) */
-        vlv_phy_reset_lanes(encoder);
+        vlv_phy_reset_lanes(encoder, old_crtc_state);
 }
 
 static void chv_hdmi_post_disable(struct intel_encoder *encoder,
@@ -1734,7 +1747,7 @@ static void chv_hdmi_post_disable(struct intel_encoder *encoder,
         mutex_lock(&dev_priv->sb_lock);
 
         /* Assert data lane reset */
-        chv_data_lane_soft_reset(encoder, true);
+        chv_data_lane_soft_reset(encoder, old_crtc_state, true);
 
         mutex_unlock(&dev_priv->sb_lock);
 }
@@ -1747,7 +1760,7 @@ static void chv_hdmi_pre_enable(struct intel_encoder *encoder,
         struct drm_device *dev = encoder->base.dev;
         struct drm_i915_private *dev_priv = to_i915(dev);
 
-        chv_phy_pre_encoder_enable(encoder);
+        chv_phy_pre_encoder_enable(encoder, pipe_config);
 
         /* FIXME: Program the support xxx V-dB */
         /* Use 800mV-0dB */
@@ -2006,7 +2019,7 @@ void intel_hdmi_init_connector(struct intel_digital_port *intel_dig_port,
         struct intel_encoder *intel_encoder = &intel_dig_port->base;
         struct drm_device *dev = intel_encoder->base.dev;
         struct drm_i915_private *dev_priv = to_i915(dev);
-        enum port port = intel_dig_port->port;
+        enum port port = intel_encoder->port;
 
         DRM_DEBUG_KMS("Adding HDMI connector on port %c\n",
                       port_name(port));
@@ -2024,7 +2037,7 @@ void intel_hdmi_init_connector(struct intel_digital_port *intel_dig_port,
         connector->doublescan_allowed = 0;
         connector->stereo_allowed = 1;
 
-        if (IS_GEMINILAKE(dev_priv))
+        if (INTEL_GEN(dev_priv) >= 10 || IS_GEMINILAKE(dev_priv))
                 connector->ycbcr_420_allowed = true;
 
         intel_hdmi->ddc_bus = intel_hdmi_ddc_pin(dev_priv, port);
@@ -2126,7 +2139,6 @@ void intel_hdmi_init(struct drm_i915_private *dev_priv,
         if (IS_G4X(dev_priv))
                 intel_encoder->cloneable |= 1 << INTEL_OUTPUT_HDMI;
 
-        intel_dig_port->port = port;
         intel_dig_port->hdmi.hdmi_reg = hdmi_reg;
         intel_dig_port->dp.output_reg = INVALID_MMIO_REG;
         intel_dig_port->max_lanes = 4;
diff --git a/drivers/gpu/drm/i915/intel_huc.c b/drivers/gpu/drm/i915/intel_huc.c
index c8a48cbc2b7d..98d17254593c 100644
--- a/drivers/gpu/drm/i915/intel_huc.c
+++ b/drivers/gpu/drm/i915/intel_huc.c
@@ -151,7 +151,7 @@ static int huc_ucode_xfer(struct intel_uc_fw *huc_fw, struct i915_vma *vma)
         I915_WRITE(DMA_CTRL, _MASKED_BIT_ENABLE(HUC_UKERNEL | START_DMA));
 
         /* Wait for DMA to finish */
-        ret = wait_for((I915_READ(DMA_CTRL) & START_DMA) == 0, 100);
+        ret = intel_wait_for_register_fw(dev_priv, DMA_CTRL, START_DMA, 0, 100);
 
         DRM_DEBUG_DRIVER("HuC DMA transfer wait over with ret %d\n", ret);
 
diff --git a/drivers/gpu/drm/i915/intel_lrc.c b/drivers/gpu/drm/i915/intel_lrc.c
index d36e25607435..be6c39adebdf 100644
--- a/drivers/gpu/drm/i915/intel_lrc.c
+++ b/drivers/gpu/drm/i915/intel_lrc.c
@@ -136,6 +136,7 @@
 #include <drm/drmP.h>
 #include <drm/i915_drm.h>
 #include "i915_drv.h"
+#include "i915_gem_render_state.h"
 #include "intel_mocs.h"
 
 #define RING_EXECLIST_QFULL             (1 << 0x2)
@@ -354,7 +355,7 @@ static void unwind_wa_tail(struct drm_i915_gem_request *rq)
         assert_ring_tail_valid(rq->ring, rq->tail);
 }
 
-static void unwind_incomplete_requests(struct intel_engine_cs *engine)
+static void __unwind_incomplete_requests(struct intel_engine_cs *engine)
 {
         struct drm_i915_gem_request *rq, *rn;
         struct i915_priolist *uninitialized_var(p);
@@ -385,6 +386,17 @@ static void unwind_incomplete_requests(struct intel_engine_cs *engine)
         }
 }
 
+void
+execlists_unwind_incomplete_requests(struct intel_engine_execlists *execlists)
+{
+        struct intel_engine_cs *engine =
+                container_of(execlists, typeof(*engine), execlists);
+
+        spin_lock_irq(&engine->timeline->lock);
+        __unwind_incomplete_requests(engine);
+        spin_unlock_irq(&engine->timeline->lock);
+}
+
 static inline void
 execlists_context_status_change(struct drm_i915_gem_request *rq,
                                 unsigned long status)
@@ -455,6 +467,11 @@ static void execlists_submit_ports(struct intel_engine_cs *engine)
                         port_set(&port[n], port_pack(rq, count));
                         desc = execlists_update_context(rq);
                         GEM_DEBUG_EXEC(port[n].context_id = upper_32_bits(desc));
+
+                        GEM_TRACE("%s in[%d]:  ctx=%d.%d, seqno=%x\n",
+                                  engine->name, n,
+                                  rq->ctx->hw_id, count,
+                                  rq->global_seqno);
                 } else {
                         GEM_BUG_ON(!n);
                         desc = 0;
@@ -509,17 +526,13 @@ static void inject_preempt_context(struct intel_engine_cs *engine)
         ce->ring->tail &= (ce->ring->size - 1);
         ce->lrc_reg_state[CTX_RING_TAIL+1] = ce->ring->tail;
 
+        GEM_TRACE("\n");
         for (n = execlists_num_ports(&engine->execlists); --n; )
                 elsp_write(0, elsp);
 
         elsp_write(ce->lrc_desc, elsp);
 }
 
-static bool can_preempt(struct intel_engine_cs *engine)
-{
-        return INTEL_INFO(engine->i915)->has_logical_ring_preemption;
-}
-
 static void execlists_dequeue(struct intel_engine_cs *engine)
 {
         struct intel_engine_execlists * const execlists = &engine->execlists;
@@ -567,7 +580,7 @@ static void execlists_dequeue(struct intel_engine_cs *engine)
                 if (port_count(&port[0]) > 1)
                         goto unlock;
 
-                if (can_preempt(engine) &&
+                if (HAS_LOGICAL_RING_PREEMPTION(engine->i915) &&
                     rb_entry(rb, struct i915_priolist, node)->priority >
                     max(last->priotree.priority, 0)) {
                         /*
@@ -690,8 +703,8 @@ unlock:
         }
 }
 
-static void
-execlist_cancel_port_requests(struct intel_engine_execlists *execlists)
+void
+execlists_cancel_port_requests(struct intel_engine_execlists * const execlists)
 {
         struct execlist_port *port = execlists->port;
         unsigned int num_ports = execlists_num_ports(execlists);
@@ -718,7 +731,7 @@ static void execlists_cancel_requests(struct intel_engine_cs *engine)
         spin_lock_irqsave(&engine->timeline->lock, flags);
 
         /* Cancel the requests on the HW and clear the ELSP tracker. */
-        execlist_cancel_port_requests(execlists);
+        execlists_cancel_port_requests(execlists);
 
         /* Mark all executing requests as skipped. */
         list_for_each_entry(rq, &engine->timeline->requests, link) {
@@ -768,7 +781,7 @@ static void execlists_cancel_requests(struct intel_engine_cs *engine)
  * Check the unread Context Status Buffers and manage the submission of new
  * contexts to the ELSP accordingly.
  */
-static void intel_lrc_irq_handler(unsigned long data)
+static void execlists_submission_tasklet(unsigned long data)
 {
         struct intel_engine_cs * const engine = (struct intel_engine_cs *)data;
         struct intel_engine_execlists * const execlists = &engine->execlists;
@@ -826,6 +839,10 @@ static void intel_lrc_irq_handler(unsigned long data)
                         head = execlists->csb_head;
                         tail = READ_ONCE(buf[write_idx]);
                 }
+                GEM_TRACE("%s cs-irq head=%d [%d], tail=%d [%d]\n",
+                          engine->name,
+                          head, GEN8_CSB_READ_PTR(readl(dev_priv->regs + i915_mmio_reg_offset(RING_CONTEXT_STATUS_PTR(engine)))),
+                          tail, GEN8_CSB_WRITE_PTR(readl(dev_priv->regs + i915_mmio_reg_offset(RING_CONTEXT_STATUS_PTR(engine)))));
 
                 while (head != tail) {
                         struct drm_i915_gem_request *rq;
@@ -853,16 +870,16 @@ static void intel_lrc_irq_handler(unsigned long data)
                          */
 
                         status = READ_ONCE(buf[2 * head]); /* maybe mmio! */
+                        GEM_TRACE("%s csb[%dd]: status=0x%08x:0x%08x\n",
+                                  engine->name, head,
+                                  status, buf[2*head + 1]);
                         if (!(status & GEN8_CTX_STATUS_COMPLETED_MASK))
                                 continue;
 
                         if (status & GEN8_CTX_STATUS_ACTIVE_IDLE &&
                             buf[2*head + 1] == PREEMPT_ID) {
-                                execlist_cancel_port_requests(execlists);
-
-                                spin_lock_irq(&engine->timeline->lock);
-                                unwind_incomplete_requests(engine);
-                                spin_unlock_irq(&engine->timeline->lock);
+                                execlists_cancel_port_requests(execlists);
+                                execlists_unwind_incomplete_requests(execlists);
 
                                 GEM_BUG_ON(!execlists_is_active(execlists,
                                                                 EXECLISTS_ACTIVE_PREEMPT));
@@ -883,6 +900,10 @@ static void intel_lrc_irq_handler(unsigned long data)
                         GEM_DEBUG_BUG_ON(buf[2 * head + 1] != port->context_id);
 
                         rq = port_unpack(port, &count);
+                        GEM_TRACE("%s out[0]: ctx=%d.%d, seqno=%x\n",
+                                  engine->name,
+                                  rq->ctx->hw_id, count,
+                                  rq->global_seqno);
                         GEM_BUG_ON(count == 0);
                         if (--count == 0) {
                                 GEM_BUG_ON(status & GEN8_CTX_STATUS_PREEMPTED);
@@ -926,7 +947,7 @@ static void insert_request(struct intel_engine_cs *engine,
 
         list_add_tail(&pt->link, &ptr_mask_bits(p, 1)->requests);
         if (ptr_unmask_bits(p, 1))
-                tasklet_hi_schedule(&engine->execlists.irq_tasklet);
+                tasklet_hi_schedule(&engine->execlists.tasklet);
 }
 
 static void execlists_submit_request(struct drm_i915_gem_request *request)
@@ -1057,12 +1078,34 @@ static void execlists_schedule(struct drm_i915_gem_request *request, int prio)
         spin_unlock_irq(&engine->timeline->lock);
 }
 
+static int __context_pin(struct i915_gem_context *ctx, struct i915_vma *vma)
+{
+        unsigned int flags;
+        int err;
+
+        /*
+         * Clear this page out of any CPU caches for coherent swap-in/out.
+         * We only want to do this on the first bind so that we do not stall
+         * on an active context (which by nature is already on the GPU).
+         */
+        if (!(vma->flags & I915_VMA_GLOBAL_BIND)) {
+                err = i915_gem_object_set_to_gtt_domain(vma->obj, true);
+                if (err)
+                        return err;
+        }
+
+        flags = PIN_GLOBAL | PIN_HIGH;
+        if (ctx->ggtt_offset_bias)
+                flags |= PIN_OFFSET_BIAS | ctx->ggtt_offset_bias;
+
+        return i915_vma_pin(vma, 0, GEN8_LR_CONTEXT_ALIGN, flags);
+}
+
 static struct intel_ring *
 execlists_context_pin(struct intel_engine_cs *engine,
                       struct i915_gem_context *ctx)
 {
         struct intel_context *ce = &ctx->engine[engine->id];
-        unsigned int flags;
         void *vaddr;
         int ret;
 
@@ -1079,11 +1122,7 @@ execlists_context_pin(struct intel_engine_cs *engine,
         }
         GEM_BUG_ON(!ce->state);
 
-        flags = PIN_GLOBAL | PIN_HIGH;
-        if (ctx->ggtt_offset_bias)
-                flags |= PIN_OFFSET_BIAS | ctx->ggtt_offset_bias;
-
-        ret = i915_vma_pin(ce->state, 0, GEN8_LR_CONTEXT_ALIGN, flags);
+        ret = __context_pin(ctx, ce->state);
         if (ret)
                 goto err;
 
@@ -1103,9 +1142,7 @@ execlists_context_pin(struct intel_engine_cs *engine,
         ce->lrc_reg_state[CTX_RING_BUFFER_START+1] =
                 i915_ggtt_offset(ce->ring->vma);
 
-        ce->state->obj->mm.dirty = true;
         ce->state->obj->pin_global++;
-
         i915_gem_context_get(ctx);
 out:
         return ce->ring;
@@ -1143,7 +1180,6 @@ static int execlists_request_alloc(struct drm_i915_gem_request *request)
 {
         struct intel_engine_cs *engine = request->engine;
         struct intel_context *ce = &request->ctx->engine[engine->id];
-        u32 *cs;
         int ret;
 
         GEM_BUG_ON(!ce->pin_count);
@@ -1154,17 +1190,9 @@ static int execlists_request_alloc(struct drm_i915_gem_request *request)
          */
         request->reserved_space += EXECLISTS_REQUEST_SIZE;
 
-        cs = intel_ring_begin(request, 0);
-        if (IS_ERR(cs))
-                return PTR_ERR(cs);
-
-        if (!ce->initialised) {
-                ret = engine->init_context(request);
-                if (ret)
-                        return ret;
-
-                ce->initialised = true;
-        }
+        ret = intel_ring_wait_for_space(request->ring, request->reserved_space);
+        if (ret)
+                return ret;
 
         /* Note that after this point, we have committed to using
          * this request as it is being used to both track the
@@ -1474,8 +1502,8 @@ static int gen8_init_common_ring(struct intel_engine_cs *engine)
         execlists->active = 0;
 
         /* After a GPU reset, we may have requests to replay */
-        if (!i915_modparams.enable_guc_submission && execlists->first)
-                tasklet_schedule(&execlists->irq_tasklet);
+        if (execlists->first)
+                tasklet_schedule(&execlists->tasklet);
 
         return 0;
 }
@@ -1531,10 +1559,10 @@ static void reset_common_ring(struct intel_engine_cs *engine,
          * guessing the missed context-switch events by looking at what
          * requests were completed.
          */
-        execlist_cancel_port_requests(execlists);
+        execlists_cancel_port_requests(execlists);
 
         /* Push back any incomplete requests for replay after the reset. */
-        unwind_incomplete_requests(engine);
+        __unwind_incomplete_requests(engine);
 
         spin_unlock_irqrestore(&engine->timeline->lock, flags);
 
@@ -1794,10 +1822,8 @@ static void gen8_emit_breadcrumb(struct drm_i915_gem_request *request, u32 *cs)
         /* w/a: bit 5 needs to be zero for MI_FLUSH_DW address. */
         BUILD_BUG_ON(I915_GEM_HWS_INDEX_ADDR & (1 << 5));
 
-        *cs++ = (MI_FLUSH_DW + 1) | MI_FLUSH_DW_OP_STOREDW;
-        *cs++ = intel_hws_seqno_address(request->engine) | MI_FLUSH_DW_USE_GTT;
-        *cs++ = 0;
-        *cs++ = request->global_seqno;
+        cs = gen8_emit_ggtt_write(cs, request->global_seqno,
+                                  intel_hws_seqno_address(request->engine));
         *cs++ = MI_USER_INTERRUPT;
         *cs++ = MI_NOOP;
         request->tail = intel_ring_offset(request, cs);
@@ -1807,24 +1833,14 @@ static void gen8_emit_breadcrumb(struct drm_i915_gem_request *request, u32 *cs)
 }
 static const int gen8_emit_breadcrumb_sz = 6 + WA_TAIL_DWORDS;
 
-static void gen8_emit_breadcrumb_render(struct drm_i915_gem_request *request,
+static void gen8_emit_breadcrumb_rcs(struct drm_i915_gem_request *request,
                                         u32 *cs)
 {
         /* We're using qword write, seqno should be aligned to 8 bytes. */
         BUILD_BUG_ON(I915_GEM_HWS_INDEX & 1);
 
-        /* w/a for post sync ops following a GPGPU operation we
-         * need a prior CS_STALL, which is emitted by the flush
-         * following the batch.
-         */
-        *cs++ = GFX_OP_PIPE_CONTROL(6);
-        *cs++ = PIPE_CONTROL_GLOBAL_GTT_IVB | PIPE_CONTROL_CS_STALL |
-                PIPE_CONTROL_QW_WRITE;
-        *cs++ = intel_hws_seqno_address(request->engine);
-        *cs++ = 0;
-        *cs++ = request->global_seqno;
-        /* We're thrashing one dword of HWS. */
-        *cs++ = 0;
+        cs = gen8_emit_ggtt_write_rcs(cs, request->global_seqno,
+                                      intel_hws_seqno_address(request->engine));
         *cs++ = MI_USER_INTERRUPT;
         *cs++ = MI_NOOP;
         request->tail = intel_ring_offset(request, cs);
@@ -1832,7 +1848,7 @@ static void gen8_emit_breadcrumb_render(struct drm_i915_gem_request *request,
 
         gen8_emit_wa_tail(request, cs);
 }
-static const int gen8_emit_breadcrumb_render_sz = 8 + WA_TAIL_DWORDS;
+static const int gen8_emit_breadcrumb_rcs_sz = 8 + WA_TAIL_DWORDS;
 
 static int gen8_init_rcs_context(struct drm_i915_gem_request *req)
 {
@@ -1865,8 +1881,9 @@ void intel_logical_ring_cleanup(struct intel_engine_cs *engine)
          * Tasklet cannot be active at this point due intel_mark_active/idle
          * so this is just for documentation.
          */
-        if (WARN_ON(test_bit(TASKLET_STATE_SCHED, &engine->execlists.irq_tasklet.state)))
-                tasklet_kill(&engine->execlists.irq_tasklet);
+        if (WARN_ON(test_bit(TASKLET_STATE_SCHED,
+                             &engine->execlists.tasklet.state)))
+                tasklet_kill(&engine->execlists.tasklet);
 
         dev_priv = engine->i915;
 
@@ -1890,7 +1907,10 @@ static void execlists_set_default_submission(struct intel_engine_cs *engine)
         engine->submit_request = execlists_submit_request;
         engine->cancel_requests = execlists_cancel_requests;
         engine->schedule = execlists_schedule;
-        engine->execlists.irq_tasklet.func = intel_lrc_irq_handler;
+        engine->execlists.tasklet.func = execlists_submission_tasklet;
+
+        engine->park = NULL;
+        engine->unpark = NULL;
 }
 
 static void
@@ -1949,8 +1969,8 @@ logical_ring_setup(struct intel_engine_cs *engine)
 
         engine->execlists.fw_domains = fw_domains;
 
-        tasklet_init(&engine->execlists.irq_tasklet,
-                     intel_lrc_irq_handler, (unsigned long)engine);
+        tasklet_init(&engine->execlists.tasklet,
+                     execlists_submission_tasklet, (unsigned long)engine);
 
         logical_ring_default_vfuncs(engine);
         logical_ring_default_irqs(engine);
@@ -1988,8 +2008,8 @@ int logical_render_ring_init(struct intel_engine_cs *engine)
                 engine->init_hw = gen8_init_render_ring;
         engine->init_context = gen8_init_rcs_context;
         engine->emit_flush = gen8_emit_flush_render;
-        engine->emit_breadcrumb = gen8_emit_breadcrumb_render;
-        engine->emit_breadcrumb_sz = gen8_emit_breadcrumb_render_sz;
+        engine->emit_breadcrumb = gen8_emit_breadcrumb_rcs;
+        engine->emit_breadcrumb_sz = gen8_emit_breadcrumb_rcs_sz;
 
         ret = intel_engine_create_scratch(engine, PAGE_SIZE);
         if (ret)
@@ -2106,7 +2126,6 @@ static void execlists_init_reg_state(u32 *regs,
 
         CTX_REG(regs, CTX_CONTEXT_CONTROL, RING_CONTEXT_CONTROL(engine),
                 _MASKED_BIT_ENABLE(CTX_CTRL_INHIBIT_SYN_CTX_SWITCH |
-                                   CTX_CTRL_ENGINE_CTX_RESTORE_INHIBIT |
                                    (HAS_RESOURCE_STREAMER(dev_priv) ?
                                    CTX_CTRL_RS_CTX_ENABLE : 0)));
         CTX_REG(regs, CTX_RING_HEAD, RING_HEAD(base), 0);
@@ -2183,6 +2202,7 @@ populate_lr_context(struct i915_gem_context *ctx,
                     struct intel_ring *ring)
 {
         void *vaddr;
+        u32 *regs;
         int ret;
 
         ret = i915_gem_object_set_to_cpu_domain(ctx_obj, true);
@@ -2199,11 +2219,31 @@ populate_lr_context(struct i915_gem_context *ctx,
         }
         ctx_obj->mm.dirty = true;
 
+        if (engine->default_state) {
+                /*
+                 * We only want to copy over the template context state;
+                 * skipping over the headers reserved for GuC communication,
+                 * leaving those as zero.
+                 */
+                const unsigned long start = LRC_HEADER_PAGES * PAGE_SIZE;
+                void *defaults;
+
+                defaults = i915_gem_object_pin_map(engine->default_state,
+                                                   I915_MAP_WB);
+                if (IS_ERR(defaults))
+                        return PTR_ERR(defaults);
+
+                memcpy(vaddr + start, defaults + start, engine->context_size);
+                i915_gem_object_unpin_map(engine->default_state);
+        }
+
         /* The second page of the context object contains some fields which must
          * be set up prior to the first execution. */
-
-        execlists_init_reg_state(vaddr + LRC_STATE_PN * PAGE_SIZE,
-                                 ctx, engine, ring);
+        regs = vaddr + LRC_STATE_PN * PAGE_SIZE;
+        execlists_init_reg_state(regs, ctx, engine, ring);
+        if (!engine->default_state)
+                regs[CTX_CONTEXT_CONTROL + 1] |=
+                        _MASKED_BIT_ENABLE(CTX_CTRL_ENGINE_CTX_RESTORE_INHIBIT);
 
         i915_gem_object_unpin_map(ctx_obj);
 
@@ -2256,7 +2296,6 @@ static int execlists_context_deferred_alloc(struct i915_gem_context *ctx,
 
         ce->ring = ring;
         ce->state = vma;
-        ce->initialised |= engine->init_context == NULL;
 
         return 0;
 
diff --git a/drivers/gpu/drm/i915/intel_lrc.h b/drivers/gpu/drm/i915/intel_lrc.h
index 689fde1a63a9..17182ce29674 100644
--- a/drivers/gpu/drm/i915/intel_lrc.h
+++ b/drivers/gpu/drm/i915/intel_lrc.h
@@ -107,7 +107,6 @@ intel_lr_context_descriptor(struct i915_gem_context *ctx,
         return ctx->engine[engine->id].lrc_desc;
 }
 
-
 /* Execlists */
 int intel_sanitize_enable_execlists(struct drm_i915_private *dev_priv,
                                     int enable_execlists);
diff --git a/drivers/gpu/drm/i915/intel_lvds.c b/drivers/gpu/drm/i915/intel_lvds.c
index 38572d65e46e..ef80499113ee 100644
--- a/drivers/gpu/drm/i915/intel_lvds.c
+++ b/drivers/gpu/drm/i915/intel_lvds.c
@@ -125,6 +125,8 @@ static void intel_lvds_get_config(struct intel_encoder *encoder,
         struct intel_lvds_encoder *lvds_encoder = to_lvds_encoder(&encoder->base);
         u32 tmp, flags = 0;
 
+        pipe_config->output_types |= BIT(INTEL_OUTPUT_LVDS);
+
         tmp = I915_READ(lvds_encoder->reg);
         if (tmp & LVDS_HSYNC_POLARITY)
                 flags |= DRM_MODE_FLAG_NHSYNC;
diff --git a/drivers/gpu/drm/i915/intel_opregion.c b/drivers/gpu/drm/i915/intel_opregion.c
index 1d946240e55f..fc65f5e451dd 100644
--- a/drivers/gpu/drm/i915/intel_opregion.c
+++ b/drivers/gpu/drm/i915/intel_opregion.c
@@ -367,7 +367,7 @@ int intel_opregion_notify_encoder(struct intel_encoder *intel_encoder,
         if (intel_encoder->type == INTEL_OUTPUT_DSI)
                 port = 0;
         else
-                port = intel_ddi_get_encoder_port(intel_encoder);
+                port = intel_encoder->port;
 
         if (port == PORT_E)  {
                 port = 0;
@@ -383,7 +383,7 @@ int intel_opregion_notify_encoder(struct intel_encoder *intel_encoder,
         case INTEL_OUTPUT_ANALOG:
                 type = DISPLAY_TYPE_CRT;
                 break;
-        case INTEL_OUTPUT_UNKNOWN:
+        case INTEL_OUTPUT_DDI:
         case INTEL_OUTPUT_DP:
         case INTEL_OUTPUT_HDMI:
         case INTEL_OUTPUT_DP_MST:
diff --git a/drivers/gpu/drm/i915/intel_pipe_crc.c b/drivers/gpu/drm/i915/intel_pipe_crc.c
index 899839f2f7c6..61641d479b93 100644
--- a/drivers/gpu/drm/i915/intel_pipe_crc.c
+++ b/drivers/gpu/drm/i915/intel_pipe_crc.c
@@ -269,7 +269,7 @@ static int i9xx_pipe_crc_auto_source(struct drm_i915_private *dev_priv,
                 case INTEL_OUTPUT_DP:
                 case INTEL_OUTPUT_EDP:
                         dig_port = enc_to_dig_port(&encoder->base);
-                        switch (dig_port->port) {
+                        switch (dig_port->base.port) {
                         case PORT_B:
                                 *source = INTEL_PIPE_CRC_SOURCE_DP_B;
                                 break;
@@ -281,7 +281,7 @@ static int i9xx_pipe_crc_auto_source(struct drm_i915_private *dev_priv,
                                 break;
                         default:
                                 WARN(1, "nonexisting DP port %c\n",
-                                     port_name(dig_port->port));
+                                     port_name(dig_port->base.port));
                                 break;
                         }
                         break;
diff --git a/drivers/gpu/drm/i915/intel_pm.c b/drivers/gpu/drm/i915/intel_pm.c
index f4a4e9496893..4d2cd432f739 100644
--- a/drivers/gpu/drm/i915/intel_pm.c
+++ b/drivers/gpu/drm/i915/intel_pm.c
@@ -75,9 +75,6 @@ static void gen9_init_clock_gating(struct drm_i915_private *dev_priv)
         I915_WRITE(CHICKEN_PAR1_1,
                    I915_READ(CHICKEN_PAR1_1) | SKL_EDP_PSR_FIX_RDWRAP);
 
-        I915_WRITE(GEN8_CONFIG0,
-                   I915_READ(GEN8_CONFIG0) | GEN9_DEFAULT_FIXES);
-
         /* WaEnableChickenDCPR:skl,bxt,kbl,glk,cfl */
         I915_WRITE(GEN8_CHICKEN_DCPR_1,
                    I915_READ(GEN8_CHICKEN_DCPR_1) | MASK_WAKEMEM);
@@ -124,7 +121,6 @@ static void bxt_init_clock_gating(struct drm_i915_private *dev_priv)
 
 static void glk_init_clock_gating(struct drm_i915_private *dev_priv)
 {
-        u32 val;
         gen9_init_clock_gating(dev_priv);
 
         /*
@@ -144,11 +140,6 @@ static void glk_init_clock_gating(struct drm_i915_private *dev_priv)
                 I915_WRITE(CHICKEN_MISC_2, val);
         }
 
-        /* Display WA #1133: WaFbcSkipSegments:glk */
-        val = I915_READ(ILK_DPFC_CHICKEN);
-        val &= ~GLK_SKIP_SEG_COUNT_MASK;
-        val |= GLK_SKIP_SEG_EN | GLK_SKIP_SEG_COUNT(1);
-        I915_WRITE(ILK_DPFC_CHICKEN, val);
 }
 
 static void i915_pineview_get_mem_freq(struct drm_i915_private *dev_priv)
@@ -928,7 +919,7 @@ static void pineview_update_wm(struct intel_crtc *unused_crtc)
  *  and the size of 8 whole lines. This adjustment is always performed
  *  in the actual pixel depth regardless of whether FBC is enabled or not."
  */
-static int g4x_tlb_miss_wa(int fifo_size, int width, int cpp)
+static unsigned int g4x_tlb_miss_wa(int fifo_size, int width, int cpp)
 {
         int tlb_miss = fifo_size * 64 - width * cpp * 8;
 
@@ -1105,8 +1096,8 @@ static uint16_t g4x_compute_wm(const struct intel_crtc_state *crtc_state,
         struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
         const struct drm_display_mode *adjusted_mode =
                 &crtc_state->base.adjusted_mode;
-        int clock, htotal, cpp, width, wm;
-        int latency = dev_priv->wm.pri_latency[level] * 10;
+        unsigned int latency = dev_priv->wm.pri_latency[level] * 10;
+        unsigned int clock, htotal, cpp, width, wm;
 
         if (latency == 0)
                 return USHRT_MAX;
@@ -1145,7 +1136,7 @@ static uint16_t g4x_compute_wm(const struct intel_crtc_state *crtc_state,
                    level == G4X_WM_LEVEL_NORMAL) {
                 wm = intel_wm_method1(clock, cpp, latency);
         } else {
-                int small, large;
+                unsigned int small, large;
 
                 small = intel_wm_method1(clock, cpp, latency);
                 large = intel_wm_method2(clock, htotal, width, cpp, latency);
@@ -1158,7 +1149,7 @@ static uint16_t g4x_compute_wm(const struct intel_crtc_state *crtc_state,
 
         wm = DIV_ROUND_UP(wm, 64) + 2;
 
-        return min_t(int, wm, USHRT_MAX);
+        return min_t(unsigned int, wm, USHRT_MAX);
 }
 
 static bool g4x_raw_plane_wm_set(struct intel_crtc_state *crtc_state,
@@ -1409,17 +1400,29 @@ static int g4x_compute_pipe_wm(struct intel_crtc_state *crtc_state)
 
 static int g4x_compute_intermediate_wm(struct drm_device *dev,
                                        struct intel_crtc *crtc,
-                                       struct intel_crtc_state *crtc_state)
+                                       struct intel_crtc_state *new_crtc_state)
 {
-        struct g4x_wm_state *intermediate = &crtc_state->wm.g4x.intermediate;
-        const struct g4x_wm_state *optimal = &crtc_state->wm.g4x.optimal;
-        const struct g4x_wm_state *active = &crtc->wm.active.g4x;
+        struct g4x_wm_state *intermediate = &new_crtc_state->wm.g4x.intermediate;
+        const struct g4x_wm_state *optimal = &new_crtc_state->wm.g4x.optimal;
+        struct intel_atomic_state *intel_state =
+                to_intel_atomic_state(new_crtc_state->base.state);
+        const struct intel_crtc_state *old_crtc_state =
+                intel_atomic_get_old_crtc_state(intel_state, crtc);
+        const struct g4x_wm_state *active = &old_crtc_state->wm.g4x.optimal;
         enum plane_id plane_id;
 
+        if (!new_crtc_state->base.active || drm_atomic_crtc_needs_modeset(&new_crtc_state->base)) {
+                *intermediate = *optimal;
+
+                intermediate->cxsr = false;
+                intermediate->hpll_en = false;
+                goto out;
+        }
+
         intermediate->cxsr = optimal->cxsr && active->cxsr &&
-                !crtc_state->disable_cxsr;
+                !new_crtc_state->disable_cxsr;
         intermediate->hpll_en = optimal->hpll_en && active->hpll_en &&
-                !crtc_state->disable_cxsr;
+                !new_crtc_state->disable_cxsr;
         intermediate->fbc_en = optimal->fbc_en && active->fbc_en;
 
         for_each_plane_id_on_crtc(crtc, plane_id) {
@@ -1461,12 +1464,13 @@ static int g4x_compute_intermediate_wm(struct drm_device *dev,
         WARN_ON(intermediate->hpll.fbc > g4x_fbc_fifo_size(2) &&
                 intermediate->fbc_en && intermediate->hpll_en);
 
+out:
         /*
          * If our intermediate WM are identical to the final WM, then we can
          * omit the post-vblank programming; only update if it's different.
          */
         if (memcmp(intermediate, optimal, sizeof(*intermediate)) != 0)
-                crtc_state->wm.need_postvbl_update = true;
+                new_crtc_state->wm.need_postvbl_update = true;
 
         return 0;
 }
@@ -1602,7 +1606,7 @@ static uint16_t vlv_compute_wm_level(const struct intel_crtc_state *crtc_state,
         struct drm_i915_private *dev_priv = to_i915(plane->base.dev);
         const struct drm_display_mode *adjusted_mode =
                 &crtc_state->base.adjusted_mode;
-        int clock, htotal, cpp, width, wm;
+        unsigned int clock, htotal, cpp, width, wm;
 
         if (dev_priv->wm.pri_latency[level] == 0)
                 return USHRT_MAX;
@@ -1628,7 +1632,7 @@ static uint16_t vlv_compute_wm_level(const struct intel_crtc_state *crtc_state,
                                     dev_priv->wm.pri_latency[level] * 10);
         }
 
-        return min_t(int, wm, USHRT_MAX);
+        return min_t(unsigned int, wm, USHRT_MAX);
 }
 
 static bool vlv_need_sprite0_fifo_workaround(unsigned int active_planes)
@@ -2029,16 +2033,27 @@ static void vlv_atomic_update_fifo(struct intel_atomic_state *state,
 
 static int vlv_compute_intermediate_wm(struct drm_device *dev,
                                        struct intel_crtc *crtc,
-                                       struct intel_crtc_state *crtc_state)
+                                       struct intel_crtc_state *new_crtc_state)
 {
-        struct vlv_wm_state *intermediate = &crtc_state->wm.vlv.intermediate;
-        const struct vlv_wm_state *optimal = &crtc_state->wm.vlv.optimal;
-        const struct vlv_wm_state *active = &crtc->wm.active.vlv;
+        struct vlv_wm_state *intermediate = &new_crtc_state->wm.vlv.intermediate;
+        const struct vlv_wm_state *optimal = &new_crtc_state->wm.vlv.optimal;
+        struct intel_atomic_state *intel_state =
+                to_intel_atomic_state(new_crtc_state->base.state);
+        const struct intel_crtc_state *old_crtc_state =
+                intel_atomic_get_old_crtc_state(intel_state, crtc);
+        const struct vlv_wm_state *active = &old_crtc_state->wm.vlv.optimal;
         int level;
 
+        if (!new_crtc_state->base.active || drm_atomic_crtc_needs_modeset(&new_crtc_state->base)) {
+                *intermediate = *optimal;
+
+                intermediate->cxsr = false;
+                goto out;
+        }
+
         intermediate->num_levels = min(optimal->num_levels, active->num_levels);
         intermediate->cxsr = optimal->cxsr && active->cxsr &&
-                !crtc_state->disable_cxsr;
+                !new_crtc_state->disable_cxsr;
 
         for (level = 0; level < intermediate->num_levels; level++) {
                 enum plane_id plane_id;
@@ -2057,12 +2072,13 @@ static int vlv_compute_intermediate_wm(struct drm_device *dev,
 
         vlv_invalidate_wms(crtc, intermediate, level);
 
+out:
         /*
          * If our intermediate WM are identical to the final WM, then we can
          * omit the post-vblank programming; only update if it's different.
          */
         if (memcmp(intermediate, optimal, sizeof(*intermediate)) != 0)
-                crtc_state->wm.need_postvbl_update = true;
+                new_crtc_state->wm.need_postvbl_update = true;
 
         return 0;
 }
@@ -3930,6 +3946,7 @@ skl_pipe_downscale_amount(const struct intel_crtc_state *crtc_state)
 int skl_check_pipe_max_pixel_rate(struct intel_crtc *intel_crtc,
                                   struct intel_crtc_state *cstate)
 {
+        struct drm_i915_private *dev_priv = to_i915(intel_crtc->base.dev);
         struct drm_crtc_state *crtc_state = &cstate->base;
         struct drm_atomic_state *state = crtc_state->state;
         struct drm_plane *plane;
@@ -3972,7 +3989,7 @@ int skl_check_pipe_max_pixel_rate(struct intel_crtc *intel_crtc,
         crtc_clock = crtc_state->adjusted_mode.crtc_clock;
         dotclk = to_intel_atomic_state(state)->cdclk.logical.cdclk;
 
-        if (IS_GEMINILAKE(to_i915(intel_crtc->base.dev)))
+        if (IS_GEMINILAKE(dev_priv) || INTEL_GEN(dev_priv) >= 10)
                 dotclk *= 2;
 
         pipe_max_pixel_rate = div_round_up_u32_fixed16(dotclk, pipe_downscale);
@@ -6620,12 +6637,19 @@ static void gen9_enable_rc6(struct drm_i915_private *dev_priv)
         I915_WRITE(GEN6_RC_CONTROL, 0);
 
         /* 2b: Program RC6 thresholds.*/
-
-        /* WaRsDoubleRc6WrlWithCoarsePowerGating: Doubling WRL only when CPG is enabled */
-        if (IS_SKYLAKE(dev_priv))
+        if (INTEL_GEN(dev_priv) >= 10) {
+                I915_WRITE(GEN6_RC6_WAKE_RATE_LIMIT, 54 << 16 | 85);
+                I915_WRITE(GEN10_MEDIA_WAKE_RATE_LIMIT, 150);
+        } else if (IS_SKYLAKE(dev_priv)) {
+                /*
+                 * WaRsDoubleRc6WrlWithCoarsePowerGating:skl Doubling WRL only
+                 * when CPG is enabled
+                 */
                 I915_WRITE(GEN6_RC6_WAKE_RATE_LIMIT, 108 << 16);
-        else
+        } else {
                 I915_WRITE(GEN6_RC6_WAKE_RATE_LIMIT, 54 << 16);
+        }
+
         I915_WRITE(GEN6_RC_EVALUATION_INTERVAL, 125000); /* 12500 * 1280ns */
         I915_WRITE(GEN6_RC_IDLE_HYSTERSIS, 25); /* 25 * 1280ns */
         for_each_engine(engine, dev_priv, id)
@@ -7910,7 +7934,6 @@ void intel_init_gt_powersave(struct drm_i915_private *dev_priv)
                 intel_runtime_pm_get(dev_priv);
         }
 
-        mutex_lock(&dev_priv->drm.struct_mutex);
         mutex_lock(&dev_priv->pcu_lock);
 
         /* Initialize RPS limits (for userspace) */
@@ -7952,9 +7975,6 @@ void intel_init_gt_powersave(struct drm_i915_private *dev_priv)
         rps->boost_freq = rps->max_freq;
 
         mutex_unlock(&dev_priv->pcu_lock);
-        mutex_unlock(&dev_priv->drm.struct_mutex);
-
-        intel_autoenable_gt_powersave(dev_priv);
 }
 
 void intel_cleanup_gt_powersave(struct drm_i915_private *dev_priv)
@@ -7979,9 +7999,6 @@ void intel_suspend_gt_powersave(struct drm_i915_private *dev_priv)
         if (INTEL_GEN(dev_priv) < 6)
                 return;
 
-        if (cancel_delayed_work_sync(&dev_priv->gt_pm.autoenable_work))
-                intel_runtime_pm_put(dev_priv);
-
         /* gen6_rps_idle() will be called later to disable interrupts */
 }
 
@@ -8140,65 +8157,6 @@ void intel_enable_gt_powersave(struct drm_i915_private *dev_priv)
         mutex_unlock(&dev_priv->pcu_lock);
 }
 
-static void __intel_autoenable_gt_powersave(struct work_struct *work)
-{
-        struct drm_i915_private *dev_priv =
-                container_of(work,
-                             typeof(*dev_priv),
-                             gt_pm.autoenable_work.work);
-        struct intel_engine_cs *rcs;
-        struct drm_i915_gem_request *req;
-
-        rcs = dev_priv->engine[RCS];
-        if (rcs->last_retired_context)
-                goto out;
-
-        if (!rcs->init_context)
-                goto out;
-
-        mutex_lock(&dev_priv->drm.struct_mutex);
-
-        req = i915_gem_request_alloc(rcs, dev_priv->kernel_context);
-        if (IS_ERR(req))
-                goto unlock;
-
-        if (!i915_modparams.enable_execlists && i915_switch_context(req) == 0)
-                rcs->init_context(req);
-
-        /* Mark the device busy, calling intel_enable_gt_powersave() */
-        i915_add_request(req);
-
-unlock:
-        mutex_unlock(&dev_priv->drm.struct_mutex);
-out:
-        intel_runtime_pm_put(dev_priv);
-}
-
-void intel_autoenable_gt_powersave(struct drm_i915_private *dev_priv)
-{
-        if (IS_IRONLAKE_M(dev_priv)) {
-                ironlake_enable_drps(dev_priv);
-                intel_init_emon(dev_priv);
-        } else if (INTEL_INFO(dev_priv)->gen >= 6) {
-                /*
-                 * PCU communication is slow and this doesn't need to be
-                 * done at any specific time, so do this out of our fast path
-                 * to make resume and init faster.
-                 *
-                 * We depend on the HW RC6 power context save/restore
-                 * mechanism when entering D3 through runtime PM suspend. So
-                 * disable RPM until RPS/RC6 is properly setup. We can only
-                 * get here via the driver load/system resume/runtime resume
-                 * paths, so the _noresume version is enough (and in case of
-                 * runtime resume it's necessary).
-                 */
-                if (queue_delayed_work(dev_priv->wq,
-                                       &dev_priv->gt_pm.autoenable_work,
-                                       round_jiffies_up_relative(HZ)))
-                        intel_runtime_pm_get_noresume(dev_priv);
-        }
-}
-
 static void ibx_init_clock_gating(struct drm_i915_private *dev_priv)
 {
         /*
@@ -8532,17 +8490,13 @@ static void cnl_init_clock_gating(struct drm_i915_private *dev_priv)
         I915_WRITE(DISP_ARB_CTL, I915_READ(DISP_ARB_CTL) |
                    DISP_FBC_MEMORY_WAKE);
 
+        val = I915_READ(SLICE_UNIT_LEVEL_CLKGATE);
+        /* ReadHitWriteOnlyDisable:cnl */
+        val |= RCCUNIT_CLKGATE_DIS;
         /* WaSarbUnitClockGatingDisable:cnl (pre-prod) */
         if (IS_CNL_REVID(dev_priv, CNL_REVID_A0, CNL_REVID_B0))
-                I915_WRITE(SLICE_UNIT_LEVEL_CLKGATE,
-                           I915_READ(SLICE_UNIT_LEVEL_CLKGATE) |
-                           SARBUNIT_CLKGATE_DIS);
-
-        /* Display WA #1133: WaFbcSkipSegments:cnl */
-        val = I915_READ(ILK_DPFC_CHICKEN);
-        val &= ~GLK_SKIP_SEG_COUNT_MASK;
-        val |= GLK_SKIP_SEG_EN | GLK_SKIP_SEG_COUNT(1);
-        I915_WRITE(ILK_DPFC_CHICKEN, val);
+                val |= SARBUNIT_CLKGATE_DIS;
+        I915_WRITE(SLICE_UNIT_LEVEL_CLKGATE, val);
 }
 
 static void cfl_init_clock_gating(struct drm_i915_private *dev_priv)
@@ -9429,8 +9383,6 @@ void intel_pm_setup(struct drm_i915_private *dev_priv)
 {
         mutex_init(&dev_priv->pcu_lock);
 
-        INIT_DELAYED_WORK(&dev_priv->gt_pm.autoenable_work,
-                          __intel_autoenable_gt_powersave);
         atomic_set(&dev_priv->gt_pm.rps.num_waiters, 0);
 
         dev_priv->runtime_pm.suspended = false;
diff --git a/drivers/gpu/drm/i915/intel_psr.c b/drivers/gpu/drm/i915/intel_psr.c
index 6e3b430fccdc..a1ad85fa5c1a 100644
--- a/drivers/gpu/drm/i915/intel_psr.c
+++ b/drivers/gpu/drm/i915/intel_psr.c
@@ -163,7 +163,7 @@ static void hsw_psr_enable_sink(struct intel_dp *intel_dp)
                 [3] = 1 - 1,
                 [4] = DP_SET_POWER_D0,
         };
-        enum port port = dig_port->port;
+        enum port port = dig_port->base.port;
         u32 aux_ctl;
         int i;
 
@@ -376,7 +376,7 @@ void intel_psr_compute_config(struct intel_dp *intel_dp,
          * ones. Since by Display design transcoder EDP is tied to port A
          * we can safely escape based on the port A.
          */
-        if (HAS_DDI(dev_priv) && dig_port->port != PORT_A) {
+        if (HAS_DDI(dev_priv) && dig_port->base.port != PORT_A) {
                 DRM_DEBUG_KMS("PSR condition failed: Port not supported\n");
                 return;
         }
diff --git a/drivers/gpu/drm/i915/intel_ringbuffer.c b/drivers/gpu/drm/i915/intel_ringbuffer.c
index 8da1bde442dd..12e734b29463 100644
--- a/drivers/gpu/drm/i915/intel_ringbuffer.c
+++ b/drivers/gpu/drm/i915/intel_ringbuffer.c
@@ -28,9 +28,12 @@
  */
 
 #include <linux/log2.h>
+
 #include <drm/drmP.h>
-#include "i915_drv.h"
 #include <drm/i915_drm.h>
+
+#include "i915_drv.h"
+#include "i915_gem_render_state.h"
 #include "i915_trace.h"
 #include "intel_drv.h"
 
@@ -480,10 +483,14 @@ static bool stop_ring(struct intel_engine_cs *engine)
                 }
         }
 
-        I915_WRITE_CTL(engine, 0);
+        I915_WRITE_HEAD(engine, I915_READ_TAIL(engine));
+
         I915_WRITE_HEAD(engine, 0);
         I915_WRITE_TAIL(engine, 0);
 
+        /* The ring must be empty before it is disabled */
+        I915_WRITE_CTL(engine, 0);
+
         return (I915_READ_HEAD(engine) & HEAD_ADDR) == 0;
 }
 
@@ -1359,12 +1366,13 @@ static int context_pin(struct i915_gem_context *ctx)
         struct i915_vma *vma = ctx->engine[RCS].state;
         int ret;
 
-        /* Clear this page out of any CPU caches for coherent swap-in/out.
+        /*
+         * Clear this page out of any CPU caches for coherent swap-in/out.
          * We only want to do this on the first bind so that we do not stall
          * on an active context (which by nature is already on the GPU).
          */
         if (!(vma->flags & I915_VMA_GLOBAL_BIND)) {
-                ret = i915_gem_object_set_to_gtt_domain(vma->obj, false);
+                ret = i915_gem_object_set_to_gtt_domain(vma->obj, true);
                 if (ret)
                         return ret;
         }
@@ -1379,11 +1387,34 @@ alloc_context_vma(struct intel_engine_cs *engine)
         struct drm_i915_private *i915 = engine->i915;
         struct drm_i915_gem_object *obj;
         struct i915_vma *vma;
+        int err;
 
         obj = i915_gem_object_create(i915, engine->context_size);
         if (IS_ERR(obj))
                 return ERR_CAST(obj);
 
+        if (engine->default_state) {
+                void *defaults, *vaddr;
+
+                vaddr = i915_gem_object_pin_map(obj, I915_MAP_WB);
+                if (IS_ERR(vaddr)) {
+                        err = PTR_ERR(vaddr);
+                        goto err_obj;
+                }
+
+                defaults = i915_gem_object_pin_map(engine->default_state,
+                                                   I915_MAP_WB);
+                if (IS_ERR(defaults)) {
+                        err = PTR_ERR(defaults);
+                        goto err_map;
+                }
+
+                memcpy(vaddr, defaults, engine->context_size);
+
+                i915_gem_object_unpin_map(engine->default_state);
+                i915_gem_object_unpin_map(obj);
+        }
+
         /*
          * Try to make the context utilize L3 as well as LLC.
          *
@@ -1405,10 +1436,18 @@ alloc_context_vma(struct intel_engine_cs *engine)
         }
 
         vma = i915_vma_instance(obj, &i915->ggtt.base, NULL);
-        if (IS_ERR(vma))
-                i915_gem_object_put(obj);
+        if (IS_ERR(vma)) {
+                err = PTR_ERR(vma);
+                goto err_obj;
+        }
 
         return vma;
+
+err_map:
+        i915_gem_object_unpin_map(obj);
+err_obj:
+        i915_gem_object_put(obj);
+        return ERR_PTR(err);
 }
 
 static struct intel_ring *
@@ -1441,20 +1480,9 @@ intel_ring_context_pin(struct intel_engine_cs *engine,
                 if (ret)
                         goto err;
 
-                ce->state->obj->mm.dirty = true;
                 ce->state->obj->pin_global++;
         }
 
-        /* The kernel context is only used as a placeholder for flushing the
-         * active context. It is never used for submitting user rendering and
-         * as such never requires the golden render context, and so we can skip
-         * emitting it when we switch to the kernel context. This is required
-         * as during eviction we cannot allocate and pin the renderstate in
-         * order to initialise the context.
-         */
-        if (i915_gem_context_is_kernel(ctx))
-                ce->initialised = true;
-
         i915_gem_context_get(ctx);
 
 out:
@@ -1550,7 +1578,7 @@ void intel_legacy_submission_resume(struct drm_i915_private *dev_priv)
 
 static int ring_request_alloc(struct drm_i915_gem_request *request)
 {
-        u32 *cs;
+        int ret;
 
         GEM_BUG_ON(!request->ctx->engine[request->engine->id].pin_count);
 
@@ -1560,37 +1588,24 @@ static int ring_request_alloc(struct drm_i915_gem_request *request)
          */
         request->reserved_space += LEGACY_REQUEST_SIZE;
 
-        cs = intel_ring_begin(request, 0);
-        if (IS_ERR(cs))
-                return PTR_ERR(cs);
+        ret = intel_ring_wait_for_space(request->ring, request->reserved_space);
+        if (ret)
+                return ret;
 
         request->reserved_space -= LEGACY_REQUEST_SIZE;
         return 0;
 }
 
-static noinline int wait_for_space(struct drm_i915_gem_request *req,
-                                   unsigned int bytes)
+static noinline int wait_for_space(struct intel_ring *ring, unsigned int bytes)
 {
-        struct intel_ring *ring = req->ring;
         struct drm_i915_gem_request *target;
         long timeout;
 
-        lockdep_assert_held(&req->i915->drm.struct_mutex);
+        lockdep_assert_held(&ring->vma->vm->i915->drm.struct_mutex);
 
         if (intel_ring_update_space(ring) >= bytes)
                 return 0;
 
-        /*
-         * Space is reserved in the ringbuffer for finalising the request,
-         * as that cannot be allowed to fail. During request finalisation,
-         * reserved_space is set to 0 to stop the overallocation and the
-         * assumption is that then we never need to wait (which has the
-         * risk of failing with EINTR).
-         *
-         * See also i915_gem_request_alloc() and i915_add_request().
-         */
-        GEM_BUG_ON(!req->reserved_space);
-
         list_for_each_entry(target, &ring->request_list, ring_link) {
                 /* Would completion of this request free enough space? */
                 if (bytes <= __intel_ring_space(target->postfix,
@@ -1614,6 +1629,22 @@ static noinline int wait_for_space(struct drm_i915_gem_request *req,
         return 0;
 }
 
+int intel_ring_wait_for_space(struct intel_ring *ring, unsigned int bytes)
+{
+        GEM_BUG_ON(bytes > ring->effective_size);
+        if (unlikely(bytes > ring->effective_size - ring->emit))
+                bytes += ring->size - ring->emit;
+
+        if (unlikely(bytes > ring->space)) {
+                int ret = wait_for_space(ring, bytes);
+                if (unlikely(ret))
+                        return ret;
+        }
+
+        GEM_BUG_ON(ring->space < bytes);
+        return 0;
+}
+
 u32 *intel_ring_begin(struct drm_i915_gem_request *req,
                       unsigned int num_dwords)
 {
@@ -1653,7 +1684,20 @@ u32 *intel_ring_begin(struct drm_i915_gem_request *req,
         }
 
         if (unlikely(total_bytes > ring->space)) {
-                int ret = wait_for_space(req, total_bytes);
+                int ret;
+
+                /*
+                 * Space is reserved in the ringbuffer for finalising the
+                 * request, as that cannot be allowed to fail. During request
+                 * finalisation, reserved_space is set to 0 to stop the
+                 * overallocation and the assumption is that then we never need
+                 * to wait (which has the risk of failing with EINTR).
+                 *
+                 * See also i915_gem_request_alloc() and i915_add_request().
+                 */
+                GEM_BUG_ON(!req->reserved_space);
+
+                ret = wait_for_space(ring, total_bytes);
                 if (unlikely(ret))
                         return ERR_PTR(ret);
         }
@@ -2028,12 +2072,15 @@ static void i9xx_set_default_submission(struct intel_engine_cs *engine)
 {
         engine->submit_request = i9xx_submit_request;
         engine->cancel_requests = cancel_requests;
+
+        engine->park = NULL;
+        engine->unpark = NULL;
 }
 
 static void gen6_bsd_set_default_submission(struct intel_engine_cs *engine)
 {
+        i9xx_set_default_submission(engine);
         engine->submit_request = gen6_bsd_submit_request;
-        engine->cancel_requests = cancel_requests;
 }
 
 static void intel_ring_default_vfuncs(struct drm_i915_private *dev_priv,
diff --git a/drivers/gpu/drm/i915/intel_ringbuffer.h b/drivers/gpu/drm/i915/intel_ringbuffer.h
index 2863d5a65187..f867aa6c31fc 100644
--- a/drivers/gpu/drm/i915/intel_ringbuffer.h
+++ b/drivers/gpu/drm/i915/intel_ringbuffer.h
@@ -166,7 +166,6 @@ struct i915_ctx_workarounds {
 };
 
 struct drm_i915_gem_request;
-struct intel_render_state;
 
 /*
  * Engine IDs definitions.
@@ -195,9 +194,9 @@ struct i915_priolist {
  */
 struct intel_engine_execlists {
         /**
-         * @irq_tasklet: softirq tasklet for bottom handler
+         * @tasklet: softirq tasklet for bottom handler
          */
-        struct tasklet_struct irq_tasklet;
+        struct tasklet_struct tasklet;
 
         /**
          * @default_priolist: priority list for I915_PRIORITY_NORMAL
@@ -290,11 +289,14 @@ struct intel_engine_execlists {
 struct intel_engine_cs {
         struct drm_i915_private *i915;
         char name[INTEL_ENGINE_CS_MAX_NAME];
+
         enum intel_engine_id id;
-        unsigned int uabi_id;
         unsigned int hw_id;
         unsigned int guc_id;
 
+        u8 uabi_id;
+        u8 uabi_class;
+
         u8 class;
         u8 instance;
         u32 context_size;
@@ -304,7 +306,7 @@ struct intel_engine_cs {
         struct intel_ring *buffer;
         struct intel_timeline *timeline;
 
-        struct intel_render_state *render_state;
+        struct drm_i915_gem_object *default_state;
 
         atomic_t irq_count;
         unsigned long irq_posted;
@@ -340,9 +342,9 @@ struct intel_engine_cs {
                 struct timer_list hangcheck; /* detect missed interrupts */
 
                 unsigned int hangcheck_interrupts;
+                unsigned int irq_enabled;
 
                 bool irq_armed : 1;
-                bool irq_enabled : 1;
                 I915_SELFTEST_DECLARE(bool mock : 1);
         } breadcrumbs;
 
@@ -366,6 +368,9 @@ struct intel_engine_cs {
         void            (*reset_hw)(struct intel_engine_cs *engine,
                                     struct drm_i915_gem_request *req);
 
+        void            (*park)(struct intel_engine_cs *engine);
+        void            (*unpark)(struct intel_engine_cs *engine);
+
         void            (*set_default_submission)(struct intel_engine_cs *engine);
 
         struct intel_ring *(*context_pin)(struct intel_engine_cs *engine,
@@ -555,6 +560,12 @@ execlists_is_active(const struct intel_engine_execlists *execlists,
         return test_bit(bit, (unsigned long *)&execlists->active);
 }
 
+void
+execlists_cancel_port_requests(struct intel_engine_execlists * const execlists);
+
+void
+execlists_unwind_incomplete_requests(struct intel_engine_execlists *execlists);
+
 static inline unsigned int
 execlists_num_ports(const struct intel_engine_execlists * const execlists)
 {
@@ -624,6 +635,8 @@ intel_write_status_page(struct intel_engine_cs *engine, int reg, u32 value)
  */
 #define I915_GEM_HWS_INDEX              0x30
 #define I915_GEM_HWS_INDEX_ADDR (I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT)
+#define I915_GEM_HWS_PREEMPT_INDEX      0x32
+#define I915_GEM_HWS_PREEMPT_ADDR (I915_GEM_HWS_PREEMPT_INDEX << MI_STORE_DWORD_INDEX_SHIFT)
 #define I915_GEM_HWS_SCRATCH_INDEX      0x40
 #define I915_GEM_HWS_SCRATCH_ADDR (I915_GEM_HWS_SCRATCH_INDEX << MI_STORE_DWORD_INDEX_SHIFT)
 
@@ -648,6 +661,7 @@ void intel_legacy_submission_resume(struct drm_i915_private *dev_priv);
 
 int __must_check intel_ring_cacheline_align(struct drm_i915_gem_request *req);
 
+int intel_ring_wait_for_space(struct intel_ring *ring, unsigned int bytes);
 u32 __must_check *intel_ring_begin(struct drm_i915_gem_request *req,
                                    unsigned int n);
 
@@ -776,6 +790,11 @@ static inline u32 intel_hws_seqno_address(struct intel_engine_cs *engine)
         return engine->status_page.ggtt_offset + I915_GEM_HWS_INDEX_ADDR;
 }
 
+static inline u32 intel_hws_preempt_done_address(struct intel_engine_cs *engine)
+{
+        return engine->status_page.ggtt_offset + I915_GEM_HWS_PREEMPT_ADDR;
+}
+
 /* intel_breadcrumbs.c -- user interrupt bottom-half for waiters */
 int intel_engine_init_breadcrumbs(struct intel_engine_cs *engine);
 
@@ -846,6 +865,9 @@ unsigned int intel_engine_wakeup(struct intel_engine_cs *engine);
 #define ENGINE_WAKEUP_WAITER BIT(0)
 #define ENGINE_WAKEUP_ASLEEP BIT(1)
 
+void intel_engine_pin_breadcrumbs_irq(struct intel_engine_cs *engine);
+void intel_engine_unpin_breadcrumbs_irq(struct intel_engine_cs *engine);
+
 void __intel_engine_disarm_breadcrumbs(struct intel_engine_cs *engine);
 void intel_engine_disarm_breadcrumbs(struct intel_engine_cs *engine);
 
@@ -864,11 +886,54 @@ static inline u32 *gen8_emit_pipe_control(u32 *batch, u32 flags, u32 offset)
         return batch + 6;
 }
 
+static inline u32 *
+gen8_emit_ggtt_write_rcs(u32 *cs, u32 value, u32 gtt_offset)
+{
+        /* We're using qword write, offset should be aligned to 8 bytes. */
+        GEM_BUG_ON(!IS_ALIGNED(gtt_offset, 8));
+
+        /* w/a for post sync ops following a GPGPU operation we
+         * need a prior CS_STALL, which is emitted by the flush
+         * following the batch.
+         */
+        *cs++ = GFX_OP_PIPE_CONTROL(6);
+        *cs++ = PIPE_CONTROL_GLOBAL_GTT_IVB | PIPE_CONTROL_CS_STALL |
+                PIPE_CONTROL_QW_WRITE;
+        *cs++ = gtt_offset;
+        *cs++ = 0;
+        *cs++ = value;
+        /* We're thrashing one dword of HWS. */
+        *cs++ = 0;
+
+        return cs;
+}
+
+static inline u32 *
+gen8_emit_ggtt_write(u32 *cs, u32 value, u32 gtt_offset)
+{
+        /* w/a: bit 5 needs to be zero for MI_FLUSH_DW address. */
+        GEM_BUG_ON(gtt_offset & (1 << 5));
+        /* Offset should be aligned to 8 bytes for both (QW/DW) write types */
+        GEM_BUG_ON(!IS_ALIGNED(gtt_offset, 8));
+
+        *cs++ = (MI_FLUSH_DW + 1) | MI_FLUSH_DW_OP_STOREDW;
+        *cs++ = gtt_offset | MI_FLUSH_DW_USE_GTT;
+        *cs++ = 0;
+        *cs++ = value;
+
+        return cs;
+}
+
 bool intel_engine_is_idle(struct intel_engine_cs *engine);
 bool intel_engines_are_idle(struct drm_i915_private *dev_priv);
 
-void intel_engines_mark_idle(struct drm_i915_private *i915);
+bool intel_engine_has_kernel_context(const struct intel_engine_cs *engine);
+
+void intel_engines_park(struct drm_i915_private *i915);
+void intel_engines_unpark(struct drm_i915_private *i915);
+
 void intel_engines_reset_default_submission(struct drm_i915_private *i915);
+unsigned int intel_engines_has_context_isolation(struct drm_i915_private *i915);
 
 bool intel_engine_can_store_dword(struct intel_engine_cs *engine);
 
diff --git a/drivers/gpu/drm/i915/intel_runtime_pm.c b/drivers/gpu/drm/i915/intel_runtime_pm.c
index 8af286c63d3b..8315499452dc 100644
--- a/drivers/gpu/drm/i915/intel_runtime_pm.c
+++ b/drivers/gpu/drm/i915/intel_runtime_pm.c
@@ -705,7 +705,8 @@ static void gen9_dc_off_power_well_enable(struct drm_i915_private *dev_priv,
         gen9_set_dc_state(dev_priv, DC_STATE_DISABLE);
 
         dev_priv->display.get_cdclk(dev_priv, &cdclk_state);
-        WARN_ON(!intel_cdclk_state_compare(&dev_priv->cdclk.hw, &cdclk_state));
+        /* Can't read out voltage_level so can't use intel_cdclk_changed() */
+        WARN_ON(intel_cdclk_needs_modeset(&dev_priv->cdclk.hw, &cdclk_state));
 
         gen9_assert_dbuf_enabled(dev_priv);
 
diff --git a/drivers/gpu/drm/i915/intel_sdvo.c b/drivers/gpu/drm/i915/intel_sdvo.c
index 7437944b388f..2b8764897d68 100644
--- a/drivers/gpu/drm/i915/intel_sdvo.c
+++ b/drivers/gpu/drm/i915/intel_sdvo.c
@@ -1429,6 +1429,8 @@ static void intel_sdvo_get_config(struct intel_encoder *encoder,
         u8 val;
         bool ret;
 
+        pipe_config->output_types |= BIT(INTEL_OUTPUT_SDVO);
+
         sdvox = I915_READ(intel_sdvo->sdvo_reg);
 
         ret = intel_sdvo_get_input_timing(intel_sdvo, &dtd);
@@ -1510,7 +1512,7 @@ static void intel_disable_sdvo(struct intel_encoder *encoder,
 {
         struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
         struct intel_sdvo *intel_sdvo = to_sdvo(encoder);
-        struct intel_crtc *crtc = to_intel_crtc(encoder->base.crtc);
+        struct intel_crtc *crtc = to_intel_crtc(old_crtc_state->base.crtc);
         u32 temp;
 
         intel_sdvo_set_active_outputs(intel_sdvo, 0);
@@ -1569,7 +1571,7 @@ static void intel_enable_sdvo(struct intel_encoder *encoder,
         struct drm_device *dev = encoder->base.dev;
         struct drm_i915_private *dev_priv = to_i915(dev);
         struct intel_sdvo *intel_sdvo = to_sdvo(encoder);
-        struct intel_crtc *intel_crtc = to_intel_crtc(encoder->base.crtc);
+        struct intel_crtc *intel_crtc = to_intel_crtc(pipe_config->base.crtc);
         u32 temp;
         bool input1, input2;
         int i;
diff --git a/drivers/gpu/drm/i915/intel_sprite.c b/drivers/gpu/drm/i915/intel_sprite.c
index 4fcf80ca91dd..ce615704982a 100644
--- a/drivers/gpu/drm/i915/intel_sprite.c
+++ b/drivers/gpu/drm/i915/intel_sprite.c
@@ -263,13 +263,9 @@ skl_update_plane(struct intel_plane *plane,
 
         spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);
 
-        if (IS_GEMINILAKE(dev_priv) || IS_CANNONLAKE(dev_priv)) {
+        if (INTEL_GEN(dev_priv) >= 10 || IS_GEMINILAKE(dev_priv))
                 I915_WRITE_FW(PLANE_COLOR_CTL(pipe, plane_id),
-                              PLANE_COLOR_PIPE_GAMMA_ENABLE |
-                              PLANE_COLOR_PIPE_CSC_ENABLE |
-                              PLANE_COLOR_PLANE_GAMMA_DISABLE);
-        }
-
+                              plane_state->color_ctl);
         if (key->flags) {
                 I915_WRITE_FW(PLANE_KEYVAL(pipe, plane_id), key->min_value);
                 I915_WRITE_FW(PLANE_KEYMAX(pipe, plane_id), key->max_value);
@@ -978,6 +974,9 @@ intel_check_sprite_plane(struct intel_plane *plane,
                 state->ctl = g4x_sprite_ctl(crtc_state, state);
         }
 
+        if (INTEL_GEN(dev_priv) >= 10 || IS_GEMINILAKE(dev_priv))
+                state->color_ctl = glk_plane_color_ctl(crtc_state, state);
+
         return 0;
 }
 
diff --git a/drivers/gpu/drm/i915/intel_tv.c b/drivers/gpu/drm/i915/intel_tv.c
index a79a7591b2cf..b3dabc219e6a 100644
--- a/drivers/gpu/drm/i915/intel_tv.c
+++ b/drivers/gpu/drm/i915/intel_tv.c
@@ -822,7 +822,7 @@ intel_enable_tv(struct intel_encoder *encoder,
 
         /* Prevents vblank waits from timing out in intel_tv_detect_type() */
         intel_wait_for_vblank(dev_priv,
-                              to_intel_crtc(encoder->base.crtc)->pipe);
+                              to_intel_crtc(pipe_config->base.crtc)->pipe);
 
         I915_WRITE(TV_CTL, I915_READ(TV_CTL) | TV_ENC_ENABLE);
 }
@@ -868,6 +868,8 @@ static void
 intel_tv_get_config(struct intel_encoder *encoder,
                     struct intel_crtc_state *pipe_config)
 {
+        pipe_config->output_types |= BIT(INTEL_OUTPUT_TVOUT);
+
         pipe_config->base.adjusted_mode.crtc_clock = pipe_config->port_clock;
 }
 
@@ -980,7 +982,7 @@ static void intel_tv_pre_enable(struct intel_encoder *encoder,
                                 const struct drm_connector_state *conn_state)
 {
         struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
-        struct intel_crtc *intel_crtc = to_intel_crtc(encoder->base.crtc);
+        struct intel_crtc *intel_crtc = to_intel_crtc(pipe_config->base.crtc);
         struct intel_tv *intel_tv = enc_to_tv(encoder);
         const struct tv_mode *tv_mode = intel_tv_mode_find(conn_state);
         u32 tv_ctl;
diff --git a/drivers/gpu/drm/i915/intel_uc.c b/drivers/gpu/drm/i915/intel_uc.c
index 25bd162f38d2..1e2a30a40ede 100644
--- a/drivers/gpu/drm/i915/intel_uc.c
+++ b/drivers/gpu/drm/i915/intel_uc.c
@@ -23,8 +23,8 @@
  */
 
 #include "intel_uc.h"
+#include "intel_guc_submission.h"
 #include "i915_drv.h"
-#include "i915_guc_submission.h"
 
 /* Reset GuC providing us with fresh state for both GuC and HuC.
  */
@@ -33,9 +33,9 @@ static int __intel_uc_reset_hw(struct drm_i915_private *dev_priv)
         int ret;
         u32 guc_status;
 
-        ret = intel_guc_reset(dev_priv);
+        ret = intel_reset_guc(dev_priv);
         if (ret) {
-                DRM_ERROR("GuC reset failed, ret = %d\n", ret);
+                DRM_ERROR("Failed to reset GuC, ret = %d\n", ret);
                 return ret;
         }
 
@@ -168,7 +168,7 @@ int intel_uc_init_hw(struct drm_i915_private *dev_priv)
                  * This is stuff we need to have available at fw load time
                  * if we are planning to enable submission later
                  */
-                ret = i915_guc_submission_init(dev_priv);
+                ret = intel_guc_submission_init(guc);
                 if (ret)
                         goto err_guc;
         }
@@ -217,7 +217,7 @@ int intel_uc_init_hw(struct drm_i915_private *dev_priv)
                 if (i915_modparams.guc_log_level >= 0)
                         gen9_enable_guc_interrupts(dev_priv);
 
-                ret = i915_guc_submission_enable(dev_priv);
+                ret = intel_guc_submission_enable(guc);
                 if (ret)
                         goto err_interrupts;
         }
@@ -246,7 +246,7 @@ err_log_capture:
         guc_capture_load_err_log(guc);
 err_submission:
         if (i915_modparams.enable_guc_submission)
-                i915_guc_submission_fini(dev_priv);
+                intel_guc_submission_fini(guc);
 err_guc:
         i915_ggtt_disable_guc(dev_priv);
 
@@ -271,19 +271,21 @@ err_guc:
 
 void intel_uc_fini_hw(struct drm_i915_private *dev_priv)
 {
-        guc_free_load_err_log(&dev_priv->guc);
+        struct intel_guc *guc = &dev_priv->guc;
+
+        guc_free_load_err_log(guc);
 
         if (!i915_modparams.enable_guc_loading)
                 return;
 
         if (i915_modparams.enable_guc_submission)
-                i915_guc_submission_disable(dev_priv);
+                intel_guc_submission_disable(guc);
 
-        guc_disable_communication(&dev_priv->guc);
+        guc_disable_communication(guc);
 
         if (i915_modparams.enable_guc_submission) {
                 gen9_disable_guc_interrupts(dev_priv);
-                i915_guc_submission_fini(dev_priv);
+                intel_guc_submission_fini(guc);
         }
 
         i915_ggtt_disable_guc(dev_priv);
diff --git a/drivers/gpu/drm/i915/intel_uc_fw.c b/drivers/gpu/drm/i915/intel_uc_fw.c
index 973888e94cba..4bc82d3005ff 100644
--- a/drivers/gpu/drm/i915/intel_uc_fw.c
+++ b/drivers/gpu/drm/i915/intel_uc_fw.c
@@ -299,7 +299,7 @@ void intel_uc_fw_fini(struct intel_uc_fw *uc_fw)
  *
  * Pretty printer for uC firmware.
  */
-void intel_uc_fw_dump(struct intel_uc_fw *uc_fw, struct drm_printer *p)
+void intel_uc_fw_dump(const struct intel_uc_fw *uc_fw, struct drm_printer *p)
 {
         drm_printf(p, "%s firmware: %s\n",
                    intel_uc_fw_type_repr(uc_fw->type), uc_fw->path);
diff --git a/drivers/gpu/drm/i915/intel_uc_fw.h b/drivers/gpu/drm/i915/intel_uc_fw.h
index 132903669391..5394d9d1e683 100644
--- a/drivers/gpu/drm/i915/intel_uc_fw.h
+++ b/drivers/gpu/drm/i915/intel_uc_fw.h
@@ -116,6 +116,6 @@ int intel_uc_fw_upload(struct intel_uc_fw *uc_fw,
                        int (*xfer)(struct intel_uc_fw *uc_fw,
                                    struct i915_vma *vma));
 void intel_uc_fw_fini(struct intel_uc_fw *uc_fw);
-void intel_uc_fw_dump(struct intel_uc_fw *uc_fw, struct drm_printer *p);
+void intel_uc_fw_dump(const struct intel_uc_fw *uc_fw, struct drm_printer *p);
 
 #endif
diff --git a/drivers/gpu/drm/i915/intel_uncore.c b/drivers/gpu/drm/i915/intel_uncore.c
index 8c2ce81f01c2..b4621271e7a2 100644
--- a/drivers/gpu/drm/i915/intel_uncore.c
+++ b/drivers/gpu/drm/i915/intel_uncore.c
@@ -69,17 +69,104 @@ fw_domain_arm_timer(struct intel_uncore_forcewake_domain *d)
                                HRTIMER_MODE_REL);
 }
 
+static inline int
+__wait_for_ack(const struct drm_i915_private *i915,
+               const struct intel_uncore_forcewake_domain *d,
+               const u32 ack,
+               const u32 value)
+{
+        return wait_for_atomic((__raw_i915_read32(i915, d->reg_ack) & ack) == value,
+                               FORCEWAKE_ACK_TIMEOUT_MS);
+}
+
+static inline int
+wait_ack_clear(const struct drm_i915_private *i915,
+               const struct intel_uncore_forcewake_domain *d,
+               const u32 ack)
+{
+        return __wait_for_ack(i915, d, ack, 0);
+}
+
+static inline int
+wait_ack_set(const struct drm_i915_private *i915,
+             const struct intel_uncore_forcewake_domain *d,
+             const u32 ack)
+{
+        return __wait_for_ack(i915, d, ack, ack);
+}
+
 static inline void
 fw_domain_wait_ack_clear(const struct drm_i915_private *i915,
                          const struct intel_uncore_forcewake_domain *d)
 {
-        if (wait_for_atomic((__raw_i915_read32(i915, d->reg_ack) &
-                             FORCEWAKE_KERNEL) == 0,
-                            FORCEWAKE_ACK_TIMEOUT_MS))
+        if (wait_ack_clear(i915, d, FORCEWAKE_KERNEL))
                 DRM_ERROR("%s: timed out waiting for forcewake ack to clear.\n",
                           intel_uncore_forcewake_domain_to_str(d->id));
 }
 
+enum ack_type {
+        ACK_CLEAR = 0,
+        ACK_SET
+};
+
+static int
+fw_domain_wait_ack_with_fallback(const struct drm_i915_private *i915,
+                                 const struct intel_uncore_forcewake_domain *d,
+                                 const enum ack_type type)
+{
+        const u32 ack_bit = FORCEWAKE_KERNEL;
+        const u32 value = type == ACK_SET ? ack_bit : 0;
+        unsigned int pass;
+        bool ack_detected;
+
+        /*
+         * There is a possibility of driver's wake request colliding
+         * with hardware's own wake requests and that can cause
+         * hardware to not deliver the driver's ack message.
+         *
+         * Use a fallback bit toggle to kick the gpu state machine
+         * in the hope that the original ack will be delivered along with
+         * the fallback ack.
+         *
+         * This workaround is described in HSDES #1604254524
+         */
+
+        pass = 1;
+        do {
+                wait_ack_clear(i915, d, FORCEWAKE_KERNEL_FALLBACK);
+
+                __raw_i915_write32(i915, d->reg_set,
+                                   _MASKED_BIT_ENABLE(FORCEWAKE_KERNEL_FALLBACK));
+                /* Give gt some time to relax before the polling frenzy */
+                udelay(10 * pass);
+                wait_ack_set(i915, d, FORCEWAKE_KERNEL_FALLBACK);
+
+                ack_detected = (__raw_i915_read32(i915, d->reg_ack) & ack_bit) == value;
+
+                __raw_i915_write32(i915, d->reg_set,
+                                   _MASKED_BIT_DISABLE(FORCEWAKE_KERNEL_FALLBACK));
+        } while (!ack_detected && pass++ < 10);
+
+        DRM_DEBUG_DRIVER("%s had to use fallback to %s ack, 0x%x (passes %u)\n",
+                         intel_uncore_forcewake_domain_to_str(d->id),
+                         type == ACK_SET ? "set" : "clear",
+                         __raw_i915_read32(i915, d->reg_ack),
+                         pass);
+
+        return ack_detected ? 0 : -ETIMEDOUT;
+}
+
+static inline void
+fw_domain_wait_ack_clear_fallback(const struct drm_i915_private *i915,
+                                  const struct intel_uncore_forcewake_domain *d)
+{
+        if (likely(!wait_ack_clear(i915, d, FORCEWAKE_KERNEL)))
+                return;
+
+        if (fw_domain_wait_ack_with_fallback(i915, d, ACK_CLEAR))
+                fw_domain_wait_ack_clear(i915, d);
+}
+
 static inline void
 fw_domain_get(struct drm_i915_private *i915,
               const struct intel_uncore_forcewake_domain *d)
@@ -88,17 +175,26 @@ fw_domain_get(struct drm_i915_private *i915,
 }
 
 static inline void
-fw_domain_wait_ack(const struct drm_i915_private *i915,
-                   const struct intel_uncore_forcewake_domain *d)
+fw_domain_wait_ack_set(const struct drm_i915_private *i915,
+                       const struct intel_uncore_forcewake_domain *d)
 {
-        if (wait_for_atomic((__raw_i915_read32(i915, d->reg_ack) &
-                             FORCEWAKE_KERNEL),
-                            FORCEWAKE_ACK_TIMEOUT_MS))
+        if (wait_ack_set(i915, d, FORCEWAKE_KERNEL))
                 DRM_ERROR("%s: timed out waiting for forcewake ack request.\n",
                           intel_uncore_forcewake_domain_to_str(d->id));
 }
 
 static inline void
+fw_domain_wait_ack_set_fallback(const struct drm_i915_private *i915,
+                                const struct intel_uncore_forcewake_domain *d)
+{
+        if (likely(!wait_ack_set(i915, d, FORCEWAKE_KERNEL)))
+                return;
+
+        if (fw_domain_wait_ack_with_fallback(i915, d, ACK_SET))
+                fw_domain_wait_ack_set(i915, d);
+}
+
+static inline void
 fw_domain_put(const struct drm_i915_private *i915,
               const struct intel_uncore_forcewake_domain *d)
 {
@@ -119,7 +215,27 @@ fw_domains_get(struct drm_i915_private *i915, enum forcewake_domains fw_domains)
         }
 
         for_each_fw_domain_masked(d, fw_domains, i915, tmp)
-                fw_domain_wait_ack(i915, d);
+                fw_domain_wait_ack_set(i915, d);
+
+        i915->uncore.fw_domains_active |= fw_domains;
+}
+
+static void
+fw_domains_get_with_fallback(struct drm_i915_private *i915,
+                             enum forcewake_domains fw_domains)
+{
+        struct intel_uncore_forcewake_domain *d;
+        unsigned int tmp;
+
+        GEM_BUG_ON(fw_domains & ~i915->uncore.fw_domains);
+
+        for_each_fw_domain_masked(d, fw_domains, i915, tmp) {
+                fw_domain_wait_ack_clear_fallback(i915, d);
+                fw_domain_get(i915, d);
+        }
+
+        for_each_fw_domain_masked(d, fw_domains, i915, tmp)
+                fw_domain_wait_ack_set_fallback(i915, d);
 
         i915->uncore.fw_domains_active |= fw_domains;
 }
@@ -229,6 +345,7 @@ intel_uncore_fw_release_timer(struct hrtimer *timer)
         return HRTIMER_NORESTART;
 }
 
+/* Note callers must have acquired the PUNIT->PMIC bus, before calling this. */
 static void intel_uncore_forcewake_reset(struct drm_i915_private *dev_priv,
                                          bool restore)
 {
@@ -237,6 +354,8 @@ static void intel_uncore_forcewake_reset(struct drm_i915_private *dev_priv,
         int retry_count = 100;
         enum forcewake_domains fw, active_domains;
 
+        iosf_mbi_assert_punit_acquired();
+
         /* Hold uncore.lock across reset to prevent any register access
          * with forcewake not set correctly. Wait until all pending
          * timers are run before holding.
@@ -416,14 +535,18 @@ static void __intel_uncore_early_sanitize(struct drm_i915_private *dev_priv,
                                    GT_FIFO_CTL_RC6_POLICY_STALL);
         }
 
+        iosf_mbi_punit_acquire();
         intel_uncore_forcewake_reset(dev_priv, restore_forcewake);
+        iosf_mbi_punit_release();
 }
 
 void intel_uncore_suspend(struct drm_i915_private *dev_priv)
 {
-        iosf_mbi_unregister_pmic_bus_access_notifier(
+        iosf_mbi_punit_acquire();
+        iosf_mbi_unregister_pmic_bus_access_notifier_unlocked(
                 &dev_priv->uncore.pmic_bus_access_nb);
         intel_uncore_forcewake_reset(dev_priv, false);
+        iosf_mbi_punit_release();
 }
 
 void intel_uncore_resume_early(struct drm_i915_private *dev_priv)
@@ -1148,7 +1271,8 @@ static void intel_uncore_fw_domains_init(struct drm_i915_private *dev_priv)
         }
 
         if (INTEL_GEN(dev_priv) >= 9) {
-                dev_priv->uncore.funcs.force_wake_get = fw_domains_get;
+                dev_priv->uncore.funcs.force_wake_get =
+                        fw_domains_get_with_fallback;
                 dev_priv->uncore.funcs.force_wake_put = fw_domains_put;
                 fw_domain_init(dev_priv, FW_DOMAIN_ID_RENDER,
                                FORCEWAKE_RENDER_GEN9,
@@ -1309,18 +1433,18 @@ void intel_uncore_init(struct drm_i915_private *dev_priv)
 
         iosf_mbi_register_pmic_bus_access_notifier(
                 &dev_priv->uncore.pmic_bus_access_nb);
-
-        i915_check_and_clear_faults(dev_priv);
 }
 
 void intel_uncore_fini(struct drm_i915_private *dev_priv)
 {
-        iosf_mbi_unregister_pmic_bus_access_notifier(
-                &dev_priv->uncore.pmic_bus_access_nb);
-
         /* Paranoia: make sure we have disabled everything before we exit. */
         intel_uncore_sanitize(dev_priv);
+
+        iosf_mbi_punit_acquire();
+        iosf_mbi_unregister_pmic_bus_access_notifier_unlocked(
+                &dev_priv->uncore.pmic_bus_access_nb);
         intel_uncore_forcewake_reset(dev_priv, false);
+        iosf_mbi_punit_release();
 }
 
 static const struct reg_whitelist {
@@ -1400,10 +1524,14 @@ static void gen3_stop_engine(struct intel_engine_cs *engine)
                 DRM_DEBUG_DRIVER("%s: timed out on STOP_RING\n",
                                  engine->name);
 
-        I915_WRITE_FW(RING_CTL(base), 0);
+        I915_WRITE_FW(RING_HEAD(base), I915_READ_FW(RING_TAIL(base)));
+
         I915_WRITE_FW(RING_HEAD(base), 0);
         I915_WRITE_FW(RING_TAIL(base), 0);
 
+        /* The ring must be empty before it is disabled */
+        I915_WRITE_FW(RING_CTL(base), 0);
+
         /* Check acts as a post */
         if (I915_READ_FW(RING_HEAD(base)) != 0)
                 DRM_DEBUG_DRIVER("%s: ring head not parked\n",
@@ -1801,18 +1929,13 @@ bool intel_has_gpu_reset(struct drm_i915_private *dev_priv)
         return intel_get_gpu_reset(dev_priv) != NULL;
 }
 
-/*
- * When GuC submission is enabled, GuC manages ELSP and can initiate the
- * engine reset too. For now, fall back to full GPU reset if it is enabled.
- */
 bool intel_has_reset_engine(struct drm_i915_private *dev_priv)
 {
         return (dev_priv->info.has_reset_engine &&
-                !dev_priv->guc.execbuf_client &&
                 i915_modparams.reset >= 2);
 }
 
-int intel_guc_reset(struct drm_i915_private *dev_priv)
+int intel_reset_guc(struct drm_i915_private *dev_priv)
 {
         int ret;
 
diff --git a/drivers/gpu/drm/i915/intel_vbt_defs.h b/drivers/gpu/drm/i915/intel_vbt_defs.h
index f225c288a121..e3d7745a9151 100644
--- a/drivers/gpu/drm/i915/intel_vbt_defs.h
+++ b/drivers/gpu/drm/i915/intel_vbt_defs.h
@@ -304,6 +304,10 @@ struct bdb_general_features {
 #define DVO_PORT_MIPIC          23                              /* 171 */
 #define DVO_PORT_MIPID          24                              /* 171 */
 
+#define HDMI_MAX_DATA_RATE_PLATFORM     0                       /* 204 */
+#define HDMI_MAX_DATA_RATE_297          1                       /* 204 */
+#define HDMI_MAX_DATA_RATE_165          2                       /* 204 */
+
 #define LEGACY_CHILD_DEVICE_CONFIG_SIZE         33
 
 /* DDC Bus DDI Type 155+ */
@@ -342,8 +346,8 @@ struct child_device_config {
                         u8 i2c_speed;
                         u8 dp_onboard_redriver;                 /* 158 */
                         u8 dp_ondock_redriver;                  /* 158 */
-                        u8 hdmi_level_shifter_value:4;          /* 169 */
-                        u8 hdmi_max_data_rate:4;                /* 204 */
+                        u8 hdmi_level_shifter_value:5;          /* 169 */
+                        u8 hdmi_max_data_rate:3;                /* 204 */
                         u16 dtd_buf_ptr;                        /* 161 */
                         u8 edidless_efp:1;                      /* 161 */
                         u8 compression_enable:1;                /* 198 */
diff --git a/drivers/gpu/drm/i915/selftests/huge_pages.c b/drivers/gpu/drm/i915/selftests/huge_pages.c
index 5cc8101bb2b1..01af540b6ef9 100644
--- a/drivers/gpu/drm/i915/selftests/huge_pages.c
+++ b/drivers/gpu/drm/i915/selftests/huge_pages.c
@@ -1159,6 +1159,9 @@ static int igt_ppgtt_exhaust_huge(void *arg)
         int n, i;
         int err = -ENODEV;
 
+        if (supported == I915_GTT_PAGE_SIZE_4K)
+                return 0;
+
         /*
          * Sanity check creating objects with a varying mix of page sizes --
          * ensuring that our writes lands in the right place.
diff --git a/drivers/gpu/drm/i915/selftests/i915_gem_coherency.c b/drivers/gpu/drm/i915/selftests/i915_gem_coherency.c
index 35d778d70626..7a0d1e17c1ad 100644
--- a/drivers/gpu/drm/i915/selftests/i915_gem_coherency.c
+++ b/drivers/gpu/drm/i915/selftests/i915_gem_coherency.c
@@ -33,7 +33,7 @@ static int cpu_set(struct drm_i915_gem_object *obj,
 {
         unsigned int needs_clflush;
         struct page *page;
-        typeof(v) *map;
+        u32 *map;
         int err;
 
         err = i915_gem_obj_prepare_shmem_write(obj, &needs_clflush);
@@ -59,7 +59,7 @@ static int cpu_get(struct drm_i915_gem_object *obj,
 {
         unsigned int needs_clflush;
         struct page *page;
-        typeof(v) map;
+        u32 *map;
         int err;
 
         err = i915_gem_obj_prepare_shmem_read(obj, &needs_clflush);
@@ -82,7 +82,7 @@ static int gtt_set(struct drm_i915_gem_object *obj,
                    u32 v)
 {
         struct i915_vma *vma;
-        typeof(v) *map;
+        u32 __iomem *map;
         int err;
 
         err = i915_gem_object_set_to_gtt_domain(obj, true);
@@ -98,7 +98,7 @@ static int gtt_set(struct drm_i915_gem_object *obj,
         if (IS_ERR(map))
                 return PTR_ERR(map);
 
-        map[offset / sizeof(*map)] = v;
+        iowrite32(v, &map[offset / sizeof(*map)]);
         i915_vma_unpin_iomap(vma);
 
         return 0;
@@ -109,7 +109,7 @@ static int gtt_get(struct drm_i915_gem_object *obj,
                    u32 *v)
 {
         struct i915_vma *vma;
-        typeof(v) map;
+        u32 __iomem *map;
         int err;
 
         err = i915_gem_object_set_to_gtt_domain(obj, false);
@@ -125,7 +125,7 @@ static int gtt_get(struct drm_i915_gem_object *obj,
         if (IS_ERR(map))
                 return PTR_ERR(map);
 
-        *v = map[offset / sizeof(*map)];
+        *v = ioread32(&map[offset / sizeof(*map)]);
         i915_vma_unpin_iomap(vma);
 
         return 0;
@@ -135,7 +135,7 @@ static int wc_set(struct drm_i915_gem_object *obj,
                   unsigned long offset,
                   u32 v)
 {
-        typeof(v) *map;
+        u32 *map;
         int err;
 
         err = i915_gem_object_set_to_wc_domain(obj, true);
@@ -156,7 +156,7 @@ static int wc_get(struct drm_i915_gem_object *obj,
                   unsigned long offset,
                   u32 *v)
 {
-        typeof(v) map;
+        u32 *map;
         int err;
 
         err = i915_gem_object_set_to_wc_domain(obj, false);
diff --git a/drivers/gpu/drm/i915/selftests/i915_gem_context.c b/drivers/gpu/drm/i915/selftests/i915_gem_context.c
index def5052862ae..c82780a9d455 100644
--- a/drivers/gpu/drm/i915/selftests/i915_gem_context.c
+++ b/drivers/gpu/drm/i915/selftests/i915_gem_context.c
@@ -325,7 +325,7 @@ static int igt_ctx_exec(void *arg)
         LIST_HEAD(objects);
         unsigned long ncontexts, ndwords, dw;
         bool first_shared_gtt = true;
-        int err;
+        int err = -ENODEV;
 
         /* Create a few different contexts (with different mm) and write
          * through each ctx/mm using the GPU making sure those writes end
diff --git a/drivers/gpu/drm/i915/selftests/i915_gem_gtt.c b/drivers/gpu/drm/i915/selftests/i915_gem_gtt.c
index 9da0c9f99916..6491cf0a4f46 100644
--- a/drivers/gpu/drm/i915/selftests/i915_gem_gtt.c
+++ b/drivers/gpu/drm/i915/selftests/i915_gem_gtt.c
@@ -216,13 +216,21 @@ static int lowlevel_hole(struct drm_i915_private *i915,
                 hole_size = (hole_end - hole_start) >> size;
                 if (hole_size > KMALLOC_MAX_SIZE / sizeof(u32))
                         hole_size = KMALLOC_MAX_SIZE / sizeof(u32);
-                count = hole_size;
+                count = hole_size >> 1;
+                if (!count) {
+                        pr_debug("%s: hole is too small [%llx - %llx] >> %d: %lld\n",
+                                 __func__, hole_start, hole_end, size, hole_size);
+                        break;
+                }
+
                 do {
-                        count >>= 1;
                         order = i915_random_order(count, &prng);
-                } while (!order && count);
-                if (!order)
-                        break;
+                        if (order)
+                                break;
+                } while (count >>= 1);
+                if (!count)
+                        return -ENOMEM;
+                GEM_BUG_ON(!order);
 
                 GEM_BUG_ON(count * BIT_ULL(size) > vm->total);
                 GEM_BUG_ON(hole_start + count * BIT_ULL(size) > hole_end);
@@ -267,7 +275,9 @@ static int lowlevel_hole(struct drm_i915_private *i915,
                         mock_vma.node.size = BIT_ULL(size);
                         mock_vma.node.start = addr;
 
+                        intel_runtime_pm_get(i915);
                         vm->insert_entries(vm, &mock_vma, I915_CACHE_NONE, 0);
+                        intel_runtime_pm_put(i915);
                 }
                 count = n;
 
@@ -697,18 +707,26 @@ static int drunk_hole(struct drm_i915_private *i915,
                 unsigned int *order, count, n;
                 struct i915_vma *vma;
                 u64 hole_size;
-                int err;
+                int err = -ENODEV;
 
                 hole_size = (hole_end - hole_start) >> size;
                 if (hole_size > KMALLOC_MAX_SIZE / sizeof(u32))
                         hole_size = KMALLOC_MAX_SIZE / sizeof(u32);
-                count = hole_size;
+                count = hole_size >> 1;
+                if (!count) {
+                        pr_debug("%s: hole is too small [%llx - %llx] >> %d: %lld\n",
+                                 __func__, hole_start, hole_end, size, hole_size);
+                        break;
+                }
+
                 do {
-                        count >>= 1;
                         order = i915_random_order(count, &prng);
-                } while (!order && count);
-                if (!order)
-                        break;
+                        if (order)
+                                break;
+                } while (count >>= 1);
+                if (!count)
+                        return -ENOMEM;
+                GEM_BUG_ON(!order);
 
                 /* Ignore allocation failures (i.e. don't report them as
                  * a test failure) as we are purposefully allocating very
@@ -956,7 +974,7 @@ static int exercise_ggtt(struct drm_i915_private *i915,
         u64 hole_start, hole_end, last = 0;
         struct drm_mm_node *node;
         IGT_TIMEOUT(end_time);
-        int err;
+        int err = 0;
 
         mutex_lock(&i915->drm.struct_mutex);
 restart:
@@ -1047,6 +1065,7 @@ static int igt_ggtt_page(void *arg)
                 goto out_remove;
         }
 
+        intel_runtime_pm_get(i915);
         for (n = 0; n < count; n++) {
                 u64 offset = tmp.start + order[n] * PAGE_SIZE;
                 u32 __iomem *vaddr;
@@ -1086,6 +1105,7 @@ static int igt_ggtt_page(void *arg)
                         break;
                 }
         }
+        intel_runtime_pm_put(i915);
 
         kfree(order);
 out_remove:
@@ -1160,7 +1180,7 @@ static int igt_gtt_reserve(void *arg)
         struct drm_i915_gem_object *obj, *on;
         LIST_HEAD(objects);
         u64 total;
-        int err;
+        int err = -ENODEV;
 
         /* i915_gem_gtt_reserve() tries to reserve the precise range
          * for the node, and evicts if it has to. So our test checks that
@@ -1351,7 +1371,7 @@ static int igt_gtt_insert(void *arg)
         }, *ii;
         LIST_HEAD(objects);
         u64 total;
-        int err;
+        int err = -ENODEV;
 
         /* i915_gem_gtt_insert() tries to allocate some free space in the GTT
          * to the node, evicting if required.
diff --git a/drivers/gpu/drm/i915/selftests/i915_gem_object.c b/drivers/gpu/drm/i915/selftests/i915_gem_object.c
index 1b8774a42e48..f32aa6bb79e2 100644
--- a/drivers/gpu/drm/i915/selftests/i915_gem_object.c
+++ b/drivers/gpu/drm/i915/selftests/i915_gem_object.c
@@ -317,6 +317,7 @@ static int igt_partial_tiling(void *arg)
         }
 
         mutex_lock(&i915->drm.struct_mutex);
+        intel_runtime_pm_get(i915);
 
         if (1) {
                 IGT_TIMEOUT(end);
@@ -418,6 +419,7 @@ next_tiling: ;
         }
 
 out_unlock:
+        intel_runtime_pm_put(i915);
         mutex_unlock(&i915->drm.struct_mutex);
         i915_gem_object_unpin_pages(obj);
 out:
diff --git a/drivers/gpu/drm/i915/selftests/i915_gem_request.c b/drivers/gpu/drm/i915/selftests/i915_gem_request.c
index a999161e8db1..6bce99050e94 100644
--- a/drivers/gpu/drm/i915/selftests/i915_gem_request.c
+++ b/drivers/gpu/drm/i915/selftests/i915_gem_request.c
@@ -332,7 +332,7 @@ static int live_nop_request(void *arg)
         struct intel_engine_cs *engine;
         struct live_test t;
         unsigned int id;
-        int err;
+        int err = -ENODEV;
 
         /* Submit various sized batches of empty requests, to each engine
          * (individually), and wait for the batch to complete. We can check
diff --git a/drivers/gpu/drm/i915/selftests/i915_gem_timeline.c b/drivers/gpu/drm/i915/selftests/i915_gem_timeline.c
index 4795877abe56..3000e6a7d82d 100644
--- a/drivers/gpu/drm/i915/selftests/i915_gem_timeline.c
+++ b/drivers/gpu/drm/i915/selftests/i915_gem_timeline.c
@@ -79,7 +79,7 @@ static int igt_sync(void *arg)
         }, *p;
         struct intel_timeline *tl;
         int order, offset;
-        int ret;
+        int ret = -ENODEV;
 
         tl = mock_timeline(0);
         if (!tl)
diff --git a/drivers/gpu/drm/i915/selftests/i915_live_selftests.h b/drivers/gpu/drm/i915/selftests/i915_live_selftests.h
index d7dd98a6acad..088f45bc6199 100644
--- a/drivers/gpu/drm/i915/selftests/i915_live_selftests.h
+++ b/drivers/gpu/drm/i915/selftests/i915_live_selftests.h
@@ -20,3 +20,4 @@ selftest(evict, i915_gem_evict_live_selftests)
 selftest(hugepages, i915_gem_huge_page_live_selftests)
 selftest(contexts, i915_gem_context_live_selftests)
 selftest(hangcheck, intel_hangcheck_live_selftests)
+selftest(guc, intel_guc_live_selftest)
diff --git a/drivers/gpu/drm/i915/selftests/i915_syncmap.c b/drivers/gpu/drm/i915/selftests/i915_syncmap.c
index bcab3d00a785..47f4ae18a1ef 100644
--- a/drivers/gpu/drm/i915/selftests/i915_syncmap.c
+++ b/drivers/gpu/drm/i915/selftests/i915_syncmap.c
@@ -333,7 +333,7 @@ static int igt_syncmap_join_below(void *arg)
 {
         struct i915_syncmap *sync;
         unsigned int step, order, idx;
-        int err;
+        int err = -ENODEV;
 
         i915_syncmap_init(&sync);
 
@@ -402,7 +402,7 @@ static int igt_syncmap_neighbours(void *arg)
         I915_RND_STATE(prng);
         IGT_TIMEOUT(end_time);
         struct i915_syncmap *sync;
-        int err;
+        int err = -ENODEV;
 
         /*
          * Each leaf holds KSYNCMAP seqno. Check that when we create KSYNCMAP
@@ -447,7 +447,7 @@ static int igt_syncmap_compact(void *arg)
 {
         struct i915_syncmap *sync;
         unsigned int idx, order;
-        int err;
+        int err = -ENODEV;
 
         i915_syncmap_init(&sync);
 
diff --git a/drivers/gpu/drm/i915/selftests/i915_vma.c b/drivers/gpu/drm/i915/selftests/i915_vma.c
index 2e86ec136b35..eb89e301b602 100644
--- a/drivers/gpu/drm/i915/selftests/i915_vma.c
+++ b/drivers/gpu/drm/i915/selftests/i915_vma.c
@@ -150,7 +150,7 @@ static int igt_vma_create(void *arg)
         IGT_TIMEOUT(end_time);
         LIST_HEAD(contexts);
         LIST_HEAD(objects);
-        int err;
+        int err = -ENOMEM;
 
         /* Exercise creating many vma amonst many objections, checking the
          * vma creation and lookup routines.
diff --git a/drivers/gpu/drm/i915/selftests/intel_guc.c b/drivers/gpu/drm/i915/selftests/intel_guc.c
new file mode 100644
index 000000000000..f10029e18820
--- /dev/null
+++ b/drivers/gpu/drm/i915/selftests/intel_guc.c
@@ -0,0 +1,367 @@
+/*
+ * Copyright © 2017 Intel Corporation
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ *
+ */
+
+#include "../i915_selftest.h"
+
+/* max doorbell number + negative test for each client type */
+#define ATTEMPTS (GUC_NUM_DOORBELLS + GUC_CLIENT_PRIORITY_NUM)
+
+struct intel_guc_client *clients[ATTEMPTS];
+
+static bool available_dbs(struct intel_guc *guc, u32 priority)
+{
+        unsigned long offset;
+        unsigned long end;
+        u16 id;
+
+        /* first half is used for normal priority, second half for high */
+        offset = 0;
+        end = GUC_NUM_DOORBELLS / 2;
+        if (priority <= GUC_CLIENT_PRIORITY_HIGH) {
+                offset = end;
+                end += offset;
+        }
+
+        id = find_next_zero_bit(guc->doorbell_bitmap, end, offset);
+        if (id < end)
+                return true;
+
+        return false;
+}
+
+static int check_all_doorbells(struct intel_guc *guc)
+{
+        u16 db_id;
+
+        pr_info_once("Max number of doorbells: %d", GUC_NUM_DOORBELLS);
+        for (db_id = 0; db_id < GUC_NUM_DOORBELLS; ++db_id) {
+                if (!doorbell_ok(guc, db_id)) {
+                        pr_err("doorbell %d, not ok\n", db_id);
+                        return -EIO;
+                }
+        }
+
+        return 0;
+}
+
+/*
+ * Basic client sanity check, handy to validate create_clients.
+ */
+static int validate_client(struct intel_guc_client *client,
+                           int client_priority,
+                           bool is_preempt_client)
+{
+        struct drm_i915_private *dev_priv = guc_to_i915(client->guc);
+        struct i915_gem_context *ctx_owner = is_preempt_client ?
+                        dev_priv->preempt_context : dev_priv->kernel_context;
+
+        if (client->owner != ctx_owner ||
+            client->engines != INTEL_INFO(dev_priv)->ring_mask ||
+            client->priority != client_priority ||
+            client->doorbell_id == GUC_DOORBELL_INVALID)
+                return -EINVAL;
+        else
+                return 0;
+}
+
+/*
+ * Check that guc_init_doorbell_hw is doing what it should.
+ *
+ * During GuC submission enable, we create GuC clients and their doorbells,
+ * but after resetting the microcontroller (resume & gpu reset), these
+ * GuC clients are still around, but the status of their doorbells may be
+ * incorrect. This is the reason behind validating that the doorbells status
+ * expected by the driver matches what the GuC/HW have.
+ */
+static int igt_guc_init_doorbell_hw(void *args)
+{
+        struct drm_i915_private *dev_priv = args;
+        struct intel_guc *guc;
+        DECLARE_BITMAP(db_bitmap_bk, GUC_NUM_DOORBELLS);
+        int i, err = 0;
+
+        GEM_BUG_ON(!HAS_GUC(dev_priv));
+        mutex_lock(&dev_priv->drm.struct_mutex);
+
+        guc = &dev_priv->guc;
+        if (!guc) {
+                pr_err("No guc object!\n");
+                err = -EINVAL;
+                goto unlock;
+        }
+
+        err = check_all_doorbells(guc);
+        if (err)
+                goto unlock;
+
+        /*
+         * Get rid of clients created during driver load because the test will
+         * recreate them.
+         */
+        guc_clients_destroy(guc);
+        if (guc->execbuf_client || guc->preempt_client) {
+                pr_err("guc_clients_destroy lied!\n");
+                err = -EINVAL;
+                goto unlock;
+        }
+
+        err = guc_clients_create(guc);
+        if (err) {
+                pr_err("Failed to create clients\n");
+                goto unlock;
+        }
+
+        err = validate_client(guc->execbuf_client,
+                              GUC_CLIENT_PRIORITY_KMD_NORMAL, false);
+        if (err) {
+                pr_err("execbug client validation failed\n");
+                goto out;
+        }
+
+        err = validate_client(guc->preempt_client,
+                              GUC_CLIENT_PRIORITY_KMD_HIGH, true);
+        if (err) {
+                pr_err("preempt client validation failed\n");
+                goto out;
+        }
+
+        /* each client should have received a doorbell during alloc */
+        if (!has_doorbell(guc->execbuf_client) ||
+            !has_doorbell(guc->preempt_client)) {
+                pr_err("guc_clients_create didn't create doorbells\n");
+                err = -EINVAL;
+                goto out;
+        }
+
+        /*
+         * Basic test - an attempt to reallocate a valid doorbell to the
+         * client it is currently assigned should not cause a failure.
+         */
+        err = guc_init_doorbell_hw(guc);
+        if (err)
+                goto out;
+
+        /*
+         * Negative test - a client with no doorbell (invalid db id).
+         * Each client gets a doorbell when it is created, after destroying
+         * the doorbell, the db id is changed to GUC_DOORBELL_INVALID and the
+         * firmware will reject any attempt to allocate a doorbell with an
+         * invalid id (db has to be reserved before allocation).
+         */
+        destroy_doorbell(guc->execbuf_client);
+        if (has_doorbell(guc->execbuf_client)) {
+                pr_err("destroy db did not work\n");
+                err = -EINVAL;
+                goto out;
+        }
+
+        err = guc_init_doorbell_hw(guc);
+        if (err != -EIO) {
+                pr_err("unexpected (err = %d)", err);
+                goto out;
+        }
+
+        if (!available_dbs(guc, guc->execbuf_client->priority)) {
+                pr_err("doorbell not available when it should\n");
+                err = -EIO;
+                goto out;
+        }
+
+        /* clean after test */
+        err = create_doorbell(guc->execbuf_client);
+        if (err) {
+                pr_err("recreate doorbell failed\n");
+                goto out;
+        }
+
+        /*
+         * Negative test - doorbell_bitmap out of sync, will trigger a few of
+         * WARN_ON(!doorbell_ok(guc, db_id)) but that's ok as long as the
+         * doorbells from our clients don't fail.
+         */
+        bitmap_copy(db_bitmap_bk, guc->doorbell_bitmap, GUC_NUM_DOORBELLS);
+        for (i = 0; i < GUC_NUM_DOORBELLS; i++)
+                if (i % 2)
+                        test_and_change_bit(i, guc->doorbell_bitmap);
+
+        err = guc_init_doorbell_hw(guc);
+        if (err) {
+                pr_err("out of sync doorbell caused an error\n");
+                goto out;
+        }
+
+        /* restore 'correct' db bitmap */
+        bitmap_copy(guc->doorbell_bitmap, db_bitmap_bk, GUC_NUM_DOORBELLS);
+        err = guc_init_doorbell_hw(guc);
+        if (err) {
+                pr_err("restored doorbell caused an error\n");
+                goto out;
+        }
+
+out:
+        /*
+         * Leave clean state for other test, plus the driver always destroy the
+         * clients during unload.
+         */
+        guc_clients_destroy(guc);
+        guc_clients_create(guc);
+unlock:
+        mutex_unlock(&dev_priv->drm.struct_mutex);
+        return err;
+}
+
+/*
+ * Create as many clients as number of doorbells. Note that there's already
+ * client(s)/doorbell(s) created during driver load, but this test creates
+ * its own and do not interact with the existing ones.
+ */
+static int igt_guc_doorbells(void *arg)
+{
+        struct drm_i915_private *dev_priv = arg;
+        struct intel_guc *guc;
+        int i, err = 0;
+        u16 db_id;
+
+        GEM_BUG_ON(!HAS_GUC(dev_priv));
+        mutex_lock(&dev_priv->drm.struct_mutex);
+
+        guc = &dev_priv->guc;
+        if (!guc) {
+                pr_err("No guc object!\n");
+                err = -EINVAL;
+                goto unlock;
+        }
+
+        err = check_all_doorbells(guc);
+        if (err)
+                goto unlock;
+
+        for (i = 0; i < ATTEMPTS; i++) {
+                clients[i] = guc_client_alloc(dev_priv,
+                                              INTEL_INFO(dev_priv)->ring_mask,
+                                              i % GUC_CLIENT_PRIORITY_NUM,
+                                              dev_priv->kernel_context);
+
+                if (!clients[i]) {
+                        pr_err("[%d] No guc client\n", i);
+                        err = -EINVAL;
+                        goto out;
+                }
+
+                if (IS_ERR(clients[i])) {
+                        if (PTR_ERR(clients[i]) != -ENOSPC) {
+                                pr_err("[%d] unexpected error\n", i);
+                                err = PTR_ERR(clients[i]);
+                                goto out;
+                        }
+
+                        if (available_dbs(guc, i % GUC_CLIENT_PRIORITY_NUM)) {
+                                pr_err("[%d] non-db related alloc fail\n", i);
+                                err = -EINVAL;
+                                goto out;
+                        }
+
+                        /* expected, ran out of dbs for this client type */
+                        continue;
+                }
+
+                /*
+                 * The check below is only valid because we keep a doorbell
+                 * assigned during the whole life of the client.
+                 */
+                if (clients[i]->stage_id >= GUC_NUM_DOORBELLS) {
+                        pr_err("[%d] more clients than doorbells (%d >= %d)\n",
+                               i, clients[i]->stage_id, GUC_NUM_DOORBELLS);
+                        err = -EINVAL;
+                        goto out;
+                }
+
+                err = validate_client(clients[i],
+                                      i % GUC_CLIENT_PRIORITY_NUM, false);
+                if (err) {
+                        pr_err("[%d] client_alloc sanity check failed!\n", i);
+                        err = -EINVAL;
+                        goto out;
+                }
+
+                db_id = clients[i]->doorbell_id;
+
+                /*
+                 * Client alloc gives us a doorbell, but we want to exercise
+                 * this ourselves (this resembles guc_init_doorbell_hw)
+                 */
+                destroy_doorbell(clients[i]);
+                if (clients[i]->doorbell_id != GUC_DOORBELL_INVALID) {
+                        pr_err("[%d] destroy db did not work!\n", i);
+                        err = -EINVAL;
+                        goto out;
+                }
+
+                err = __reserve_doorbell(clients[i]);
+                if (err) {
+                        pr_err("[%d] Failed to reserve a doorbell\n", i);
+                        goto out;
+                }
+
+                __update_doorbell_desc(clients[i], clients[i]->doorbell_id);
+                err = __create_doorbell(clients[i]);
+                if (err) {
+                        pr_err("[%d] Failed to create a doorbell\n", i);
+                        goto out;
+                }
+
+                /* doorbell id shouldn't change, we are holding the mutex */
+                if (db_id != clients[i]->doorbell_id) {
+                        pr_err("[%d] doorbell id changed (%d != %d)\n",
+                               i, db_id, clients[i]->doorbell_id);
+                        err = -EINVAL;
+                        goto out;
+                }
+
+                err = check_all_doorbells(guc);
+                if (err)
+                        goto out;
+        }
+
+out:
+        for (i = 0; i < ATTEMPTS; i++)
+                if (!IS_ERR_OR_NULL(clients[i]))
+                        guc_client_free(clients[i]);
+unlock:
+        mutex_unlock(&dev_priv->drm.struct_mutex);
+        return err;
+}
+
+int intel_guc_live_selftest(struct drm_i915_private *dev_priv)
+{
+        static const struct i915_subtest tests[] = {
+                SUBTEST(igt_guc_init_doorbell_hw),
+                SUBTEST(igt_guc_doorbells),
+        };
+
+        if (!i915_modparams.enable_guc_submission)
+                return 0;
+
+        return i915_subtests(tests, dev_priv);
+}
diff --git a/drivers/gpu/drm/i915/selftests/intel_uncore.c b/drivers/gpu/drm/i915/selftests/intel_uncore.c
index 3cac22eb47ce..2f6367643171 100644
--- a/drivers/gpu/drm/i915/selftests/intel_uncore.c
+++ b/drivers/gpu/drm/i915/selftests/intel_uncore.c
@@ -120,10 +120,10 @@ static int intel_uncore_check_forcewake_domains(struct drm_i915_private *dev_pri
             !IS_CHERRYVIEW(dev_priv))
                 return 0;
 
-        if (IS_VALLEYVIEW(dev_priv)) /* XXX system lockup! */
-                return 0;
-
-        if (IS_BROADWELL(dev_priv)) /* XXX random GPU hang afterwards! */
+        /*
+         * This test may lockup the machine or cause GPU hangs afterwards.
+         */
+        if (!IS_ENABLED(CONFIG_DRM_I915_SELFTEST_BROKEN))
                 return 0;
 
         valid = kzalloc(BITS_TO_LONGS(FW_RANGE) * sizeof(*valid),
@@ -148,7 +148,10 @@ static int intel_uncore_check_forcewake_domains(struct drm_i915_private *dev_pri
         for_each_set_bit(offset, valid, FW_RANGE) {
                 i915_reg_t reg = { offset };
 
+                iosf_mbi_punit_acquire();
                 intel_uncore_forcewake_reset(dev_priv, false);
+                iosf_mbi_punit_release();
+
                 check_for_unclaimed_mmio(dev_priv);
 
                 (void)I915_READ(reg);
diff --git a/drivers/gpu/drm/i915/selftests/mock_engine.c b/drivers/gpu/drm/i915/selftests/mock_engine.c
index 331c2b09869e..55c0e2c15782 100644
--- a/drivers/gpu/drm/i915/selftests/mock_engine.c
+++ b/drivers/gpu/drm/i915/selftests/mock_engine.c
@@ -32,6 +32,13 @@ static struct mock_request *first_request(struct mock_engine *engine)
                                         link);
 }
 
+static void advance(struct mock_engine *engine,
+                    struct mock_request *request)
+{
+        list_del_init(&request->link);
+        mock_seqno_advance(&engine->base, request->base.global_seqno);
+}
+
 static void hw_delay_complete(struct timer_list *t)
 {
         struct mock_engine *engine = from_timer(engine, t, hw_delay);
@@ -39,15 +46,23 @@ static void hw_delay_complete(struct timer_list *t)
 
         spin_lock(&engine->hw_lock);
 
-        request = first_request(engine);
-        if (request) {
-                list_del_init(&request->link);
-                mock_seqno_advance(&engine->base, request->base.global_seqno);
-        }
-
+        /* Timer fired, first request is complete */
         request = first_request(engine);
         if (request)
-                mod_timer(&engine->hw_delay, jiffies + request->delay);
+                advance(engine, request);
+
+        /*
+         * Also immediately signal any subsequent 0-delay requests, but
+         * requeue the timer for the next delayed request.
+         */
+        while ((request = first_request(engine))) {
+                if (request->delay) {
+                        mod_timer(&engine->hw_delay, jiffies + request->delay);
+                        break;
+                }
+
+                advance(engine, request);
+        }
 
         spin_unlock(&engine->hw_lock);
 }
@@ -98,16 +113,22 @@ static void mock_submit_request(struct drm_i915_gem_request *request)
 
         spin_lock_irq(&engine->hw_lock);
         list_add_tail(&mock->link, &engine->hw_queue);
-        if (mock->link.prev == &engine->hw_queue)
-                mod_timer(&engine->hw_delay, jiffies + mock->delay);
+        if (mock->link.prev == &engine->hw_queue) {
+                if (mock->delay)
+                        mod_timer(&engine->hw_delay, jiffies + mock->delay);
+                else
+                        advance(engine, mock);
+        }
         spin_unlock_irq(&engine->hw_lock);
 }
 
 static struct intel_ring *mock_ring(struct intel_engine_cs *engine)
 {
-        const unsigned long sz = roundup_pow_of_two(sizeof(struct intel_ring));
+        const unsigned long sz = PAGE_SIZE / 2;
         struct intel_ring *ring;
 
+        BUILD_BUG_ON(MIN_SPACE_FOR_ADD_REQUEST > sz);
+
         ring = kzalloc(sizeof(*ring) + sz, GFP_KERNEL);
         if (!ring)
                 return NULL;
diff --git a/drivers/gpu/drm/i915/selftests/mock_gem_device.c b/drivers/gpu/drm/i915/selftests/mock_gem_device.c
index 04eb9362f4f8..80f152aaedf9 100644
--- a/drivers/gpu/drm/i915/selftests/mock_gem_device.c
+++ b/drivers/gpu/drm/i915/selftests/mock_gem_device.c
@@ -179,8 +179,8 @@ struct drm_i915_private *mock_gem_device(void)
                 I915_GTT_PAGE_SIZE_64K |
                 I915_GTT_PAGE_SIZE_2M;
 
-        spin_lock_init(&i915->mm.object_stat_lock);
         mock_uncore_init(i915);
+        i915_gem_init__mm(i915);
 
         init_waitqueue_head(&i915->gpu_error.wait_queue);
         init_waitqueue_head(&i915->gpu_error.reset_queue);
@@ -189,11 +189,6 @@ struct drm_i915_private *mock_gem_device(void)
         if (!i915->wq)
                 goto put_device;
 
-        INIT_WORK(&i915->mm.free_work, __i915_gem_free_work);
-        init_llist_head(&i915->mm.free_list);
-        INIT_LIST_HEAD(&i915->mm.unbound_list);
-        INIT_LIST_HEAD(&i915->mm.bound_list);
-
         mock_init_contexts(i915);
 
         INIT_DELAYED_WORK(&i915->gt.retire_work, mock_retire_work_handler);
diff --git a/include/uapi/drm/i915_drm.h b/include/uapi/drm/i915_drm.h
index ac3c6503ca27..b57985929553 100644
--- a/include/uapi/drm/i915_drm.h
+++ b/include/uapi/drm/i915_drm.h
@@ -86,6 +86,22 @@ enum i915_mocs_table_index {
         I915_MOCS_CACHED,
 };
 
+/*
+ * Different engines serve different roles, and there may be more than one
+ * engine serving each role. enum drm_i915_gem_engine_class provides a
+ * classification of the role of the engine, which may be used when requesting
+ * operations to be performed on a certain subset of engines, or for providing
+ * information about that group.
+ */
+enum drm_i915_gem_engine_class {
+        I915_ENGINE_CLASS_RENDER        = 0,
+        I915_ENGINE_CLASS_COPY          = 1,
+        I915_ENGINE_CLASS_VIDEO         = 2,
+        I915_ENGINE_CLASS_VIDEO_ENHANCE = 3,
+
+        I915_ENGINE_CLASS_INVALID       = -1
+};
+
 /* Each region is a minimum of 16k, and there are at most 255 of them.
  */
 #define I915_NR_TEX_REGIONS 255 /* table size 2k - maximum due to use
@@ -450,6 +466,27 @@ typedef struct drm_i915_irq_wait {
  */
 #define I915_PARAM_HAS_EXEC_FENCE_ARRAY  49
 
+/*
+ * Query whether every context (both per-file default and user created) is
+ * isolated (insofar as HW supports). If this parameter is not true, then
+ * freshly created contexts may inherit values from an existing context,
+ * rather than default HW values. If true, it also ensures (insofar as HW
+ * supports) that all state set by this context will not leak to any other
+ * context.
+ *
+ * As not every engine across every gen support contexts, the returned
+ * value reports the support of context isolation for individual engines by
+ * returning a bitmask of each engine class set to true if that class supports
+ * isolation.
+ */
+#define I915_PARAM_HAS_CONTEXT_ISOLATION 50
+
+/* Frequency of the command streamer timestamps given by the *_TIMESTAMP
+ * registers. This used to be fixed per platform but from CNL onwards, this
+ * might vary depending on the parts.
+ */
+#define I915_PARAM_CS_TIMESTAMP_FREQUENCY 51
+
 typedef struct drm_i915_getparam {
         __s32 param;
         /*