Merge tag 'drm-intel-next-2018-07-09' of git://anongit.freedesktop.org/drm/drm-intel into drm-next

Higlights here goes to many PSR fixes and improvements; to the Ice lake work with
power well support and begin of DSI support addition. Also there were many improvements
on execlists and interrupts for minimal latency on command submission; and many fixes
on selftests, mostly caught by our CI.

General driver:
- Clean-up on aux irq (Lucas)
- Mark expected switch fall-through for dealing with static analysis tools (Gustavo)

Gem:
- Different fixes for GuC (Chris, Anusha, Michal)
- Avoid self-relocation BIAS if no relocation (Chris)
- Improve debugging cases in on EINVAL return and vma allocation (Chris)
- Fixes and improvements on context destroying and freeing (Chris)
- Wait for engines to idle before retiring (Chris)
- Many improvements on execlists and interrupts for minimal latency on command submission (Chris)
- Many fixes in selftests, specially on cases highlighted on CI (Chris)
- Other fixes and improvements around GGTT (Chris)
- Prevent background reaping of active objects (Chris)

Display:
- Parallel modeset cleanup to fix driver reset (Chris)
- Get AUX power domain for DP main link (Imre)
- Clean-up on PSR unused func pointers (Rodrigo)
- Many PSR/PSR2 fixes and improvements (DK, Jose, Tarun)
- Add a PSR1 live status (Vathsala)
- Replace old drm_*_{un/reference} with put,get functions (Thomas)
- FBC fixes (Maarten)
- Abstract and document the usage of picking macros (Jani)
- Remove unnecessary check for unsupported modifiers for NV12. (DK)
- Interrupt fixes for display (Ville)
- Clean up on sdvo code (Ville)
- Clean up on current DSI code (Jani)
- Remove support for legacy debugfs crc interface (Maarten)
- Simplify get_encoder_power_domains (Imre)

Icelake:
- MG PLL fixes (Imre)
- Add hw workaround for alpha blending (Vandita)
- Add power well support (Imre)
- Add Interrupt Support (Anusha)
- Start to add support for DSI on Ice Lake (Madhav)

Signed-off-by: Dave Airlie <airlied@redhat.com>

# gpg: Signature made Tue 10 Jul 2018 08:41:37 AM AEST
# gpg:                using RSA key FA625F640EEB13CA
# gpg: Good signature from "Rodrigo Vivi <rodrigo.vivi@intel.com>"
# gpg:                 aka "Rodrigo Vivi <rodrigo.vivi@gmail.com>"
# gpg: WARNING: This key is not certified with a trusted signature!
# gpg:          There is no indication that the signature belongs to the owner.
# Primary key fingerprint: 6D20 7068 EEDD 6509 1C2C  E2A3 FA62 5F64 0EEB 13CA
Link: https://patchwork.freedesktop.org/patch/msgid/20180710234349.GA16562@intel.com

76 files changed, 2728 insertions, 1868 deletions

diff --git a/drivers/gpu/drm/i915/Kconfig.debug b/drivers/gpu/drm/i915/Kconfig.debug
index 9de8b1c51a5c..459f8f88a34c 100644
--- a/drivers/gpu/drm/i915/Kconfig.debug
+++ b/drivers/gpu/drm/i915/Kconfig.debug
@@ -51,6 +51,18 @@ config DRM_I915_DEBUG_GEM
 
           If in doubt, say "N".
 
+config DRM_I915_ERRLOG_GEM
+        bool "Insert extra logging (very verbose) for common GEM errors"
+        default n
+        depends on DRM_I915_DEBUG_GEM
+        help
+          Enable additional logging that may help track down the cause of
+          principally userspace errors.
+
+          Recommended for driver developers only.
+
+          If in doubt, say "N".
+
 config DRM_I915_TRACE_GEM
         bool "Insert extra ftrace output from the GEM internals"
         depends on DRM_I915_DEBUG_GEM
diff --git a/drivers/gpu/drm/i915/Makefile b/drivers/gpu/drm/i915/Makefile
index 4c6adae23e18..5794f102f9b8 100644
--- a/drivers/gpu/drm/i915/Makefile
+++ b/drivers/gpu/drm/i915/Makefile
@@ -135,15 +135,14 @@ i915-y += dvo_ch7017.o \
           dvo_ns2501.o \
           dvo_sil164.o \
           dvo_tfp410.o \
+          icl_dsi.o \
           intel_crt.o \
           intel_ddi.o \
           intel_dp_aux_backlight.o \
           intel_dp_link_training.o \
           intel_dp_mst.o \
           intel_dp.o \
-          intel_dsi.o \
           intel_dsi_dcs_backlight.o \
-          intel_dsi_pll.o \
           intel_dsi_vbt.o \
           intel_dvo.o \
           intel_hdmi.o \
@@ -152,7 +151,9 @@ i915-y += dvo_ch7017.o \
           intel_lvds.o \
           intel_panel.o \
           intel_sdvo.o \
-          intel_tv.o
+          intel_tv.o \
+          vlv_dsi.o \
+          vlv_dsi_pll.o
 
 # Post-mortem debug and GPU hang state capture
 i915-$(CONFIG_DRM_I915_CAPTURE_ERROR) += i915_gpu_error.o
diff --git a/drivers/gpu/drm/i915/gvt/scheduler.c b/drivers/gpu/drm/i915/gvt/scheduler.c
index 928818f218f7..b0e566956b8d 100644
--- a/drivers/gpu/drm/i915/gvt/scheduler.c
+++ b/drivers/gpu/drm/i915/gvt/scheduler.c
@@ -476,7 +476,11 @@ static int prepare_shadow_batch_buffer(struct intel_vgpu_workload *workload)
                         i915_gem_obj_finish_shmem_access(bb->obj);
                         bb->accessing = false;
 
-                        i915_vma_move_to_active(bb->vma, workload->req, 0);
+                        ret = i915_vma_move_to_active(bb->vma,
+                                                      workload->req,
+                                                      0);
+                        if (ret)
+                                goto err;
                 }
         }
         return 0;
diff --git a/drivers/gpu/drm/i915/i915_debugfs.c b/drivers/gpu/drm/i915/i915_debugfs.c
index c400f42a54ec..099f97ef2303 100644
--- a/drivers/gpu/drm/i915/i915_debugfs.c
+++ b/drivers/gpu/drm/i915/i915_debugfs.c
@@ -1659,11 +1659,6 @@ static int i915_fbc_status(struct seq_file *m, void *unused)
         else
                 seq_printf(m, "FBC disabled: %s\n", fbc->no_fbc_reason);
 
-        if (fbc->work.scheduled)
-                seq_printf(m, "FBC worker scheduled on vblank %llu, now %llu\n",
-                           fbc->work.scheduled_vblank,
-                           drm_crtc_vblank_count(&fbc->crtc->base));
-
         if (intel_fbc_is_active(dev_priv)) {
                 u32 mask;
 
@@ -2597,27 +2592,55 @@ static const struct file_operations i915_guc_log_relay_fops = {
         .release = i915_guc_log_relay_release,
 };
 
-static const char *psr2_live_status(u32 val)
-{
-        static const char * const live_status[] = {
-                "IDLE",
-                "CAPTURE",
-                "CAPTURE_FS",
-                "SLEEP",
-                "BUFON_FW",
-                "ML_UP",
-                "SU_STANDBY",
-                "FAST_SLEEP",
-                "DEEP_SLEEP",
-                "BUF_ON",
-                "TG_ON"
-        };
+static void
+psr_source_status(struct drm_i915_private *dev_priv, struct seq_file *m)
+{
+        u32 val, psr_status;
 
-        val = (val & EDP_PSR2_STATUS_STATE_MASK) >> EDP_PSR2_STATUS_STATE_SHIFT;
-        if (val < ARRAY_SIZE(live_status))
-                return live_status[val];
+        if (dev_priv->psr.psr2_enabled) {
+                static const char * const live_status[] = {
+                        "IDLE",
+                        "CAPTURE",
+                        "CAPTURE_FS",
+                        "SLEEP",
+                        "BUFON_FW",
+                        "ML_UP",
+                        "SU_STANDBY",
+                        "FAST_SLEEP",
+                        "DEEP_SLEEP",
+                        "BUF_ON",
+                        "TG_ON"
+                };
+                psr_status = I915_READ(EDP_PSR2_STATUS);
+                val = (psr_status & EDP_PSR2_STATUS_STATE_MASK) >>
+                        EDP_PSR2_STATUS_STATE_SHIFT;
+                if (val < ARRAY_SIZE(live_status)) {
+                        seq_printf(m, "Source PSR status: 0x%x [%s]\n",
+                                   psr_status, live_status[val]);
+                        return;
+                }
+        } else {
+                static const char * const live_status[] = {
+                        "IDLE",
+                        "SRDONACK",
+                        "SRDENT",
+                        "BUFOFF",
+                        "BUFON",
+                        "AUXACK",
+                        "SRDOFFACK",
+                        "SRDENT_ON",
+                };
+                psr_status = I915_READ(EDP_PSR_STATUS);
+                val = (psr_status & EDP_PSR_STATUS_STATE_MASK) >>
+                        EDP_PSR_STATUS_STATE_SHIFT;
+                if (val < ARRAY_SIZE(live_status)) {
+                        seq_printf(m, "Source PSR status: 0x%x [%s]\n",
+                                   psr_status, live_status[val]);
+                        return;
+                }
+        }
 
-        return "unknown";
+        seq_printf(m, "Source PSR status: 0x%x [%s]\n", psr_status, "unknown");
 }
 
 static const char *psr_sink_status(u8 val)
@@ -2681,12 +2704,8 @@ static int i915_edp_psr_status(struct seq_file *m, void *data)
 
                 seq_printf(m, "Performance_Counter: %u\n", psrperf);
         }
-        if (dev_priv->psr.psr2_enabled) {
-                u32 psr2 = I915_READ(EDP_PSR2_STATUS);
 
-                seq_printf(m, "EDP_PSR2_STATUS: %x [%s]\n",
-                           psr2, psr2_live_status(psr2));
-        }
+        psr_source_status(dev_priv, m);
 
         if (dev_priv->psr.enabled) {
                 struct drm_dp_aux *aux = &dev_priv->psr.enabled->aux;
@@ -4086,7 +4105,8 @@ fault_irq_set(struct drm_i915_private *i915,
 
         err = i915_gem_wait_for_idle(i915,
                                      I915_WAIT_LOCKED |
-                                     I915_WAIT_INTERRUPTIBLE);
+                                     I915_WAIT_INTERRUPTIBLE,
+                                     MAX_SCHEDULE_TIMEOUT);
         if (err)
                 goto err_unlock;
 
@@ -4191,7 +4211,8 @@ i915_drop_caches_set(void *data, u64 val)
                 if (val & DROP_ACTIVE)
                         ret = i915_gem_wait_for_idle(dev_priv,
                                                      I915_WAIT_INTERRUPTIBLE |
-                                                     I915_WAIT_LOCKED);
+                                                     I915_WAIT_LOCKED,
+                                                     MAX_SCHEDULE_TIMEOUT);
 
                 if (val & DROP_RETIRE)
                         i915_retire_requests(dev_priv);
@@ -4799,7 +4820,6 @@ static const struct i915_debugfs_files {
 #endif
         {"i915_fifo_underrun_reset", &i915_fifo_underrun_reset_ops},
         {"i915_next_seqno", &i915_next_seqno_fops},
-        {"i915_display_crc_ctl", &i915_display_crc_ctl_fops},
         {"i915_pri_wm_latency", &i915_pri_wm_latency_fops},
         {"i915_spr_wm_latency", &i915_spr_wm_latency_fops},
         {"i915_cur_wm_latency", &i915_cur_wm_latency_fops},
@@ -4819,7 +4839,7 @@ int i915_debugfs_register(struct drm_i915_private *dev_priv)
 {
         struct drm_minor *minor = dev_priv->drm.primary;
         struct dentry *ent;
-        int ret, i;
+        int i;
 
         ent = debugfs_create_file("i915_forcewake_user", S_IRUSR,
                                   minor->debugfs_root, to_i915(minor->dev),
@@ -4827,10 +4847,6 @@ int i915_debugfs_register(struct drm_i915_private *dev_priv)
         if (!ent)
                 return -ENOMEM;
 
-        ret = intel_pipe_crc_create(minor);
-        if (ret)
-                return ret;
-
         for (i = 0; i < ARRAY_SIZE(i915_debugfs_files); i++) {
                 ent = debugfs_create_file(i915_debugfs_files[i].name,
                                           S_IRUGO | S_IWUSR,
diff --git a/drivers/gpu/drm/i915/i915_drv.c b/drivers/gpu/drm/i915/i915_drv.c
index beb0951001ce..0db3c83cce29 100644
--- a/drivers/gpu/drm/i915/i915_drv.c
+++ b/drivers/gpu/drm/i915/i915_drv.c
@@ -1165,6 +1165,12 @@ static int i915_driver_init_hw(struct drm_i915_private *dev_priv)
          * get lost on g4x as well, and interrupt delivery seems to stay
          * properly dead afterwards. So we'll just disable them for all
          * pre-gen5 chipsets.
+         *
+         * dp aux and gmbus irq on gen4 seems to be able to generate legacy
+         * interrupts even when in MSI mode. This results in spurious
+         * interrupt warnings if the legacy irq no. is shared with another
+         * device. The kernel then disables that interrupt source and so
+         * prevents the other device from working properly.
          */
         if (INTEL_GEN(dev_priv) >= 5) {
                 if (pci_enable_msi(pdev) < 0)
diff --git a/drivers/gpu/drm/i915/i915_drv.h b/drivers/gpu/drm/i915/i915_drv.h
index f4751b383858..eeb002a47032 100644
--- a/drivers/gpu/drm/i915/i915_drv.h
+++ b/drivers/gpu/drm/i915/i915_drv.h
@@ -86,8 +86,8 @@
 
 #define DRIVER_NAME             "i915"
 #define DRIVER_DESC             "Intel Graphics"
-#define DRIVER_DATE             "20180620"
-#define DRIVER_TIMESTAMP        1529529048
+#define DRIVER_DATE             "20180709"
+#define DRIVER_TIMESTAMP        1531175967
 
 /* Use I915_STATE_WARN(x) and I915_STATE_WARN_ON() (rather than WARN() and
  * WARN_ON()) for hw state sanity checks to check for unexpected conditions
@@ -512,6 +512,7 @@ struct intel_fbc {
 
         bool enabled;
         bool active;
+        bool flip_pending;
 
         bool underrun_detected;
         struct work_struct underrun_work;
@@ -579,12 +580,6 @@ struct intel_fbc {
                 unsigned int gen9_wa_cfb_stride;
         } params;
 
-        struct intel_fbc_work {
-                bool scheduled;
-                u64 scheduled_vblank;
-                struct work_struct work;
-        } work;
-
         const char *no_fbc_reason;
 };
 
@@ -631,14 +626,6 @@ struct i915_psr {
         bool debug;
         ktime_t last_entry_attempt;
         ktime_t last_exit;
-
-        void (*enable_source)(struct intel_dp *,
-                              const struct intel_crtc_state *);
-        void (*disable_source)(struct intel_dp *,
-                               const struct intel_crtc_state *);
-        void (*enable_sink)(struct intel_dp *);
-        void (*activate)(struct intel_dp *);
-        void (*setup_vsc)(struct intel_dp *, const struct intel_crtc_state *);
 };
 
 enum intel_pch {
@@ -965,7 +952,7 @@ struct i915_gem_mm {
         /**
          * Small stash of WC pages
          */
-        struct pagevec wc_stash;
+        struct pagestash wc_stash;
 
         /**
          * tmpfs instance used for shmem backed objects
@@ -1284,20 +1271,11 @@ enum intel_pipe_crc_source {
         INTEL_PIPE_CRC_SOURCE_MAX,
 };
 
-struct intel_pipe_crc_entry {
-        uint32_t frame;
-        uint32_t crc[5];
-};
-
 #define INTEL_PIPE_CRC_ENTRIES_NR       128
 struct intel_pipe_crc {
         spinlock_t lock;
-        bool opened;            /* exclusive access to the result file */
-        struct intel_pipe_crc_entry *entries;
-        enum intel_pipe_crc_source source;
-        int head, tail;
-        wait_queue_head_t wq;
         int skipped;
+        enum intel_pipe_crc_source source;
 };
 
 struct i915_frontbuffer_tracking {
@@ -1757,7 +1735,6 @@ struct drm_i915_private {
         struct drm_atomic_state *modeset_restore_state;
         struct drm_modeset_acquire_ctx reset_ctx;
 
-        struct list_head vm_list; /* Global list of all address spaces */
         struct i915_ggtt ggtt; /* VM representing the global address space */
 
         struct i915_gem_mm mm;
@@ -2326,6 +2303,7 @@ intel_info(const struct drm_i915_private *dev_priv)
 }
 
 #define INTEL_INFO(dev_priv)    intel_info((dev_priv))
+#define DRIVER_CAPS(dev_priv)   (&(dev_priv)->caps)
 
 #define INTEL_GEN(dev_priv)     ((dev_priv)->info.gen)
 #define INTEL_DEVID(dev_priv)   ((dev_priv)->info.device_id)
@@ -2578,16 +2556,6 @@ intel_info(const struct drm_i915_private *dev_priv)
         (IS_CANNONLAKE(dev_priv) || \
          IS_SKL_GT3(dev_priv) || IS_SKL_GT4(dev_priv))
 
-/*
- * dp aux and gmbus irq on gen4 seems to be able to generate legacy interrupts
- * even when in MSI mode. This results in spurious interrupt warnings if the
- * legacy irq no. is shared with another device. The kernel then disables that
- * interrupt source and so prevents the other device from working properly.
- *
- * Since we don't enable MSI anymore on gen4, we can always use GMBUS/AUX
- * interrupts.
- */
-#define HAS_AUX_IRQ(dev_priv)   true
 #define HAS_GMBUS_IRQ(dev_priv) (INTEL_GEN(dev_priv) >= 4)
 
 /* With the 945 and later, Y tiling got adjusted so that it was 32 128-byte
@@ -3119,9 +3087,6 @@ i915_gem_obj_finish_shmem_access(struct drm_i915_gem_object *obj)
 }
 
 int __must_check i915_mutex_lock_interruptible(struct drm_device *dev);
-void i915_vma_move_to_active(struct i915_vma *vma,
-                             struct i915_request *rq,
-                             unsigned int flags);
 int i915_gem_dumb_create(struct drm_file *file_priv,
                          struct drm_device *dev,
                          struct drm_mode_create_dumb *args);
@@ -3189,7 +3154,7 @@ void i915_gem_init_swizzling(struct drm_i915_private *dev_priv);
 void i915_gem_fini(struct drm_i915_private *dev_priv);
 void i915_gem_cleanup_engines(struct drm_i915_private *dev_priv);
 int i915_gem_wait_for_idle(struct drm_i915_private *dev_priv,
-                           unsigned int flags);
+                           unsigned int flags, long timeout);
 int __must_check i915_gem_suspend(struct drm_i915_private *dev_priv);
 void i915_gem_suspend_late(struct drm_i915_private *dev_priv);
 void i915_gem_resume(struct drm_i915_private *dev_priv);
diff --git a/drivers/gpu/drm/i915/i915_gem.c b/drivers/gpu/drm/i915/i915_gem.c
index 858d188dd33b..b35cbfd16c9c 100644
--- a/drivers/gpu/drm/i915/i915_gem.c
+++ b/drivers/gpu/drm/i915/i915_gem.c
@@ -837,6 +837,10 @@ flush_write_domain(struct drm_i915_gem_object *obj, unsigned int flush_domains)
                 }
                 break;
 
+        case I915_GEM_DOMAIN_WC:
+                wmb();
+                break;
+
         case I915_GEM_DOMAIN_CPU:
                 i915_gem_clflush_object(obj, I915_CLFLUSH_SYNC);
                 break;
@@ -2006,7 +2010,6 @@ vm_fault_t i915_gem_fault(struct vm_fault *vmf)
         bool write = !!(vmf->flags & FAULT_FLAG_WRITE);
         struct i915_vma *vma;
         pgoff_t page_offset;
-        unsigned int flags;
         int ret;
 
         /* We don't use vmf->pgoff since that has the fake offset */
@@ -2042,27 +2045,34 @@ vm_fault_t i915_gem_fault(struct vm_fault *vmf)
                 goto err_unlock;
         }
 
-        /* If the object is smaller than a couple of partial vma, it is
-         * not worth only creating a single partial vma - we may as well
-         * clear enough space for the full object.
-         */
-        flags = PIN_MAPPABLE;
-        if (obj->base.size > 2 * MIN_CHUNK_PAGES << PAGE_SHIFT)
-                flags |= PIN_NONBLOCK | PIN_NONFAULT;
 
         /* Now pin it into the GTT as needed */
-        vma = i915_gem_object_ggtt_pin(obj, NULL, 0, 0, flags);
+        vma = i915_gem_object_ggtt_pin(obj, NULL, 0, 0,
+                                       PIN_MAPPABLE |
+                                       PIN_NONBLOCK |
+                                       PIN_NONFAULT);
         if (IS_ERR(vma)) {
                 /* Use a partial view if it is bigger than available space */
                 struct i915_ggtt_view view =
                         compute_partial_view(obj, page_offset, MIN_CHUNK_PAGES);
+                unsigned int flags;
 
-                /* Userspace is now writing through an untracked VMA, abandon
+                flags = PIN_MAPPABLE;
+                if (view.type == I915_GGTT_VIEW_NORMAL)
+                        flags |= PIN_NONBLOCK; /* avoid warnings for pinned */
+
+                /*
+                 * Userspace is now writing through an untracked VMA, abandon
                  * all hope that the hardware is able to track future writes.
                  */
                 obj->frontbuffer_ggtt_origin = ORIGIN_CPU;
 
-                vma = i915_gem_object_ggtt_pin(obj, &view, 0, 0, PIN_MAPPABLE);
+                vma = i915_gem_object_ggtt_pin(obj, &view, 0, 0, flags);
+                if (IS_ERR(vma) && !view.type) {
+                        flags = PIN_MAPPABLE;
+                        view.type = I915_GGTT_VIEW_PARTIAL;
+                        vma = i915_gem_object_ggtt_pin(obj, &view, 0, 0, flags);
+                }
         }
         if (IS_ERR(vma)) {
                 ret = PTR_ERR(vma);
@@ -2114,6 +2124,7 @@ err:
                  */
                 if (!i915_terminally_wedged(&dev_priv->gpu_error))
                         return VM_FAULT_SIGBUS;
+                /* else: fall through */
         case -EAGAIN:
                 /*
                  * EAGAIN means the gpu is hung and we'll wait for the error
@@ -2256,7 +2267,9 @@ static int i915_gem_object_create_mmap_offset(struct drm_i915_gem_object *obj)
 
         /* Attempt to reap some mmap space from dead objects */
         do {
-                err = i915_gem_wait_for_idle(dev_priv, I915_WAIT_INTERRUPTIBLE);
+                err = i915_gem_wait_for_idle(dev_priv,
+                                             I915_WAIT_INTERRUPTIBLE,
+                                             MAX_SCHEDULE_TIMEOUT);
                 if (err)
                         break;
 
@@ -3074,25 +3087,6 @@ int i915_gem_reset_prepare(struct drm_i915_private *dev_priv)
         return err;
 }
 
-static void skip_request(struct i915_request *request)
-{
-        void *vaddr = request->ring->vaddr;
-        u32 head;
-
-        /* As this request likely depends on state from the lost
-         * context, clear out all the user operations leaving the
-         * breadcrumb at the end (so we get the fence notifications).
-         */
-        head = request->head;
-        if (request->postfix < head) {
-                memset(vaddr + head, 0, request->ring->size - head);
-                head = 0;
-        }
-        memset(vaddr + head, 0, request->postfix - head);
-
-        dma_fence_set_error(&request->fence, -EIO);
-}
-
 static void engine_skip_context(struct i915_request *request)
 {
         struct intel_engine_cs *engine = request->engine;
@@ -3103,14 +3097,14 @@ static void engine_skip_context(struct i915_request *request)
         GEM_BUG_ON(timeline == &engine->timeline);
 
         spin_lock_irqsave(&engine->timeline.lock, flags);
-        spin_lock_nested(&timeline->lock, SINGLE_DEPTH_NESTING);
+        spin_lock(&timeline->lock);
 
         list_for_each_entry_continue(request, &engine->timeline.requests, link)
                 if (request->gem_context == hung_ctx)
-                        skip_request(request);
+                        i915_request_skip(request, -EIO);
 
         list_for_each_entry(request, &timeline->requests, link)
-                skip_request(request);
+                i915_request_skip(request, -EIO);
 
         spin_unlock(&timeline->lock);
         spin_unlock_irqrestore(&engine->timeline.lock, flags);
@@ -3153,7 +3147,7 @@ i915_gem_reset_request(struct intel_engine_cs *engine,
 
         if (stalled) {
                 i915_gem_context_mark_guilty(request->gem_context);
-                skip_request(request);
+                i915_request_skip(request, -EIO);
 
                 /* If this context is now banned, skip all pending requests. */
                 if (i915_gem_context_is_banned(request->gem_context))
@@ -3750,14 +3744,14 @@ i915_gem_wait_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
         return ret;
 }
 
-static int wait_for_timeline(struct i915_timeline *tl, unsigned int flags)
+static long wait_for_timeline(struct i915_timeline *tl,
+                              unsigned int flags, long timeout)
 {
         struct i915_request *rq;
-        long ret;
 
         rq = i915_gem_active_get_unlocked(&tl->last_request);
         if (!rq)
-                return 0;
+                return timeout;
 
         /*
          * "Race-to-idle".
@@ -3771,10 +3765,10 @@ static int wait_for_timeline(struct i915_timeline *tl, unsigned int flags)
         if (flags & I915_WAIT_FOR_IDLE_BOOST)
                 gen6_rps_boost(rq, NULL);
 
-        ret = i915_request_wait(rq, flags, MAX_SCHEDULE_TIMEOUT);
+        timeout = i915_request_wait(rq, flags, timeout);
         i915_request_put(rq);
 
-        return ret < 0 ? ret : 0;
+        return timeout;
 }
 
 static int wait_for_engines(struct drm_i915_private *i915)
@@ -3790,10 +3784,12 @@ static int wait_for_engines(struct drm_i915_private *i915)
         return 0;
 }
 
-int i915_gem_wait_for_idle(struct drm_i915_private *i915, unsigned int flags)
+int i915_gem_wait_for_idle(struct drm_i915_private *i915,
+                           unsigned int flags, long timeout)
 {
-        GEM_TRACE("flags=%x (%s)\n",
-                  flags, flags & I915_WAIT_LOCKED ? "locked" : "unlocked");
+        GEM_TRACE("flags=%x (%s), timeout=%ld%s\n",
+                  flags, flags & I915_WAIT_LOCKED ? "locked" : "unlocked",
+                  timeout, timeout == MAX_SCHEDULE_TIMEOUT ? " (forever)" : "");
 
         /* If the device is asleep, we have no requests outstanding */
         if (!READ_ONCE(i915->gt.awake))
@@ -3806,27 +3802,31 @@ int i915_gem_wait_for_idle(struct drm_i915_private *i915, unsigned int flags)
                 lockdep_assert_held(&i915->drm.struct_mutex);
 
                 list_for_each_entry(tl, &i915->gt.timelines, link) {
-                        err = wait_for_timeline(tl, flags);
-                        if (err)
-                                return err;
+                        timeout = wait_for_timeline(tl, flags, timeout);
+                        if (timeout < 0)
+                                return timeout;
                 }
+
+                err = wait_for_engines(i915);
+                if (err)
+                        return err;
+
                 i915_retire_requests(i915);
                 GEM_BUG_ON(i915->gt.active_requests);
-
-                return wait_for_engines(i915);
         } else {
                 struct intel_engine_cs *engine;
                 enum intel_engine_id id;
-                int err;
 
                 for_each_engine(engine, i915, id) {
-                        err = wait_for_timeline(&engine->timeline, flags);
-                        if (err)
-                                return err;
-                }
+                        struct i915_timeline *tl = &engine->timeline;
 
-                return 0;
+                        timeout = wait_for_timeline(tl, flags, timeout);
+                        if (timeout < 0)
+                                return timeout;
+                }
         }
+
+        return 0;
 }
 
 static void __i915_gem_object_flush_for_display(struct drm_i915_gem_object *obj)
@@ -5057,7 +5057,8 @@ int i915_gem_suspend(struct drm_i915_private *dev_priv)
                 ret = i915_gem_wait_for_idle(dev_priv,
                                              I915_WAIT_INTERRUPTIBLE |
                                              I915_WAIT_LOCKED |
-                                             I915_WAIT_FOR_IDLE_BOOST);
+                                             I915_WAIT_FOR_IDLE_BOOST,
+                                             MAX_SCHEDULE_TIMEOUT);
                 if (ret && ret != -EIO)
                         goto err_unlock;
 
@@ -5361,9 +5362,11 @@ static int __intel_engines_record_defaults(struct drm_i915_private *i915)
         if (err)
                 goto err_active;
 
-        err = i915_gem_wait_for_idle(i915, I915_WAIT_LOCKED);
-        if (err)
+        if (i915_gem_wait_for_idle(i915, I915_WAIT_LOCKED, HZ / 5)) {
+                i915_gem_set_wedged(i915);
+                err = -EIO; /* Caller will declare us wedged */
                 goto err_active;
+        }
 
         assert_kernel_context_is_current(i915);
 
@@ -5426,7 +5429,9 @@ err_active:
         if (WARN_ON(i915_gem_switch_to_kernel_context(i915)))
                 goto out_ctx;
 
-        if (WARN_ON(i915_gem_wait_for_idle(i915, I915_WAIT_LOCKED)))
+        if (WARN_ON(i915_gem_wait_for_idle(i915,
+                                           I915_WAIT_LOCKED,
+                                           MAX_SCHEDULE_TIMEOUT)))
                 goto out_ctx;
 
         i915_gem_contexts_lost(i915);
@@ -5456,13 +5461,13 @@ int i915_gem_init(struct drm_i915_private *dev_priv)
         if (ret)
                 return ret;
 
-        ret = intel_wopcm_init(&dev_priv->wopcm);
+        ret = intel_uc_init_misc(dev_priv);
         if (ret)
                 return ret;
 
-        ret = intel_uc_init_misc(dev_priv);
+        ret = intel_wopcm_init(&dev_priv->wopcm);
         if (ret)
-                return ret;
+                goto err_uc_misc;
 
         /* This is just a security blanket to placate dragons.
          * On some systems, we very sporadically observe that the first TLBs
@@ -5560,6 +5565,7 @@ err_unlock:
         intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
         mutex_unlock(&dev_priv->drm.struct_mutex);
 
+err_uc_misc:
         intel_uc_fini_misc(dev_priv);
 
         if (ret != -EIO)
diff --git a/drivers/gpu/drm/i915/i915_gem.h b/drivers/gpu/drm/i915/i915_gem.h
index 261da577829a..e46592956872 100644
--- a/drivers/gpu/drm/i915/i915_gem.h
+++ b/drivers/gpu/drm/i915/i915_gem.h
@@ -88,4 +88,9 @@ static inline void __tasklet_enable_sync_once(struct tasklet_struct *t)
                 tasklet_kill(t);
 }
 
+static inline bool __tasklet_is_enabled(const struct tasklet_struct *t)
+{
+        return !atomic_read(&t->count);
+}
+
 #endif /* __I915_GEM_H__ */
diff --git a/drivers/gpu/drm/i915/i915_gem_context.c b/drivers/gpu/drm/i915/i915_gem_context.c
index ccf463ab6562..b10770cfccd2 100644
--- a/drivers/gpu/drm/i915/i915_gem_context.c
+++ b/drivers/gpu/drm/i915/i915_gem_context.c
@@ -374,7 +374,7 @@ i915_gem_create_context(struct drm_i915_private *dev_priv,
         if (USES_FULL_PPGTT(dev_priv)) {
                 struct i915_hw_ppgtt *ppgtt;
 
-                ppgtt = i915_ppgtt_create(dev_priv, file_priv, ctx->name);
+                ppgtt = i915_ppgtt_create(dev_priv, file_priv);
                 if (IS_ERR(ppgtt)) {
                         DRM_DEBUG_DRIVER("PPGTT setup failed (%ld)\n",
                                          PTR_ERR(ppgtt));
@@ -512,8 +512,8 @@ int i915_gem_contexts_init(struct drm_i915_private *dev_priv)
         }
 
         DRM_DEBUG_DRIVER("%s context support initialized\n",
-                         dev_priv->engine[RCS]->context_size ? "logical" :
-                         "fake");
+                         DRIVER_CAPS(dev_priv)->has_logical_contexts ?
+                         "logical" : "fake");
         return 0;
 }
 
@@ -720,7 +720,7 @@ int i915_gem_context_create_ioctl(struct drm_device *dev, void *data,
         struct i915_gem_context *ctx;
         int ret;
 
-        if (!dev_priv->engine[RCS]->context_size)
+        if (!DRIVER_CAPS(dev_priv)->has_logical_contexts)
                 return -ENODEV;
 
         if (args->pad != 0)
diff --git a/drivers/gpu/drm/i915/i915_gem_evict.c b/drivers/gpu/drm/i915/i915_gem_evict.c
index 54814a196ee4..02b83a5ed96c 100644
--- a/drivers/gpu/drm/i915/i915_gem_evict.c
+++ b/drivers/gpu/drm/i915/i915_gem_evict.c
@@ -69,7 +69,8 @@ static int ggtt_flush(struct drm_i915_private *i915)
 
         err = i915_gem_wait_for_idle(i915,
                                      I915_WAIT_INTERRUPTIBLE |
-                                     I915_WAIT_LOCKED);
+                                     I915_WAIT_LOCKED,
+                                     MAX_SCHEDULE_TIMEOUT);
         if (err)
                 return err;
 
diff --git a/drivers/gpu/drm/i915/i915_gem_execbuffer.c b/drivers/gpu/drm/i915/i915_gem_execbuffer.c
index 60dc2a865f5f..3f0c612d42e7 100644
--- a/drivers/gpu/drm/i915/i915_gem_execbuffer.c
+++ b/drivers/gpu/drm/i915/i915_gem_execbuffer.c
@@ -66,6 +66,15 @@ enum {
 #define __I915_EXEC_ILLEGAL_FLAGS \
         (__I915_EXEC_UNKNOWN_FLAGS | I915_EXEC_CONSTANTS_MASK)
 
+/* Catch emission of unexpected errors for CI! */
+#if IS_ENABLED(CONFIG_DRM_I915_DEBUG_GEM)
+#undef EINVAL
+#define EINVAL ({ \
+        DRM_DEBUG_DRIVER("EINVAL at %s:%d\n", __func__, __LINE__); \
+        22; \
+})
+#endif
+
 /**
  * DOC: User command execution
  *
@@ -534,7 +543,8 @@ eb_add_vma(struct i915_execbuffer *eb,
          * paranoia do it everywhere.
          */
         if (i == batch_idx) {
-                if (!(eb->flags[i] & EXEC_OBJECT_PINNED))
+                if (entry->relocation_count &&
+                    !(eb->flags[i] & EXEC_OBJECT_PINNED))
                         eb->flags[i] |= __EXEC_OBJECT_NEEDS_BIAS;
                 if (eb->reloc_cache.has_fence)
                         eb->flags[i] |= EXEC_OBJECT_NEEDS_FENCE;
@@ -1155,18 +1165,16 @@ static int __reloc_gpu_alloc(struct i915_execbuffer *eb,
                 goto err_request;
 
         GEM_BUG_ON(!reservation_object_test_signaled_rcu(batch->resv, true));
-        i915_vma_move_to_active(batch, rq, 0);
-        reservation_object_lock(batch->resv, NULL);
-        reservation_object_add_excl_fence(batch->resv, &rq->fence);
-        reservation_object_unlock(batch->resv);
-        i915_vma_unpin(batch);
+        err = i915_vma_move_to_active(batch, rq, 0);
+        if (err)
+                goto skip_request;
 
-        i915_vma_move_to_active(vma, rq, EXEC_OBJECT_WRITE);
-        reservation_object_lock(vma->resv, NULL);
-        reservation_object_add_excl_fence(vma->resv, &rq->fence);
-        reservation_object_unlock(vma->resv);
+        err = i915_vma_move_to_active(vma, rq, EXEC_OBJECT_WRITE);
+        if (err)
+                goto skip_request;
 
         rq->batch = batch;
+        i915_vma_unpin(batch);
 
         cache->rq = rq;
         cache->rq_cmd = cmd;
@@ -1175,6 +1183,8 @@ static int __reloc_gpu_alloc(struct i915_execbuffer *eb,
         /* Return with batch mapping (cmd) still pinned */
         return 0;
 
+skip_request:
+        i915_request_skip(rq, err);
 err_request:
         i915_request_add(rq);
 err_unpin:
@@ -1761,25 +1771,6 @@ slow:
         return eb_relocate_slow(eb);
 }
 
-static void eb_export_fence(struct i915_vma *vma,
-                            struct i915_request *rq,
-                            unsigned int flags)
-{
-        struct reservation_object *resv = vma->resv;
-
-        /*
-         * Ignore errors from failing to allocate the new fence, we can't
-         * handle an error right now. Worst case should be missed
-         * synchronisation leading to rendering corruption.
-         */
-        reservation_object_lock(resv, NULL);
-        if (flags & EXEC_OBJECT_WRITE)
-                reservation_object_add_excl_fence(resv, &rq->fence);
-        else if (reservation_object_reserve_shared(resv) == 0)
-                reservation_object_add_shared_fence(resv, &rq->fence);
-        reservation_object_unlock(resv);
-}
-
 static int eb_move_to_gpu(struct i915_execbuffer *eb)
 {
         const unsigned int count = eb->buffer_count;
@@ -1833,8 +1824,11 @@ static int eb_move_to_gpu(struct i915_execbuffer *eb)
                 unsigned int flags = eb->flags[i];
                 struct i915_vma *vma = eb->vma[i];
 
-                i915_vma_move_to_active(vma, eb->request, flags);
-                eb_export_fence(vma, eb->request, flags);
+                err = i915_vma_move_to_active(vma, eb->request, flags);
+                if (unlikely(err)) {
+                        i915_request_skip(eb->request, err);
+                        return err;
+                }
 
                 __eb_unreserve_vma(vma, flags);
                 vma->exec_flags = NULL;
@@ -1874,45 +1868,6 @@ static bool i915_gem_check_execbuffer(struct drm_i915_gem_execbuffer2 *exec)
         return true;
 }
 
-void i915_vma_move_to_active(struct i915_vma *vma,
-                             struct i915_request *rq,
-                             unsigned int flags)
-{
-        struct drm_i915_gem_object *obj = vma->obj;
-        const unsigned int idx = rq->engine->id;
-
-        lockdep_assert_held(&rq->i915->drm.struct_mutex);
-        GEM_BUG_ON(!drm_mm_node_allocated(&vma->node));
-
-        /*
-         * Add a reference if we're newly entering the active list.
-         * The order in which we add operations to the retirement queue is
-         * vital here: mark_active adds to the start of the callback list,
-         * such that subsequent callbacks are called first. Therefore we
-         * add the active reference first and queue for it to be dropped
-         * *last*.
-         */
-        if (!i915_vma_is_active(vma))
-                obj->active_count++;
-        i915_vma_set_active(vma, idx);
-        i915_gem_active_set(&vma->last_read[idx], rq);
-        list_move_tail(&vma->vm_link, &vma->vm->active_list);
-
-        obj->write_domain = 0;
-        if (flags & EXEC_OBJECT_WRITE) {
-                obj->write_domain = I915_GEM_DOMAIN_RENDER;
-
-                if (intel_fb_obj_invalidate(obj, ORIGIN_CS))
-                        i915_gem_active_set(&obj->frontbuffer_write, rq);
-
-                obj->read_domains = 0;
-        }
-        obj->read_domains |= I915_GEM_GPU_DOMAINS;
-
-        if (flags & EXEC_OBJECT_NEEDS_FENCE)
-                i915_gem_active_set(&vma->last_fence, rq);
-}
-
 static int i915_reset_gen7_sol_offsets(struct i915_request *rq)
 {
         u32 *cs;
diff --git a/drivers/gpu/drm/i915/i915_gem_gtt.c b/drivers/gpu/drm/i915/i915_gem_gtt.c
index c6aa761ca085..abd81fb9b0b6 100644
--- a/drivers/gpu/drm/i915/i915_gem_gtt.c
+++ b/drivers/gpu/drm/i915/i915_gem_gtt.c
@@ -375,37 +375,70 @@ static gen6_pte_t iris_pte_encode(dma_addr_t addr,
         return pte;
 }
 
+static void stash_init(struct pagestash *stash)
+{
+        pagevec_init(&stash->pvec);
+        spin_lock_init(&stash->lock);
+}
+
+static struct page *stash_pop_page(struct pagestash *stash)
+{
+        struct page *page = NULL;
+
+        spin_lock(&stash->lock);
+        if (likely(stash->pvec.nr))
+                page = stash->pvec.pages[--stash->pvec.nr];
+        spin_unlock(&stash->lock);
+
+        return page;
+}
+
+static void stash_push_pagevec(struct pagestash *stash, struct pagevec *pvec)
+{
+        int nr;
+
+        spin_lock_nested(&stash->lock, SINGLE_DEPTH_NESTING);
+
+        nr = min_t(int, pvec->nr, pagevec_space(&stash->pvec));
+        memcpy(stash->pvec.pages + stash->pvec.nr,
+               pvec->pages + pvec->nr - nr,
+               sizeof(pvec->pages[0]) * nr);
+        stash->pvec.nr += nr;
+
+        spin_unlock(&stash->lock);
+
+        pvec->nr -= nr;
+}
+
 static struct page *vm_alloc_page(struct i915_address_space *vm, gfp_t gfp)
 {
-        struct pagevec *pvec = &vm->free_pages;
-        struct pagevec stash;
+        struct pagevec stack;
+        struct page *page;
 
         if (I915_SELFTEST_ONLY(should_fail(&vm->fault_attr, 1)))
                 i915_gem_shrink_all(vm->i915);
 
-        if (likely(pvec->nr))
-                return pvec->pages[--pvec->nr];
+        page = stash_pop_page(&vm->free_pages);
+        if (page)
+                return page;
 
         if (!vm->pt_kmap_wc)
                 return alloc_page(gfp);
 
-        /* A placeholder for a specific mutex to guard the WC stash */
-        lockdep_assert_held(&vm->i915->drm.struct_mutex);
-
         /* Look in our global stash of WC pages... */
-        pvec = &vm->i915->mm.wc_stash;
-        if (likely(pvec->nr))
-                return pvec->pages[--pvec->nr];
+        page = stash_pop_page(&vm->i915->mm.wc_stash);
+        if (page)
+                return page;
 
         /*
-         * Otherwise batch allocate pages to amoritize cost of set_pages_wc.
+         * Otherwise batch allocate pages to amortize cost of set_pages_wc.
          *
          * We have to be careful as page allocation may trigger the shrinker
          * (via direct reclaim) which will fill up the WC stash underneath us.
          * So we add our WB pages into a temporary pvec on the stack and merge
          * them into the WC stash after all the allocations are complete.
          */
-        pagevec_init(&stash);
+        pagevec_init(&stack);
         do {
                 struct page *page;
 
@@ -413,59 +446,67 @@ static struct page *vm_alloc_page(struct i915_address_space *vm, gfp_t gfp)
                 if (unlikely(!page))
                         break;
 
-                stash.pages[stash.nr++] = page;
-        } while (stash.nr < pagevec_space(pvec));
+                stack.pages[stack.nr++] = page;
+        } while (pagevec_space(&stack));
 
-        if (stash.nr) {
-                int nr = min_t(int, stash.nr, pagevec_space(pvec));
-                struct page **pages = stash.pages + stash.nr - nr;
+        if (stack.nr && !set_pages_array_wc(stack.pages, stack.nr)) {
+                page = stack.pages[--stack.nr];
 
-                if (nr && !set_pages_array_wc(pages, nr)) {
-                        memcpy(pvec->pages + pvec->nr,
-                               pages, sizeof(pages[0]) * nr);
-                        pvec->nr += nr;
-                        stash.nr -= nr;
-                }
+                /* Merge spare WC pages to the global stash */
+                stash_push_pagevec(&vm->i915->mm.wc_stash, &stack);
 
-                pagevec_release(&stash);
+                /* Push any surplus WC pages onto the local VM stash */
+                if (stack.nr)
+                        stash_push_pagevec(&vm->free_pages, &stack);
         }
 
-        return likely(pvec->nr) ? pvec->pages[--pvec->nr] : NULL;
+        /* Return unwanted leftovers */
+        if (unlikely(stack.nr)) {
+                WARN_ON_ONCE(set_pages_array_wb(stack.pages, stack.nr));
+                __pagevec_release(&stack);
+        }
+
+        return page;
 }
 
 static void vm_free_pages_release(struct i915_address_space *vm,
                                   bool immediate)
 {
-        struct pagevec *pvec = &vm->free_pages;
+        struct pagevec *pvec = &vm->free_pages.pvec;
+        struct pagevec stack;
 
+        lockdep_assert_held(&vm->free_pages.lock);
         GEM_BUG_ON(!pagevec_count(pvec));
 
         if (vm->pt_kmap_wc) {
-                struct pagevec *stash = &vm->i915->mm.wc_stash;
-
-                /* When we use WC, first fill up the global stash and then
+                /*
+                 * When we use WC, first fill up the global stash and then
                  * only if full immediately free the overflow.
                  */
+                stash_push_pagevec(&vm->i915->mm.wc_stash, pvec);
 
-                lockdep_assert_held(&vm->i915->drm.struct_mutex);
-                if (pagevec_space(stash)) {
-                        do {
-                                stash->pages[stash->nr++] =
-                                        pvec->pages[--pvec->nr];
-                                if (!pvec->nr)
-                                        return;
-                        } while (pagevec_space(stash));
-
-                        /* As we have made some room in the VM's free_pages,
-                         * we can wait for it to fill again. Unless we are
-                         * inside i915_address_space_fini() and must
-                         * immediately release the pages!
-                         */
-                        if (!immediate)
-                                return;
-                }
+                /*
+                 * As we have made some room in the VM's free_pages,
+                 * we can wait for it to fill again. Unless we are
+                 * inside i915_address_space_fini() and must
+                 * immediately release the pages!
+                 */
+                if (pvec->nr <= (immediate ? 0 : PAGEVEC_SIZE - 1))
+                        return;
+
+                /*
+                 * We have to drop the lock to allow ourselves to sleep,
+                 * so take a copy of the pvec and clear the stash for
+                 * others to use it as we sleep.
+                 */
+                stack = *pvec;
+                pagevec_reinit(pvec);
+                spin_unlock(&vm->free_pages.lock);
 
+                pvec = &stack;
                 set_pages_array_wb(pvec->pages, pvec->nr);
+
+                spin_lock(&vm->free_pages.lock);
         }
 
         __pagevec_release(pvec);
@@ -481,8 +522,35 @@ static void vm_free_page(struct i915_address_space *vm, struct page *page)
          * unconditional might_sleep() for everybody.
          */
         might_sleep();
-        if (!pagevec_add(&vm->free_pages, page))
+        spin_lock(&vm->free_pages.lock);
+        if (!pagevec_add(&vm->free_pages.pvec, page))
                 vm_free_pages_release(vm, false);
+        spin_unlock(&vm->free_pages.lock);
+}
+
+static void i915_address_space_init(struct i915_address_space *vm,
+                                    struct drm_i915_private *dev_priv)
+{
+        GEM_BUG_ON(!vm->total);
+        drm_mm_init(&vm->mm, 0, vm->total);
+        vm->mm.head_node.color = I915_COLOR_UNEVICTABLE;
+
+        stash_init(&vm->free_pages);
+
+        INIT_LIST_HEAD(&vm->active_list);
+        INIT_LIST_HEAD(&vm->inactive_list);
+        INIT_LIST_HEAD(&vm->unbound_list);
+}
+
+static void i915_address_space_fini(struct i915_address_space *vm)
+{
+        spin_lock(&vm->free_pages.lock);
+        if (pagevec_count(&vm->free_pages.pvec))
+                vm_free_pages_release(vm, true);
+        GEM_BUG_ON(pagevec_count(&vm->free_pages.pvec));
+        spin_unlock(&vm->free_pages.lock);
+
+        drm_mm_takedown(&vm->mm);
 }
 
 static int __setup_page_dma(struct i915_address_space *vm,
@@ -493,8 +561,11 @@ static int __setup_page_dma(struct i915_address_space *vm,
         if (unlikely(!p->page))
                 return -ENOMEM;
 
-        p->daddr = dma_map_page(vm->dma, p->page, 0, PAGE_SIZE,
-                                PCI_DMA_BIDIRECTIONAL);
+        p->daddr = dma_map_page_attrs(vm->dma,
+                                      p->page, 0, PAGE_SIZE,
+                                      PCI_DMA_BIDIRECTIONAL,
+                                      DMA_ATTR_SKIP_CPU_SYNC |
+                                      DMA_ATTR_NO_WARN);
         if (unlikely(dma_mapping_error(vm->dma, p->daddr))) {
                 vm_free_page(vm, p->page);
                 return -ENOMEM;
@@ -575,8 +646,11 @@ setup_scratch_page(struct i915_address_space *vm, gfp_t gfp)
                 if (unlikely(!page))
                         goto skip;
 
-                addr = dma_map_page(vm->dma, page, 0, size,
-                                    PCI_DMA_BIDIRECTIONAL);
+                addr = dma_map_page_attrs(vm->dma,
+                                          page, 0, size,
+                                          PCI_DMA_BIDIRECTIONAL,
+                                          DMA_ATTR_SKIP_CPU_SYNC |
+                                          DMA_ATTR_NO_WARN);
                 if (unlikely(dma_mapping_error(vm->dma, addr)))
                         goto free_page;
 
@@ -1562,6 +1636,8 @@ static struct i915_hw_ppgtt *gen8_ppgtt_create(struct drm_i915_private *i915)
         if (!ppgtt)
                 return ERR_PTR(-ENOMEM);
 
+        kref_init(&ppgtt->ref);
+
         ppgtt->vm.i915 = i915;
         ppgtt->vm.dma = &i915->drm.pdev->dev;
 
@@ -1569,6 +1645,8 @@ static struct i915_hw_ppgtt *gen8_ppgtt_create(struct drm_i915_private *i915)
                 1ULL << 48 :
                 1ULL << 32;
 
+        i915_address_space_init(&ppgtt->vm, i915);
+
         /* There are only few exceptions for gen >=6. chv and bxt.
          * And we are not sure about the latter so play safe for now.
          */
@@ -1996,7 +2074,6 @@ static struct i915_vma *pd_vma_create(struct gen6_hw_ppgtt *ppgtt, int size)
         struct drm_i915_private *i915 = ppgtt->base.vm.i915;
         struct i915_ggtt *ggtt = &i915->ggtt;
         struct i915_vma *vma;
-        int i;
 
         GEM_BUG_ON(!IS_ALIGNED(size, I915_GTT_PAGE_SIZE));
         GEM_BUG_ON(size > ggtt->vm.total);
@@ -2005,14 +2082,14 @@ static struct i915_vma *pd_vma_create(struct gen6_hw_ppgtt *ppgtt, int size)
         if (!vma)
                 return ERR_PTR(-ENOMEM);
 
-        for (i = 0; i < ARRAY_SIZE(vma->last_read); i++)
-                init_request_active(&vma->last_read[i], NULL);
         init_request_active(&vma->last_fence, NULL);
 
         vma->vm = &ggtt->vm;
         vma->ops = &pd_vma_ops;
         vma->private = ppgtt;
 
+        vma->active = RB_ROOT;
+
         vma->size = size;
         vma->fence_size = size;
         vma->flags = I915_VMA_GGTT;
@@ -2068,11 +2145,15 @@ static struct i915_hw_ppgtt *gen6_ppgtt_create(struct drm_i915_private *i915)
         if (!ppgtt)
                 return ERR_PTR(-ENOMEM);
 
+        kref_init(&ppgtt->base.ref);
+
         ppgtt->base.vm.i915 = i915;
         ppgtt->base.vm.dma = &i915->drm.pdev->dev;
 
         ppgtt->base.vm.total = I915_PDES * GEN6_PTES * PAGE_SIZE;
 
+        i915_address_space_init(&ppgtt->base.vm, i915);
+
         ppgtt->base.vm.allocate_va_range = gen6_alloc_va_range;
         ppgtt->base.vm.clear_range = gen6_ppgtt_clear_range;
         ppgtt->base.vm.insert_entries = gen6_ppgtt_insert_entries;
@@ -2105,30 +2186,6 @@ err_free:
         return ERR_PTR(err);
 }
 
-static void i915_address_space_init(struct i915_address_space *vm,
-                                    struct drm_i915_private *dev_priv,
-                                    const char *name)
-{
-        drm_mm_init(&vm->mm, 0, vm->total);
-        vm->mm.head_node.color = I915_COLOR_UNEVICTABLE;
-
-        INIT_LIST_HEAD(&vm->active_list);
-        INIT_LIST_HEAD(&vm->inactive_list);
-        INIT_LIST_HEAD(&vm->unbound_list);
-
-        list_add_tail(&vm->global_link, &dev_priv->vm_list);
-        pagevec_init(&vm->free_pages);
-}
-
-static void i915_address_space_fini(struct i915_address_space *vm)
-{
-        if (pagevec_count(&vm->free_pages))
-                vm_free_pages_release(vm, true);
-
-        drm_mm_takedown(&vm->mm);
-        list_del(&vm->global_link);
-}
-
 static void gtt_write_workarounds(struct drm_i915_private *dev_priv)
 {
         /* This function is for gtt related workarounds. This function is
@@ -2199,8 +2256,7 @@ __hw_ppgtt_create(struct drm_i915_private *i915)
 
 struct i915_hw_ppgtt *
 i915_ppgtt_create(struct drm_i915_private *i915,
-                  struct drm_i915_file_private *fpriv,
-                  const char *name)
+                  struct drm_i915_file_private *fpriv)
 {
         struct i915_hw_ppgtt *ppgtt;
 
@@ -2208,8 +2264,6 @@ i915_ppgtt_create(struct drm_i915_private *i915,
         if (IS_ERR(ppgtt))
                 return ppgtt;
 
-        kref_init(&ppgtt->ref);
-        i915_address_space_init(&ppgtt->vm, i915, name);
         ppgtt->vm.file = fpriv;
 
         trace_i915_ppgtt_create(&ppgtt->vm);
@@ -2739,7 +2793,7 @@ void i915_gem_gtt_finish_pages(struct drm_i915_gem_object *obj,
         struct i915_ggtt *ggtt = &dev_priv->ggtt;
 
         if (unlikely(ggtt->do_idle_maps)) {
-                if (i915_gem_wait_for_idle(dev_priv, 0)) {
+                if (i915_gem_wait_for_idle(dev_priv, 0, MAX_SCHEDULE_TIMEOUT)) {
                         DRM_ERROR("Failed to wait for idle; VT'd may hang.\n");
                         /* Wait a bit, in hopes it avoids the hang */
                         udelay(10);
@@ -2788,7 +2842,7 @@ int i915_gem_init_aliasing_ppgtt(struct drm_i915_private *i915)
         struct i915_hw_ppgtt *ppgtt;
         int err;
 
-        ppgtt = i915_ppgtt_create(i915, ERR_PTR(-EPERM), "[alias]");
+        ppgtt = i915_ppgtt_create(i915, ERR_PTR(-EPERM));
         if (IS_ERR(ppgtt))
                 return PTR_ERR(ppgtt);
 
@@ -2918,7 +2972,7 @@ void i915_ggtt_cleanup_hw(struct drm_i915_private *dev_priv)
 
         ggtt->vm.cleanup(&ggtt->vm);
 
-        pvec = &dev_priv->mm.wc_stash;
+        pvec = &dev_priv->mm.wc_stash.pvec;
         if (pvec->nr) {
                 set_pages_array_wb(pvec->pages, pvec->nr);
                 __pagevec_release(pvec);
@@ -3518,7 +3572,7 @@ int i915_ggtt_init_hw(struct drm_i915_private *dev_priv)
         struct i915_ggtt *ggtt = &dev_priv->ggtt;
         int ret;
 
-        INIT_LIST_HEAD(&dev_priv->vm_list);
+        stash_init(&dev_priv->mm.wc_stash);
 
         /* Note that we use page colouring to enforce a guard page at the
          * end of the address space. This is required as the CS may prefetch
@@ -3526,7 +3580,7 @@ int i915_ggtt_init_hw(struct drm_i915_private *dev_priv)
          * and beyond the end of the GTT if we do not provide a guard.
          */
         mutex_lock(&dev_priv->drm.struct_mutex);
-        i915_address_space_init(&ggtt->vm, dev_priv, "[global]");
+        i915_address_space_init(&ggtt->vm, dev_priv);
         if (!HAS_LLC(dev_priv) && !USES_PPGTT(dev_priv))
                 ggtt->vm.mm.color_adjust = i915_gtt_color_adjust;
         mutex_unlock(&dev_priv->drm.struct_mutex);
diff --git a/drivers/gpu/drm/i915/i915_gem_gtt.h b/drivers/gpu/drm/i915/i915_gem_gtt.h
index 9a4824cae68d..feda45dfd481 100644
--- a/drivers/gpu/drm/i915/i915_gem_gtt.h
+++ b/drivers/gpu/drm/i915/i915_gem_gtt.h
@@ -270,6 +270,11 @@ struct i915_vma_ops {
         void (*clear_pages)(struct i915_vma *vma);
 };
 
+struct pagestash {
+        spinlock_t lock;
+        struct pagevec pvec;
+};
+
 struct i915_address_space {
         struct drm_mm mm;
         struct drm_i915_private *i915;
@@ -283,7 +288,6 @@ struct i915_address_space {
          * assign blame.
          */
         struct drm_i915_file_private *file;
-        struct list_head global_link;
         u64 total;              /* size addr space maps (ex. 2GB for ggtt) */
         u64 reserved;           /* size addr space reserved */
 
@@ -324,7 +328,7 @@ struct i915_address_space {
          */
         struct list_head unbound_list;
 
-        struct pagevec free_pages;
+        struct pagestash free_pages;
         bool pt_kmap_wc;
 
         /* FIXME: Need a more generic return type */
@@ -615,8 +619,7 @@ void i915_ggtt_cleanup_hw(struct drm_i915_private *dev_priv);
 int i915_ppgtt_init_hw(struct drm_i915_private *dev_priv);
 void i915_ppgtt_release(struct kref *kref);
 struct i915_hw_ppgtt *i915_ppgtt_create(struct drm_i915_private *dev_priv,
-                                        struct drm_i915_file_private *fpriv,
-                                        const char *name);
+                                        struct drm_i915_file_private *fpriv);
 void i915_ppgtt_close(struct i915_address_space *vm);
 static inline void i915_ppgtt_get(struct i915_hw_ppgtt *ppgtt)
 {
diff --git a/drivers/gpu/drm/i915/i915_gem_object.h b/drivers/gpu/drm/i915/i915_gem_object.h
index 54f00b350779..c3c6f2e588fb 100644
--- a/drivers/gpu/drm/i915/i915_gem_object.h
+++ b/drivers/gpu/drm/i915/i915_gem_object.h
@@ -337,26 +337,17 @@ __attribute__((nonnull))
 static inline struct drm_i915_gem_object *
 i915_gem_object_get(struct drm_i915_gem_object *obj)
 {
-        drm_gem_object_reference(&obj->base);
+        drm_gem_object_get(&obj->base);
         return obj;
 }
 
-__deprecated
-extern void drm_gem_object_reference(struct drm_gem_object *);
-
 __attribute__((nonnull))
 static inline void
 i915_gem_object_put(struct drm_i915_gem_object *obj)
 {
-        __drm_gem_object_unreference(&obj->base);
+        __drm_gem_object_put(&obj->base);
 }
 
-__deprecated
-extern void drm_gem_object_unreference(struct drm_gem_object *);
-
-__deprecated
-extern void drm_gem_object_unreference_unlocked(struct drm_gem_object *);
-
 static inline void i915_gem_object_lock(struct drm_i915_gem_object *obj)
 {
         reservation_object_lock(obj->resv, NULL);
diff --git a/drivers/gpu/drm/i915/i915_gem_render_state.c b/drivers/gpu/drm/i915/i915_gem_render_state.c
index 3210cedfa46c..90baf9086d0a 100644
--- a/drivers/gpu/drm/i915/i915_gem_render_state.c
+++ b/drivers/gpu/drm/i915/i915_gem_render_state.c
@@ -222,7 +222,7 @@ int i915_gem_render_state_emit(struct i915_request *rq)
                         goto err_unpin;
         }
 
-        i915_vma_move_to_active(so.vma, rq, 0);
+        err = i915_vma_move_to_active(so.vma, rq, 0);
 err_unpin:
         i915_vma_unpin(so.vma);
 err_vma:
diff --git a/drivers/gpu/drm/i915/i915_gem_shrinker.c b/drivers/gpu/drm/i915/i915_gem_shrinker.c
index 55e84e71f526..c61f5b80fee3 100644
--- a/drivers/gpu/drm/i915/i915_gem_shrinker.c
+++ b/drivers/gpu/drm/i915/i915_gem_shrinker.c
@@ -172,7 +172,9 @@ i915_gem_shrink(struct drm_i915_private *i915,
          * we will free as much as we can and hope to get a second chance.
          */
         if (flags & I915_SHRINK_ACTIVE)
-                i915_gem_wait_for_idle(i915, I915_WAIT_LOCKED);
+                i915_gem_wait_for_idle(i915,
+                                       I915_WAIT_LOCKED,
+                                       MAX_SCHEDULE_TIMEOUT);
 
         trace_i915_gem_shrink(i915, target, flags);
         i915_retire_requests(i915);
@@ -392,7 +394,8 @@ shrinker_lock_uninterruptible(struct drm_i915_private *i915, bool *unlock,
         unsigned long timeout = jiffies + msecs_to_jiffies_timeout(timeout_ms);
 
         do {
-                if (i915_gem_wait_for_idle(i915, 0) == 0 &&
+                if (i915_gem_wait_for_idle(i915,
+                                           0, MAX_SCHEDULE_TIMEOUT) == 0 &&
                     shrinker_lock(i915, unlock))
                         break;
 
@@ -466,7 +469,9 @@ i915_gem_shrinker_vmap(struct notifier_block *nb, unsigned long event, void *ptr
                 return NOTIFY_DONE;
 
         /* Force everything onto the inactive lists */
-        ret = i915_gem_wait_for_idle(i915, I915_WAIT_LOCKED);
+        ret = i915_gem_wait_for_idle(i915,
+                                     I915_WAIT_LOCKED,
+                                     MAX_SCHEDULE_TIMEOUT);
         if (ret)
                 goto out;
 
diff --git a/drivers/gpu/drm/i915/i915_gem_stolen.c b/drivers/gpu/drm/i915/i915_gem_stolen.c
index 79a347295e00..055f8687776d 100644
--- a/drivers/gpu/drm/i915/i915_gem_stolen.c
+++ b/drivers/gpu/drm/i915/i915_gem_stolen.c
@@ -254,6 +254,7 @@ static void vlv_get_stolen_reserved(struct drm_i915_private *dev_priv,
         switch (reg_val & GEN7_STOLEN_RESERVED_SIZE_MASK) {
         default:
                 MISSING_CASE(reg_val & GEN7_STOLEN_RESERVED_SIZE_MASK);
+                /* fall through */
         case GEN7_STOLEN_RESERVED_1M:
                 *size = 1024 * 1024;
                 break;
diff --git a/drivers/gpu/drm/i915/i915_gpu_error.c b/drivers/gpu/drm/i915/i915_gpu_error.c
index df524c9cad40..8c81cf3aa182 100644
--- a/drivers/gpu/drm/i915/i915_gpu_error.c
+++ b/drivers/gpu/drm/i915/i915_gpu_error.c
@@ -335,21 +335,16 @@ static void print_error_buffers(struct drm_i915_error_state_buf *m,
                                 struct drm_i915_error_buffer *err,
                                 int count)
 {
-        int i;
-
         err_printf(m, "%s [%d]:\n", name, count);
 
         while (count--) {
-                err_printf(m, "    %08x_%08x %8u %02x %02x [ ",
+                err_printf(m, "    %08x_%08x %8u %02x %02x %02x",
                            upper_32_bits(err->gtt_offset),
                            lower_32_bits(err->gtt_offset),
                            err->size,
                            err->read_domains,
-                           err->write_domain);
-                for (i = 0; i < I915_NUM_ENGINES; i++)
-                        err_printf(m, "%02x ", err->rseqno[i]);
-
-                err_printf(m, "] %02x", err->wseqno);
+                           err->write_domain,
+                           err->wseqno);
                 err_puts(m, tiling_flag(err->tiling));
                 err_puts(m, dirty_flag(err->dirty));
                 err_puts(m, purgeable_flag(err->purgeable));
@@ -1021,13 +1016,10 @@ static void capture_bo(struct drm_i915_error_buffer *err,
                        struct i915_vma *vma)
 {
         struct drm_i915_gem_object *obj = vma->obj;
-        int i;
 
         err->size = obj->base.size;
         err->name = obj->base.name;
 
-        for (i = 0; i < I915_NUM_ENGINES; i++)
-                err->rseqno[i] = __active_get_seqno(&vma->last_read[i]);
         err->wseqno = __active_get_seqno(&obj->frontbuffer_write);
         err->engine = __active_get_engine_id(&obj->frontbuffer_write);
 
diff --git a/drivers/gpu/drm/i915/i915_gpu_error.h b/drivers/gpu/drm/i915/i915_gpu_error.h
index 58910f1dc67c..f893a4e8b783 100644
--- a/drivers/gpu/drm/i915/i915_gpu_error.h
+++ b/drivers/gpu/drm/i915/i915_gpu_error.h
@@ -177,7 +177,7 @@ struct i915_gpu_state {
         struct drm_i915_error_buffer {
                 u32 size;
                 u32 name;
-                u32 rseqno[I915_NUM_ENGINES], wseqno;
+                u32 wseqno;
                 u64 gtt_offset;
                 u32 read_domains;
                 u32 write_domain;
diff --git a/drivers/gpu/drm/i915/i915_irq.c b/drivers/gpu/drm/i915/i915_irq.c
index 46aaef5c1851..495b9d27990e 100644
--- a/drivers/gpu/drm/i915/i915_irq.c
+++ b/drivers/gpu/drm/i915/i915_irq.c
@@ -122,6 +122,15 @@ static const u32 hpd_gen11[HPD_NUM_PINS] = {
         [HPD_PORT_F] = GEN11_TC4_HOTPLUG | GEN11_TBT4_HOTPLUG
 };
 
+static const u32 hpd_icp[HPD_NUM_PINS] = {
+        [HPD_PORT_A] = SDE_DDIA_HOTPLUG_ICP,
+        [HPD_PORT_B] = SDE_DDIB_HOTPLUG_ICP,
+        [HPD_PORT_C] = SDE_TC1_HOTPLUG_ICP,
+        [HPD_PORT_D] = SDE_TC2_HOTPLUG_ICP,
+        [HPD_PORT_E] = SDE_TC3_HOTPLUG_ICP,
+        [HPD_PORT_F] = SDE_TC4_HOTPLUG_ICP
+};
+
 /* IIR can theoretically queue up two events. Be paranoid. */
 #define GEN8_IRQ_RESET_NDX(type, which) do { \
         I915_WRITE(GEN8_##type##_IMR(which), 0xffffffff); \
@@ -1145,21 +1154,21 @@ static void ironlake_rps_change_irq_handler(struct drm_i915_private *dev_priv)
 
 static void notify_ring(struct intel_engine_cs *engine)
 {
+        const u32 seqno = intel_engine_get_seqno(engine);
         struct i915_request *rq = NULL;
+        struct task_struct *tsk = NULL;
         struct intel_wait *wait;
 
-        if (!engine->breadcrumbs.irq_armed)
+        if (unlikely(!engine->breadcrumbs.irq_armed))
                 return;
 
-        atomic_inc(&engine->irq_count);
-        set_bit(ENGINE_IRQ_BREADCRUMB, &engine->irq_posted);
+        rcu_read_lock();
 
         spin_lock(&engine->breadcrumbs.irq_lock);
         wait = engine->breadcrumbs.irq_wait;
         if (wait) {
-                bool wakeup = engine->irq_seqno_barrier;
-
-                /* We use a callback from the dma-fence to submit
+                /*
+                 * We use a callback from the dma-fence to submit
                  * requests after waiting on our own requests. To
                  * ensure minimum delay in queuing the next request to
                  * hardware, signal the fence now rather than wait for
@@ -1170,19 +1179,26 @@ static void notify_ring(struct intel_engine_cs *engine)
                  * and to handle coalescing of multiple seqno updates
                  * and many waiters.
                  */
-                if (i915_seqno_passed(intel_engine_get_seqno(engine),
-                                      wait->seqno)) {
+                if (i915_seqno_passed(seqno, wait->seqno)) {
                         struct i915_request *waiter = wait->request;
 
-                        wakeup = true;
-                        if (!test_bit(DMA_FENCE_FLAG_SIGNALED_BIT,
+                        if (waiter &&
+                            !test_bit(DMA_FENCE_FLAG_SIGNALED_BIT,
                                       &waiter->fence.flags) &&
                             intel_wait_check_request(wait, waiter))
                                 rq = i915_request_get(waiter);
+
+                        tsk = wait->tsk;
+                } else {
+                        if (engine->irq_seqno_barrier &&
+                            i915_seqno_passed(seqno, wait->seqno - 1)) {
+                                set_bit(ENGINE_IRQ_BREADCRUMB,
+                                        &engine->irq_posted);
+                                tsk = wait->tsk;
+                        }
                 }
 
-                if (wakeup)
-                        wake_up_process(wait->tsk);
+                engine->breadcrumbs.irq_count++;
         } else {
                 if (engine->breadcrumbs.irq_armed)
                         __intel_engine_disarm_breadcrumbs(engine);
@@ -1190,11 +1206,19 @@ static void notify_ring(struct intel_engine_cs *engine)
         spin_unlock(&engine->breadcrumbs.irq_lock);
 
         if (rq) {
-                dma_fence_signal(&rq->fence);
+                spin_lock(&rq->lock);
+                dma_fence_signal_locked(&rq->fence);
                 GEM_BUG_ON(!i915_request_completed(rq));
+                spin_unlock(&rq->lock);
+
                 i915_request_put(rq);
         }
 
+        if (tsk && tsk->state & TASK_NORMAL)
+                wake_up_process(tsk);
+
+        rcu_read_unlock();
+
         trace_intel_engine_notify(engine, wait);
 }
 
@@ -1469,14 +1493,10 @@ static void snb_gt_irq_handler(struct drm_i915_private *dev_priv,
 static void
 gen8_cs_irq_handler(struct intel_engine_cs *engine, u32 iir)
 {
-        struct intel_engine_execlists * const execlists = &engine->execlists;
         bool tasklet = false;
 
-        if (iir & GT_CONTEXT_SWITCH_INTERRUPT) {
-                if (READ_ONCE(engine->execlists.active))
-                        tasklet = !test_and_set_bit(ENGINE_IRQ_EXECLIST,
-                                                    &engine->irq_posted);
-        }
+        if (iir & GT_CONTEXT_SWITCH_INTERRUPT)
+                tasklet = true;
 
         if (iir & GT_RENDER_USER_INTERRUPT) {
                 notify_ring(engine);
@@ -1484,7 +1504,7 @@ gen8_cs_irq_handler(struct intel_engine_cs *engine, u32 iir)
         }
 
         if (tasklet)
-                tasklet_hi_schedule(&execlists->tasklet);
+                tasklet_hi_schedule(&engine->execlists.tasklet);
 }
 
 static void gen8_gt_irq_ack(struct drm_i915_private *i915,
@@ -1586,6 +1606,34 @@ static bool bxt_port_hotplug_long_detect(enum port port, u32 val)
         }
 }
 
+static bool icp_ddi_port_hotplug_long_detect(enum port port, u32 val)
+{
+        switch (port) {
+        case PORT_A:
+                return val & ICP_DDIA_HPD_LONG_DETECT;
+        case PORT_B:
+                return val & ICP_DDIB_HPD_LONG_DETECT;
+        default:
+                return false;
+        }
+}
+
+static bool icp_tc_port_hotplug_long_detect(enum port port, u32 val)
+{
+        switch (port) {
+        case PORT_C:
+                return val & ICP_TC_HPD_LONG_DETECT(PORT_TC1);
+        case PORT_D:
+                return val & ICP_TC_HPD_LONG_DETECT(PORT_TC2);
+        case PORT_E:
+                return val & ICP_TC_HPD_LONG_DETECT(PORT_TC3);
+        case PORT_F:
+                return val & ICP_TC_HPD_LONG_DETECT(PORT_TC4);
+        default:
+                return false;
+        }
+}
+
 static bool spt_port_hotplug2_long_detect(enum port port, u32 val)
 {
         switch (port) {
@@ -1703,69 +1751,34 @@ static void display_pipe_crc_irq_handler(struct drm_i915_private *dev_priv,
                                          uint32_t crc4)
 {
         struct intel_pipe_crc *pipe_crc = &dev_priv->pipe_crc[pipe];
-        struct intel_pipe_crc_entry *entry;
         struct intel_crtc *crtc = intel_get_crtc_for_pipe(dev_priv, pipe);
-        struct drm_driver *driver = dev_priv->drm.driver;
         uint32_t crcs[5];
-        int head, tail;
 
         spin_lock(&pipe_crc->lock);
-        if (pipe_crc->source && !crtc->base.crc.opened) {
-                if (!pipe_crc->entries) {
-                        spin_unlock(&pipe_crc->lock);
-                        DRM_DEBUG_KMS("spurious interrupt\n");
-                        return;
-                }
-
-                head = pipe_crc->head;
-                tail = pipe_crc->tail;
-
-                if (CIRC_SPACE(head, tail, INTEL_PIPE_CRC_ENTRIES_NR) < 1) {
-                        spin_unlock(&pipe_crc->lock);
-                        DRM_ERROR("CRC buffer overflowing\n");
-                        return;
-                }
-
-                entry = &pipe_crc->entries[head];
-
-                entry->frame = driver->get_vblank_counter(&dev_priv->drm, pipe);
-                entry->crc[0] = crc0;
-                entry->crc[1] = crc1;
-                entry->crc[2] = crc2;
-                entry->crc[3] = crc3;
-                entry->crc[4] = crc4;
-
-                head = (head + 1) & (INTEL_PIPE_CRC_ENTRIES_NR - 1);
-                pipe_crc->head = head;
-
-                spin_unlock(&pipe_crc->lock);
-
-                wake_up_interruptible(&pipe_crc->wq);
-        } else {
-                /*
-                 * For some not yet identified reason, the first CRC is
-                 * bonkers. So let's just wait for the next vblank and read
-                 * out the buggy result.
-                 *
-                 * On GEN8+ sometimes the second CRC is bonkers as well, so
-                 * don't trust that one either.
-                 */
-                if (pipe_crc->skipped <= 0 ||
-                    (INTEL_GEN(dev_priv) >= 8 && pipe_crc->skipped == 1)) {
-                        pipe_crc->skipped++;
-                        spin_unlock(&pipe_crc->lock);
-                        return;
-                }
+        /*
+         * For some not yet identified reason, the first CRC is
+         * bonkers. So let's just wait for the next vblank and read
+         * out the buggy result.
+         *
+         * On GEN8+ sometimes the second CRC is bonkers as well, so
+         * don't trust that one either.
+         */
+        if (pipe_crc->skipped <= 0 ||
+            (INTEL_GEN(dev_priv) >= 8 && pipe_crc->skipped == 1)) {
+                pipe_crc->skipped++;
                 spin_unlock(&pipe_crc->lock);
-                crcs[0] = crc0;
-                crcs[1] = crc1;
-                crcs[2] = crc2;
-                crcs[3] = crc3;
-                crcs[4] = crc4;
-                drm_crtc_add_crc_entry(&crtc->base, true,
-                                       drm_crtc_accurate_vblank_count(&crtc->base),
-                                       crcs);
+                return;
         }
+        spin_unlock(&pipe_crc->lock);
+
+        crcs[0] = crc0;
+        crcs[1] = crc1;
+        crcs[2] = crc2;
+        crcs[3] = crc3;
+        crcs[4] = crc4;
+        drm_crtc_add_crc_entry(&crtc->base, true,
+                                drm_crtc_accurate_vblank_count(&crtc->base),
+                                crcs);
 }
 #else
 static inline void
@@ -2021,10 +2034,38 @@ static void valleyview_pipestat_irq_handler(struct drm_i915_private *dev_priv,
 
 static u32 i9xx_hpd_irq_ack(struct drm_i915_private *dev_priv)
 {
-        u32 hotplug_status = I915_READ(PORT_HOTPLUG_STAT);
+        u32 hotplug_status = 0, hotplug_status_mask;
+        int i;
+
+        if (IS_G4X(dev_priv) ||
+            IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
+                hotplug_status_mask = HOTPLUG_INT_STATUS_G4X |
+                        DP_AUX_CHANNEL_MASK_INT_STATUS_G4X;
+        else
+                hotplug_status_mask = HOTPLUG_INT_STATUS_I915;
+
+        /*
+         * We absolutely have to clear all the pending interrupt
+         * bits in PORT_HOTPLUG_STAT. Otherwise the ISR port
+         * interrupt bit won't have an edge, and the i965/g4x
+         * edge triggered IIR will not notice that an interrupt
+         * is still pending. We can't use PORT_HOTPLUG_EN to
+         * guarantee the edge as the act of toggling the enable
+         * bits can itself generate a new hotplug interrupt :(
+         */
+        for (i = 0; i < 10; i++) {
+                u32 tmp = I915_READ(PORT_HOTPLUG_STAT) & hotplug_status_mask;
+
+                if (tmp == 0)
+                        return hotplug_status;
 
-        if (hotplug_status)
+                hotplug_status |= tmp;
                 I915_WRITE(PORT_HOTPLUG_STAT, hotplug_status);
+        }
+
+        WARN_ONCE(1,
+                  "PORT_HOTPLUG_STAT did not clear (0x%08x)\n",
+                  I915_READ(PORT_HOTPLUG_STAT));
 
         return hotplug_status;
 }
@@ -2131,7 +2172,6 @@ static irqreturn_t valleyview_irq_handler(int irq, void *arg)
 
                 I915_WRITE(VLV_IER, ier);
                 I915_WRITE(VLV_MASTER_IER, MASTER_INTERRUPT_ENABLE);
-                POSTING_READ(VLV_MASTER_IER);
 
                 if (gt_iir)
                         snb_gt_irq_handler(dev_priv, gt_iir);
@@ -2216,7 +2256,6 @@ static irqreturn_t cherryview_irq_handler(int irq, void *arg)
 
                 I915_WRITE(VLV_IER, ier);
                 I915_WRITE(GEN8_MASTER_IRQ, GEN8_MASTER_IRQ_CONTROL);
-                POSTING_READ(GEN8_MASTER_IRQ);
 
                 gen8_gt_irq_handler(dev_priv, master_ctl, gt_iir);
 
@@ -2385,6 +2424,43 @@ static void cpt_irq_handler(struct drm_i915_private *dev_priv, u32 pch_iir)
                 cpt_serr_int_handler(dev_priv);
 }
 
+static void icp_irq_handler(struct drm_i915_private *dev_priv, u32 pch_iir)
+{
+        u32 ddi_hotplug_trigger = pch_iir & SDE_DDI_MASK_ICP;
+        u32 tc_hotplug_trigger = pch_iir & SDE_TC_MASK_ICP;
+        u32 pin_mask = 0, long_mask = 0;
+
+        if (ddi_hotplug_trigger) {
+                u32 dig_hotplug_reg;
+
+                dig_hotplug_reg = I915_READ(SHOTPLUG_CTL_DDI);
+                I915_WRITE(SHOTPLUG_CTL_DDI, dig_hotplug_reg);
+
+                intel_get_hpd_pins(dev_priv, &pin_mask, &long_mask,
+                                   ddi_hotplug_trigger,
+                                   dig_hotplug_reg, hpd_icp,
+                                   icp_ddi_port_hotplug_long_detect);
+        }
+
+        if (tc_hotplug_trigger) {
+                u32 dig_hotplug_reg;
+
+                dig_hotplug_reg = I915_READ(SHOTPLUG_CTL_TC);
+                I915_WRITE(SHOTPLUG_CTL_TC, dig_hotplug_reg);
+
+                intel_get_hpd_pins(dev_priv, &pin_mask, &long_mask,
+                                   tc_hotplug_trigger,
+                                   dig_hotplug_reg, hpd_icp,
+                                   icp_tc_port_hotplug_long_detect);
+        }
+
+        if (pin_mask)
+                intel_hpd_irq_handler(dev_priv, pin_mask, long_mask);
+
+        if (pch_iir & SDE_GMBUS_ICP)
+                gmbus_irq_handler(dev_priv);
+}
+
 static void spt_irq_handler(struct drm_i915_private *dev_priv, u32 pch_iir)
 {
         u32 hotplug_trigger = pch_iir & SDE_HOTPLUG_MASK_SPT &
@@ -2548,7 +2624,6 @@ static irqreturn_t ironlake_irq_handler(int irq, void *arg)
         /* disable master interrupt before clearing iir  */
         de_ier = I915_READ(DEIER);
         I915_WRITE(DEIER, de_ier & ~DE_MASTER_IRQ_CONTROL);
-        POSTING_READ(DEIER);
 
         /* Disable south interrupts. We'll only write to SDEIIR once, so further
          * interrupts will will be stored on its back queue, and then we'll be
@@ -2558,7 +2633,6 @@ static irqreturn_t ironlake_irq_handler(int irq, void *arg)
         if (!HAS_PCH_NOP(dev_priv)) {
                 sde_ier = I915_READ(SDEIER);
                 I915_WRITE(SDEIER, 0);
-                POSTING_READ(SDEIER);
         }
 
         /* Find, clear, then process each source of interrupt */
@@ -2593,11 +2667,8 @@ static irqreturn_t ironlake_irq_handler(int irq, void *arg)
         }
 
         I915_WRITE(DEIER, de_ier);
-        POSTING_READ(DEIER);
-        if (!HAS_PCH_NOP(dev_priv)) {
+        if (!HAS_PCH_NOP(dev_priv))
                 I915_WRITE(SDEIER, sde_ier);
-                POSTING_READ(SDEIER);
-        }
 
         /* IRQs are synced during runtime_suspend, we don't require a wakeref */
         enable_rpm_wakeref_asserts(dev_priv);
@@ -2804,8 +2875,11 @@ gen8_de_irq_handler(struct drm_i915_private *dev_priv, u32 master_ctl)
                         I915_WRITE(SDEIIR, iir);
                         ret = IRQ_HANDLED;
 
-                        if (HAS_PCH_SPT(dev_priv) || HAS_PCH_KBP(dev_priv) ||
-                            HAS_PCH_CNP(dev_priv))
+                        if (HAS_PCH_ICP(dev_priv))
+                                icp_irq_handler(dev_priv, iir);
+                        else if (HAS_PCH_SPT(dev_priv) ||
+                                 HAS_PCH_KBP(dev_priv) ||
+                                 HAS_PCH_CNP(dev_priv))
                                 spt_irq_handler(dev_priv, iir);
                         else
                                 cpt_irq_handler(dev_priv, iir);
@@ -3170,7 +3244,7 @@ static void i915_clear_error_registers(struct drm_i915_private *dev_priv)
                  */
                 DRM_DEBUG_DRIVER("EIR stuck: 0x%08x, masking\n", eir);
                 I915_WRITE(EMR, I915_READ(EMR) | eir);
-                I915_WRITE(IIR, I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT);
+                I915_WRITE(IIR, I915_MASTER_ERROR_INTERRUPT);
         }
 }
 
@@ -3584,6 +3658,9 @@ static void gen11_irq_reset(struct drm_device *dev)
         GEN3_IRQ_RESET(GEN11_DE_HPD_);
         GEN3_IRQ_RESET(GEN11_GU_MISC_);
         GEN3_IRQ_RESET(GEN8_PCU_);
+
+        if (HAS_PCH_ICP(dev_priv))
+                GEN3_IRQ_RESET(SDE);
 }
 
 void gen8_irq_power_well_post_enable(struct drm_i915_private *dev_priv,
@@ -3700,6 +3777,35 @@ static void ibx_hpd_irq_setup(struct drm_i915_private *dev_priv)
         ibx_hpd_detection_setup(dev_priv);
 }
 
+static void icp_hpd_detection_setup(struct drm_i915_private *dev_priv)
+{
+        u32 hotplug;
+
+        hotplug = I915_READ(SHOTPLUG_CTL_DDI);
+        hotplug |= ICP_DDIA_HPD_ENABLE |
+                   ICP_DDIB_HPD_ENABLE;
+        I915_WRITE(SHOTPLUG_CTL_DDI, hotplug);
+
+        hotplug = I915_READ(SHOTPLUG_CTL_TC);
+        hotplug |= ICP_TC_HPD_ENABLE(PORT_TC1) |
+                   ICP_TC_HPD_ENABLE(PORT_TC2) |
+                   ICP_TC_HPD_ENABLE(PORT_TC3) |
+                   ICP_TC_HPD_ENABLE(PORT_TC4);
+        I915_WRITE(SHOTPLUG_CTL_TC, hotplug);
+}
+
+static void icp_hpd_irq_setup(struct drm_i915_private *dev_priv)
+{
+        u32 hotplug_irqs, enabled_irqs;
+
+        hotplug_irqs = SDE_DDI_MASK_ICP | SDE_TC_MASK_ICP;
+        enabled_irqs = intel_hpd_enabled_irqs(dev_priv, hpd_icp);
+
+        ibx_display_interrupt_update(dev_priv, hotplug_irqs, enabled_irqs);
+
+        icp_hpd_detection_setup(dev_priv);
+}
+
 static void gen11_hpd_detection_setup(struct drm_i915_private *dev_priv)
 {
         u32 hotplug;
@@ -3733,6 +3839,9 @@ static void gen11_hpd_irq_setup(struct drm_i915_private *dev_priv)
         POSTING_READ(GEN11_DE_HPD_IMR);
 
         gen11_hpd_detection_setup(dev_priv);
+
+        if (HAS_PCH_ICP(dev_priv))
+                icp_hpd_irq_setup(dev_priv);
 }
 
 static void spt_hpd_detection_setup(struct drm_i915_private *dev_priv)
@@ -4168,11 +4277,29 @@ static void gen11_gt_irq_postinstall(struct drm_i915_private *dev_priv)
         I915_WRITE(GEN11_GPM_WGBOXPERF_INTR_MASK,  ~0);
 }
 
+static void icp_irq_postinstall(struct drm_device *dev)
+{
+        struct drm_i915_private *dev_priv = to_i915(dev);
+        u32 mask = SDE_GMBUS_ICP;
+
+        WARN_ON(I915_READ(SDEIER) != 0);
+        I915_WRITE(SDEIER, 0xffffffff);
+        POSTING_READ(SDEIER);
+
+        gen3_assert_iir_is_zero(dev_priv, SDEIIR);
+        I915_WRITE(SDEIMR, ~mask);
+
+        icp_hpd_detection_setup(dev_priv);
+}
+
 static int gen11_irq_postinstall(struct drm_device *dev)
 {
         struct drm_i915_private *dev_priv = dev->dev_private;
         u32 gu_misc_masked = GEN11_GU_MISC_GSE;
 
+        if (HAS_PCH_ICP(dev_priv))
+                icp_irq_postinstall(dev);
+
         gen11_gt_irq_postinstall(dev_priv);
         gen8_de_irq_postinstall(dev_priv);
 
@@ -4225,11 +4352,13 @@ static int i8xx_irq_postinstall(struct drm_device *dev)
         /* Unmask the interrupts that we always want on. */
         dev_priv->irq_mask =
                 ~(I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
-                  I915_DISPLAY_PIPE_B_EVENT_INTERRUPT);
+                  I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
+                  I915_MASTER_ERROR_INTERRUPT);
 
         enable_mask =
                 I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
                 I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
+                I915_MASTER_ERROR_INTERRUPT |
                 I915_USER_INTERRUPT;
 
         GEN2_IRQ_INIT(, dev_priv->irq_mask, enable_mask);
@@ -4244,6 +4373,81 @@ static int i8xx_irq_postinstall(struct drm_device *dev)
         return 0;
 }
 
+static void i8xx_error_irq_ack(struct drm_i915_private *dev_priv,
+                               u16 *eir, u16 *eir_stuck)
+{
+        u16 emr;
+
+        *eir = I915_READ16(EIR);
+
+        if (*eir)
+                I915_WRITE16(EIR, *eir);
+
+        *eir_stuck = I915_READ16(EIR);
+        if (*eir_stuck == 0)
+                return;
+
+        /*
+         * Toggle all EMR bits to make sure we get an edge
+         * in the ISR master error bit if we don't clear
+         * all the EIR bits. Otherwise the edge triggered
+         * IIR on i965/g4x wouldn't notice that an interrupt
+         * is still pending. Also some EIR bits can't be
+         * cleared except by handling the underlying error
+         * (or by a GPU reset) so we mask any bit that
+         * remains set.
+         */
+        emr = I915_READ16(EMR);
+        I915_WRITE16(EMR, 0xffff);
+        I915_WRITE16(EMR, emr | *eir_stuck);
+}
+
+static void i8xx_error_irq_handler(struct drm_i915_private *dev_priv,
+                                   u16 eir, u16 eir_stuck)
+{
+        DRM_DEBUG("Master Error: EIR 0x%04x\n", eir);
+
+        if (eir_stuck)
+                DRM_DEBUG_DRIVER("EIR stuck: 0x%04x, masked\n", eir_stuck);
+}
+
+static void i9xx_error_irq_ack(struct drm_i915_private *dev_priv,
+                               u32 *eir, u32 *eir_stuck)
+{
+        u32 emr;
+
+        *eir = I915_READ(EIR);
+
+        I915_WRITE(EIR, *eir);
+
+        *eir_stuck = I915_READ(EIR);
+        if (*eir_stuck == 0)
+                return;
+
+        /*
+         * Toggle all EMR bits to make sure we get an edge
+         * in the ISR master error bit if we don't clear
+         * all the EIR bits. Otherwise the edge triggered
+         * IIR on i965/g4x wouldn't notice that an interrupt
+         * is still pending. Also some EIR bits can't be
+         * cleared except by handling the underlying error
+         * (or by a GPU reset) so we mask any bit that
+         * remains set.
+         */
+        emr = I915_READ(EMR);
+        I915_WRITE(EMR, 0xffffffff);
+        I915_WRITE(EMR, emr | *eir_stuck);
+}
+
+static void i9xx_error_irq_handler(struct drm_i915_private *dev_priv,
+                                   u32 eir, u32 eir_stuck)
+{
+        DRM_DEBUG("Master Error, EIR 0x%08x\n", eir);
+
+        if (eir_stuck)
+                DRM_DEBUG_DRIVER("EIR stuck: 0x%08x, masked\n", eir_stuck);
+}
+
 static irqreturn_t i8xx_irq_handler(int irq, void *arg)
 {
         struct drm_device *dev = arg;
@@ -4258,6 +4462,7 @@ static irqreturn_t i8xx_irq_handler(int irq, void *arg)
 
         do {
                 u32 pipe_stats[I915_MAX_PIPES] = {};
+                u16 eir = 0, eir_stuck = 0;
                 u16 iir;
 
                 iir = I915_READ16(IIR);
@@ -4270,13 +4475,16 @@ static irqreturn_t i8xx_irq_handler(int irq, void *arg)
                  * signalled in iir */
                 i9xx_pipestat_irq_ack(dev_priv, iir, pipe_stats);
 
+                if (iir & I915_MASTER_ERROR_INTERRUPT)
+                        i8xx_error_irq_ack(dev_priv, &eir, &eir_stuck);
+
                 I915_WRITE16(IIR, iir);
 
                 if (iir & I915_USER_INTERRUPT)
                         notify_ring(dev_priv->engine[RCS]);
 
-                if (iir & I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT)
-                        DRM_DEBUG("Command parser error, iir 0x%08x\n", iir);
+                if (iir & I915_MASTER_ERROR_INTERRUPT)
+                        i8xx_error_irq_handler(dev_priv, eir, eir_stuck);
 
                 i8xx_pipestat_irq_handler(dev_priv, iir, pipe_stats);
         } while (0);
@@ -4314,12 +4522,14 @@ static int i915_irq_postinstall(struct drm_device *dev)
         dev_priv->irq_mask =
                 ~(I915_ASLE_INTERRUPT |
                   I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
-                  I915_DISPLAY_PIPE_B_EVENT_INTERRUPT);
+                  I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
+                  I915_MASTER_ERROR_INTERRUPT);
 
         enable_mask =
                 I915_ASLE_INTERRUPT |
                 I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
                 I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
+                I915_MASTER_ERROR_INTERRUPT |
                 I915_USER_INTERRUPT;
 
         if (I915_HAS_HOTPLUG(dev_priv)) {
@@ -4357,6 +4567,7 @@ static irqreturn_t i915_irq_handler(int irq, void *arg)
 
         do {
                 u32 pipe_stats[I915_MAX_PIPES] = {};
+                u32 eir = 0, eir_stuck = 0;
                 u32 hotplug_status = 0;
                 u32 iir;
 
@@ -4374,13 +4585,16 @@ static irqreturn_t i915_irq_handler(int irq, void *arg)
                  * signalled in iir */
                 i9xx_pipestat_irq_ack(dev_priv, iir, pipe_stats);
 
+                if (iir & I915_MASTER_ERROR_INTERRUPT)
+                        i9xx_error_irq_ack(dev_priv, &eir, &eir_stuck);
+
                 I915_WRITE(IIR, iir);
 
                 if (iir & I915_USER_INTERRUPT)
                         notify_ring(dev_priv->engine[RCS]);
 
-                if (iir & I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT)
-                        DRM_DEBUG("Command parser error, iir 0x%08x\n", iir);
+                if (iir & I915_MASTER_ERROR_INTERRUPT)
+                        i9xx_error_irq_handler(dev_priv, eir, eir_stuck);
 
                 if (hotplug_status)
                         i9xx_hpd_irq_handler(dev_priv, hotplug_status);
@@ -4434,14 +4648,14 @@ static int i965_irq_postinstall(struct drm_device *dev)
                   I915_DISPLAY_PORT_INTERRUPT |
                   I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
                   I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
-                  I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT);
+                  I915_MASTER_ERROR_INTERRUPT);
 
         enable_mask =
                 I915_ASLE_INTERRUPT |
                 I915_DISPLAY_PORT_INTERRUPT |
                 I915_DISPLAY_PIPE_A_EVENT_INTERRUPT |
                 I915_DISPLAY_PIPE_B_EVENT_INTERRUPT |
-                I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT |
+                I915_MASTER_ERROR_INTERRUPT |
                 I915_USER_INTERRUPT;
 
         if (IS_G4X(dev_priv))
@@ -4501,6 +4715,7 @@ static irqreturn_t i965_irq_handler(int irq, void *arg)
 
         do {
                 u32 pipe_stats[I915_MAX_PIPES] = {};
+                u32 eir = 0, eir_stuck = 0;
                 u32 hotplug_status = 0;
                 u32 iir;
 
@@ -4517,6 +4732,9 @@ static irqreturn_t i965_irq_handler(int irq, void *arg)
                  * signalled in iir */
                 i9xx_pipestat_irq_ack(dev_priv, iir, pipe_stats);
 
+                if (iir & I915_MASTER_ERROR_INTERRUPT)
+                        i9xx_error_irq_ack(dev_priv, &eir, &eir_stuck);
+
                 I915_WRITE(IIR, iir);
 
                 if (iir & I915_USER_INTERRUPT)
@@ -4525,8 +4743,8 @@ static irqreturn_t i965_irq_handler(int irq, void *arg)
                 if (iir & I915_BSD_USER_INTERRUPT)
                         notify_ring(dev_priv->engine[VCS]);
 
-                if (iir & I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT)
-                        DRM_DEBUG("Command parser error, iir 0x%08x\n", iir);
+                if (iir & I915_MASTER_ERROR_INTERRUPT)
+                        i9xx_error_irq_handler(dev_priv, eir, eir_stuck);
 
                 if (hotplug_status)
                         i9xx_hpd_irq_handler(dev_priv, hotplug_status);
diff --git a/drivers/gpu/drm/i915/i915_perf.c b/drivers/gpu/drm/i915/i915_perf.c
index 447407fee3b8..6bf10952c724 100644
--- a/drivers/gpu/drm/i915/i915_perf.c
+++ b/drivers/gpu/drm/i915/i915_perf.c
@@ -1836,7 +1836,9 @@ static int gen8_configure_all_contexts(struct drm_i915_private *dev_priv,
          * So far the best way to work around this issue seems to be draining
          * the GPU from any submitted work.
          */
-        ret = i915_gem_wait_for_idle(dev_priv, wait_flags);
+        ret = i915_gem_wait_for_idle(dev_priv,
+                                     wait_flags,
+                                     MAX_SCHEDULE_TIMEOUT);
         if (ret)
                 goto out;
 
diff --git a/drivers/gpu/drm/i915/i915_reg.h b/drivers/gpu/drm/i915/i915_reg.h
index 4bfd7a9bd75f..0424e45f88db 100644
--- a/drivers/gpu/drm/i915/i915_reg.h
+++ b/drivers/gpu/drm/i915/i915_reg.h
@@ -139,19 +139,35 @@ static inline bool i915_mmio_reg_valid(i915_reg_t reg)
         return !i915_mmio_reg_equal(reg, INVALID_MMIO_REG);
 }
 
+/*
+ * Given the first two numbers __a and __b of arbitrarily many evenly spaced
+ * numbers, pick the 0-based __index'th value.
+ *
+ * Always prefer this over _PICK() if the numbers are evenly spaced.
+ */
+#define _PICK_EVEN(__index, __a, __b) ((__a) + (__index) * ((__b) - (__a)))
+
+/*
+ * Given the arbitrary numbers in varargs, pick the 0-based __index'th number.
+ *
+ * Always prefer _PICK_EVEN() over this if the numbers are evenly spaced.
+ */
 #define _PICK(__index, ...) (((const u32 []){ __VA_ARGS__ })[__index])
 
-#define _PIPE(pipe, a, b) ((a) + (pipe) * ((b) - (a)))
+/*
+ * Named helper wrappers around _PICK_EVEN() and _PICK().
+ */
+#define _PIPE(pipe, a, b) _PICK_EVEN(pipe, a, b)
 #define _MMIO_PIPE(pipe, a, b) _MMIO(_PIPE(pipe, a, b))
-#define _PLANE(plane, a, b) _PIPE(plane, a, b)
+#define _PLANE(plane, a, b) _PICK_EVEN(plane, a, b)
 #define _MMIO_PLANE(plane, a, b) _MMIO_PIPE(plane, a, b)
-#define _TRANS(tran, a, b) ((a) + (tran) * ((b) - (a)))
+#define _TRANS(tran, a, b) _PICK_EVEN(tran, a, b)
 #define _MMIO_TRANS(tran, a, b) _MMIO(_TRANS(tran, a, b))
-#define _PORT(port, a, b) ((a) + (port) * ((b) - (a)))
+#define _PORT(port, a, b) _PICK_EVEN(port, a, b)
 #define _MMIO_PORT(port, a, b) _MMIO(_PORT(port, a, b))
 #define _MMIO_PIPE3(pipe, a, b, c) _MMIO(_PICK(pipe, a, b, c))
 #define _MMIO_PORT3(pipe, a, b, c) _MMIO(_PICK(pipe, a, b, c))
-#define _PLL(pll, a, b) ((a) + (pll) * ((b) - (a)))
+#define _PLL(pll, a, b) _PICK_EVEN(pll, a, b)
 #define _MMIO_PLL(pll, a, b) _MMIO(_PLL(pll, a, b))
 #define _PHY3(phy, ...) _PICK(phy, __VA_ARGS__)
 #define _MMIO_PHY3(phy, a, b, c) _MMIO(_PHY3(phy, a, b, c))
@@ -1045,13 +1061,13 @@ enum i915_power_well_id {
 
         /*
          * HSW/BDW
-         *  - HSW_PWR_WELL_CTL_DRIVER(0) (status bit: id*2, req bit: id*2+1)
+         *  - _HSW_PWR_WELL_CTL1-4 (status bit: id*2, req bit: id*2+1)
          */
         HSW_DISP_PW_GLOBAL = 15,
 
         /*
          * GEN9+
-         *  - HSW_PWR_WELL_CTL_DRIVER(0) (status bit: id*2, req bit: id*2+1)
+         *  - _HSW_PWR_WELL_CTL1-4 (status bit: id*2, req bit: id*2+1)
          */
         SKL_DISP_PW_MISC_IO = 0,
         SKL_DISP_PW_DDI_A_E,
@@ -1075,17 +1091,54 @@ enum i915_power_well_id {
         SKL_DISP_PW_2,
 
         /* - custom power wells */
-        SKL_DISP_PW_DC_OFF,
         BXT_DPIO_CMN_A,
         BXT_DPIO_CMN_BC,
-        GLK_DPIO_CMN_C,                 /* 19 */
+        GLK_DPIO_CMN_C,                 /* 18 */
+
+        /*
+         * GEN11+
+         *  - _HSW_PWR_WELL_CTL1-4
+         *    (status bit: (id&15)*2, req bit:(id&15)*2+1)
+         */
+        ICL_DISP_PW_1 = 0,
+        ICL_DISP_PW_2,
+        ICL_DISP_PW_3,
+        ICL_DISP_PW_4,
+
+        /*
+         *  - _HSW_PWR_WELL_CTL_AUX1/2/4
+         *    (status bit: (id&15)*2, req bit:(id&15)*2+1)
+         */
+        ICL_DISP_PW_AUX_A = 16,
+        ICL_DISP_PW_AUX_B,
+        ICL_DISP_PW_AUX_C,
+        ICL_DISP_PW_AUX_D,
+        ICL_DISP_PW_AUX_E,
+        ICL_DISP_PW_AUX_F,
+
+        ICL_DISP_PW_AUX_TBT1 = 24,
+        ICL_DISP_PW_AUX_TBT2,
+        ICL_DISP_PW_AUX_TBT3,
+        ICL_DISP_PW_AUX_TBT4,
+
+        /*
+         *  - _HSW_PWR_WELL_CTL_DDI1/2/4
+         *    (status bit: (id&15)*2, req bit:(id&15)*2+1)
+         */
+        ICL_DISP_PW_DDI_A = 32,
+        ICL_DISP_PW_DDI_B,
+        ICL_DISP_PW_DDI_C,
+        ICL_DISP_PW_DDI_D,
+        ICL_DISP_PW_DDI_E,
+        ICL_DISP_PW_DDI_F,                      /* 37 */
 
         /*
          * Multiple platforms.
          * Must start following the highest ID of any platform.
          * - custom power wells
          */
-        I915_DISP_PW_ALWAYS_ON = 20,
+        SKL_DISP_PW_DC_OFF = 38,
+        I915_DISP_PW_ALWAYS_ON,
 };
 
 #define PUNIT_REG_PWRGT_CTRL                    0x60
@@ -1667,6 +1720,26 @@ enum i915_power_well_id {
 #define ICL_PORT_CL_DW5(port)   _MMIO_PORT(port, _ICL_PORT_CL_DW5_A, \
                                                  _ICL_PORT_CL_DW5_B)
 
+#define _CNL_PORT_CL_DW10_A             0x162028
+#define _ICL_PORT_CL_DW10_B             0x6c028
+#define ICL_PORT_CL_DW10(port)          _MMIO_PORT(port,        \
+                                                   _CNL_PORT_CL_DW10_A, \
+                                                   _ICL_PORT_CL_DW10_B)
+#define  PG_SEQ_DELAY_OVERRIDE_MASK     (3 << 25)
+#define  PG_SEQ_DELAY_OVERRIDE_SHIFT    25
+#define  PG_SEQ_DELAY_OVERRIDE_ENABLE   (1 << 24)
+#define  PWR_UP_ALL_LANES               (0x0 << 4)
+#define  PWR_DOWN_LN_3_2_1              (0xe << 4)
+#define  PWR_DOWN_LN_3_2                (0xc << 4)
+#define  PWR_DOWN_LN_3                  (0x8 << 4)
+#define  PWR_DOWN_LN_2_1_0              (0x7 << 4)
+#define  PWR_DOWN_LN_1_0                (0x3 << 4)
+#define  PWR_DOWN_LN_1                  (0x2 << 4)
+#define  PWR_DOWN_LN_3_1                (0xa << 4)
+#define  PWR_DOWN_LN_3_1_0              (0xb << 4)
+#define  PWR_DOWN_LN_MASK               (0xf << 4)
+#define  PWR_DOWN_LN_SHIFT              4
+
 #define _PORT_CL1CM_DW9_A               0x162024
 #define _PORT_CL1CM_DW9_BC              0x6C024
 #define   IREF0RC_OFFSET_SHIFT          8
@@ -1679,6 +1752,13 @@ enum i915_power_well_id {
 #define   IREF1RC_OFFSET_MASK           (0xFF << IREF1RC_OFFSET_SHIFT)
 #define BXT_PORT_CL1CM_DW10(phy)        _BXT_PHY((phy), _PORT_CL1CM_DW10_BC)
 
+#define _ICL_PORT_CL_DW12_A             0x162030
+#define _ICL_PORT_CL_DW12_B             0x6C030
+#define   ICL_LANE_ENABLE_AUX           (1 << 0)
+#define ICL_PORT_CL_DW12(port)          _MMIO_PORT((port),              \
+                                                   _ICL_PORT_CL_DW12_A, \
+                                                   _ICL_PORT_CL_DW12_B)
+
 #define _PORT_CL1CM_DW28_A              0x162070
 #define _PORT_CL1CM_DW28_BC             0x6C070
 #define   OCL1_POWER_DOWN_EN            (1 << 23)
@@ -1716,16 +1796,22 @@ enum i915_power_well_id {
                                                     _CNL_PORT_PCS_DW1_LN0_D, \
                                                     _CNL_PORT_PCS_DW1_LN0_AE, \
                                                     _CNL_PORT_PCS_DW1_LN0_F))
+
 #define _ICL_PORT_PCS_DW1_GRP_A         0x162604
 #define _ICL_PORT_PCS_DW1_GRP_B         0x6C604
 #define _ICL_PORT_PCS_DW1_LN0_A         0x162804
 #define _ICL_PORT_PCS_DW1_LN0_B         0x6C804
+#define _ICL_PORT_PCS_DW1_AUX_A         0x162304
+#define _ICL_PORT_PCS_DW1_AUX_B         0x6c304
 #define ICL_PORT_PCS_DW1_GRP(port)      _MMIO_PORT(port,\
                                                    _ICL_PORT_PCS_DW1_GRP_A, \
                                                    _ICL_PORT_PCS_DW1_GRP_B)
 #define ICL_PORT_PCS_DW1_LN0(port)      _MMIO_PORT(port, \
                                                    _ICL_PORT_PCS_DW1_LN0_A, \
                                                    _ICL_PORT_PCS_DW1_LN0_B)
+#define ICL_PORT_PCS_DW1_AUX(port)      _MMIO_PORT(port, \
+                                                   _ICL_PORT_PCS_DW1_AUX_A, \
+                                                   _ICL_PORT_PCS_DW1_AUX_B)
 #define   COMMON_KEEPER_EN              (1 << 26)
 
 /* CNL Port TX registers */
@@ -1762,16 +1848,23 @@ enum i915_power_well_id {
 #define _ICL_PORT_TX_DW2_GRP_B          0x6C688
 #define _ICL_PORT_TX_DW2_LN0_A          0x162888
 #define _ICL_PORT_TX_DW2_LN0_B          0x6C888
+#define _ICL_PORT_TX_DW2_AUX_A          0x162388
+#define _ICL_PORT_TX_DW2_AUX_B          0x6c388
 #define ICL_PORT_TX_DW2_GRP(port)       _MMIO_PORT(port, \
                                                    _ICL_PORT_TX_DW2_GRP_A, \
                                                    _ICL_PORT_TX_DW2_GRP_B)
 #define ICL_PORT_TX_DW2_LN0(port)       _MMIO_PORT(port, \
                                                    _ICL_PORT_TX_DW2_LN0_A, \
                                                    _ICL_PORT_TX_DW2_LN0_B)
+#define ICL_PORT_TX_DW2_AUX(port)       _MMIO_PORT(port, \
+                                                   _ICL_PORT_TX_DW2_AUX_A, \
+                                                   _ICL_PORT_TX_DW2_AUX_B)
 #define   SWING_SEL_UPPER(x)            (((x) >> 3) << 15)
 #define   SWING_SEL_UPPER_MASK          (1 << 15)
 #define   SWING_SEL_LOWER(x)            (((x) & 0x7) << 11)
 #define   SWING_SEL_LOWER_MASK          (0x7 << 11)
+#define   FRC_LATENCY_OPTIM_MASK        (0x7 << 8)
+#define   FRC_LATENCY_OPTIM_VAL(x)      ((x) << 8)
 #define   RCOMP_SCALAR(x)               ((x) << 0)
 #define   RCOMP_SCALAR_MASK             (0xFF << 0)
 
@@ -1787,6 +1880,8 @@ enum i915_power_well_id {
 #define _ICL_PORT_TX_DW4_LN0_A          0x162890
 #define _ICL_PORT_TX_DW4_LN1_A          0x162990
 #define _ICL_PORT_TX_DW4_LN0_B          0x6C890
+#define _ICL_PORT_TX_DW4_AUX_A          0x162390
+#define _ICL_PORT_TX_DW4_AUX_B          0x6c390
 #define ICL_PORT_TX_DW4_GRP(port)       _MMIO_PORT(port, \
                                                    _ICL_PORT_TX_DW4_GRP_A, \
                                                    _ICL_PORT_TX_DW4_GRP_B)
@@ -1795,6 +1890,9 @@ enum i915_power_well_id {
                                                    _ICL_PORT_TX_DW4_LN0_B) + \
                                              ((ln) * (_ICL_PORT_TX_DW4_LN1_A - \
                                                       _ICL_PORT_TX_DW4_LN0_A)))
+#define ICL_PORT_TX_DW4_AUX(port)       _MMIO_PORT(port, \
+                                                   _ICL_PORT_TX_DW4_AUX_A, \
+                                                   _ICL_PORT_TX_DW4_AUX_B)
 #define   LOADGEN_SELECT                (1 << 31)
 #define   POST_CURSOR_1(x)              ((x) << 12)
 #define   POST_CURSOR_1_MASK            (0x3F << 12)
@@ -1809,12 +1907,17 @@ enum i915_power_well_id {
 #define _ICL_PORT_TX_DW5_GRP_B          0x6C694
 #define _ICL_PORT_TX_DW5_LN0_A          0x162894
 #define _ICL_PORT_TX_DW5_LN0_B          0x6C894
+#define _ICL_PORT_TX_DW5_AUX_A          0x162394
+#define _ICL_PORT_TX_DW5_AUX_B          0x6c394
 #define ICL_PORT_TX_DW5_GRP(port)       _MMIO_PORT(port, \
                                                    _ICL_PORT_TX_DW5_GRP_A, \
                                                    _ICL_PORT_TX_DW5_GRP_B)
 #define ICL_PORT_TX_DW5_LN0(port)       _MMIO_PORT(port, \
                                                    _ICL_PORT_TX_DW5_LN0_A, \
                                                    _ICL_PORT_TX_DW5_LN0_B)
+#define ICL_PORT_TX_DW5_AUX(port)       _MMIO_PORT(port, \
+                                                   _ICL_PORT_TX_DW5_AUX_A, \
+                                                   _ICL_PORT_TX_DW5_AUX_B)
 #define   TX_TRAINING_EN                (1 << 31)
 #define   TAP2_DISABLE                  (1 << 30)
 #define   TAP3_DISABLE                  (1 << 29)
@@ -2811,7 +2914,6 @@ enum i915_power_well_id {
 #define I915_DISPLAY_PORT_INTERRUPT                     (1 << 17)
 #define I915_DISPLAY_PIPE_C_HBLANK_INTERRUPT            (1 << 16)
 #define I915_MASTER_ERROR_INTERRUPT                     (1 << 15)
-#define I915_RENDER_COMMAND_PARSER_ERROR_INTERRUPT      (1 << 15)
 #define I915_DISPLAY_PIPE_B_HBLANK_INTERRUPT            (1 << 14)
 #define I915_GMCH_THERMAL_SENSOR_EVENT_INTERRUPT        (1 << 14) /* p-state */
 #define I915_DISPLAY_PIPE_A_HBLANK_INTERRUPT            (1 << 13)
@@ -4044,6 +4146,7 @@ enum {
 #define   EDP_PSR_SKIP_AUX_EXIT                 (1 << 12)
 #define   EDP_PSR_TP1_TP2_SEL                   (0 << 11)
 #define   EDP_PSR_TP1_TP3_SEL                   (1 << 11)
+#define   EDP_PSR_CRC_ENABLE                    (1 << 10) /* BDW+ */
 #define   EDP_PSR_TP2_TP3_TIME_500us            (0 << 8)
 #define   EDP_PSR_TP2_TP3_TIME_100us            (1 << 8)
 #define   EDP_PSR_TP2_TP3_TIME_2500us           (2 << 8)
@@ -4072,6 +4175,7 @@ enum {
 
 #define EDP_PSR_STATUS                          _MMIO(dev_priv->psr_mmio_base + 0x40)
 #define   EDP_PSR_STATUS_STATE_MASK             (7 << 29)
+#define   EDP_PSR_STATUS_STATE_SHIFT            29
 #define   EDP_PSR_STATUS_STATE_IDLE             (0 << 29)
 #define   EDP_PSR_STATUS_STATE_SRDONACK         (1 << 29)
 #define   EDP_PSR_STATUS_STATE_SRDENT           (2 << 29)
@@ -6829,7 +6933,7 @@ enum {
 #define _PS_ECC_STAT_2B     0x68AD0
 #define _PS_ECC_STAT_1C     0x691D0
 
-#define _ID(id, a, b) ((a) + (id) * ((b) - (a)))
+#define _ID(id, a, b) _PICK_EVEN(id, a, b)
 #define SKL_PS_CTRL(pipe, id) _MMIO_PIPE(pipe,        \
                         _ID(id, _PS_1A_CTRL, _PS_2A_CTRL),       \
                         _ID(id, _PS_1B_CTRL, _PS_2B_CTRL))
@@ -7366,6 +7470,14 @@ enum {
 #define BDW_SCRATCH1                                    _MMIO(0xb11c)
 #define  GEN9_LBS_SLA_RETRY_TIMER_DECREMENT_ENABLE      (1 << 2)
 
+/*GEN11 chicken */
+#define _PIPEA_CHICKEN                  0x70038
+#define _PIPEB_CHICKEN                  0x71038
+#define _PIPEC_CHICKEN                  0x72038
+#define  PER_PIXEL_ALPHA_BYPASS_EN      (1 << 7)
+#define PIPE_CHICKEN(pipe)              _MMIO_PIPE(pipe, _PIPEA_CHICKEN,\
+                                                   _PIPEB_CHICKEN)
+
 /* PCH */
 
 /* south display engine interrupt: IBX */
@@ -7409,7 +7521,7 @@ enum {
 #define SDE_TRANSA_FIFO_UNDER   (1 << 0)
 #define SDE_TRANS_MASK          (0x3f)
 
-/* south display engine interrupt: CPT/PPT */
+/* south display engine interrupt: CPT - CNP */
 #define SDE_AUDIO_POWER_D_CPT   (1 << 31)
 #define SDE_AUDIO_POWER_C_CPT   (1 << 30)
 #define SDE_AUDIO_POWER_B_CPT   (1 << 29)
@@ -7457,6 +7569,21 @@ enum {
                                  SDE_FDI_RXB_CPT | \
                                  SDE_FDI_RXA_CPT)
 
+/* south display engine interrupt: ICP */
+#define SDE_TC4_HOTPLUG_ICP             (1 << 27)
+#define SDE_TC3_HOTPLUG_ICP             (1 << 26)
+#define SDE_TC2_HOTPLUG_ICP             (1 << 25)
+#define SDE_TC1_HOTPLUG_ICP             (1 << 24)
+#define SDE_GMBUS_ICP                   (1 << 23)
+#define SDE_DDIB_HOTPLUG_ICP            (1 << 17)
+#define SDE_DDIA_HOTPLUG_ICP            (1 << 16)
+#define SDE_DDI_MASK_ICP                (SDE_DDIB_HOTPLUG_ICP | \
+                                         SDE_DDIA_HOTPLUG_ICP)
+#define SDE_TC_MASK_ICP                 (SDE_TC4_HOTPLUG_ICP |  \
+                                         SDE_TC3_HOTPLUG_ICP |  \
+                                         SDE_TC2_HOTPLUG_ICP |  \
+                                         SDE_TC1_HOTPLUG_ICP)
+
 #define SDEISR  _MMIO(0xc4000)
 #define SDEIMR  _MMIO(0xc4004)
 #define SDEIIR  _MMIO(0xc4008)
@@ -7517,6 +7644,30 @@ enum {
 #define  PORTE_HOTPLUG_SHORT_DETECT     (1 << 0)
 #define  PORTE_HOTPLUG_LONG_DETECT      (2 << 0)
 
+/* This register is a reuse of PCH_PORT_HOTPLUG register. The
+ * functionality covered in PCH_PORT_HOTPLUG is split into
+ * SHOTPLUG_CTL_DDI and SHOTPLUG_CTL_TC.
+ */
+
+#define SHOTPLUG_CTL_DDI                        _MMIO(0xc4030)
+#define   ICP_DDIB_HPD_ENABLE                   (1 << 7)
+#define   ICP_DDIB_HPD_STATUS_MASK              (3 << 4)
+#define   ICP_DDIB_HPD_NO_DETECT                (0 << 4)
+#define   ICP_DDIB_HPD_SHORT_DETECT             (1 << 4)
+#define   ICP_DDIB_HPD_LONG_DETECT              (2 << 4)
+#define   ICP_DDIB_HPD_SHORT_LONG_DETECT        (3 << 4)
+#define   ICP_DDIA_HPD_ENABLE                   (1 << 3)
+#define   ICP_DDIA_HPD_STATUS_MASK              (3 << 0)
+#define   ICP_DDIA_HPD_NO_DETECT                (0 << 0)
+#define   ICP_DDIA_HPD_SHORT_DETECT             (1 << 0)
+#define   ICP_DDIA_HPD_LONG_DETECT              (2 << 0)
+#define   ICP_DDIA_HPD_SHORT_LONG_DETECT        (3 << 0)
+
+#define SHOTPLUG_CTL_TC                         _MMIO(0xc4034)
+#define   ICP_TC_HPD_ENABLE(tc_port)            (8 << (tc_port) * 4)
+#define   ICP_TC_HPD_LONG_DETECT(tc_port)       (2 << (tc_port) * 4)
+#define   ICP_TC_HPD_SHORT_DETECT(tc_port)      (1 << (tc_port) * 4)
+
 #define PCH_GPIOA               _MMIO(0xc5010)
 #define PCH_GPIOB               _MMIO(0xc5014)
 #define PCH_GPIOC               _MMIO(0xc5018)
@@ -8555,6 +8706,14 @@ enum {
 #define _HSW_PWR_WELL_CTL3                      0x45408
 #define _HSW_PWR_WELL_CTL4                      0x4540C
 
+#define _ICL_PWR_WELL_CTL_AUX1                  0x45440
+#define _ICL_PWR_WELL_CTL_AUX2                  0x45444
+#define _ICL_PWR_WELL_CTL_AUX4                  0x4544C
+
+#define _ICL_PWR_WELL_CTL_DDI1                  0x45450
+#define _ICL_PWR_WELL_CTL_DDI2                  0x45454
+#define _ICL_PWR_WELL_CTL_DDI4                  0x4545C
+
 /*
  * Each power well control register contains up to 16 (request, status) HW
  * flag tuples. The register index and HW flag shift is determined by the
@@ -8564,14 +8723,20 @@ enum {
  */
 #define _HSW_PW_REG_IDX(pw)                     ((pw) >> 4)
 #define _HSW_PW_SHIFT(pw)                       (((pw) & 0xf) * 2)
-/* TODO: Add all PWR_WELL_CTL registers below for new platforms */
 #define HSW_PWR_WELL_CTL_BIOS(pw)       _MMIO(_PICK(_HSW_PW_REG_IDX(pw),       \
-                                                    _HSW_PWR_WELL_CTL1))
+                                                    _HSW_PWR_WELL_CTL1,        \
+                                                    _ICL_PWR_WELL_CTL_AUX1,    \
+                                                    _ICL_PWR_WELL_CTL_DDI1))
 #define HSW_PWR_WELL_CTL_DRIVER(pw)     _MMIO(_PICK(_HSW_PW_REG_IDX(pw),       \
-                                                    _HSW_PWR_WELL_CTL2))
+                                                    _HSW_PWR_WELL_CTL2,        \
+                                                    _ICL_PWR_WELL_CTL_AUX2,    \
+                                                    _ICL_PWR_WELL_CTL_DDI2))
+/* KVMR doesn't have a reg for AUX or DDI power well control */
 #define HSW_PWR_WELL_CTL_KVMR           _MMIO(_HSW_PWR_WELL_CTL3)
 #define HSW_PWR_WELL_CTL_DEBUG(pw)      _MMIO(_PICK(_HSW_PW_REG_IDX(pw),       \
-                                                    _HSW_PWR_WELL_CTL4))
+                                                    _HSW_PWR_WELL_CTL4,        \
+                                                    _ICL_PWR_WELL_CTL_AUX4,    \
+                                                    _ICL_PWR_WELL_CTL_DDI4))
 
 #define   HSW_PWR_WELL_CTL_REQ(pw)              (1 << (_HSW_PW_SHIFT(pw) + 1))
 #define   HSW_PWR_WELL_CTL_STATE(pw)            (1 << _HSW_PW_SHIFT(pw))
@@ -8592,6 +8757,8 @@ enum skl_power_gate {
 #define  SKL_FUSE_DOWNLOAD_STATUS               (1 << 31)
 /* PG0 (HW control->no power well ID), PG1..PG2 (SKL_DISP_PW1..SKL_DISP_PW2) */
 #define  SKL_PW_TO_PG(pw)                       ((pw) - SKL_DISP_PW_1 + SKL_PG1)
+/* PG0 (HW control->no power well ID), PG1..PG4 (ICL_DISP_PW1..ICL_DISP_PW4) */
+#define  ICL_PW_TO_PG(pw)                       ((pw) - ICL_DISP_PW_1 + SKL_PG1)
 #define  SKL_FUSE_PG_DIST_STATUS(pg)            (1 << (27 - (pg)))
 
 #define _CNL_AUX_REG_IDX(pw)            ((pw) - 9)
@@ -9047,6 +9214,7 @@ enum skl_power_gate {
 #define _MG_REFCLKIN_CTL_PORT3                          0x16A92C
 #define _MG_REFCLKIN_CTL_PORT4                          0x16B92C
 #define   MG_REFCLKIN_CTL_OD_2_MUX(x)                   ((x) << 8)
+#define   MG_REFCLKIN_CTL_OD_2_MUX_MASK                 (0x7 << 8)
 #define MG_REFCLKIN_CTL(port) _MMIO_PORT((port) - PORT_C, \
                                          _MG_REFCLKIN_CTL_PORT1, \
                                          _MG_REFCLKIN_CTL_PORT2)
@@ -9056,7 +9224,9 @@ enum skl_power_gate {
 #define _MG_CLKTOP2_CORECLKCTL1_PORT3                   0x16A8D8
 #define _MG_CLKTOP2_CORECLKCTL1_PORT4                   0x16B8D8
 #define   MG_CLKTOP2_CORECLKCTL1_B_DIVRATIO(x)          ((x) << 16)
+#define   MG_CLKTOP2_CORECLKCTL1_B_DIVRATIO_MASK        (0xff << 16)
 #define   MG_CLKTOP2_CORECLKCTL1_A_DIVRATIO(x)          ((x) << 8)
+#define   MG_CLKTOP2_CORECLKCTL1_A_DIVRATIO_MASK        (0xff << 8)
 #define MG_CLKTOP2_CORECLKCTL1(port) _MMIO_PORT((port) - PORT_C, \
                                                 _MG_CLKTOP2_CORECLKCTL1_PORT1, \
                                                 _MG_CLKTOP2_CORECLKCTL1_PORT2)
@@ -9066,9 +9236,13 @@ enum skl_power_gate {
 #define _MG_CLKTOP2_HSCLKCTL_PORT3                      0x16A8D4
 #define _MG_CLKTOP2_HSCLKCTL_PORT4                      0x16B8D4
 #define   MG_CLKTOP2_HSCLKCTL_CORE_INPUTSEL(x)          ((x) << 16)
+#define   MG_CLKTOP2_HSCLKCTL_CORE_INPUTSEL_MASK        (0x1 << 16)
 #define   MG_CLKTOP2_HSCLKCTL_TLINEDRV_CLKSEL(x)        ((x) << 14)
+#define   MG_CLKTOP2_HSCLKCTL_TLINEDRV_CLKSEL_MASK      (0x3 << 14)
 #define   MG_CLKTOP2_HSCLKCTL_HSDIV_RATIO(x)            ((x) << 12)
+#define   MG_CLKTOP2_HSCLKCTL_HSDIV_RATIO_MASK          (0x3 << 12)
 #define   MG_CLKTOP2_HSCLKCTL_DSDIV_RATIO(x)            ((x) << 8)
+#define   MG_CLKTOP2_HSCLKCTL_DSDIV_RATIO_MASK          (0xf << 8)
 #define MG_CLKTOP2_HSCLKCTL(port) _MMIO_PORT((port) - PORT_C, \
                                              _MG_CLKTOP2_HSCLKCTL_PORT1, \
                                              _MG_CLKTOP2_HSCLKCTL_PORT2)
@@ -9142,12 +9316,18 @@ enum skl_power_gate {
 #define _MG_PLL_BIAS_PORT3                              0x16AA14
 #define _MG_PLL_BIAS_PORT4                              0x16BA14
 #define   MG_PLL_BIAS_BIAS_GB_SEL(x)                    ((x) << 30)
+#define   MG_PLL_BIAS_BIAS_GB_SEL_MASK                  (0x3 << 30)
 #define   MG_PLL_BIAS_INIT_DCOAMP(x)                    ((x) << 24)
+#define   MG_PLL_BIAS_INIT_DCOAMP_MASK                  (0x3f << 24)
 #define   MG_PLL_BIAS_BIAS_BONUS(x)                     ((x) << 16)
+#define   MG_PLL_BIAS_BIAS_BONUS_MASK                   (0xff << 16)
 #define   MG_PLL_BIAS_BIASCAL_EN                        (1 << 15)
 #define   MG_PLL_BIAS_CTRIM(x)                          ((x) << 8)
+#define   MG_PLL_BIAS_CTRIM_MASK                        (0x1f << 8)
 #define   MG_PLL_BIAS_VREF_RDAC(x)                      ((x) << 5)
+#define   MG_PLL_BIAS_VREF_RDAC_MASK                    (0x7 << 5)
 #define   MG_PLL_BIAS_IREFTRIM(x)                       ((x) << 0)
+#define   MG_PLL_BIAS_IREFTRIM_MASK                     (0x1f << 0)
 #define MG_PLL_BIAS(port) _MMIO_PORT((port) - PORT_C, _MG_PLL_BIAS_PORT1, \
                                      _MG_PLL_BIAS_PORT2)
 
@@ -9401,6 +9581,22 @@ enum skl_power_gate {
 #define MIPIO_TXESC_CLK_DIV2                    _MMIO(0x160008)
 #define  GLK_TX_ESC_CLK_DIV2_MASK                       0x3FF
 
+#define _ICL_DSI_ESC_CLK_DIV0           0x6b090
+#define _ICL_DSI_ESC_CLK_DIV1           0x6b890
+#define ICL_DSI_ESC_CLK_DIV(port)       _MMIO_PORT((port),      \
+                                                        _ICL_DSI_ESC_CLK_DIV0, \
+                                                        _ICL_DSI_ESC_CLK_DIV1)
+#define _ICL_DPHY_ESC_CLK_DIV0          0x162190
+#define _ICL_DPHY_ESC_CLK_DIV1          0x6C190
+#define ICL_DPHY_ESC_CLK_DIV(port)      _MMIO_PORT((port),      \
+                                                _ICL_DPHY_ESC_CLK_DIV0, \
+                                                _ICL_DPHY_ESC_CLK_DIV1)
+#define  ICL_BYTE_CLK_PER_ESC_CLK_MASK          (0x1f << 16)
+#define  ICL_BYTE_CLK_PER_ESC_CLK_SHIFT 16
+#define  ICL_ESC_CLK_DIV_MASK                   0x1ff
+#define  ICL_ESC_CLK_DIV_SHIFT                  0
+#define DSI_MAX_ESC_CLK                 20000           /* in KHz */
+
 /* Gen4+ Timestamp and Pipe Frame time stamp registers */
 #define GEN4_TIMESTAMP          _MMIO(0x2358)
 #define ILK_TIMESTAMP_HI        _MMIO(0x70070)
@@ -9535,6 +9731,14 @@ enum skl_power_gate {
 #define _BXT_MIPIC_PORT_CTRL                            0x6B8C0
 #define BXT_MIPI_PORT_CTRL(tc)  _MMIO_MIPI(tc, _BXT_MIPIA_PORT_CTRL, _BXT_MIPIC_PORT_CTRL)
 
+/* ICL DSI MODE control */
+#define _ICL_DSI_IO_MODECTL_0                           0x6B094
+#define _ICL_DSI_IO_MODECTL_1                           0x6B894
+#define ICL_DSI_IO_MODECTL(port)        _MMIO_PORT(port,        \
+                                                    _ICL_DSI_IO_MODECTL_0, \
+                                                    _ICL_DSI_IO_MODECTL_1)
+#define  COMBO_PHY_MODE_DSI                             (1 << 0)
+
 #define BXT_P_DSI_REGULATOR_CFG                 _MMIO(0x160020)
 #define  STAP_SELECT                                    (1 << 0)
 
diff --git a/drivers/gpu/drm/i915/i915_request.c b/drivers/gpu/drm/i915/i915_request.c
index e1dbb544046f..5c2c93cbab12 100644
--- a/drivers/gpu/drm/i915/i915_request.c
+++ b/drivers/gpu/drm/i915/i915_request.c
@@ -206,7 +206,8 @@ static int reset_all_global_seqno(struct drm_i915_private *i915, u32 seqno)
         /* Carefully retire all requests without writing to the rings */
         ret = i915_gem_wait_for_idle(i915,
                                      I915_WAIT_INTERRUPTIBLE |
-                                     I915_WAIT_LOCKED);
+                                     I915_WAIT_LOCKED,
+                                     MAX_SCHEDULE_TIMEOUT);
         if (ret)
                 return ret;
 
@@ -503,7 +504,7 @@ static void move_to_timeline(struct i915_request *request,
         GEM_BUG_ON(request->timeline == &request->engine->timeline);
         lockdep_assert_held(&request->engine->timeline.lock);
 
-        spin_lock_nested(&request->timeline->lock, SINGLE_DEPTH_NESTING);
+        spin_lock(&request->timeline->lock);
         list_move_tail(&request->link, &timeline->requests);
         spin_unlock(&request->timeline->lock);
 }
@@ -735,7 +736,8 @@ i915_request_alloc(struct intel_engine_cs *engine, struct i915_gem_context *ctx)
                 /* Ratelimit ourselves to prevent oom from malicious clients */
                 ret = i915_gem_wait_for_idle(i915,
                                              I915_WAIT_LOCKED |
-                                             I915_WAIT_INTERRUPTIBLE);
+                                             I915_WAIT_INTERRUPTIBLE,
+                                             MAX_SCHEDULE_TIMEOUT);
                 if (ret)
                         goto err_unreserve;
 
@@ -1013,6 +1015,27 @@ i915_request_await_object(struct i915_request *to,
         return ret;
 }
 
+void i915_request_skip(struct i915_request *rq, int error)
+{
+        void *vaddr = rq->ring->vaddr;
+        u32 head;
+
+        GEM_BUG_ON(!IS_ERR_VALUE((long)error));
+        dma_fence_set_error(&rq->fence, error);
+
+        /*
+         * As this request likely depends on state from the lost
+         * context, clear out all the user operations leaving the
+         * breadcrumb at the end (so we get the fence notifications).
+         */
+        head = rq->infix;
+        if (rq->postfix < head) {
+                memset(vaddr + head, 0, rq->ring->size - head);
+                head = 0;
+        }
+        memset(vaddr + head, 0, rq->postfix - head);
+}
+
 /*
  * NB: This function is not allowed to fail. Doing so would mean the the
  * request is not being tracked for completion but the work itself is
@@ -1196,7 +1219,7 @@ static bool __i915_spin_request(const struct i915_request *rq,
          * takes to sleep on a request, on the order of a microsecond.
          */
 
-        irq = atomic_read(&engine->irq_count);
+        irq = READ_ONCE(engine->breadcrumbs.irq_count);
         timeout_us += local_clock_us(&cpu);
         do {
                 if (i915_seqno_passed(intel_engine_get_seqno(engine), seqno))
@@ -1208,7 +1231,7 @@ static bool __i915_spin_request(const struct i915_request *rq,
                  * assume we won't see one in the near future but require
                  * the engine->seqno_barrier() to fixup coherency.
                  */
-                if (atomic_read(&engine->irq_count) != irq)
+                if (READ_ONCE(engine->breadcrumbs.irq_count) != irq)
                         break;
 
                 if (signal_pending_state(state, current))
@@ -1285,7 +1308,7 @@ long i915_request_wait(struct i915_request *rq,
         if (flags & I915_WAIT_LOCKED)
                 add_wait_queue(errq, &reset);
 
-        intel_wait_init(&wait, rq);
+        intel_wait_init(&wait);
 
 restart:
         do {
diff --git a/drivers/gpu/drm/i915/i915_request.h b/drivers/gpu/drm/i915/i915_request.h
index 7ee220ded9c9..e1c9365dfefb 100644
--- a/drivers/gpu/drm/i915/i915_request.h
+++ b/drivers/gpu/drm/i915/i915_request.h
@@ -258,6 +258,8 @@ void i915_request_add(struct i915_request *rq);
 void __i915_request_submit(struct i915_request *request);
 void i915_request_submit(struct i915_request *request);
 
+void i915_request_skip(struct i915_request *request, int error);
+
 void __i915_request_unsubmit(struct i915_request *request);
 void i915_request_unsubmit(struct i915_request *request);
 
@@ -378,6 +380,7 @@ static inline void
 init_request_active(struct i915_gem_active *active,
                     i915_gem_retire_fn retire)
 {
+        RCU_INIT_POINTER(active->request, NULL);
         INIT_LIST_HEAD(&active->link);
         active->retire = retire ?: i915_gem_retire_noop;
 }
diff --git a/drivers/gpu/drm/i915/i915_timeline.h b/drivers/gpu/drm/i915/i915_timeline.h
index dc2a4632faa7..a2c2c3ab5fb0 100644
--- a/drivers/gpu/drm/i915/i915_timeline.h
+++ b/drivers/gpu/drm/i915/i915_timeline.h
@@ -37,6 +37,8 @@ struct i915_timeline {
         u32 seqno;
 
         spinlock_t lock;
+#define TIMELINE_CLIENT 0 /* default subclass */
+#define TIMELINE_ENGINE 1
 
         /**
          * List of breadcrumbs associated with GPU requests currently
diff --git a/drivers/gpu/drm/i915/i915_vma.c b/drivers/gpu/drm/i915/i915_vma.c
index e82aa804cdba..ed4e0fb558f7 100644
--- a/drivers/gpu/drm/i915/i915_vma.c
+++ b/drivers/gpu/drm/i915/i915_vma.c
@@ -21,7 +21,7 @@
  * IN THE SOFTWARE.
  *
  */
- 
+
 #include "i915_vma.h"
 
 #include "i915_drv.h"
@@ -30,18 +30,53 @@
 
 #include <drm/drm_gem.h>
 
+#if IS_ENABLED(CONFIG_DRM_I915_ERRLOG_GEM) && IS_ENABLED(CONFIG_DRM_DEBUG_MM)
+
+#include <linux/stackdepot.h>
+
+static void vma_print_allocator(struct i915_vma *vma, const char *reason)
+{
+        unsigned long entries[12];
+        struct stack_trace trace = {
+                .entries = entries,
+                .max_entries = ARRAY_SIZE(entries),
+        };
+        char buf[512];
+
+        if (!vma->node.stack) {
+                DRM_DEBUG_DRIVER("vma.node [%08llx + %08llx] %s: unknown owner\n",
+                                 vma->node.start, vma->node.size, reason);
+                return;
+        }
+
+        depot_fetch_stack(vma->node.stack, &trace);
+        snprint_stack_trace(buf, sizeof(buf), &trace, 0);
+        DRM_DEBUG_DRIVER("vma.node [%08llx + %08llx] %s: inserted at %s\n",
+                         vma->node.start, vma->node.size, reason, buf);
+}
+
+#else
+
+static void vma_print_allocator(struct i915_vma *vma, const char *reason)
+{
+}
+
+#endif
+
+struct i915_vma_active {
+        struct i915_gem_active base;
+        struct i915_vma *vma;
+        struct rb_node node;
+        u64 timeline;
+};
+
 static void
-i915_vma_retire(struct i915_gem_active *active, struct i915_request *rq)
+__i915_vma_retire(struct i915_vma *vma, struct i915_request *rq)
 {
-        const unsigned int idx = rq->engine->id;
-        struct i915_vma *vma =
-                container_of(active, struct i915_vma, last_read[idx]);
         struct drm_i915_gem_object *obj = vma->obj;
 
-        GEM_BUG_ON(!i915_vma_has_active_engine(vma, idx));
-
-        i915_vma_clear_active(vma, idx);
-        if (i915_vma_is_active(vma))
+        GEM_BUG_ON(!i915_vma_is_active(vma));
+        if (--vma->active_count)
                 return;
 
         GEM_BUG_ON(!drm_mm_node_allocated(&vma->node));
@@ -75,6 +110,21 @@ i915_vma_retire(struct i915_gem_active *active, struct i915_request *rq)
         }
 }
 
+static void
+i915_vma_retire(struct i915_gem_active *base, struct i915_request *rq)
+{
+        struct i915_vma_active *active =
+                container_of(base, typeof(*active), base);
+
+        __i915_vma_retire(active->vma, rq);
+}
+
+static void
+i915_vma_last_retire(struct i915_gem_active *base, struct i915_request *rq)
+{
+        __i915_vma_retire(container_of(base, struct i915_vma, last_active), rq);
+}
+
 static struct i915_vma *
 vma_create(struct drm_i915_gem_object *obj,
            struct i915_address_space *vm,
@@ -82,7 +132,6 @@ vma_create(struct drm_i915_gem_object *obj,
 {
         struct i915_vma *vma;
         struct rb_node *rb, **p;
-        int i;
 
         /* The aliasing_ppgtt should never be used directly! */
         GEM_BUG_ON(vm == &vm->i915->mm.aliasing_ppgtt->vm);
@@ -91,8 +140,9 @@ vma_create(struct drm_i915_gem_object *obj,
         if (vma == NULL)
                 return ERR_PTR(-ENOMEM);
 
-        for (i = 0; i < ARRAY_SIZE(vma->last_read); i++)
-                init_request_active(&vma->last_read[i], i915_vma_retire);
+        vma->active = RB_ROOT;
+
+        init_request_active(&vma->last_active, i915_vma_last_retire);
         init_request_active(&vma->last_fence, NULL);
         vma->vm = vm;
         vma->ops = &vm->vma_ops;
@@ -110,7 +160,7 @@ vma_create(struct drm_i915_gem_object *obj,
                                                      obj->base.size >> PAGE_SHIFT));
                         vma->size = view->partial.size;
                         vma->size <<= PAGE_SHIFT;
-                        GEM_BUG_ON(vma->size >= obj->base.size);
+                        GEM_BUG_ON(vma->size > obj->base.size);
                 } else if (view->type == I915_GGTT_VIEW_ROTATED) {
                         vma->size = intel_rotation_info_size(&view->rotated);
                         vma->size <<= PAGE_SHIFT;
@@ -745,13 +795,11 @@ void i915_vma_reopen(struct i915_vma *vma)
 static void __i915_vma_destroy(struct i915_vma *vma)
 {
         struct drm_i915_private *i915 = vma->vm->i915;
-        int i;
+        struct i915_vma_active *iter, *n;
 
         GEM_BUG_ON(vma->node.allocated);
         GEM_BUG_ON(vma->fence);
 
-        for (i = 0; i < ARRAY_SIZE(vma->last_read); i++)
-                GEM_BUG_ON(i915_gem_active_isset(&vma->last_read[i]));
         GEM_BUG_ON(i915_gem_active_isset(&vma->last_fence));
 
         list_del(&vma->obj_link);
@@ -762,6 +810,11 @@ static void __i915_vma_destroy(struct i915_vma *vma)
         if (!i915_vma_is_ggtt(vma))
                 i915_ppgtt_put(i915_vm_to_ppgtt(vma->vm));
 
+        rbtree_postorder_for_each_entry_safe(iter, n, &vma->active, node) {
+                GEM_BUG_ON(i915_gem_active_isset(&iter->base));
+                kfree(iter);
+        }
+
         kmem_cache_free(i915->vmas, vma);
 }
 
@@ -826,9 +879,151 @@ void i915_vma_revoke_mmap(struct i915_vma *vma)
                 list_del(&vma->obj->userfault_link);
 }
 
+static void export_fence(struct i915_vma *vma,
+                         struct i915_request *rq,
+                         unsigned int flags)
+{
+        struct reservation_object *resv = vma->resv;
+
+        /*
+         * Ignore errors from failing to allocate the new fence, we can't
+         * handle an error right now. Worst case should be missed
+         * synchronisation leading to rendering corruption.
+         */
+        reservation_object_lock(resv, NULL);
+        if (flags & EXEC_OBJECT_WRITE)
+                reservation_object_add_excl_fence(resv, &rq->fence);
+        else if (reservation_object_reserve_shared(resv) == 0)
+                reservation_object_add_shared_fence(resv, &rq->fence);
+        reservation_object_unlock(resv);
+}
+
+static struct i915_gem_active *active_instance(struct i915_vma *vma, u64 idx)
+{
+        struct i915_vma_active *active;
+        struct rb_node **p, *parent;
+        struct i915_request *old;
+
+        /*
+         * We track the most recently used timeline to skip a rbtree search
+         * for the common case, under typical loads we never need the rbtree
+         * at all. We can reuse the last_active slot if it is empty, that is
+         * after the previous activity has been retired, or if the active
+         * matches the current timeline.
+         *
+         * Note that we allow the timeline to be active simultaneously in
+         * the rbtree and the last_active cache. We do this to avoid having
+         * to search and replace the rbtree element for a new timeline, with
+         * the cost being that we must be aware that the vma may be retired
+         * twice for the same timeline (as the older rbtree element will be
+         * retired before the new request added to last_active).
+         */
+        old = i915_gem_active_raw(&vma->last_active,
+                                  &vma->vm->i915->drm.struct_mutex);
+        if (!old || old->fence.context == idx)
+                goto out;
+
+        /* Move the currently active fence into the rbtree */
+        idx = old->fence.context;
+
+        parent = NULL;
+        p = &vma->active.rb_node;
+        while (*p) {
+                parent = *p;
+
+                active = rb_entry(parent, struct i915_vma_active, node);
+                if (active->timeline == idx)
+                        goto replace;
+
+                if (active->timeline < idx)
+                        p = &parent->rb_right;
+                else
+                        p = &parent->rb_left;
+        }
+
+        active = kmalloc(sizeof(*active), GFP_KERNEL);
+        if (unlikely(!active))
+                return ERR_PTR(-ENOMEM);
+
+        init_request_active(&active->base, i915_vma_retire);
+        active->vma = vma;
+        active->timeline = idx;
+
+        rb_link_node(&active->node, parent, p);
+        rb_insert_color(&active->node, &vma->active);
+
+replace:
+        /*
+         * Overwrite the previous active slot in the rbtree with last_active,
+         * leaving last_active zeroed. If the previous slot is still active,
+         * we must be careful as we now only expect to receive one retire
+         * callback not two, and so much undo the active counting for the
+         * overwritten slot.
+         */
+        if (i915_gem_active_isset(&active->base)) {
+                /* Retire ourselves from the old rq->active_list */
+                __list_del_entry(&active->base.link);
+                vma->active_count--;
+                GEM_BUG_ON(!vma->active_count);
+        }
+        GEM_BUG_ON(list_empty(&vma->last_active.link));
+        list_replace_init(&vma->last_active.link, &active->base.link);
+        active->base.request = fetch_and_zero(&vma->last_active.request);
+
+out:
+        return &vma->last_active;
+}
+
+int i915_vma_move_to_active(struct i915_vma *vma,
+                            struct i915_request *rq,
+                            unsigned int flags)
+{
+        struct drm_i915_gem_object *obj = vma->obj;
+        struct i915_gem_active *active;
+
+        lockdep_assert_held(&rq->i915->drm.struct_mutex);
+        GEM_BUG_ON(!drm_mm_node_allocated(&vma->node));
+
+        active = active_instance(vma, rq->fence.context);
+        if (IS_ERR(active))
+                return PTR_ERR(active);
+
+        /*
+         * Add a reference if we're newly entering the active list.
+         * The order in which we add operations to the retirement queue is
+         * vital here: mark_active adds to the start of the callback list,
+         * such that subsequent callbacks are called first. Therefore we
+         * add the active reference first and queue for it to be dropped
+         * *last*.
+         */
+        if (!i915_gem_active_isset(active) && !vma->active_count++) {
+                list_move_tail(&vma->vm_link, &vma->vm->active_list);
+                obj->active_count++;
+        }
+        i915_gem_active_set(active, rq);
+        GEM_BUG_ON(!i915_vma_is_active(vma));
+        GEM_BUG_ON(!obj->active_count);
+
+        obj->write_domain = 0;
+        if (flags & EXEC_OBJECT_WRITE) {
+                obj->write_domain = I915_GEM_DOMAIN_RENDER;
+
+                if (intel_fb_obj_invalidate(obj, ORIGIN_CS))
+                        i915_gem_active_set(&obj->frontbuffer_write, rq);
+
+                obj->read_domains = 0;
+        }
+        obj->read_domains |= I915_GEM_GPU_DOMAINS;
+
+        if (flags & EXEC_OBJECT_NEEDS_FENCE)
+                i915_gem_active_set(&vma->last_fence, rq);
+
+        export_fence(vma, rq, flags);
+        return 0;
+}
+
 int i915_vma_unbind(struct i915_vma *vma)
 {
-        unsigned long active;
         int ret;
 
         lockdep_assert_held(&vma->vm->i915->drm.struct_mutex);
@@ -838,9 +1033,8 @@ int i915_vma_unbind(struct i915_vma *vma)
          * have side-effects such as unpinning or even unbinding this vma.
          */
         might_sleep();
-        active = i915_vma_get_active(vma);
-        if (active) {
-                int idx;
+        if (i915_vma_is_active(vma)) {
+                struct i915_vma_active *active, *n;
 
                 /*
                  * When a closed VMA is retired, it is unbound - eek.
@@ -857,26 +1051,32 @@ int i915_vma_unbind(struct i915_vma *vma)
                  */
                 __i915_vma_pin(vma);
 
-                for_each_active(active, idx) {
-                        ret = i915_gem_active_retire(&vma->last_read[idx],
-                                                     &vma->vm->i915->drm.struct_mutex);
-                        if (ret)
-                                break;
-                }
+                ret = i915_gem_active_retire(&vma->last_active,
+                                             &vma->vm->i915->drm.struct_mutex);
+                if (ret)
+                        goto unpin;
 
-                if (!ret) {
-                        ret = i915_gem_active_retire(&vma->last_fence,
+                rbtree_postorder_for_each_entry_safe(active, n,
+                                                     &vma->active, node) {
+                        ret = i915_gem_active_retire(&active->base,
                                                      &vma->vm->i915->drm.struct_mutex);
+                        if (ret)
+                                goto unpin;
                 }
 
+                ret = i915_gem_active_retire(&vma->last_fence,
+                                             &vma->vm->i915->drm.struct_mutex);
+unpin:
                 __i915_vma_unpin(vma);
                 if (ret)
                         return ret;
         }
         GEM_BUG_ON(i915_vma_is_active(vma));
 
-        if (i915_vma_is_pinned(vma))
+        if (i915_vma_is_pinned(vma)) {
+                vma_print_allocator(vma, "is pinned");
                 return -EBUSY;
+        }
 
         if (!drm_mm_node_allocated(&vma->node))
                 return 0;
diff --git a/drivers/gpu/drm/i915/i915_vma.h b/drivers/gpu/drm/i915/i915_vma.h
index 66a228931517..f06d66377107 100644
--- a/drivers/gpu/drm/i915/i915_vma.h
+++ b/drivers/gpu/drm/i915/i915_vma.h
@@ -26,6 +26,7 @@
 #define __I915_VMA_H__
 
 #include <linux/io-mapping.h>
+#include <linux/rbtree.h>
 
 #include <drm/drm_mm.h>
 
@@ -94,8 +95,9 @@ struct i915_vma {
 #define I915_VMA_USERFAULT      BIT(I915_VMA_USERFAULT_BIT)
 #define I915_VMA_GGTT_WRITE     BIT(12)
 
-        unsigned int active;
-        struct i915_gem_active last_read[I915_NUM_ENGINES];
+        unsigned int active_count;
+        struct rb_root active;
+        struct i915_gem_active last_active;
         struct i915_gem_active last_fence;
 
         /**
@@ -138,6 +140,15 @@ i915_vma_instance(struct drm_i915_gem_object *obj,
 
 void i915_vma_unpin_and_release(struct i915_vma **p_vma);
 
+static inline bool i915_vma_is_active(struct i915_vma *vma)
+{
+        return vma->active_count;
+}
+
+int __must_check i915_vma_move_to_active(struct i915_vma *vma,
+                                         struct i915_request *rq,
+                                         unsigned int flags);
+
 static inline bool i915_vma_is_ggtt(const struct i915_vma *vma)
 {
         return vma->flags & I915_VMA_GGTT;
@@ -187,34 +198,6 @@ static inline bool i915_vma_has_userfault(const struct i915_vma *vma)
         return test_bit(I915_VMA_USERFAULT_BIT, &vma->flags);
 }
 
-static inline unsigned int i915_vma_get_active(const struct i915_vma *vma)
-{
-        return vma->active;
-}
-
-static inline bool i915_vma_is_active(const struct i915_vma *vma)
-{
-        return i915_vma_get_active(vma);
-}
-
-static inline void i915_vma_set_active(struct i915_vma *vma,
-                                       unsigned int engine)
-{
-        vma->active |= BIT(engine);
-}
-
-static inline void i915_vma_clear_active(struct i915_vma *vma,
-                                         unsigned int engine)
-{
-        vma->active &= ~BIT(engine);
-}
-
-static inline bool i915_vma_has_active_engine(const struct i915_vma *vma,
-                                              unsigned int engine)
-{
-        return vma->active & BIT(engine);
-}
-
 static inline u32 i915_ggtt_offset(const struct i915_vma *vma)
 {
         GEM_BUG_ON(!i915_vma_is_ggtt(vma));
diff --git a/drivers/gpu/drm/i915/icl_dsi.c b/drivers/gpu/drm/i915/icl_dsi.c
new file mode 100644
index 000000000000..13830e43a4d1
--- /dev/null
+++ b/drivers/gpu/drm/i915/icl_dsi.c
@@ -0,0 +1,127 @@
+/*
+ * Copyright © 2018 Intel Corporation
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ *
+ * Authors:
+ *   Madhav Chauhan <madhav.chauhan@intel.com>
+ *   Jani Nikula <jani.nikula@intel.com>
+ */
+
+#include "intel_dsi.h"
+
+static void gen11_dsi_program_esc_clk_div(struct intel_encoder *encoder)
+{
+        struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
+        struct intel_dsi *intel_dsi = enc_to_intel_dsi(&encoder->base);
+        enum port port;
+        u32 bpp = mipi_dsi_pixel_format_to_bpp(intel_dsi->pixel_format);
+        u32 afe_clk_khz; /* 8X Clock */
+        u32 esc_clk_div_m;
+
+        afe_clk_khz = DIV_ROUND_CLOSEST(intel_dsi->pclk * bpp,
+                                        intel_dsi->lane_count);
+
+        esc_clk_div_m = DIV_ROUND_UP(afe_clk_khz, DSI_MAX_ESC_CLK);
+
+        for_each_dsi_port(port, intel_dsi->ports) {
+                I915_WRITE(ICL_DSI_ESC_CLK_DIV(port),
+                           esc_clk_div_m & ICL_ESC_CLK_DIV_MASK);
+                POSTING_READ(ICL_DSI_ESC_CLK_DIV(port));
+        }
+
+        for_each_dsi_port(port, intel_dsi->ports) {
+                I915_WRITE(ICL_DPHY_ESC_CLK_DIV(port),
+                           esc_clk_div_m & ICL_ESC_CLK_DIV_MASK);
+                POSTING_READ(ICL_DPHY_ESC_CLK_DIV(port));
+        }
+}
+
+static void gen11_dsi_enable_io_power(struct intel_encoder *encoder)
+{
+        struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
+        struct intel_dsi *intel_dsi = enc_to_intel_dsi(&encoder->base);
+        enum port port;
+        u32 tmp;
+
+        for_each_dsi_port(port, intel_dsi->ports) {
+                tmp = I915_READ(ICL_DSI_IO_MODECTL(port));
+                tmp |= COMBO_PHY_MODE_DSI;
+                I915_WRITE(ICL_DSI_IO_MODECTL(port), tmp);
+        }
+
+        for_each_dsi_port(port, intel_dsi->ports) {
+                intel_display_power_get(dev_priv, port == PORT_A ?
+                                        POWER_DOMAIN_PORT_DDI_A_IO :
+                                        POWER_DOMAIN_PORT_DDI_B_IO);
+        }
+}
+
+static void gen11_dsi_power_up_lanes(struct intel_encoder *encoder)
+{
+        struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
+        struct intel_dsi *intel_dsi = enc_to_intel_dsi(&encoder->base);
+        enum port port;
+        u32 tmp;
+        u32 lane_mask;
+
+        switch (intel_dsi->lane_count) {
+        case 1:
+                lane_mask = PWR_DOWN_LN_3_1_0;
+                break;
+        case 2:
+                lane_mask = PWR_DOWN_LN_3_1;
+                break;
+        case 3:
+                lane_mask = PWR_DOWN_LN_3;
+                break;
+        case 4:
+        default:
+                lane_mask = PWR_UP_ALL_LANES;
+                break;
+        }
+
+        for_each_dsi_port(port, intel_dsi->ports) {
+                tmp = I915_READ(ICL_PORT_CL_DW10(port));
+                tmp &= ~PWR_DOWN_LN_MASK;
+                I915_WRITE(ICL_PORT_CL_DW10(port), tmp | lane_mask);
+        }
+}
+
+static void gen11_dsi_enable_port_and_phy(struct intel_encoder *encoder)
+{
+        /* step 4a: power up all lanes of the DDI used by DSI */
+        gen11_dsi_power_up_lanes(encoder);
+}
+
+static void __attribute__((unused))
+gen11_dsi_pre_enable(struct intel_encoder *encoder,
+                     const struct intel_crtc_state *pipe_config,
+                     const struct drm_connector_state *conn_state)
+{
+        /* step2: enable IO power */
+        gen11_dsi_enable_io_power(encoder);
+
+        /* step3: enable DSI PLL */
+        gen11_dsi_program_esc_clk_div(encoder);
+
+        /* step4: enable DSI port and DPHY */
+        gen11_dsi_enable_port_and_phy(encoder);
+}
diff --git a/drivers/gpu/drm/i915/intel_breadcrumbs.c b/drivers/gpu/drm/i915/intel_breadcrumbs.c
index 86a987b8ac66..1db6ba7d926e 100644
--- a/drivers/gpu/drm/i915/intel_breadcrumbs.c
+++ b/drivers/gpu/drm/i915/intel_breadcrumbs.c
@@ -98,12 +98,14 @@ static void intel_breadcrumbs_hangcheck(struct timer_list *t)
         struct intel_engine_cs *engine =
                 from_timer(engine, t, breadcrumbs.hangcheck);
         struct intel_breadcrumbs *b = &engine->breadcrumbs;
+        unsigned int irq_count;
 
         if (!b->irq_armed)
                 return;
 
-        if (b->hangcheck_interrupts != atomic_read(&engine->irq_count)) {
-                b->hangcheck_interrupts = atomic_read(&engine->irq_count);
+        irq_count = READ_ONCE(b->irq_count);
+        if (b->hangcheck_interrupts != irq_count) {
+                b->hangcheck_interrupts = irq_count;
                 mod_timer(&b->hangcheck, wait_timeout());
                 return;
         }
@@ -272,13 +274,14 @@ static bool use_fake_irq(const struct intel_breadcrumbs *b)
         if (!test_bit(engine->id, &engine->i915->gpu_error.missed_irq_rings))
                 return false;
 
-        /* Only start with the heavy weight fake irq timer if we have not
+        /*
+         * Only start with the heavy weight fake irq timer if we have not
          * seen any interrupts since enabling it the first time. If the
          * interrupts are still arriving, it means we made a mistake in our
          * engine->seqno_barrier(), a timing error that should be transient
          * and unlikely to reoccur.
          */
-        return atomic_read(&engine->irq_count) == b->hangcheck_interrupts;
+        return READ_ONCE(b->irq_count) == b->hangcheck_interrupts;
 }
 
 static void enable_fake_irq(struct intel_breadcrumbs *b)
diff --git a/drivers/gpu/drm/i915/intel_cdclk.c b/drivers/gpu/drm/i915/intel_cdclk.c
index bf9433d7964d..29075c763428 100644
--- a/drivers/gpu/drm/i915/intel_cdclk.c
+++ b/drivers/gpu/drm/i915/intel_cdclk.c
@@ -316,6 +316,7 @@ static void pnv_get_cdclk(struct drm_i915_private *dev_priv,
                 break;
         default:
                 DRM_ERROR("Unknown pnv display core clock 0x%04x\n", gcfgc);
+                /* fall through */
         case GC_DISPLAY_CLOCK_133_MHZ_PNV:
                 cdclk_state->cdclk = 133333;
                 break;
@@ -1797,6 +1798,7 @@ static int icl_calc_cdclk(int min_cdclk, unsigned int ref)
         switch (ref) {
         default:
                 MISSING_CASE(ref);
+                /* fall through */
         case 24000:
                 ranges = ranges_24;
                 break;
@@ -1824,6 +1826,7 @@ static int icl_calc_cdclk_pll_vco(struct drm_i915_private *dev_priv, int cdclk)
         switch (cdclk) {
         default:
                 MISSING_CASE(cdclk);
+                /* fall through */
         case 307200:
         case 556800:
         case 652800:
@@ -1896,6 +1899,7 @@ static u8 icl_calc_voltage_level(int cdclk)
                 return 1;
         default:
                 MISSING_CASE(cdclk);
+                /* fall through */
         case 652800:
         case 648000:
                 return 2;
@@ -1913,6 +1917,7 @@ static void icl_get_cdclk(struct drm_i915_private *dev_priv,
         switch (val & ICL_DSSM_CDCLK_PLL_REFCLK_MASK) {
         default:
                 MISSING_CASE(val);
+                /* fall through */
         case ICL_DSSM_CDCLK_PLL_REFCLK_24MHz:
                 cdclk_state->ref = 24000;
                 break;
diff --git a/drivers/gpu/drm/i915/intel_ddi.c b/drivers/gpu/drm/i915/intel_ddi.c
index 044fe1fb9872..32838ed89ee7 100644
--- a/drivers/gpu/drm/i915/intel_ddi.c
+++ b/drivers/gpu/drm/i915/intel_ddi.c
@@ -1069,6 +1069,7 @@ static uint32_t icl_pll_to_ddi_pll_sel(struct intel_encoder *encoder,
         switch (id) {
         default:
                 MISSING_CASE(id);
+                /* fall through */
         case DPLL_ID_ICL_DPLL0:
         case DPLL_ID_ICL_DPLL1:
                 return DDI_CLK_SEL_NONE;
@@ -1983,15 +1984,50 @@ out:
         return ret;
 }
 
-static u64 intel_ddi_get_power_domains(struct intel_encoder *encoder)
+static inline enum intel_display_power_domain
+intel_ddi_main_link_aux_domain(struct intel_dp *intel_dp)
+{
+        /* CNL HW requires corresponding AUX IOs to be powered up for PSR with
+         * DC states enabled at the same time, while for driver initiated AUX
+         * transfers we need the same AUX IOs to be powered but with DC states
+         * disabled. Accordingly use the AUX power domain here which leaves DC
+         * states enabled.
+         * However, for non-A AUX ports the corresponding non-EDP transcoders
+         * would have already enabled power well 2 and DC_OFF. This means we can
+         * acquire a wider POWER_DOMAIN_AUX_{B,C,D,F} reference instead of a
+         * specific AUX_IO reference without powering up any extra wells.
+         * Note that PSR is enabled only on Port A even though this function
+         * returns the correct domain for other ports too.
+         */
+        return intel_dp->aux_ch == AUX_CH_A ? POWER_DOMAIN_AUX_IO_A :
+                                              intel_dp->aux_power_domain;
+}
+
+static u64 intel_ddi_get_power_domains(struct intel_encoder *encoder,
+                                       struct intel_crtc_state *crtc_state)
 {
-        struct intel_digital_port *dig_port = enc_to_dig_port(&encoder->base);
-        enum pipe pipe;
+        struct intel_digital_port *dig_port;
+        u64 domains;
 
-        if (intel_ddi_get_hw_state(encoder, &pipe))
-                return BIT_ULL(dig_port->ddi_io_power_domain);
+        /*
+         * TODO: Add support for MST encoders. Atm, the following should never
+         * happen since fake-MST encoders don't set their get_power_domains()
+         * hook.
+         */
+        if (WARN_ON(intel_crtc_has_type(crtc_state, INTEL_OUTPUT_DP_MST)))
+                return 0;
 
-        return 0;
+        dig_port = enc_to_dig_port(&encoder->base);
+        domains = BIT_ULL(dig_port->ddi_io_power_domain);
+
+        /* AUX power is only needed for (e)DP mode, not for HDMI. */
+        if (intel_crtc_has_dp_encoder(crtc_state)) {
+                struct intel_dp *intel_dp = &dig_port->dp;
+
+                domains |= BIT_ULL(intel_ddi_main_link_aux_domain(intel_dp));
+        }
+
+        return domains;
 }
 
 void intel_ddi_enable_pipe_clock(const struct intel_crtc_state *crtc_state)
@@ -2631,6 +2667,9 @@ static void intel_ddi_pre_enable_dp(struct intel_encoder *encoder,
 
         WARN_ON(is_mst && (port == PORT_A || port == PORT_E));
 
+        intel_display_power_get(dev_priv,
+                                intel_ddi_main_link_aux_domain(intel_dp));
+
         intel_dp_set_link_params(intel_dp, crtc_state->port_clock,
                                  crtc_state->lane_count, is_mst);
 
@@ -2775,6 +2814,9 @@ static void intel_ddi_post_disable_dp(struct intel_encoder *encoder,
         intel_display_power_put(dev_priv, dig_port->ddi_io_power_domain);
 
         intel_ddi_clk_disable(encoder);
+
+        intel_display_power_put(dev_priv,
+                                intel_ddi_main_link_aux_domain(intel_dp));
 }
 
 static void intel_ddi_post_disable_hdmi(struct intel_encoder *encoder,
diff --git a/drivers/gpu/drm/i915/intel_device_info.c b/drivers/gpu/drm/i915/intel_device_info.c
index 0fd13df424cf..0ef0c6448d53 100644
--- a/drivers/gpu/drm/i915/intel_device_info.c
+++ b/drivers/gpu/drm/i915/intel_device_info.c
@@ -858,6 +858,8 @@ void intel_device_info_runtime_init(struct intel_device_info *info)
 void intel_driver_caps_print(const struct intel_driver_caps *caps,
                              struct drm_printer *p)
 {
+        drm_printf(p, "Has logical contexts? %s\n",
+                   yesno(caps->has_logical_contexts));
         drm_printf(p, "scheduler: %x\n", caps->scheduler);
 }
 
diff --git a/drivers/gpu/drm/i915/intel_device_info.h b/drivers/gpu/drm/i915/intel_device_info.h
index 933e31669557..633f9fbf72ea 100644
--- a/drivers/gpu/drm/i915/intel_device_info.h
+++ b/drivers/gpu/drm/i915/intel_device_info.h
@@ -186,6 +186,7 @@ struct intel_device_info {
 
 struct intel_driver_caps {
         unsigned int scheduler;
+        bool has_logical_contexts:1;
 };
 
 static inline unsigned int sseu_subslice_total(const struct sseu_dev_info *sseu)
diff --git a/drivers/gpu/drm/i915/intel_display.c b/drivers/gpu/drm/i915/intel_display.c
index 694a4703042f..8f3199b06d1f 100644
--- a/drivers/gpu/drm/i915/intel_display.c
+++ b/drivers/gpu/drm/i915/intel_display.c
@@ -5632,6 +5632,7 @@ static void haswell_crtc_enable(struct intel_crtc_state *pipe_config,
         struct intel_atomic_state *old_intel_state =
                 to_intel_atomic_state(old_state);
         bool psl_clkgate_wa;
+        u32 pipe_chicken;
 
         if (WARN_ON(intel_crtc->active))
                 return;
@@ -5691,6 +5692,17 @@ static void haswell_crtc_enable(struct intel_crtc_state *pipe_config,
          */
         intel_color_load_luts(&pipe_config->base);
 
+        /*
+         * Display WA #1153: enable hardware to bypass the alpha math
+         * and rounding for per-pixel values 00 and 0xff
+         */
+        if (INTEL_GEN(dev_priv) >= 11) {
+                pipe_chicken = I915_READ(PIPE_CHICKEN(pipe));
+                if (!(pipe_chicken & PER_PIXEL_ALPHA_BYPASS_EN))
+                        I915_WRITE_FW(PIPE_CHICKEN(pipe),
+                                      pipe_chicken | PER_PIXEL_ALPHA_BYPASS_EN);
+        }
+
         intel_ddi_set_pipe_settings(pipe_config);
         if (!transcoder_is_dsi(cpu_transcoder))
                 intel_ddi_enable_transcoder_func(pipe_config);
@@ -9347,6 +9359,7 @@ static bool hsw_get_transcoder_state(struct intel_crtc *crtc,
                 switch (tmp & TRANS_DDI_EDP_INPUT_MASK) {
                 default:
                         WARN(1, "unknown pipe linked to edp transcoder\n");
+                        /* fall through */
                 case TRANS_DDI_EDP_INPUT_A_ONOFF:
                 case TRANS_DDI_EDP_INPUT_A_ON:
                         trans_edp_pipe = PIPE_A;
@@ -9402,7 +9415,7 @@ static bool bxt_get_dsi_transcoder_state(struct intel_crtc *crtc,
                  * registers/MIPI[BXT]. We can break out here early, since we
                  * need the same DSI PLL to be enabled for both DSI ports.
                  */
-                if (!intel_dsi_pll_is_enabled(dev_priv))
+                if (!bxt_dsi_pll_is_enabled(dev_priv))
                         break;
 
                 /* XXX: this works for video mode only */
@@ -10724,7 +10737,7 @@ static void intel_modeset_update_connector_atomic_state(struct drm_device *dev)
         drm_connector_list_iter_begin(dev, &conn_iter);
         for_each_intel_connector_iter(connector, &conn_iter) {
                 if (connector->base.state->crtc)
-                        drm_connector_unreference(&connector->base);
+                        drm_connector_put(&connector->base);
 
                 if (connector->base.encoder) {
                         connector->base.state->best_encoder =
@@ -10732,7 +10745,7 @@ static void intel_modeset_update_connector_atomic_state(struct drm_device *dev)
                         connector->base.state->crtc =
                                 connector->base.encoder->crtc;
 
-                        drm_connector_reference(&connector->base);
+                        drm_connector_get(&connector->base);
                 } else {
                         connector->base.state->best_encoder = NULL;
                         connector->base.state->crtc = NULL;
@@ -11011,6 +11024,7 @@ static bool check_digital_port_conflicts(struct drm_atomic_state *state)
                 case INTEL_OUTPUT_DDI:
                         if (WARN_ON(!HAS_DDI(to_i915(dev))))
                                 break;
+                        /* else: fall through */
                 case INTEL_OUTPUT_DP:
                 case INTEL_OUTPUT_HDMI:
                 case INTEL_OUTPUT_EDP:
@@ -12542,6 +12556,19 @@ static void intel_atomic_commit_fence_wait(struct intel_atomic_state *intel_stat
         finish_wait(&dev_priv->gpu_error.wait_queue, &wait_reset);
 }
 
+static void intel_atomic_cleanup_work(struct work_struct *work)
+{
+        struct drm_atomic_state *state =
+                container_of(work, struct drm_atomic_state, commit_work);
+        struct drm_i915_private *i915 = to_i915(state->dev);
+
+        drm_atomic_helper_cleanup_planes(&i915->drm, state);
+        drm_atomic_helper_commit_cleanup_done(state);
+        drm_atomic_state_put(state);
+
+        intel_atomic_helper_free_state(i915);
+}
+
 static void intel_atomic_commit_tail(struct drm_atomic_state *state)
 {
         struct drm_device *dev = state->dev;
@@ -12702,13 +12729,16 @@ static void intel_atomic_commit_tail(struct drm_atomic_state *state)
                 intel_display_power_put(dev_priv, POWER_DOMAIN_MODESET);
         }
 
-        drm_atomic_helper_cleanup_planes(dev, state);
-
-        drm_atomic_helper_commit_cleanup_done(state);
-
-        drm_atomic_state_put(state);
-
-        intel_atomic_helper_free_state(dev_priv);
+        /*
+         * Defer the cleanup of the old state to a separate worker to not
+         * impede the current task (userspace for blocking modesets) that
+         * are executed inline. For out-of-line asynchronous modesets/flips,
+         * deferring to a new worker seems overkill, but we would place a
+         * schedule point (cond_resched()) here anyway to keep latencies
+         * down.
+         */
+        INIT_WORK(&state->commit_work, intel_atomic_cleanup_work);
+        schedule_work(&state->commit_work);
 }
 
 static void intel_atomic_commit_work(struct work_struct *work)
@@ -14105,7 +14135,7 @@ static void intel_setup_outputs(struct drm_i915_private *dev_priv)
                 intel_ddi_init(dev_priv, PORT_B);
                 intel_ddi_init(dev_priv, PORT_C);
 
-                intel_dsi_init(dev_priv);
+                vlv_dsi_init(dev_priv);
         } else if (HAS_DDI(dev_priv)) {
                 int found;
 
@@ -14211,7 +14241,7 @@ static void intel_setup_outputs(struct drm_i915_private *dev_priv)
                                 intel_hdmi_init(dev_priv, CHV_HDMID, PORT_D);
                 }
 
-                intel_dsi_init(dev_priv);
+                vlv_dsi_init(dev_priv);
         } else if (!IS_GEN2(dev_priv) && !IS_PINEVIEW(dev_priv)) {
                 bool found = false;
 
@@ -14493,11 +14523,6 @@ static int intel_framebuffer_init(struct intel_framebuffer *intel_fb,
                 }
                 break;
         case DRM_FORMAT_NV12:
-                if (mode_cmd->modifier[0] == I915_FORMAT_MOD_Y_TILED_CCS ||
-                    mode_cmd->modifier[0] == I915_FORMAT_MOD_Yf_TILED_CCS) {
-                        DRM_DEBUG_KMS("RC not to be enabled with NV12\n");
-                        goto err;
-                }
                 if (INTEL_GEN(dev_priv) < 9 || IS_SKYLAKE(dev_priv) ||
                     IS_BROXTON(dev_priv)) {
                         DRM_DEBUG_KMS("unsupported pixel format: %s\n",
@@ -15676,11 +15701,20 @@ get_encoder_power_domains(struct drm_i915_private *dev_priv)
         for_each_intel_encoder(&dev_priv->drm, encoder) {
                 u64 get_domains;
                 enum intel_display_power_domain domain;
+                struct intel_crtc_state *crtc_state;
 
                 if (!encoder->get_power_domains)
                         continue;
 
-                get_domains = encoder->get_power_domains(encoder);
+                /*
+                 * MST-primary and inactive encoders don't have a crtc state
+                 * and neither of these require any power domain references.
+                 */
+                if (!encoder->base.crtc)
+                        continue;
+
+                crtc_state = to_intel_crtc_state(encoder->base.crtc->state);
+                get_domains = encoder->get_power_domains(encoder, crtc_state);
                 for_each_power_domain(domain, get_domains)
                         intel_display_power_get(dev_priv, domain);
         }
diff --git a/drivers/gpu/drm/i915/intel_display.h b/drivers/gpu/drm/i915/intel_display.h
index dd30cae5eb00..ca5a10f3400d 100644
--- a/drivers/gpu/drm/i915/intel_display.h
+++ b/drivers/gpu/drm/i915/intel_display.h
@@ -199,6 +199,10 @@ enum intel_display_power_domain {
         POWER_DOMAIN_AUX_E,
         POWER_DOMAIN_AUX_F,
         POWER_DOMAIN_AUX_IO_A,
+        POWER_DOMAIN_AUX_TBT1,
+        POWER_DOMAIN_AUX_TBT2,
+        POWER_DOMAIN_AUX_TBT3,
+        POWER_DOMAIN_AUX_TBT4,
         POWER_DOMAIN_GMBUS,
         POWER_DOMAIN_MODESET,
         POWER_DOMAIN_GT_IRQ,
diff --git a/drivers/gpu/drm/i915/intel_dp.c b/drivers/gpu/drm/i915/intel_dp.c
index 6ac6c8787dcf..5be07e1d816d 100644
--- a/drivers/gpu/drm/i915/intel_dp.c
+++ b/drivers/gpu/drm/i915/intel_dp.c
@@ -953,7 +953,7 @@ intel_dp_check_edp(struct intel_dp *intel_dp)
 }
 
 static uint32_t
-intel_dp_aux_wait_done(struct intel_dp *intel_dp, bool has_aux_irq)
+intel_dp_aux_wait_done(struct intel_dp *intel_dp)
 {
         struct drm_i915_private *dev_priv = to_i915(intel_dp_to_dev(intel_dp));
         i915_reg_t ch_ctl = intel_dp->aux_ch_ctl_reg(intel_dp);
@@ -961,14 +961,10 @@ intel_dp_aux_wait_done(struct intel_dp *intel_dp, bool has_aux_irq)
         bool done;
 
 #define C (((status = I915_READ_NOTRACE(ch_ctl)) & DP_AUX_CH_CTL_SEND_BUSY) == 0)
-        if (has_aux_irq)
-                done = wait_event_timeout(dev_priv->gmbus_wait_queue, C,
-                                          msecs_to_jiffies_timeout(10));
-        else
-                done = wait_for(C, 10) == 0;
+        done = wait_event_timeout(dev_priv->gmbus_wait_queue, C,
+                                  msecs_to_jiffies_timeout(10));
         if (!done)
-                DRM_ERROR("dp aux hw did not signal timeout (has irq: %i)!\n",
-                          has_aux_irq);
+                DRM_ERROR("dp aux hw did not signal timeout!\n");
 #undef C
 
         return status;
@@ -1033,7 +1029,6 @@ static uint32_t skl_get_aux_clock_divider(struct intel_dp *intel_dp, int index)
 }
 
 static uint32_t g4x_get_aux_send_ctl(struct intel_dp *intel_dp,
-                                     bool has_aux_irq,
                                      int send_bytes,
                                      uint32_t aux_clock_divider)
 {
@@ -1054,7 +1049,7 @@ static uint32_t g4x_get_aux_send_ctl(struct intel_dp *intel_dp,
 
         return DP_AUX_CH_CTL_SEND_BUSY |
                DP_AUX_CH_CTL_DONE |
-               (has_aux_irq ? DP_AUX_CH_CTL_INTERRUPT : 0) |
+               DP_AUX_CH_CTL_INTERRUPT |
                DP_AUX_CH_CTL_TIME_OUT_ERROR |
                timeout |
                DP_AUX_CH_CTL_RECEIVE_ERROR |
@@ -1064,13 +1059,12 @@ static uint32_t g4x_get_aux_send_ctl(struct intel_dp *intel_dp,
 }
 
 static uint32_t skl_get_aux_send_ctl(struct intel_dp *intel_dp,
-                                      bool has_aux_irq,
                                       int send_bytes,
                                       uint32_t unused)
 {
         return DP_AUX_CH_CTL_SEND_BUSY |
                DP_AUX_CH_CTL_DONE |
-               (has_aux_irq ? DP_AUX_CH_CTL_INTERRUPT : 0) |
+               DP_AUX_CH_CTL_INTERRUPT |
                DP_AUX_CH_CTL_TIME_OUT_ERROR |
                DP_AUX_CH_CTL_TIME_OUT_MAX |
                DP_AUX_CH_CTL_RECEIVE_ERROR |
@@ -1093,7 +1087,6 @@ intel_dp_aux_xfer(struct intel_dp *intel_dp,
         int i, ret, recv_bytes;
         uint32_t status;
         int try, clock = 0;
-        bool has_aux_irq = HAS_AUX_IRQ(dev_priv);
         bool vdd;
 
         ch_ctl = intel_dp->aux_ch_ctl_reg(intel_dp);
@@ -1148,7 +1141,6 @@ intel_dp_aux_xfer(struct intel_dp *intel_dp,
 
         while ((aux_clock_divider = intel_dp->get_aux_clock_divider(intel_dp, clock++))) {
                 u32 send_ctl = intel_dp->get_aux_send_ctl(intel_dp,
-                                                          has_aux_irq,
                                                           send_bytes,
                                                           aux_clock_divider);
 
@@ -1165,7 +1157,7 @@ intel_dp_aux_xfer(struct intel_dp *intel_dp,
                         /* Send the command and wait for it to complete */
                         I915_WRITE(ch_ctl, send_ctl);
 
-                        status = intel_dp_aux_wait_done(intel_dp, has_aux_irq);
+                        status = intel_dp_aux_wait_done(intel_dp);
 
                         /* Clear done status and any errors */
                         I915_WRITE(ch_ctl,
@@ -4499,6 +4491,8 @@ intel_dp_short_pulse(struct intel_dp *intel_dp)
         if (intel_dp_needs_link_retrain(intel_dp))
                 return false;
 
+        intel_psr_short_pulse(intel_dp);
+
         if (intel_dp->compliance.test_type == DP_TEST_LINK_TRAINING) {
                 DRM_DEBUG_KMS("Link Training Compliance Test requested\n");
                 /* Send a Hotplug Uevent to userspace to start modeset */
diff --git a/drivers/gpu/drm/i915/intel_dp_mst.c b/drivers/gpu/drm/i915/intel_dp_mst.c
index 0f012fbe34eb..85ecf41eeabb 100644
--- a/drivers/gpu/drm/i915/intel_dp_mst.c
+++ b/drivers/gpu/drm/i915/intel_dp_mst.c
@@ -514,7 +514,7 @@ static void intel_dp_destroy_mst_connector(struct drm_dp_mst_topology_mgr *mgr,
         intel_connector->mst_port = NULL;
         drm_modeset_unlock(&connector->dev->mode_config.connection_mutex);
 
-        drm_connector_unreference(connector);
+        drm_connector_put(connector);
 }
 
 static void intel_dp_mst_hotplug(struct drm_dp_mst_topology_mgr *mgr)
diff --git a/drivers/gpu/drm/i915/intel_dpll_mgr.c b/drivers/gpu/drm/i915/intel_dpll_mgr.c
index 156f8e4cbe4c..b51ad2917dbe 100644
--- a/drivers/gpu/drm/i915/intel_dpll_mgr.c
+++ b/drivers/gpu/drm/i915/intel_dpll_mgr.c
@@ -2566,6 +2566,7 @@ int icl_calc_dp_combo_pll_link(struct drm_i915_private *dev_priv,
         switch (index) {
         default:
                 MISSING_CASE(index);
+                /* fall through */
         case 0:
                 link_clock = 540000;
                 break;
@@ -2639,6 +2640,7 @@ static bool icl_mg_pll_find_divisors(int clock_khz, bool is_dp, bool use_ssc,
                         switch (div1) {
                         default:
                                 MISSING_CASE(div1);
+                                /* fall through */
                         case 2:
                                 hsdiv = 0;
                                 break;
@@ -2812,25 +2814,31 @@ static bool icl_calc_mg_pll_state(struct intel_crtc_state *crtc_state,
                                 MG_PLL_SSC_FLLEN |
                                 MG_PLL_SSC_STEPSIZE(ssc_stepsize);
 
-        pll_state->mg_pll_tdc_coldst_bias = MG_PLL_TDC_COLDST_COLDSTART;
-
-        if (refclk_khz != 38400) {
-                pll_state->mg_pll_tdc_coldst_bias |=
-                        MG_PLL_TDC_COLDST_IREFINT_EN |
-                        MG_PLL_TDC_COLDST_REFBIAS_START_PULSE_W(iref_pulse_w) |
-                        MG_PLL_TDC_COLDST_COLDSTART |
-                        MG_PLL_TDC_TDCOVCCORR_EN |
-                        MG_PLL_TDC_TDCSEL(3);
-
-                pll_state->mg_pll_bias = MG_PLL_BIAS_BIAS_GB_SEL(3) |
-                                         MG_PLL_BIAS_INIT_DCOAMP(0x3F) |
-                                         MG_PLL_BIAS_BIAS_BONUS(10) |
-                                         MG_PLL_BIAS_BIASCAL_EN |
-                                         MG_PLL_BIAS_CTRIM(12) |
-                                         MG_PLL_BIAS_VREF_RDAC(4) |
-                                         MG_PLL_BIAS_IREFTRIM(iref_trim);
+        pll_state->mg_pll_tdc_coldst_bias = MG_PLL_TDC_COLDST_COLDSTART |
+                                            MG_PLL_TDC_COLDST_IREFINT_EN |
+                                            MG_PLL_TDC_COLDST_REFBIAS_START_PULSE_W(iref_pulse_w) |
+                                            MG_PLL_TDC_TDCOVCCORR_EN |
+                                            MG_PLL_TDC_TDCSEL(3);
+
+        pll_state->mg_pll_bias = MG_PLL_BIAS_BIAS_GB_SEL(3) |
+                                 MG_PLL_BIAS_INIT_DCOAMP(0x3F) |
+                                 MG_PLL_BIAS_BIAS_BONUS(10) |
+                                 MG_PLL_BIAS_BIASCAL_EN |
+                                 MG_PLL_BIAS_CTRIM(12) |
+                                 MG_PLL_BIAS_VREF_RDAC(4) |
+                                 MG_PLL_BIAS_IREFTRIM(iref_trim);
+
+        if (refclk_khz == 38400) {
+                pll_state->mg_pll_tdc_coldst_bias_mask = MG_PLL_TDC_COLDST_COLDSTART;
+                pll_state->mg_pll_bias_mask = 0;
+        } else {
+                pll_state->mg_pll_tdc_coldst_bias_mask = -1U;
+                pll_state->mg_pll_bias_mask = -1U;
         }
 
+        pll_state->mg_pll_tdc_coldst_bias &= pll_state->mg_pll_tdc_coldst_bias_mask;
+        pll_state->mg_pll_bias &= pll_state->mg_pll_bias_mask;
+
         return true;
 }
 
@@ -2897,6 +2905,7 @@ static i915_reg_t icl_pll_id_to_enable_reg(enum intel_dpll_id id)
         switch (id) {
         default:
                 MISSING_CASE(id);
+                /* fall through */
         case DPLL_ID_ICL_DPLL0:
         case DPLL_ID_ICL_DPLL1:
                 return CNL_DPLL_ENABLE(id);
@@ -2939,18 +2948,41 @@ static bool icl_pll_get_hw_state(struct drm_i915_private *dev_priv,
         case DPLL_ID_ICL_MGPLL4:
                 port = icl_mg_pll_id_to_port(id);
                 hw_state->mg_refclkin_ctl = I915_READ(MG_REFCLKIN_CTL(port));
+                hw_state->mg_refclkin_ctl &= MG_REFCLKIN_CTL_OD_2_MUX_MASK;
+
                 hw_state->mg_clktop2_coreclkctl1 =
                         I915_READ(MG_CLKTOP2_CORECLKCTL1(port));
+                hw_state->mg_clktop2_coreclkctl1 &=
+                        MG_CLKTOP2_CORECLKCTL1_A_DIVRATIO_MASK;
+
                 hw_state->mg_clktop2_hsclkctl =
                         I915_READ(MG_CLKTOP2_HSCLKCTL(port));
+                hw_state->mg_clktop2_hsclkctl &=
+                        MG_CLKTOP2_HSCLKCTL_TLINEDRV_CLKSEL_MASK |
+                        MG_CLKTOP2_HSCLKCTL_CORE_INPUTSEL_MASK |
+                        MG_CLKTOP2_HSCLKCTL_HSDIV_RATIO_MASK |
+                        MG_CLKTOP2_HSCLKCTL_DSDIV_RATIO_MASK;
+
                 hw_state->mg_pll_div0 = I915_READ(MG_PLL_DIV0(port));
                 hw_state->mg_pll_div1 = I915_READ(MG_PLL_DIV1(port));
                 hw_state->mg_pll_lf = I915_READ(MG_PLL_LF(port));
                 hw_state->mg_pll_frac_lock = I915_READ(MG_PLL_FRAC_LOCK(port));
                 hw_state->mg_pll_ssc = I915_READ(MG_PLL_SSC(port));
+
                 hw_state->mg_pll_bias = I915_READ(MG_PLL_BIAS(port));
                 hw_state->mg_pll_tdc_coldst_bias =
                         I915_READ(MG_PLL_TDC_COLDST_BIAS(port));
+
+                if (dev_priv->cdclk.hw.ref == 38400) {
+                        hw_state->mg_pll_tdc_coldst_bias_mask = MG_PLL_TDC_COLDST_COLDSTART;
+                        hw_state->mg_pll_bias_mask = 0;
+                } else {
+                        hw_state->mg_pll_tdc_coldst_bias_mask = -1U;
+                        hw_state->mg_pll_bias_mask = -1U;
+                }
+
+                hw_state->mg_pll_tdc_coldst_bias &= hw_state->mg_pll_tdc_coldst_bias_mask;
+                hw_state->mg_pll_bias &= hw_state->mg_pll_bias_mask;
                 break;
         default:
                 MISSING_CASE(id);
@@ -2978,19 +3010,48 @@ static void icl_mg_pll_write(struct drm_i915_private *dev_priv,
 {
         struct intel_dpll_hw_state *hw_state = &pll->state.hw_state;
         enum port port = icl_mg_pll_id_to_port(pll->info->id);
+        u32 val;
+
+        /*
+         * Some of the following registers have reserved fields, so program
+         * these with RMW based on a mask. The mask can be fixed or generated
+         * during the calc/readout phase if the mask depends on some other HW
+         * state like refclk, see icl_calc_mg_pll_state().
+         */
+        val = I915_READ(MG_REFCLKIN_CTL(port));
+        val &= ~MG_REFCLKIN_CTL_OD_2_MUX_MASK;
+        val |= hw_state->mg_refclkin_ctl;
+        I915_WRITE(MG_REFCLKIN_CTL(port), val);
+
+        val = I915_READ(MG_CLKTOP2_CORECLKCTL1(port));
+        val &= ~MG_CLKTOP2_CORECLKCTL1_A_DIVRATIO_MASK;
+        val |= hw_state->mg_clktop2_coreclkctl1;
+        I915_WRITE(MG_CLKTOP2_CORECLKCTL1(port), val);
+
+        val = I915_READ(MG_CLKTOP2_HSCLKCTL(port));
+        val &= ~(MG_CLKTOP2_HSCLKCTL_TLINEDRV_CLKSEL_MASK |
+                 MG_CLKTOP2_HSCLKCTL_CORE_INPUTSEL_MASK |
+                 MG_CLKTOP2_HSCLKCTL_HSDIV_RATIO_MASK |
+                 MG_CLKTOP2_HSCLKCTL_DSDIV_RATIO_MASK);
+        val |= hw_state->mg_clktop2_hsclkctl;
+        I915_WRITE(MG_CLKTOP2_HSCLKCTL(port), val);
 
-        I915_WRITE(MG_REFCLKIN_CTL(port), hw_state->mg_refclkin_ctl);
-        I915_WRITE(MG_CLKTOP2_CORECLKCTL1(port),
-                   hw_state->mg_clktop2_coreclkctl1);
-        I915_WRITE(MG_CLKTOP2_HSCLKCTL(port), hw_state->mg_clktop2_hsclkctl);
         I915_WRITE(MG_PLL_DIV0(port), hw_state->mg_pll_div0);
         I915_WRITE(MG_PLL_DIV1(port), hw_state->mg_pll_div1);
         I915_WRITE(MG_PLL_LF(port), hw_state->mg_pll_lf);
         I915_WRITE(MG_PLL_FRAC_LOCK(port), hw_state->mg_pll_frac_lock);
         I915_WRITE(MG_PLL_SSC(port), hw_state->mg_pll_ssc);
-        I915_WRITE(MG_PLL_BIAS(port), hw_state->mg_pll_bias);
-        I915_WRITE(MG_PLL_TDC_COLDST_BIAS(port),
-                   hw_state->mg_pll_tdc_coldst_bias);
+
+        val = I915_READ(MG_PLL_BIAS(port));
+        val &= ~hw_state->mg_pll_bias_mask;
+        val |= hw_state->mg_pll_bias;
+        I915_WRITE(MG_PLL_BIAS(port), val);
+
+        val = I915_READ(MG_PLL_TDC_COLDST_BIAS(port));
+        val &= ~hw_state->mg_pll_tdc_coldst_bias_mask;
+        val |= hw_state->mg_pll_tdc_coldst_bias;
+        I915_WRITE(MG_PLL_TDC_COLDST_BIAS(port), val);
+
         POSTING_READ(MG_PLL_TDC_COLDST_BIAS(port));
 }
 
diff --git a/drivers/gpu/drm/i915/intel_dpll_mgr.h b/drivers/gpu/drm/i915/intel_dpll_mgr.h
index ba925c7ee482..7e522cf4f13f 100644
--- a/drivers/gpu/drm/i915/intel_dpll_mgr.h
+++ b/drivers/gpu/drm/i915/intel_dpll_mgr.h
@@ -180,6 +180,8 @@ struct intel_dpll_hw_state {
         uint32_t mg_pll_ssc;
         uint32_t mg_pll_bias;
         uint32_t mg_pll_tdc_coldst_bias;
+        uint32_t mg_pll_bias_mask;
+        uint32_t mg_pll_tdc_coldst_bias_mask;
 };
 
 /**
diff --git a/drivers/gpu/drm/i915/intel_drv.h b/drivers/gpu/drm/i915/intel_drv.h
index 0c3ac0eafde0..61e715ddd0d5 100644
--- a/drivers/gpu/drm/i915/intel_drv.h
+++ b/drivers/gpu/drm/i915/intel_drv.h
@@ -254,7 +254,8 @@ struct intel_encoder {
                            struct intel_crtc_state *pipe_config);
         /* Returns a mask of power domains that need to be referenced as part
          * of the hardware state readout code. */
-        u64 (*get_power_domains)(struct intel_encoder *encoder);
+        u64 (*get_power_domains)(struct intel_encoder *encoder,
+                                 struct intel_crtc_state *crtc_state);
         /*
          * Called during system suspend after all pending requests for the
          * encoder are flushed (for example for DP AUX transactions) and
@@ -1133,7 +1134,6 @@ struct intel_dp {
          * register with to kick off an AUX transaction.
          */
         uint32_t (*get_aux_send_ctl)(struct intel_dp *dp,
-                                     bool has_aux_irq,
                                      int send_bytes,
                                      uint32_t aux_clock_divider);
 
@@ -1254,6 +1254,7 @@ enc_to_dig_port(struct drm_encoder *encoder)
         switch (intel_encoder->type) {
         case INTEL_OUTPUT_DDI:
                 WARN_ON(!HAS_DDI(to_i915(encoder->dev)));
+                /* fall through */
         case INTEL_OUTPUT_DP:
         case INTEL_OUTPUT_EDP:
         case INTEL_OUTPUT_HDMI:
@@ -1730,8 +1731,8 @@ int intel_dp_aux_init_backlight_funcs(struct intel_connector *intel_connector);
 /* intel_dp_mst.c */
 int intel_dp_mst_encoder_init(struct intel_digital_port *intel_dig_port, int conn_id);
 void intel_dp_mst_encoder_cleanup(struct intel_digital_port *intel_dig_port);
-/* intel_dsi.c */
-void intel_dsi_init(struct drm_i915_private *dev_priv);
+/* vlv_dsi.c */
+void vlv_dsi_init(struct drm_i915_private *dev_priv);
 
 /* intel_dsi_dcs_backlight.c */
 int intel_dsi_dcs_init_backlight_funcs(struct intel_connector *intel_connector);
@@ -1921,6 +1922,8 @@ void intel_psr_compute_config(struct intel_dp *intel_dp,
                               struct intel_crtc_state *crtc_state);
 void intel_psr_irq_control(struct drm_i915_private *dev_priv, bool debug);
 void intel_psr_irq_handler(struct drm_i915_private *dev_priv, u32 psr_iir);
+void intel_psr_short_pulse(struct intel_dp *intel_dp);
+int intel_psr_wait_for_idle(struct drm_i915_private *dev_priv);
 
 /* intel_runtime_pm.c */
 int intel_power_domains_init(struct drm_i915_private *);
@@ -2151,7 +2154,6 @@ void lspcon_resume(struct intel_lspcon *lspcon);
 void lspcon_wait_pcon_mode(struct intel_lspcon *lspcon);
 
 /* intel_pipe_crc.c */
-int intel_pipe_crc_create(struct drm_minor *minor);
 #ifdef CONFIG_DEBUG_FS
 int intel_crtc_set_crc_source(struct drm_crtc *crtc, const char *source_name,
                               size_t *values_cnt);
@@ -2167,5 +2169,4 @@ static inline void intel_crtc_enable_pipe_crc(struct intel_crtc *crtc)
 {
 }
 #endif
-extern const struct file_operations i915_display_crc_ctl_fops;
 #endif /* __INTEL_DRV_H__ */
diff --git a/drivers/gpu/drm/i915/intel_dsi.h b/drivers/gpu/drm/i915/intel_dsi.h
index 7afeb9580f41..ad7c1cb32983 100644
--- a/drivers/gpu/drm/i915/intel_dsi.h
+++ b/drivers/gpu/drm/i915/intel_dsi.h
@@ -129,21 +129,29 @@ static inline struct intel_dsi *enc_to_intel_dsi(struct drm_encoder *encoder)
         return container_of(encoder, struct intel_dsi, base.base);
 }
 
-/* intel_dsi.c */
-void wait_for_dsi_fifo_empty(struct intel_dsi *intel_dsi, enum port port);
+/* vlv_dsi.c */
+void vlv_dsi_wait_for_fifo_empty(struct intel_dsi *intel_dsi, enum port port);
 enum mipi_dsi_pixel_format pixel_format_from_register_bits(u32 fmt);
 
-/* intel_dsi_pll.c */
-bool intel_dsi_pll_is_enabled(struct drm_i915_private *dev_priv);
-int intel_compute_dsi_pll(struct intel_encoder *encoder,
-                          struct intel_crtc_state *config);
-void intel_enable_dsi_pll(struct intel_encoder *encoder,
-                          const struct intel_crtc_state *config);
-void intel_disable_dsi_pll(struct intel_encoder *encoder);
-u32 intel_dsi_get_pclk(struct intel_encoder *encoder, int pipe_bpp,
-                       struct intel_crtc_state *config);
-void intel_dsi_reset_clocks(struct intel_encoder *encoder,
-                            enum port port);
+/* vlv_dsi_pll.c */
+int vlv_dsi_pll_compute(struct intel_encoder *encoder,
+                        struct intel_crtc_state *config);
+void vlv_dsi_pll_enable(struct intel_encoder *encoder,
+                        const struct intel_crtc_state *config);
+void vlv_dsi_pll_disable(struct intel_encoder *encoder);
+u32 vlv_dsi_get_pclk(struct intel_encoder *encoder, int pipe_bpp,
+                     struct intel_crtc_state *config);
+void vlv_dsi_reset_clocks(struct intel_encoder *encoder, enum port port);
+
+bool bxt_dsi_pll_is_enabled(struct drm_i915_private *dev_priv);
+int bxt_dsi_pll_compute(struct intel_encoder *encoder,
+                        struct intel_crtc_state *config);
+void bxt_dsi_pll_enable(struct intel_encoder *encoder,
+                        const struct intel_crtc_state *config);
+void bxt_dsi_pll_disable(struct intel_encoder *encoder);
+u32 bxt_dsi_get_pclk(struct intel_encoder *encoder, int pipe_bpp,
+                     struct intel_crtc_state *config);
+void bxt_dsi_reset_clocks(struct intel_encoder *encoder, enum port port);
 
 /* intel_dsi_vbt.c */
 bool intel_dsi_vbt_init(struct intel_dsi *intel_dsi, u16 panel_id);
diff --git a/drivers/gpu/drm/i915/intel_dsi_vbt.c b/drivers/gpu/drm/i915/intel_dsi_vbt.c
index 4d6ffa7b3e7b..ac83d6b89ae0 100644
--- a/drivers/gpu/drm/i915/intel_dsi_vbt.c
+++ b/drivers/gpu/drm/i915/intel_dsi_vbt.c
@@ -181,7 +181,7 @@ static const u8 *mipi_exec_send_packet(struct intel_dsi *intel_dsi,
                 break;
         }
 
-        wait_for_dsi_fifo_empty(intel_dsi, port);
+        vlv_dsi_wait_for_fifo_empty(intel_dsi, port);
 
 out:
         data += len;
diff --git a/drivers/gpu/drm/i915/intel_engine_cs.c b/drivers/gpu/drm/i915/intel_engine_cs.c
index 32bf3a408d46..0ac497275a51 100644
--- a/drivers/gpu/drm/i915/intel_engine_cs.c
+++ b/drivers/gpu/drm/i915/intel_engine_cs.c
@@ -25,7 +25,6 @@
 #include <drm/drm_print.h>
 
 #include "i915_drv.h"
-#include "i915_vgpu.h"
 #include "intel_ringbuffer.h"
 #include "intel_lrc.h"
 
@@ -230,6 +229,7 @@ __intel_engine_context_size(struct drm_i915_private *dev_priv, u8 class)
                 break;
         default:
                 MISSING_CASE(class);
+                /* fall through */
         case VIDEO_DECODE_CLASS:
         case VIDEO_ENHANCEMENT_CLASS:
         case COPY_ENGINE_CLASS:
@@ -302,6 +302,8 @@ intel_engine_setup(struct drm_i915_private *dev_priv,
                                                            engine->class);
         if (WARN_ON(engine->context_size > BIT(20)))
                 engine->context_size = 0;
+        if (engine->context_size)
+                DRIVER_CAPS(dev_priv)->has_logical_contexts = true;
 
         /* Nothing to do here, execute in order of dependencies */
         engine->schedule = NULL;
@@ -456,21 +458,10 @@ static void intel_engine_init_batch_pool(struct intel_engine_cs *engine)
         i915_gem_batch_pool_init(&engine->batch_pool, engine);
 }
 
-static bool csb_force_mmio(struct drm_i915_private *i915)
-{
-        /* Older GVT emulation depends upon intercepting CSB mmio */
-        if (intel_vgpu_active(i915) && !intel_vgpu_has_hwsp_emulation(i915))
-                return true;
-
-        return false;
-}
-
 static void intel_engine_init_execlist(struct intel_engine_cs *engine)
 {
         struct intel_engine_execlists * const execlists = &engine->execlists;
 
-        execlists->csb_use_mmio = csb_force_mmio(engine->i915);
-
         execlists->port_mask = 1;
         BUILD_BUG_ON_NOT_POWER_OF_2(execlists_num_ports(execlists));
         GEM_BUG_ON(execlists_num_ports(execlists) > EXECLIST_MAX_PORTS);
@@ -492,6 +483,7 @@ static void intel_engine_init_execlist(struct intel_engine_cs *engine)
 void intel_engine_setup_common(struct intel_engine_cs *engine)
 {
         i915_timeline_init(engine->i915, &engine->timeline, engine->name);
+        lockdep_set_subclass(&engine->timeline.lock, TIMELINE_ENGINE);
 
         intel_engine_init_execlist(engine);
         intel_engine_init_hangcheck(engine);
@@ -1000,10 +992,12 @@ bool intel_engine_is_idle(struct intel_engine_cs *engine)
         if (READ_ONCE(engine->execlists.active)) {
                 struct intel_engine_execlists *execlists = &engine->execlists;
 
+                local_bh_disable();
                 if (tasklet_trylock(&execlists->tasklet)) {
                         execlists->tasklet.func(execlists->tasklet.data);
                         tasklet_unlock(&execlists->tasklet);
                 }
+                local_bh_enable();
 
                 if (READ_ONCE(execlists->active))
                         return false;
@@ -1363,12 +1357,10 @@ static void intel_engine_print_registers(const struct intel_engine_cs *engine,
                 ptr = I915_READ(RING_CONTEXT_STATUS_PTR(engine));
                 read = GEN8_CSB_READ_PTR(ptr);
                 write = GEN8_CSB_WRITE_PTR(ptr);
-                drm_printf(m, "\tExeclist CSB read %d [%d cached], write %d [%d from hws], interrupt posted? %s, tasklet queued? %s (%s)\n",
+                drm_printf(m, "\tExeclist CSB read %d [%d cached], write %d [%d from hws], tasklet queued? %s (%s)\n",
                            read, execlists->csb_head,
                            write,
                            intel_read_status_page(engine, intel_hws_csb_write_index(engine->i915)),
-                           yesno(test_bit(ENGINE_IRQ_EXECLIST,
-                                          &engine->irq_posted)),
                            yesno(test_bit(TASKLET_STATE_SCHED,
                                           &engine->execlists.tasklet.state)),
                            enableddisabled(!atomic_read(&engine->execlists.tasklet.count)));
@@ -1580,11 +1572,9 @@ void intel_engine_dump(struct intel_engine_cs *engine,
         spin_unlock(&b->rb_lock);
         local_irq_restore(flags);
 
-        drm_printf(m, "IRQ? 0x%lx (breadcrumbs? %s) (execlists? %s)\n",
+        drm_printf(m, "IRQ? 0x%lx (breadcrumbs? %s)\n",
                    engine->irq_posted,
                    yesno(test_bit(ENGINE_IRQ_BREADCRUMB,
-                                  &engine->irq_posted)),
-                   yesno(test_bit(ENGINE_IRQ_EXECLIST,
                                   &engine->irq_posted)));
 
         drm_printf(m, "HWSP:\n");
@@ -1633,8 +1623,8 @@ int intel_enable_engine_stats(struct intel_engine_cs *engine)
         if (!intel_engine_supports_stats(engine))
                 return -ENODEV;
 
-        tasklet_disable(&execlists->tasklet);
-        write_seqlock_irqsave(&engine->stats.lock, flags);
+        spin_lock_irqsave(&engine->timeline.lock, flags);
+        write_seqlock(&engine->stats.lock);
 
         if (unlikely(engine->stats.enabled == ~0)) {
                 err = -EBUSY;
@@ -1658,8 +1648,8 @@ int intel_enable_engine_stats(struct intel_engine_cs *engine)
         }
 
 unlock:
-        write_sequnlock_irqrestore(&engine->stats.lock, flags);
-        tasklet_enable(&execlists->tasklet);
+        write_sequnlock(&engine->stats.lock);
+        spin_unlock_irqrestore(&engine->timeline.lock, flags);
 
         return err;
 }
diff --git a/drivers/gpu/drm/i915/intel_fbc.c b/drivers/gpu/drm/i915/intel_fbc.c
index b431b6733cc1..01d1d2088f04 100644
--- a/drivers/gpu/drm/i915/intel_fbc.c
+++ b/drivers/gpu/drm/i915/intel_fbc.c
@@ -399,89 +399,6 @@ bool intel_fbc_is_active(struct drm_i915_private *dev_priv)
         return dev_priv->fbc.active;
 }
 
-static void intel_fbc_work_fn(struct work_struct *__work)
-{
-        struct drm_i915_private *dev_priv =
-                container_of(__work, struct drm_i915_private, fbc.work.work);
-        struct intel_fbc *fbc = &dev_priv->fbc;
-        struct intel_fbc_work *work = &fbc->work;
-        struct intel_crtc *crtc = fbc->crtc;
-        struct drm_vblank_crtc *vblank = &dev_priv->drm.vblank[crtc->pipe];
-
-        if (drm_crtc_vblank_get(&crtc->base)) {
-                /* CRTC is now off, leave FBC deactivated */
-                mutex_lock(&fbc->lock);
-                work->scheduled = false;
-                mutex_unlock(&fbc->lock);
-                return;
-        }
-
-retry:
-        /* Delay the actual enabling to let pageflipping cease and the
-         * display to settle before starting the compression. Note that
-         * this delay also serves a second purpose: it allows for a
-         * vblank to pass after disabling the FBC before we attempt
-         * to modify the control registers.
-         *
-         * WaFbcWaitForVBlankBeforeEnable:ilk,snb
-         *
-         * It is also worth mentioning that since work->scheduled_vblank can be
-         * updated multiple times by the other threads, hitting the timeout is
-         * not an error condition. We'll just end up hitting the "goto retry"
-         * case below.
-         */
-        wait_event_timeout(vblank->queue,
-                drm_crtc_vblank_count(&crtc->base) != work->scheduled_vblank,
-                msecs_to_jiffies(50));
-
-        mutex_lock(&fbc->lock);
-
-        /* Were we cancelled? */
-        if (!work->scheduled)
-                goto out;
-
-        /* Were we delayed again while this function was sleeping? */
-        if (drm_crtc_vblank_count(&crtc->base) == work->scheduled_vblank) {
-                mutex_unlock(&fbc->lock);
-                goto retry;
-        }
-
-        intel_fbc_hw_activate(dev_priv);
-
-        work->scheduled = false;
-
-out:
-        mutex_unlock(&fbc->lock);
-        drm_crtc_vblank_put(&crtc->base);
-}
-
-static void intel_fbc_schedule_activation(struct intel_crtc *crtc)
-{
-        struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
-        struct intel_fbc *fbc = &dev_priv->fbc;
-        struct intel_fbc_work *work = &fbc->work;
-
-        WARN_ON(!mutex_is_locked(&fbc->lock));
-        if (WARN_ON(!fbc->enabled))
-                return;
-
-        if (drm_crtc_vblank_get(&crtc->base)) {
-                DRM_ERROR("vblank not available for FBC on pipe %c\n",
-                          pipe_name(crtc->pipe));
-                return;
-        }
-
-        /* It is useless to call intel_fbc_cancel_work() or cancel_work() in
-         * this function since we're not releasing fbc.lock, so it won't have an
-         * opportunity to grab it to discover that it was cancelled. So we just
-         * update the expected jiffy count. */
-        work->scheduled = true;
-        work->scheduled_vblank = drm_crtc_vblank_count(&crtc->base);
-        drm_crtc_vblank_put(&crtc->base);
-
-        schedule_work(&work->work);
-}
-
 static void intel_fbc_deactivate(struct drm_i915_private *dev_priv,
                                  const char *reason)
 {
@@ -489,11 +406,6 @@ static void intel_fbc_deactivate(struct drm_i915_private *dev_priv,
 
         WARN_ON(!mutex_is_locked(&fbc->lock));
 
-        /* Calling cancel_work() here won't help due to the fact that the work
-         * function grabs fbc->lock. Just set scheduled to false so the work
-         * function can know it was cancelled. */
-        fbc->work.scheduled = false;
-
         if (fbc->active)
                 intel_fbc_hw_deactivate(dev_priv);
 
@@ -924,13 +836,6 @@ static void intel_fbc_get_reg_params(struct intel_crtc *crtc,
                                                 32 * fbc->threshold) * 8;
 }
 
-static bool intel_fbc_reg_params_equal(struct intel_fbc_reg_params *params1,
-                                       struct intel_fbc_reg_params *params2)
-{
-        /* We can use this since intel_fbc_get_reg_params() does a memset. */
-        return memcmp(params1, params2, sizeof(*params1)) == 0;
-}
-
 void intel_fbc_pre_update(struct intel_crtc *crtc,
                           struct intel_crtc_state *crtc_state,
                           struct intel_plane_state *plane_state)
@@ -953,6 +858,7 @@ void intel_fbc_pre_update(struct intel_crtc *crtc,
                 goto unlock;
 
         intel_fbc_update_state_cache(crtc, crtc_state, plane_state);
+        fbc->flip_pending = true;
 
 deactivate:
         intel_fbc_deactivate(dev_priv, reason);
@@ -988,13 +894,15 @@ static void __intel_fbc_post_update(struct intel_crtc *crtc)
 {
         struct drm_i915_private *dev_priv = to_i915(crtc->base.dev);
         struct intel_fbc *fbc = &dev_priv->fbc;
-        struct intel_fbc_reg_params old_params;
 
         WARN_ON(!mutex_is_locked(&fbc->lock));
 
         if (!fbc->enabled || fbc->crtc != crtc)
                 return;
 
+        fbc->flip_pending = false;
+        WARN_ON(fbc->active);
+
         if (!i915_modparams.enable_fbc) {
                 intel_fbc_deactivate(dev_priv, "disabled at runtime per module param");
                 __intel_fbc_disable(dev_priv);
@@ -1002,25 +910,16 @@ static void __intel_fbc_post_update(struct intel_crtc *crtc)
                 return;
         }
 
-        if (!intel_fbc_can_activate(crtc)) {
-                WARN_ON(fbc->active);
-                return;
-        }
-
-        old_params = fbc->params;
         intel_fbc_get_reg_params(crtc, &fbc->params);
 
-        /* If the scanout has not changed, don't modify the FBC settings.
-         * Note that we make the fundamental assumption that the fb->obj
-         * cannot be unpinned (and have its GTT offset and fence revoked)
-         * without first being decoupled from the scanout and FBC disabled.
-         */
-        if (fbc->active &&
-            intel_fbc_reg_params_equal(&old_params, &fbc->params))
+        if (!intel_fbc_can_activate(crtc))
                 return;
 
-        intel_fbc_deactivate(dev_priv, "FBC enabled (active or scheduled)");
-        intel_fbc_schedule_activation(crtc);
+        if (!fbc->busy_bits) {
+                intel_fbc_deactivate(dev_priv, "FBC enabled (active or scheduled)");
+                intel_fbc_hw_activate(dev_priv);
+        } else
+                intel_fbc_deactivate(dev_priv, "frontbuffer write");
 }
 
 void intel_fbc_post_update(struct intel_crtc *crtc)
@@ -1085,7 +984,7 @@ void intel_fbc_flush(struct drm_i915_private *dev_priv,
             (frontbuffer_bits & intel_fbc_get_frontbuffer_bit(fbc))) {
                 if (fbc->active)
                         intel_fbc_recompress(dev_priv);
-                else
+                else if (!fbc->flip_pending)
                         __intel_fbc_post_update(fbc->crtc);
         }
 
@@ -1225,8 +1124,6 @@ void intel_fbc_disable(struct intel_crtc *crtc)
         if (fbc->crtc == crtc)
                 __intel_fbc_disable(dev_priv);
         mutex_unlock(&fbc->lock);
-
-        cancel_work_sync(&fbc->work.work);
 }
 
 /**
@@ -1248,8 +1145,6 @@ void intel_fbc_global_disable(struct drm_i915_private *dev_priv)
                 __intel_fbc_disable(dev_priv);
         }
         mutex_unlock(&fbc->lock);
-
-        cancel_work_sync(&fbc->work.work);
 }
 
 static void intel_fbc_underrun_work_fn(struct work_struct *work)
@@ -1400,12 +1295,10 @@ void intel_fbc_init(struct drm_i915_private *dev_priv)
 {
         struct intel_fbc *fbc = &dev_priv->fbc;
 
-        INIT_WORK(&fbc->work.work, intel_fbc_work_fn);
         INIT_WORK(&fbc->underrun_work, intel_fbc_underrun_work_fn);
         mutex_init(&fbc->lock);
         fbc->enabled = false;
         fbc->active = false;
-        fbc->work.scheduled = false;
 
         if (need_fbc_vtd_wa(dev_priv))
                 mkwrite_device_info(dev_priv)->has_fbc = false;
diff --git a/drivers/gpu/drm/i915/intel_guc.c b/drivers/gpu/drm/i915/intel_guc.c
index 1aff30b0870c..e12bd259df17 100644
--- a/drivers/gpu/drm/i915/intel_guc.c
+++ b/drivers/gpu/drm/i915/intel_guc.c
@@ -27,6 +27,8 @@
 #include "intel_guc_submission.h"
 #include "i915_drv.h"
 
+static void guc_init_ggtt_pin_bias(struct intel_guc *guc);
+
 static void gen8_guc_raise_irq(struct intel_guc *guc)
 {
         struct drm_i915_private *dev_priv = guc_to_i915(guc);
@@ -73,7 +75,7 @@ void intel_guc_init_early(struct intel_guc *guc)
         guc->notify = gen8_guc_raise_irq;
 }
 
-int intel_guc_init_wq(struct intel_guc *guc)
+static int guc_init_wq(struct intel_guc *guc)
 {
         struct drm_i915_private *dev_priv = guc_to_i915(guc);
 
@@ -124,7 +126,7 @@ int intel_guc_init_wq(struct intel_guc *guc)
         return 0;
 }
 
-void intel_guc_fini_wq(struct intel_guc *guc)
+static void guc_fini_wq(struct intel_guc *guc)
 {
         struct drm_i915_private *dev_priv = guc_to_i915(guc);
 
@@ -135,6 +137,28 @@ void intel_guc_fini_wq(struct intel_guc *guc)
         destroy_workqueue(guc->log.relay.flush_wq);
 }
 
+int intel_guc_init_misc(struct intel_guc *guc)
+{
+        struct drm_i915_private *i915 = guc_to_i915(guc);
+        int ret;
+
+        guc_init_ggtt_pin_bias(guc);
+
+        ret = guc_init_wq(guc);
+        if (ret)
+                return ret;
+
+        intel_uc_fw_fetch(i915, &guc->fw);
+
+        return 0;
+}
+
+void intel_guc_fini_misc(struct intel_guc *guc)
+{
+        intel_uc_fw_fini(&guc->fw);
+        guc_fini_wq(guc);
+}
+
 static int guc_shared_data_create(struct intel_guc *guc)
 {
         struct i915_vma *vma;
@@ -169,7 +193,7 @@ int intel_guc_init(struct intel_guc *guc)
 
         ret = guc_shared_data_create(guc);
         if (ret)
-                return ret;
+                goto err_fetch;
         GEM_BUG_ON(!guc->shared_data);
 
         ret = intel_guc_log_create(&guc->log);
@@ -190,6 +214,8 @@ err_log:
         intel_guc_log_destroy(&guc->log);
 err_shared:
         guc_shared_data_destroy(guc);
+err_fetch:
+        intel_uc_fw_fini(&guc->fw);
         return ret;
 }
 
@@ -201,12 +227,17 @@ void intel_guc_fini(struct intel_guc *guc)
         intel_guc_ads_destroy(guc);
         intel_guc_log_destroy(&guc->log);
         guc_shared_data_destroy(guc);
+        intel_uc_fw_fini(&guc->fw);
 }
 
 static u32 guc_ctl_debug_flags(struct intel_guc *guc)
 {
         u32 level = intel_guc_log_get_level(&guc->log);
-        u32 flags = 0;
+        u32 flags;
+        u32 ads;
+
+        ads = intel_guc_ggtt_offset(guc, guc->ads_vma) >> PAGE_SHIFT;
+        flags = ads << GUC_ADS_ADDR_SHIFT | GUC_ADS_ENABLED;
 
         if (!GUC_LOG_LEVEL_IS_ENABLED(level))
                 flags |= GUC_LOG_DEFAULT_DISABLED;
@@ -217,13 +248,6 @@ static u32 guc_ctl_debug_flags(struct intel_guc *guc)
                 flags |= GUC_LOG_LEVEL_TO_VERBOSITY(level) <<
                          GUC_LOG_VERBOSITY_SHIFT;
 
-        if (USES_GUC_SUBMISSION(guc_to_i915(guc))) {
-                u32 ads = intel_guc_ggtt_offset(guc, guc->ads_vma)
-                        >> PAGE_SHIFT;
-
-                flags |= ads << GUC_ADS_ADDR_SHIFT | GUC_ADS_ENABLED;
-        }
-
         return flags;
 }
 
@@ -327,6 +351,9 @@ void intel_guc_init_params(struct intel_guc *guc)
         params[GUC_CTL_DEBUG] = guc_ctl_debug_flags(guc);
         params[GUC_CTL_CTXINFO] = guc_ctl_ctxinfo_flags(guc);
 
+        for (i = 0; i < GUC_CTL_MAX_DWORDS; i++)
+                DRM_DEBUG_DRIVER("param[%2d] = %#x\n", i, params[i]);
+
         /*
          * All SOFT_SCRATCH registers are in FORCEWAKE_BLITTER domain and
          * they are power context saved so it's ok to release forcewake
@@ -585,13 +612,13 @@ int intel_guc_resume(struct intel_guc *guc)
  */
 
 /**
- * intel_guc_init_ggtt_pin_bias() - Initialize the GuC ggtt_pin_bias value.
+ * guc_init_ggtt_pin_bias() - Initialize the GuC ggtt_pin_bias value.
  * @guc: intel_guc structure.
  *
  * This function will calculate and initialize the ggtt_pin_bias value based on
  * overall WOPCM size and GuC WOPCM size.
  */
-void intel_guc_init_ggtt_pin_bias(struct intel_guc *guc)
+static void guc_init_ggtt_pin_bias(struct intel_guc *guc)
 {
         struct drm_i915_private *i915 = guc_to_i915(guc);
 
diff --git a/drivers/gpu/drm/i915/intel_guc.h b/drivers/gpu/drm/i915/intel_guc.h
index f1265e122d30..4121928a495e 100644
--- a/drivers/gpu/drm/i915/intel_guc.h
+++ b/drivers/gpu/drm/i915/intel_guc.h
@@ -151,11 +151,10 @@ static inline u32 intel_guc_ggtt_offset(struct intel_guc *guc,
 void intel_guc_init_early(struct intel_guc *guc);
 void intel_guc_init_send_regs(struct intel_guc *guc);
 void intel_guc_init_params(struct intel_guc *guc);
-void intel_guc_init_ggtt_pin_bias(struct intel_guc *guc);
-int intel_guc_init_wq(struct intel_guc *guc);
-void intel_guc_fini_wq(struct intel_guc *guc);
+int intel_guc_init_misc(struct intel_guc *guc);
 int intel_guc_init(struct intel_guc *guc);
 void intel_guc_fini(struct intel_guc *guc);
+void intel_guc_fini_misc(struct intel_guc *guc);
 int intel_guc_send_nop(struct intel_guc *guc, const u32 *action, u32 len,
                        u32 *response_buf, u32 response_buf_size);
 int intel_guc_send_mmio(struct intel_guc *guc, const u32 *action, u32 len,
diff --git a/drivers/gpu/drm/i915/intel_huc.c b/drivers/gpu/drm/i915/intel_huc.c
index 291285277403..ffcad5fad6a7 100644
--- a/drivers/gpu/drm/i915/intel_huc.c
+++ b/drivers/gpu/drm/i915/intel_huc.c
@@ -32,6 +32,14 @@ void intel_huc_init_early(struct intel_huc *huc)
         intel_huc_fw_init_early(huc);
 }
 
+int intel_huc_init_misc(struct intel_huc *huc)
+{
+        struct drm_i915_private *i915 = huc_to_i915(huc);
+
+        intel_uc_fw_fetch(i915, &huc->fw);
+        return 0;
+}
+
 /**
  * intel_huc_auth() - Authenticate HuC uCode
  * @huc: intel_huc structure
diff --git a/drivers/gpu/drm/i915/intel_huc.h b/drivers/gpu/drm/i915/intel_huc.h
index aa854907abac..7e41d870b509 100644
--- a/drivers/gpu/drm/i915/intel_huc.h
+++ b/drivers/gpu/drm/i915/intel_huc.h
@@ -36,9 +36,15 @@ struct intel_huc {
 };
 
 void intel_huc_init_early(struct intel_huc *huc);
+int intel_huc_init_misc(struct intel_huc *huc);
 int intel_huc_auth(struct intel_huc *huc);
 int intel_huc_check_status(struct intel_huc *huc);
 
+static inline void intel_huc_fini_misc(struct intel_huc *huc)
+{
+        intel_uc_fw_fini(&huc->fw);
+}
+
 static inline int intel_huc_sanitize(struct intel_huc *huc)
 {
         intel_uc_fw_sanitize(&huc->fw);
diff --git a/drivers/gpu/drm/i915/intel_lrc.c b/drivers/gpu/drm/i915/intel_lrc.c
index 33bc914c2ef5..ab89dabc2965 100644
--- a/drivers/gpu/drm/i915/intel_lrc.c
+++ b/drivers/gpu/drm/i915/intel_lrc.c
@@ -137,6 +137,7 @@
 #include <drm/i915_drm.h>
 #include "i915_drv.h"
 #include "i915_gem_render_state.h"
+#include "i915_vgpu.h"
 #include "intel_lrc_reg.h"
 #include "intel_mocs.h"
 #include "intel_workarounds.h"
@@ -562,12 +563,14 @@ static void complete_preempt_context(struct intel_engine_execlists *execlists)
         GEM_BUG_ON(!execlists_is_active(execlists, EXECLISTS_ACTIVE_PREEMPT));
 
         execlists_cancel_port_requests(execlists);
-        execlists_unwind_incomplete_requests(execlists);
+        __unwind_incomplete_requests(container_of(execlists,
+                                                  struct intel_engine_cs,
+                                                  execlists));
 
         execlists_clear_active(execlists, EXECLISTS_ACTIVE_PREEMPT);
 }
 
-static bool __execlists_dequeue(struct intel_engine_cs *engine)
+static void execlists_dequeue(struct intel_engine_cs *engine)
 {
         struct intel_engine_execlists * const execlists = &engine->execlists;
         struct execlist_port *port = execlists->port;
@@ -577,9 +580,8 @@ static bool __execlists_dequeue(struct intel_engine_cs *engine)
         struct rb_node *rb;
         bool submit = false;
 
-        lockdep_assert_held(&engine->timeline.lock);
-
-        /* Hardware submission is through 2 ports. Conceptually each port
+        /*
+         * Hardware submission is through 2 ports. Conceptually each port
          * has a (RING_START, RING_HEAD, RING_TAIL) tuple. RING_START is
          * static for a context, and unique to each, so we only execute
          * requests belonging to a single context from each ring. RING_HEAD
@@ -622,11 +624,11 @@ static bool __execlists_dequeue(struct intel_engine_cs *engine)
                  * the HW to indicate that it has had a chance to respond.
                  */
                 if (!execlists_is_active(execlists, EXECLISTS_ACTIVE_HWACK))
-                        return false;
+                        return;
 
                 if (need_preempt(engine, last, execlists->queue_priority)) {
                         inject_preempt_context(engine);
-                        return false;
+                        return;
                 }
 
                 /*
@@ -651,7 +653,7 @@ static bool __execlists_dequeue(struct intel_engine_cs *engine)
                  * priorities of the ports haven't been switch.
                  */
                 if (port_count(&port[1]))
-                        return false;
+                        return;
 
                 /*
                  * WaIdleLiteRestore:bdw,skl
@@ -751,8 +753,10 @@ done:
                 port != execlists->port ? rq_prio(last) : INT_MIN;
 
         execlists->first = rb;
-        if (submit)
+        if (submit) {
                 port_assign(port, last);
+                execlists_submit_ports(engine);
+        }
 
         /* We must always keep the beast fed if we have work piled up */
         GEM_BUG_ON(execlists->first && !port_isset(execlists->port));
@@ -761,24 +765,10 @@ done:
         if (last)
                 execlists_user_begin(execlists, execlists->port);
 
-        return submit;
-}
-
-static void execlists_dequeue(struct intel_engine_cs *engine)
-{
-        struct intel_engine_execlists * const execlists = &engine->execlists;
-        unsigned long flags;
-        bool submit;
-
-        spin_lock_irqsave(&engine->timeline.lock, flags);
-        submit = __execlists_dequeue(engine);
-        spin_unlock_irqrestore(&engine->timeline.lock, flags);
-
-        if (submit)
-                execlists_submit_ports(engine);
-
-        GEM_BUG_ON(port_isset(execlists->port) &&
-                   !execlists_is_active(execlists, EXECLISTS_ACTIVE_USER));
+        /* If the engine is now idle, so should be the flag; and vice versa. */
+        GEM_BUG_ON(execlists_is_active(&engine->execlists,
+                                       EXECLISTS_ACTIVE_USER) ==
+                   !port_isset(engine->execlists.port));
 }
 
 void
@@ -874,17 +864,23 @@ static void reset_irq(struct intel_engine_cs *engine)
 {
         /* Mark all CS interrupts as complete */
         smp_store_mb(engine->execlists.active, 0);
-        synchronize_hardirq(engine->i915->drm.irq);
 
         clear_gtiir(engine);
+}
 
+static void reset_csb_pointers(struct intel_engine_execlists *execlists)
+{
         /*
-         * The port is checked prior to scheduling a tasklet, but
-         * just in case we have suspended the tasklet to do the
-         * wedging make sure that when it wakes, it decides there
-         * is no work to do by clearing the irq_posted bit.
+         * After a reset, the HW starts writing into CSB entry [0]. We
+         * therefore have to set our HEAD pointer back one entry so that
+         * the *first* entry we check is entry 0. To complicate this further,
+         * as we don't wait for the first interrupt after reset, we have to
+         * fake the HW write to point back to the last entry so that our
+         * inline comparison of our cached head position against the last HW
+         * write works even before the first interrupt.
          */
-        clear_bit(ENGINE_IRQ_EXECLIST, &engine->irq_posted);
+        execlists->csb_head = execlists->csb_write_reset;
+        WRITE_ONCE(*execlists->csb_write, execlists->csb_write_reset);
 }
 
 static void execlists_cancel_requests(struct intel_engine_cs *engine)
@@ -911,14 +907,12 @@ static void execlists_cancel_requests(struct intel_engine_cs *engine)
          * submission's irq state, we also wish to remind ourselves that
          * it is irq state.)
          */
-        local_irq_save(flags);
+        spin_lock_irqsave(&engine->timeline.lock, flags);
 
         /* Cancel the requests on the HW and clear the ELSP tracker. */
         execlists_cancel_port_requests(execlists);
         reset_irq(engine);
 
-        spin_lock(&engine->timeline.lock);
-
         /* Mark all executing requests as skipped. */
         list_for_each_entry(rq, &engine->timeline.requests, link) {
                 GEM_BUG_ON(!rq->global_seqno);
@@ -952,194 +946,169 @@ static void execlists_cancel_requests(struct intel_engine_cs *engine)
         execlists->first = NULL;
         GEM_BUG_ON(port_isset(execlists->port));
 
-        spin_unlock(&engine->timeline.lock);
+        spin_unlock_irqrestore(&engine->timeline.lock, flags);
+}
 
-        local_irq_restore(flags);
+static inline bool
+reset_in_progress(const struct intel_engine_execlists *execlists)
+{
+        return unlikely(!__tasklet_is_enabled(&execlists->tasklet));
 }
 
 static void process_csb(struct intel_engine_cs *engine)
 {
         struct intel_engine_execlists * const execlists = &engine->execlists;
         struct execlist_port *port = execlists->port;
-        struct drm_i915_private *i915 = engine->i915;
-        bool fw = false;
+        const u32 * const buf = execlists->csb_status;
+        u8 head, tail;
+
+        /*
+         * Note that csb_write, csb_status may be either in HWSP or mmio.
+         * When reading from the csb_write mmio register, we have to be
+         * careful to only use the GEN8_CSB_WRITE_PTR portion, which is
+         * the low 4bits. As it happens we know the next 4bits are always
+         * zero and so we can simply masked off the low u8 of the register
+         * and treat it identically to reading from the HWSP (without having
+         * to use explicit shifting and masking, and probably bifurcating
+         * the code to handle the legacy mmio read).
+         */
+        head = execlists->csb_head;
+        tail = READ_ONCE(*execlists->csb_write);
+        GEM_TRACE("%s cs-irq head=%d, tail=%d\n", engine->name, head, tail);
+        if (unlikely(head == tail))
+                return;
+
+        /*
+         * Hopefully paired with a wmb() in HW!
+         *
+         * We must complete the read of the write pointer before any reads
+         * from the CSB, so that we do not see stale values. Without an rmb
+         * (lfence) the HW may speculatively perform the CSB[] reads *before*
+         * we perform the READ_ONCE(*csb_write).
+         */
+        rmb();
 
         do {
-                /* The HWSP contains a (cacheable) mirror of the CSB */
-                const u32 *buf =
-                        &engine->status_page.page_addr[I915_HWS_CSB_BUF0_INDEX];
-                unsigned int head, tail;
+                struct i915_request *rq;
+                unsigned int status;
+                unsigned int count;
 
-                /* Clear before reading to catch new interrupts */
-                clear_bit(ENGINE_IRQ_EXECLIST, &engine->irq_posted);
-                smp_mb__after_atomic();
+                if (++head == GEN8_CSB_ENTRIES)
+                        head = 0;
 
-                if (unlikely(execlists->csb_use_mmio)) {
-                        if (!fw) {
-                                intel_uncore_forcewake_get(i915, execlists->fw_domains);
-                                fw = true;
-                        }
+                /*
+                 * We are flying near dragons again.
+                 *
+                 * We hold a reference to the request in execlist_port[]
+                 * but no more than that. We are operating in softirq
+                 * context and so cannot hold any mutex or sleep. That
+                 * prevents us stopping the requests we are processing
+                 * in port[] from being retired simultaneously (the
+                 * breadcrumb will be complete before we see the
+                 * context-switch). As we only hold the reference to the
+                 * request, any pointer chasing underneath the request
+                 * is subject to a potential use-after-free. Thus we
+                 * store all of the bookkeeping within port[] as
+                 * required, and avoid using unguarded pointers beneath
+                 * request itself. The same applies to the atomic
+                 * status notifier.
+                 */
 
-                        buf = (u32 * __force)
-                                (i915->regs + i915_mmio_reg_offset(RING_CONTEXT_STATUS_BUF_LO(engine, 0)));
+                GEM_TRACE("%s csb[%d]: status=0x%08x:0x%08x, active=0x%x\n",
+                          engine->name, head,
+                          buf[2 * head + 0], buf[2 * head + 1],
+                          execlists->active);
+
+                status = buf[2 * head];
+                if (status & (GEN8_CTX_STATUS_IDLE_ACTIVE |
+                              GEN8_CTX_STATUS_PREEMPTED))
+                        execlists_set_active(execlists,
+                                             EXECLISTS_ACTIVE_HWACK);
+                if (status & GEN8_CTX_STATUS_ACTIVE_IDLE)
+                        execlists_clear_active(execlists,
+                                               EXECLISTS_ACTIVE_HWACK);
+
+                if (!(status & GEN8_CTX_STATUS_COMPLETED_MASK))
+                        continue;
 
-                        head = readl(i915->regs + i915_mmio_reg_offset(RING_CONTEXT_STATUS_PTR(engine)));
-                        tail = GEN8_CSB_WRITE_PTR(head);
-                        head = GEN8_CSB_READ_PTR(head);
-                        execlists->csb_head = head;
-                } else {
-                        const int write_idx =
-                                intel_hws_csb_write_index(i915) -
-                                I915_HWS_CSB_BUF0_INDEX;
+                /* We should never get a COMPLETED | IDLE_ACTIVE! */
+                GEM_BUG_ON(status & GEN8_CTX_STATUS_IDLE_ACTIVE);
 
-                        head = execlists->csb_head;
-                        tail = READ_ONCE(buf[write_idx]);
-                        rmb(); /* Hopefully paired with a wmb() in HW */
+                if (status & GEN8_CTX_STATUS_COMPLETE &&
+                    buf[2*head + 1] == execlists->preempt_complete_status) {
+                        GEM_TRACE("%s preempt-idle\n", engine->name);
+                        complete_preempt_context(execlists);
+                        continue;
                 }
-                GEM_TRACE("%s cs-irq head=%d [%d%s], tail=%d [%d%s]\n",
-                          engine->name,
-                          head, GEN8_CSB_READ_PTR(readl(i915->regs + i915_mmio_reg_offset(RING_CONTEXT_STATUS_PTR(engine)))), fw ? "" : "?",
-                          tail, GEN8_CSB_WRITE_PTR(readl(i915->regs + i915_mmio_reg_offset(RING_CONTEXT_STATUS_PTR(engine)))), fw ? "" : "?");
 
-                while (head != tail) {
-                        struct i915_request *rq;
-                        unsigned int status;
-                        unsigned int count;
+                if (status & GEN8_CTX_STATUS_PREEMPTED &&
+                    execlists_is_active(execlists,
+                                        EXECLISTS_ACTIVE_PREEMPT))
+                        continue;
 
-                        if (++head == GEN8_CSB_ENTRIES)
-                                head = 0;
+                GEM_BUG_ON(!execlists_is_active(execlists,
+                                                EXECLISTS_ACTIVE_USER));
 
+                rq = port_unpack(port, &count);
+                GEM_TRACE("%s out[0]: ctx=%d.%d, global=%d (fence %llx:%d) (current %d), prio=%d\n",
+                          engine->name,
+                          port->context_id, count,
+                          rq ? rq->global_seqno : 0,
+                          rq ? rq->fence.context : 0,
+                          rq ? rq->fence.seqno : 0,
+                          intel_engine_get_seqno(engine),
+                          rq ? rq_prio(rq) : 0);
+
+                /* Check the context/desc id for this event matches */
+                GEM_DEBUG_BUG_ON(buf[2 * head + 1] != port->context_id);
+
+                GEM_BUG_ON(count == 0);
+                if (--count == 0) {
                         /*
-                         * We are flying near dragons again.
-                         *
-                         * We hold a reference to the request in execlist_port[]
-                         * but no more than that. We are operating in softirq
-                         * context and so cannot hold any mutex or sleep. That
-                         * prevents us stopping the requests we are processing
-                         * in port[] from being retired simultaneously (the
-                         * breadcrumb will be complete before we see the
-                         * context-switch). As we only hold the reference to the
-                         * request, any pointer chasing underneath the request
-                         * is subject to a potential use-after-free. Thus we
-                         * store all of the bookkeeping within port[] as
-                         * required, and avoid using unguarded pointers beneath
-                         * request itself. The same applies to the atomic
-                         * status notifier.
+                         * On the final event corresponding to the
+                         * submission of this context, we expect either
+                         * an element-switch event or a completion
+                         * event (and on completion, the active-idle
+                         * marker). No more preemptions, lite-restore
+                         * or otherwise.
                          */
+                        GEM_BUG_ON(status & GEN8_CTX_STATUS_PREEMPTED);
+                        GEM_BUG_ON(port_isset(&port[1]) &&
+                                   !(status & GEN8_CTX_STATUS_ELEMENT_SWITCH));
+                        GEM_BUG_ON(!port_isset(&port[1]) &&
+                                   !(status & GEN8_CTX_STATUS_ACTIVE_IDLE));
 
-                        status = READ_ONCE(buf[2 * head]); /* maybe mmio! */
-                        GEM_TRACE("%s csb[%d]: status=0x%08x:0x%08x, active=0x%x\n",
-                                  engine->name, head,
-                                  status, buf[2*head + 1],
-                                  execlists->active);
-
-                        if (status & (GEN8_CTX_STATUS_IDLE_ACTIVE |
-                                      GEN8_CTX_STATUS_PREEMPTED))
-                                execlists_set_active(execlists,
-                                                     EXECLISTS_ACTIVE_HWACK);
-                        if (status & GEN8_CTX_STATUS_ACTIVE_IDLE)
-                                execlists_clear_active(execlists,
-                                                       EXECLISTS_ACTIVE_HWACK);
-
-                        if (!(status & GEN8_CTX_STATUS_COMPLETED_MASK))
-                                continue;
-
-                        /* We should never get a COMPLETED | IDLE_ACTIVE! */
-                        GEM_BUG_ON(status & GEN8_CTX_STATUS_IDLE_ACTIVE);
-
-                        if (status & GEN8_CTX_STATUS_COMPLETE &&
-                            buf[2*head + 1] == execlists->preempt_complete_status) {
-                                GEM_TRACE("%s preempt-idle\n", engine->name);
-                                complete_preempt_context(execlists);
-                                continue;
-                        }
-
-                        if (status & GEN8_CTX_STATUS_PREEMPTED &&
-                            execlists_is_active(execlists,
-                                                EXECLISTS_ACTIVE_PREEMPT))
-                                continue;
-
-                        GEM_BUG_ON(!execlists_is_active(execlists,
-                                                        EXECLISTS_ACTIVE_USER));
-
-                        rq = port_unpack(port, &count);
-                        GEM_TRACE("%s out[0]: ctx=%d.%d, global=%d (fence %llx:%d) (current %d), prio=%d\n",
-                                  engine->name,
-                                  port->context_id, count,
-                                  rq ? rq->global_seqno : 0,
-                                  rq ? rq->fence.context : 0,
-                                  rq ? rq->fence.seqno : 0,
-                                  intel_engine_get_seqno(engine),
-                                  rq ? rq_prio(rq) : 0);
-
-                        /* Check the context/desc id for this event matches */
-                        GEM_DEBUG_BUG_ON(buf[2 * head + 1] != port->context_id);
+                        /*
+                         * We rely on the hardware being strongly
+                         * ordered, that the breadcrumb write is
+                         * coherent (visible from the CPU) before the
+                         * user interrupt and CSB is processed.
+                         */
+                        GEM_BUG_ON(!i915_request_completed(rq));
 
-                        GEM_BUG_ON(count == 0);
-                        if (--count == 0) {
-                                /*
-                                 * On the final event corresponding to the
-                                 * submission of this context, we expect either
-                                 * an element-switch event or a completion
-                                 * event (and on completion, the active-idle
-                                 * marker). No more preemptions, lite-restore
-                                 * or otherwise.
-                                 */
-                                GEM_BUG_ON(status & GEN8_CTX_STATUS_PREEMPTED);
-                                GEM_BUG_ON(port_isset(&port[1]) &&
-                                           !(status & GEN8_CTX_STATUS_ELEMENT_SWITCH));
-                                GEM_BUG_ON(!port_isset(&port[1]) &&
-                                           !(status & GEN8_CTX_STATUS_ACTIVE_IDLE));
+                        execlists_context_schedule_out(rq,
+                                                       INTEL_CONTEXT_SCHEDULE_OUT);
+                        i915_request_put(rq);
 
-                                /*
-                                 * We rely on the hardware being strongly
-                                 * ordered, that the breadcrumb write is
-                                 * coherent (visible from the CPU) before the
-                                 * user interrupt and CSB is processed.
-                                 */
-                                GEM_BUG_ON(!i915_request_completed(rq));
-
-                                execlists_context_schedule_out(rq,
-                                                               INTEL_CONTEXT_SCHEDULE_OUT);
-                                i915_request_put(rq);
-
-                                GEM_TRACE("%s completed ctx=%d\n",
-                                          engine->name, port->context_id);
-
-                                port = execlists_port_complete(execlists, port);
-                                if (port_isset(port))
-                                        execlists_user_begin(execlists, port);
-                                else
-                                        execlists_user_end(execlists);
-                        } else {
-                                port_set(port, port_pack(rq, count));
-                        }
-                }
+                        GEM_TRACE("%s completed ctx=%d\n",
+                                  engine->name, port->context_id);
 
-                if (head != execlists->csb_head) {
-                        execlists->csb_head = head;
-                        writel(_MASKED_FIELD(GEN8_CSB_READ_PTR_MASK, head << 8),
-                               i915->regs + i915_mmio_reg_offset(RING_CONTEXT_STATUS_PTR(engine)));
+                        port = execlists_port_complete(execlists, port);
+                        if (port_isset(port))
+                                execlists_user_begin(execlists, port);
+                        else
+                                execlists_user_end(execlists);
+                } else {
+                        port_set(port, port_pack(rq, count));
                 }
-        } while (test_bit(ENGINE_IRQ_EXECLIST, &engine->irq_posted));
+        } while (head != tail);
 
-        if (unlikely(fw))
-                intel_uncore_forcewake_put(i915, execlists->fw_domains);
+        execlists->csb_head = head;
 }
 
-/*
- * Check the unread Context Status Buffers and manage the submission of new
- * contexts to the ELSP accordingly.
- */
-static void execlists_submission_tasklet(unsigned long data)
+static void __execlists_submission_tasklet(struct intel_engine_cs *const engine)
 {
-        struct intel_engine_cs * const engine = (struct intel_engine_cs *)data;
-
-        GEM_TRACE("%s awake?=%d, active=%x, irq-posted?=%d\n",
-                  engine->name,
-                  engine->i915->gt.awake,
-                  engine->execlists.active,
-                  test_bit(ENGINE_IRQ_EXECLIST, &engine->irq_posted));
+        lockdep_assert_held(&engine->timeline.lock);
 
         /*
          * We can skip acquiring intel_runtime_pm_get() here as it was taken
@@ -1151,21 +1120,31 @@ static void execlists_submission_tasklet(unsigned long data)
          */
         GEM_BUG_ON(!engine->i915->gt.awake);
 
-        /*
-         * Prefer doing test_and_clear_bit() as a two stage operation to avoid
-         * imposing the cost of a locked atomic transaction when submitting a
-         * new request (outside of the context-switch interrupt).
-         */
-        if (test_bit(ENGINE_IRQ_EXECLIST, &engine->irq_posted))
-                process_csb(engine);
-
+        process_csb(engine);
         if (!execlists_is_active(&engine->execlists, EXECLISTS_ACTIVE_PREEMPT))
                 execlists_dequeue(engine);
+}
 
-        /* If the engine is now idle, so should be the flag; and vice versa. */
-        GEM_BUG_ON(execlists_is_active(&engine->execlists,
-                                       EXECLISTS_ACTIVE_USER) ==
-                   !port_isset(engine->execlists.port));
+/*
+ * Check the unread Context Status Buffers and manage the submission of new
+ * contexts to the ELSP accordingly.
+ */
+static void execlists_submission_tasklet(unsigned long data)
+{
+        struct intel_engine_cs * const engine = (struct intel_engine_cs *)data;
+        unsigned long flags;
+
+        GEM_TRACE("%s awake?=%d, active=%x\n",
+                  engine->name,
+                  engine->i915->gt.awake,
+                  engine->execlists.active);
+
+        spin_lock_irqsave(&engine->timeline.lock, flags);
+
+        if (engine->i915->gt.awake) /* we may be delayed until after we idle! */
+                __execlists_submission_tasklet(engine);
+
+        spin_unlock_irqrestore(&engine->timeline.lock, flags);
 }
 
 static void queue_request(struct intel_engine_cs *engine,
@@ -1176,16 +1155,30 @@ static void queue_request(struct intel_engine_cs *engine,
                       &lookup_priolist(engine, prio)->requests);
 }
 
-static void __submit_queue(struct intel_engine_cs *engine, int prio)
+static void __update_queue(struct intel_engine_cs *engine, int prio)
 {
         engine->execlists.queue_priority = prio;
-        tasklet_hi_schedule(&engine->execlists.tasklet);
+}
+
+static void __submit_queue_imm(struct intel_engine_cs *engine)
+{
+        struct intel_engine_execlists * const execlists = &engine->execlists;
+
+        if (reset_in_progress(execlists))
+                return; /* defer until we restart the engine following reset */
+
+        if (execlists->tasklet.func == execlists_submission_tasklet)
+                __execlists_submission_tasklet(engine);
+        else
+                tasklet_hi_schedule(&execlists->tasklet);
 }
 
 static void submit_queue(struct intel_engine_cs *engine, int prio)
 {
-        if (prio > engine->execlists.queue_priority)
-                __submit_queue(engine, prio);
+        if (prio > engine->execlists.queue_priority) {
+                __update_queue(engine, prio);
+                __submit_queue_imm(engine);
+        }
 }
 
 static void execlists_submit_request(struct i915_request *request)
@@ -1197,11 +1190,12 @@ static void execlists_submit_request(struct i915_request *request)
         spin_lock_irqsave(&engine->timeline.lock, flags);
 
         queue_request(engine, &request->sched, rq_prio(request));
-        submit_queue(engine, rq_prio(request));
 
         GEM_BUG_ON(!engine->execlists.first);
         GEM_BUG_ON(list_empty(&request->sched.link));
 
+        submit_queue(engine, rq_prio(request));
+
         spin_unlock_irqrestore(&engine->timeline.lock, flags);
 }
 
@@ -1328,8 +1322,11 @@ static void execlists_schedule(struct i915_request *request,
                 }
 
                 if (prio > engine->execlists.queue_priority &&
-                    i915_sw_fence_done(&sched_to_request(node)->submit))
-                        __submit_queue(engine, prio);
+                    i915_sw_fence_done(&sched_to_request(node)->submit)) {
+                        /* defer submission until after all of our updates */
+                        __update_queue(engine, prio);
+                        tasklet_hi_schedule(&engine->execlists.tasklet);
+                }
         }
 
         spin_unlock_irq(&engine->timeline.lock);
@@ -1337,11 +1334,15 @@ static void execlists_schedule(struct i915_request *request,
 
 static void execlists_context_destroy(struct intel_context *ce)
 {
-        GEM_BUG_ON(!ce->state);
         GEM_BUG_ON(ce->pin_count);
 
+        if (!ce->state)
+                return;
+
         intel_ring_free(ce->ring);
-        __i915_gem_object_release_unless_active(ce->state->obj);
+
+        GEM_BUG_ON(i915_gem_object_is_active(ce->state->obj));
+        i915_gem_object_put(ce->state->obj);
 }
 
 static void execlists_context_unpin(struct intel_context *ce)
@@ -1906,6 +1907,7 @@ execlists_reset_prepare(struct intel_engine_cs *engine)
 {
         struct intel_engine_execlists * const execlists = &engine->execlists;
         struct i915_request *request, *active;
+        unsigned long flags;
 
         GEM_TRACE("%s\n", engine->name);
 
@@ -1920,6 +1922,8 @@ execlists_reset_prepare(struct intel_engine_cs *engine)
          */
         __tasklet_disable_sync_once(&execlists->tasklet);
 
+        spin_lock_irqsave(&engine->timeline.lock, flags);
+
         /*
          * We want to flush the pending context switches, having disabled
          * the tasklet above, we can assume exclusive access to the execlists.
@@ -1927,8 +1931,7 @@ execlists_reset_prepare(struct intel_engine_cs *engine)
          * and avoid blaming an innocent request if the stall was due to the
          * preemption itself.
          */
-        if (test_bit(ENGINE_IRQ_EXECLIST, &engine->irq_posted))
-                process_csb(engine);
+        process_csb(engine);
 
         /*
          * The last active request can then be no later than the last request
@@ -1938,15 +1941,12 @@ execlists_reset_prepare(struct intel_engine_cs *engine)
         active = NULL;
         request = port_request(execlists->port);
         if (request) {
-                unsigned long flags;
-
                 /*
                  * Prevent the breadcrumb from advancing before we decide
                  * which request is currently active.
                  */
                 intel_engine_stop_cs(engine);
 
-                spin_lock_irqsave(&engine->timeline.lock, flags);
                 list_for_each_entry_from_reverse(request,
                                                  &engine->timeline.requests,
                                                  link) {
@@ -1956,9 +1956,10 @@ execlists_reset_prepare(struct intel_engine_cs *engine)
 
                         active = request;
                 }
-                spin_unlock_irqrestore(&engine->timeline.lock, flags);
         }
 
+        spin_unlock_irqrestore(&engine->timeline.lock, flags);
+
         return active;
 }
 
@@ -1973,8 +1974,7 @@ static void execlists_reset(struct intel_engine_cs *engine,
                   engine->name, request ? request->global_seqno : 0,
                   intel_engine_get_seqno(engine));
 
-        /* See execlists_cancel_requests() for the irq/spinlock split. */
-        local_irq_save(flags);
+        spin_lock_irqsave(&engine->timeline.lock, flags);
 
         /*
          * Catch up with any missed context-switch interrupts.
@@ -1989,14 +1989,12 @@ static void execlists_reset(struct intel_engine_cs *engine,
         reset_irq(engine);
 
         /* Push back any incomplete requests for replay after the reset. */
-        spin_lock(&engine->timeline.lock);
         __unwind_incomplete_requests(engine);
-        spin_unlock(&engine->timeline.lock);
 
         /* Following the reset, we need to reload the CSB read/write pointers */
-        engine->execlists.csb_head = GEN8_CSB_ENTRIES - 1;
+        reset_csb_pointers(&engine->execlists);
 
-        local_irq_restore(flags);
+        spin_unlock_irqrestore(&engine->timeline.lock, flags);
 
         /*
          * If the request was innocent, we leave the request in the ELSP
@@ -2446,28 +2444,11 @@ logical_ring_default_irqs(struct intel_engine_cs *engine)
 static void
 logical_ring_setup(struct intel_engine_cs *engine)
 {
-        struct drm_i915_private *dev_priv = engine->i915;
-        enum forcewake_domains fw_domains;
-
         intel_engine_setup_common(engine);
 
         /* Intentionally left blank. */
         engine->buffer = NULL;
 
-        fw_domains = intel_uncore_forcewake_for_reg(dev_priv,
-                                                    RING_ELSP(engine),
-                                                    FW_REG_WRITE);
-
-        fw_domains |= intel_uncore_forcewake_for_reg(dev_priv,
-                                                     RING_CONTEXT_STATUS_PTR(engine),
-                                                     FW_REG_READ | FW_REG_WRITE);
-
-        fw_domains |= intel_uncore_forcewake_for_reg(dev_priv,
-                                                     RING_CONTEXT_STATUS_BUF_BASE(engine),
-                                                     FW_REG_READ);
-
-        engine->execlists.fw_domains = fw_domains;
-
         tasklet_init(&engine->execlists.tasklet,
                      execlists_submission_tasklet, (unsigned long)engine);
 
@@ -2475,34 +2456,60 @@ logical_ring_setup(struct intel_engine_cs *engine)
         logical_ring_default_irqs(engine);
 }
 
+static bool csb_force_mmio(struct drm_i915_private *i915)
+{
+        /* Older GVT emulation depends upon intercepting CSB mmio */
+        return intel_vgpu_active(i915) && !intel_vgpu_has_hwsp_emulation(i915);
+}
+
 static int logical_ring_init(struct intel_engine_cs *engine)
 {
+        struct drm_i915_private *i915 = engine->i915;
+        struct intel_engine_execlists * const execlists = &engine->execlists;
         int ret;
 
         ret = intel_engine_init_common(engine);
         if (ret)
                 goto error;
 
-        if (HAS_LOGICAL_RING_ELSQ(engine->i915)) {
-                engine->execlists.submit_reg = engine->i915->regs +
+        if (HAS_LOGICAL_RING_ELSQ(i915)) {
+                execlists->submit_reg = i915->regs +
                         i915_mmio_reg_offset(RING_EXECLIST_SQ_CONTENTS(engine));
-                engine->execlists.ctrl_reg = engine->i915->regs +
+                execlists->ctrl_reg = i915->regs +
                         i915_mmio_reg_offset(RING_EXECLIST_CONTROL(engine));
         } else {
-                engine->execlists.submit_reg = engine->i915->regs +
+                execlists->submit_reg = i915->regs +
                         i915_mmio_reg_offset(RING_ELSP(engine));
         }
 
-        engine->execlists.preempt_complete_status = ~0u;
-        if (engine->i915->preempt_context) {
+        execlists->preempt_complete_status = ~0u;
+        if (i915->preempt_context) {
                 struct intel_context *ce =
-                        to_intel_context(engine->i915->preempt_context, engine);
+                        to_intel_context(i915->preempt_context, engine);
 
-                engine->execlists.preempt_complete_status =
+                execlists->preempt_complete_status =
                         upper_32_bits(ce->lrc_desc);
         }
 
-        engine->execlists.csb_head = GEN8_CSB_ENTRIES - 1;
+        execlists->csb_read =
+                i915->regs + i915_mmio_reg_offset(RING_CONTEXT_STATUS_PTR(engine));
+        if (csb_force_mmio(i915)) {
+                execlists->csb_status = (u32 __force *)
+                        (i915->regs + i915_mmio_reg_offset(RING_CONTEXT_STATUS_BUF_LO(engine, 0)));
+
+                execlists->csb_write = (u32 __force *)execlists->csb_read;
+                execlists->csb_write_reset =
+                        _MASKED_FIELD(GEN8_CSB_WRITE_PTR_MASK,
+                                      GEN8_CSB_ENTRIES - 1);
+        } else {
+                execlists->csb_status =
+                        &engine->status_page.page_addr[I915_HWS_CSB_BUF0_INDEX];
+
+                execlists->csb_write =
+                        &engine->status_page.page_addr[intel_hws_csb_write_index(i915)];
+                execlists->csb_write_reset = GEN8_CSB_ENTRIES - 1;
+        }
+        reset_csb_pointers(execlists);
 
         return 0;
 
diff --git a/drivers/gpu/drm/i915/intel_pipe_crc.c b/drivers/gpu/drm/i915/intel_pipe_crc.c
index 39a4e4edda07..849e1b69ba73 100644
--- a/drivers/gpu/drm/i915/intel_pipe_crc.c
+++ b/drivers/gpu/drm/i915/intel_pipe_crc.c
@@ -30,160 +30,6 @@
 #include <linux/debugfs.h>
 #include "intel_drv.h"
 
-struct pipe_crc_info {
-        const char *name;
-        struct drm_i915_private *dev_priv;
-        enum pipe pipe;
-};
-
-static int i915_pipe_crc_open(struct inode *inode, struct file *filep)
-{
-        struct pipe_crc_info *info = inode->i_private;
-        struct drm_i915_private *dev_priv = info->dev_priv;
-        struct intel_pipe_crc *pipe_crc = &dev_priv->pipe_crc[info->pipe];
-
-        if (info->pipe >= INTEL_INFO(dev_priv)->num_pipes)
-                return -ENODEV;
-
-        spin_lock_irq(&pipe_crc->lock);
-
-        if (pipe_crc->opened) {
-                spin_unlock_irq(&pipe_crc->lock);
-                return -EBUSY; /* already open */
-        }
-
-        pipe_crc->opened = true;
-        filep->private_data = inode->i_private;
-
-        spin_unlock_irq(&pipe_crc->lock);
-
-        return 0;
-}
-
-static int i915_pipe_crc_release(struct inode *inode, struct file *filep)
-{
-        struct pipe_crc_info *info = inode->i_private;
-        struct drm_i915_private *dev_priv = info->dev_priv;
-        struct intel_pipe_crc *pipe_crc = &dev_priv->pipe_crc[info->pipe];
-
-        spin_lock_irq(&pipe_crc->lock);
-        pipe_crc->opened = false;
-        spin_unlock_irq(&pipe_crc->lock);
-
-        return 0;
-}
-
-/* (6 fields, 8 chars each, space separated (5) + '\n') */
-#define PIPE_CRC_LINE_LEN       (6 * 8 + 5 + 1)
-/* account for \'0' */
-#define PIPE_CRC_BUFFER_LEN     (PIPE_CRC_LINE_LEN + 1)
-
-static int pipe_crc_data_count(struct intel_pipe_crc *pipe_crc)
-{
-        lockdep_assert_held(&pipe_crc->lock);
-        return CIRC_CNT(pipe_crc->head, pipe_crc->tail,
-                        INTEL_PIPE_CRC_ENTRIES_NR);
-}
-
-static ssize_t
-i915_pipe_crc_read(struct file *filep, char __user *user_buf, size_t count,
-                   loff_t *pos)
-{
-        struct pipe_crc_info *info = filep->private_data;
-        struct drm_i915_private *dev_priv = info->dev_priv;
-        struct intel_pipe_crc *pipe_crc = &dev_priv->pipe_crc[info->pipe];
-        char buf[PIPE_CRC_BUFFER_LEN];
-        int n_entries;
-        ssize_t bytes_read;
-
-        /*
-         * Don't allow user space to provide buffers not big enough to hold
-         * a line of data.
-         */
-        if (count < PIPE_CRC_LINE_LEN)
-                return -EINVAL;
-
-        if (pipe_crc->source == INTEL_PIPE_CRC_SOURCE_NONE)
-                return 0;
-
-        /* nothing to read */
-        spin_lock_irq(&pipe_crc->lock);
-        while (pipe_crc_data_count(pipe_crc) == 0) {
-                int ret;
-
-                if (filep->f_flags & O_NONBLOCK) {
-                        spin_unlock_irq(&pipe_crc->lock);
-                        return -EAGAIN;
-                }
-
-                ret = wait_event_interruptible_lock_irq(pipe_crc->wq,
-                                pipe_crc_data_count(pipe_crc), pipe_crc->lock);
-                if (ret) {
-                        spin_unlock_irq(&pipe_crc->lock);
-                        return ret;
-                }
-        }
-
-        /* We now have one or more entries to read */
-        n_entries = count / PIPE_CRC_LINE_LEN;
-
-        bytes_read = 0;
-        while (n_entries > 0) {
-                struct intel_pipe_crc_entry *entry =
-                        &pipe_crc->entries[pipe_crc->tail];
-
-                if (CIRC_CNT(pipe_crc->head, pipe_crc->tail,
-                             INTEL_PIPE_CRC_ENTRIES_NR) < 1)
-                        break;
-
-                BUILD_BUG_ON_NOT_POWER_OF_2(INTEL_PIPE_CRC_ENTRIES_NR);
-                pipe_crc->tail = (pipe_crc->tail + 1) &
-                                 (INTEL_PIPE_CRC_ENTRIES_NR - 1);
-
-                bytes_read += snprintf(buf, PIPE_CRC_BUFFER_LEN,
-                                       "%8u %8x %8x %8x %8x %8x\n",
-                                       entry->frame, entry->crc[0],
-                                       entry->crc[1], entry->crc[2],
-                                       entry->crc[3], entry->crc[4]);
-
-                spin_unlock_irq(&pipe_crc->lock);
-
-                if (copy_to_user(user_buf, buf, PIPE_CRC_LINE_LEN))
-                        return -EFAULT;
-
-                user_buf += PIPE_CRC_LINE_LEN;
-                n_entries--;
-
-                spin_lock_irq(&pipe_crc->lock);
-        }
-
-        spin_unlock_irq(&pipe_crc->lock);
-
-        return bytes_read;
-}
-
-static const struct file_operations i915_pipe_crc_fops = {
-        .owner = THIS_MODULE,
-        .open = i915_pipe_crc_open,
-        .read = i915_pipe_crc_read,
-        .release = i915_pipe_crc_release,
-};
-
-static struct pipe_crc_info i915_pipe_crc_data[I915_MAX_PIPES] = {
-        {
-                .name = "i915_pipe_A_crc",
-                .pipe = PIPE_A,
-        },
-        {
-                .name = "i915_pipe_B_crc",
-                .pipe = PIPE_B,
-        },
-        {
-                .name = "i915_pipe_C_crc",
-                .pipe = PIPE_C,
-        },
-};
-
 static const char * const pipe_crc_sources[] = {
         "none",
         "plane1",
@@ -197,29 +43,6 @@ static const char * const pipe_crc_sources[] = {
         "auto",
 };
 
-static const char *pipe_crc_source_name(enum intel_pipe_crc_source source)
-{
-        BUILD_BUG_ON(ARRAY_SIZE(pipe_crc_sources) != INTEL_PIPE_CRC_SOURCE_MAX);
-        return pipe_crc_sources[source];
-}
-
-static int display_crc_ctl_show(struct seq_file *m, void *data)
-{
-        struct drm_i915_private *dev_priv = m->private;
-        enum pipe pipe;
-
-        for_each_pipe(dev_priv, pipe)
-                seq_printf(m, "%c %s\n", pipe_name(pipe),
-                           pipe_crc_source_name(dev_priv->pipe_crc[pipe].source));
-
-        return 0;
-}
-
-static int display_crc_ctl_open(struct inode *inode, struct file *file)
-{
-        return single_open(file, display_crc_ctl_show, inode->i_private);
-}
-
 static int i8xx_pipe_crc_ctl_reg(enum intel_pipe_crc_source *source,
                                  uint32_t *val)
 {
@@ -616,177 +439,6 @@ static int get_new_crc_ctl_reg(struct drm_i915_private *dev_priv,
                 return ivb_pipe_crc_ctl_reg(dev_priv, pipe, source, val, set_wa);
 }
 
-static int pipe_crc_set_source(struct drm_i915_private *dev_priv,
-                               enum pipe pipe,
-                               enum intel_pipe_crc_source source)
-{
-        struct intel_pipe_crc *pipe_crc = &dev_priv->pipe_crc[pipe];
-        enum intel_display_power_domain power_domain;
-        u32 val = 0; /* shut up gcc */
-        int ret;
-
-        if (pipe_crc->source == source)
-                return 0;
-
-        /* forbid changing the source without going back to 'none' */
-        if (pipe_crc->source && source)
-                return -EINVAL;
-
-        power_domain = POWER_DOMAIN_PIPE(pipe);
-        if (!intel_display_power_get_if_enabled(dev_priv, power_domain)) {
-                DRM_DEBUG_KMS("Trying to capture CRC while pipe is off\n");
-                return -EIO;
-        }
-
-        ret = get_new_crc_ctl_reg(dev_priv, pipe, &source, &val, true);
-        if (ret != 0)
-                goto out;
-
-        /* none -> real source transition */
-        if (source) {
-                struct intel_pipe_crc_entry *entries;
-
-                DRM_DEBUG_DRIVER("collecting CRCs for pipe %c, %s\n",
-                                 pipe_name(pipe), pipe_crc_source_name(source));
-
-                entries = kcalloc(INTEL_PIPE_CRC_ENTRIES_NR,
-                                  sizeof(pipe_crc->entries[0]),
-                                  GFP_KERNEL);
-                if (!entries) {
-                        ret = -ENOMEM;
-                        goto out;
-                }
-
-                spin_lock_irq(&pipe_crc->lock);
-                kfree(pipe_crc->entries);
-                pipe_crc->entries = entries;
-                pipe_crc->head = 0;
-                pipe_crc->tail = 0;
-                spin_unlock_irq(&pipe_crc->lock);
-        }
-
-        pipe_crc->source = source;
-
-        I915_WRITE(PIPE_CRC_CTL(pipe), val);
-        POSTING_READ(PIPE_CRC_CTL(pipe));
-
-        /* real source -> none transition */
-        if (!source) {
-                struct intel_pipe_crc_entry *entries;
-                struct intel_crtc *crtc = intel_get_crtc_for_pipe(dev_priv,
-                                                                  pipe);
-
-                DRM_DEBUG_DRIVER("stopping CRCs for pipe %c\n",
-                                 pipe_name(pipe));
-
-                drm_modeset_lock(&crtc->base.mutex, NULL);
-                if (crtc->base.state->active)
-                        intel_wait_for_vblank(dev_priv, pipe);
-                drm_modeset_unlock(&crtc->base.mutex);
-
-                spin_lock_irq(&pipe_crc->lock);
-                entries = pipe_crc->entries;
-                pipe_crc->entries = NULL;
-                pipe_crc->head = 0;
-                pipe_crc->tail = 0;
-                spin_unlock_irq(&pipe_crc->lock);
-
-                kfree(entries);
-
-                if (IS_G4X(dev_priv))
-                        g4x_undo_pipe_scramble_reset(dev_priv, pipe);
-                else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
-                        vlv_undo_pipe_scramble_reset(dev_priv, pipe);
-                else if ((IS_HASWELL(dev_priv) ||
-                          IS_BROADWELL(dev_priv)) && pipe == PIPE_A)
-                        hsw_pipe_A_crc_wa(dev_priv, false);
-        }
-
-        ret = 0;
-
-out:
-        intel_display_power_put(dev_priv, power_domain);
-
-        return ret;
-}
-
-/*
- * Parse pipe CRC command strings:
- *   command: wsp* object wsp+ name wsp+ source wsp*
- *   object: 'pipe'
- *   name: (A | B | C)
- *   source: (none | plane1 | plane2 | pf)
- *   wsp: (#0x20 | #0x9 | #0xA)+
- *
- * eg.:
- *  "pipe A plane1"  ->  Start CRC computations on plane1 of pipe A
- *  "pipe A none"    ->  Stop CRC
- */
-static int display_crc_ctl_tokenize(char *buf, char *words[], int max_words)
-{
-        int n_words = 0;
-
-        while (*buf) {
-                char *end;
-
-                /* skip leading white space */
-                buf = skip_spaces(buf);
-                if (!*buf)
-                        break;  /* end of buffer */
-
-                /* find end of word */
-                for (end = buf; *end && !isspace(*end); end++)
-                        ;
-
-                if (n_words == max_words) {
-                        DRM_DEBUG_DRIVER("too many words, allowed <= %d\n",
-                                         max_words);
-                        return -EINVAL; /* ran out of words[] before bytes */
-                }
-
-                if (*end)
-                        *end++ = '\0';
-                words[n_words++] = buf;
-                buf = end;
-        }
-
-        return n_words;
-}
-
-enum intel_pipe_crc_object {
-        PIPE_CRC_OBJECT_PIPE,
-};
-
-static const char * const pipe_crc_objects[] = {
-        "pipe",
-};
-
-static int
-display_crc_ctl_parse_object(const char *buf, enum intel_pipe_crc_object *o)
-{
-        int i;
-
-        i = match_string(pipe_crc_objects, ARRAY_SIZE(pipe_crc_objects), buf);
-        if (i < 0)
-                return i;
-
-        *o = i;
-        return 0;
-}
-
-static int display_crc_ctl_parse_pipe(struct drm_i915_private *dev_priv,
-                                      const char *buf, enum pipe *pipe)
-{
-        const char name = buf[0];
-
-        if (name < 'A' || name >= pipe_name(INTEL_INFO(dev_priv)->num_pipes))
-                return -EINVAL;
-
-        *pipe = name - 'A';
-
-        return 0;
-}
-
 static int
 display_crc_ctl_parse_source(const char *buf, enum intel_pipe_crc_source *s)
 {
@@ -805,81 +457,6 @@ display_crc_ctl_parse_source(const char *buf, enum intel_pipe_crc_source *s)
         return 0;
 }
 
-static int display_crc_ctl_parse(struct drm_i915_private *dev_priv,
-                                 char *buf, size_t len)
-{
-#define N_WORDS 3
-        int n_words;
-        char *words[N_WORDS];
-        enum pipe pipe;
-        enum intel_pipe_crc_object object;
-        enum intel_pipe_crc_source source;
-
-        n_words = display_crc_ctl_tokenize(buf, words, N_WORDS);
-        if (n_words != N_WORDS) {
-                DRM_DEBUG_DRIVER("tokenize failed, a command is %d words\n",
-                                 N_WORDS);
-                return -EINVAL;
-        }
-
-        if (display_crc_ctl_parse_object(words[0], &object) < 0) {
-                DRM_DEBUG_DRIVER("unknown object %s\n", words[0]);
-                return -EINVAL;
-        }
-
-        if (display_crc_ctl_parse_pipe(dev_priv, words[1], &pipe) < 0) {
-                DRM_DEBUG_DRIVER("unknown pipe %s\n", words[1]);
-                return -EINVAL;
-        }
-
-        if (display_crc_ctl_parse_source(words[2], &source) < 0) {
-                DRM_DEBUG_DRIVER("unknown source %s\n", words[2]);
-                return -EINVAL;
-        }
-
-        return pipe_crc_set_source(dev_priv, pipe, source);
-}
-
-static ssize_t display_crc_ctl_write(struct file *file, const char __user *ubuf,
-                                     size_t len, loff_t *offp)
-{
-        struct seq_file *m = file->private_data;
-        struct drm_i915_private *dev_priv = m->private;
-        char *tmpbuf;
-        int ret;
-
-        if (len == 0)
-                return 0;
-
-        if (len > PAGE_SIZE - 1) {
-                DRM_DEBUG_DRIVER("expected <%lu bytes into pipe crc control\n",
-                                 PAGE_SIZE);
-                return -E2BIG;
-        }
-
-        tmpbuf = memdup_user_nul(ubuf, len);
-        if (IS_ERR(tmpbuf))
-                return PTR_ERR(tmpbuf);
-
-        ret = display_crc_ctl_parse(dev_priv, tmpbuf, len);
-
-        kfree(tmpbuf);
-        if (ret < 0)
-                return ret;
-
-        *offp += len;
-        return len;
-}
-
-const struct file_operations i915_display_crc_ctl_fops = {
-        .owner = THIS_MODULE,
-        .open = display_crc_ctl_open,
-        .read = seq_read,
-        .llseek = seq_lseek,
-        .release = single_release,
-        .write = display_crc_ctl_write
-};
-
 void intel_display_crc_init(struct drm_i915_private *dev_priv)
 {
         enum pipe pipe;
@@ -887,30 +464,8 @@ void intel_display_crc_init(struct drm_i915_private *dev_priv)
         for_each_pipe(dev_priv, pipe) {
                 struct intel_pipe_crc *pipe_crc = &dev_priv->pipe_crc[pipe];
 
-                pipe_crc->opened = false;
                 spin_lock_init(&pipe_crc->lock);
-                init_waitqueue_head(&pipe_crc->wq);
-        }
-}
-
-int intel_pipe_crc_create(struct drm_minor *minor)
-{
-        struct drm_i915_private *dev_priv = to_i915(minor->dev);
-        struct dentry *ent;
-        int i;
-
-        for (i = 0; i < ARRAY_SIZE(i915_pipe_crc_data); i++) {
-                struct pipe_crc_info *info = &i915_pipe_crc_data[i];
-
-                info->dev_priv = dev_priv;
-                ent = debugfs_create_file(info->name, S_IRUGO,
-                                          minor->debugfs_root, info,
-                                          &i915_pipe_crc_fops);
-                if (!ent)
-                        return -ENOMEM;
         }
-
-        return 0;
 }
 
 int intel_crtc_set_crc_source(struct drm_crtc *crtc, const char *source_name,
diff --git a/drivers/gpu/drm/i915/intel_psr.c b/drivers/gpu/drm/i915/intel_psr.c
index d4cd19fea148..23acc9ac8d4d 100644
--- a/drivers/gpu/drm/i915/intel_psr.c
+++ b/drivers/gpu/drm/i915/intel_psr.c
@@ -56,43 +56,6 @@
 #include "intel_drv.h"
 #include "i915_drv.h"
 
-static inline enum intel_display_power_domain
-psr_aux_domain(struct intel_dp *intel_dp)
-{
-        /* CNL HW requires corresponding AUX IOs to be powered up for PSR.
-         * However, for non-A AUX ports the corresponding non-EDP transcoders
-         * would have already enabled power well 2 and DC_OFF. This means we can
-         * acquire a wider POWER_DOMAIN_AUX_{B,C,D,F} reference instead of a
-         * specific AUX_IO reference without powering up any extra wells.
-         * Note that PSR is enabled only on Port A even though this function
-         * returns the correct domain for other ports too.
-         */
-        return intel_dp->aux_ch == AUX_CH_A ? POWER_DOMAIN_AUX_IO_A :
-                                              intel_dp->aux_power_domain;
-}
-
-static void psr_aux_io_power_get(struct intel_dp *intel_dp)
-{
-        struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp);
-        struct drm_i915_private *dev_priv = to_i915(intel_dig_port->base.base.dev);
-
-        if (INTEL_GEN(dev_priv) < 10)
-                return;
-
-        intel_display_power_get(dev_priv, psr_aux_domain(intel_dp));
-}
-
-static void psr_aux_io_power_put(struct intel_dp *intel_dp)
-{
-        struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp);
-        struct drm_i915_private *dev_priv = to_i915(intel_dig_port->base.base.dev);
-
-        if (INTEL_GEN(dev_priv) < 10)
-                return;
-
-        intel_display_power_put(dev_priv, psr_aux_domain(intel_dp));
-}
-
 void intel_psr_irq_control(struct drm_i915_private *dev_priv, bool debug)
 {
         u32 debug_mask, mask;
@@ -278,8 +241,8 @@ void intel_psr_init_dpcd(struct intel_dp *intel_dp)
         }
 }
 
-static void hsw_psr_setup_vsc(struct intel_dp *intel_dp,
-                              const struct intel_crtc_state *crtc_state)
+static void intel_psr_setup_vsc(struct intel_dp *intel_dp,
+                                const struct intel_crtc_state *crtc_state)
 {
         struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp);
         struct drm_i915_private *dev_priv = to_i915(intel_dig_port->base.base.dev);
@@ -336,7 +299,7 @@ static void hsw_psr_setup_aux(struct intel_dp *intel_dp)
         aux_clock_divider = intel_dp->get_aux_clock_divider(intel_dp, 0);
 
         /* Start with bits set for DDI_AUX_CTL register */
-        aux_ctl = intel_dp->get_aux_send_ctl(intel_dp, 0, sizeof(aux_msg),
+        aux_ctl = intel_dp->get_aux_send_ctl(intel_dp, sizeof(aux_msg),
                                              aux_clock_divider);
 
         /* Select only valid bits for SRD_AUX_CTL */
@@ -344,7 +307,7 @@ static void hsw_psr_setup_aux(struct intel_dp *intel_dp)
         I915_WRITE(EDP_PSR_AUX_CTL, aux_ctl);
 }
 
-static void hsw_psr_enable_sink(struct intel_dp *intel_dp)
+static void intel_psr_enable_sink(struct intel_dp *intel_dp)
 {
         struct intel_digital_port *dig_port = dp_to_dig_port(intel_dp);
         struct drm_device *dev = dig_port->base.base.dev;
@@ -360,6 +323,8 @@ static void hsw_psr_enable_sink(struct intel_dp *intel_dp)
 
         if (dev_priv->psr.link_standby)
                 dpcd_val |= DP_PSR_MAIN_LINK_ACTIVE;
+        if (!dev_priv->psr.psr2_enabled && INTEL_GEN(dev_priv) >= 8)
+                dpcd_val |= DP_PSR_CRC_VERIFICATION;
         drm_dp_dpcd_writeb(&intel_dp->aux, DP_PSR_EN_CFG, dpcd_val);
 
         drm_dp_dpcd_writeb(&intel_dp->aux, DP_SET_POWER, DP_SET_POWER_D0);
@@ -415,6 +380,9 @@ static void hsw_activate_psr1(struct intel_dp *intel_dp)
         else
                 val |= EDP_PSR_TP1_TP2_SEL;
 
+        if (INTEL_GEN(dev_priv) >= 8)
+                val |= EDP_PSR_CRC_ENABLE;
+
         val |= I915_READ(EDP_PSR_CTL) & EDP_PSR_RESTORE_PSR_ACTIVE_CTX_MASK;
         I915_WRITE(EDP_PSR_CTL, val);
 }
@@ -456,24 +424,6 @@ static void hsw_activate_psr2(struct intel_dp *intel_dp)
         I915_WRITE(EDP_PSR2_CTL, val);
 }
 
-static void hsw_psr_activate(struct intel_dp *intel_dp)
-{
-        struct intel_digital_port *dig_port = dp_to_dig_port(intel_dp);
-        struct drm_device *dev = dig_port->base.base.dev;
-        struct drm_i915_private *dev_priv = to_i915(dev);
-
-        /* On HSW+ after we enable PSR on source it will activate it
-         * as soon as it match configure idle_frame count. So
-         * we just actually enable it here on activation time.
-         */
-
-        /* psr1 and psr2 are mutually exclusive.*/
-        if (dev_priv->psr.psr2_enabled)
-                hsw_activate_psr2(intel_dp);
-        else
-                hsw_activate_psr1(intel_dp);
-}
-
 static bool intel_psr2_config_valid(struct intel_dp *intel_dp,
                                     struct intel_crtc_state *crtc_state)
 {
@@ -576,27 +526,29 @@ static void intel_psr_activate(struct intel_dp *intel_dp)
         struct drm_device *dev = intel_dig_port->base.base.dev;
         struct drm_i915_private *dev_priv = to_i915(dev);
 
-        if (dev_priv->psr.psr2_enabled)
+        if (INTEL_GEN(dev_priv) >= 9)
                 WARN_ON(I915_READ(EDP_PSR2_CTL) & EDP_PSR2_ENABLE);
-        else
-                WARN_ON(I915_READ(EDP_PSR_CTL) & EDP_PSR_ENABLE);
+        WARN_ON(I915_READ(EDP_PSR_CTL) & EDP_PSR_ENABLE);
         WARN_ON(dev_priv->psr.active);
         lockdep_assert_held(&dev_priv->psr.lock);
 
-        dev_priv->psr.activate(intel_dp);
+        /* psr1 and psr2 are mutually exclusive.*/
+        if (dev_priv->psr.psr2_enabled)
+                hsw_activate_psr2(intel_dp);
+        else
+                hsw_activate_psr1(intel_dp);
+
         dev_priv->psr.active = true;
 }
 
-static void hsw_psr_enable_source(struct intel_dp *intel_dp,
-                                  const struct intel_crtc_state *crtc_state)
+static void intel_psr_enable_source(struct intel_dp *intel_dp,
+                                    const struct intel_crtc_state *crtc_state)
 {
         struct intel_digital_port *dig_port = dp_to_dig_port(intel_dp);
         struct drm_device *dev = dig_port->base.base.dev;
         struct drm_i915_private *dev_priv = to_i915(dev);
         enum transcoder cpu_transcoder = crtc_state->cpu_transcoder;
 
-        psr_aux_io_power_get(intel_dp);
-
         /* Only HSW and BDW have PSR AUX registers that need to be setup. SKL+
          * use hardcoded values PSR AUX transactions
          */
@@ -632,7 +584,8 @@ static void hsw_psr_enable_source(struct intel_dp *intel_dp,
                            EDP_PSR_DEBUG_MASK_MEMUP |
                            EDP_PSR_DEBUG_MASK_HPD |
                            EDP_PSR_DEBUG_MASK_LPSP |
-                           EDP_PSR_DEBUG_MASK_DISP_REG_WRITE);
+                           EDP_PSR_DEBUG_MASK_DISP_REG_WRITE |
+                           EDP_PSR_DEBUG_MASK_MAX_SLEEP);
         }
 }
 
@@ -666,9 +619,9 @@ void intel_psr_enable(struct intel_dp *intel_dp,
         dev_priv->psr.psr2_enabled = crtc_state->has_psr2;
         dev_priv->psr.busy_frontbuffer_bits = 0;
 
-        dev_priv->psr.setup_vsc(intel_dp, crtc_state);
-        dev_priv->psr.enable_sink(intel_dp);
-        dev_priv->psr.enable_source(intel_dp, crtc_state);
+        intel_psr_setup_vsc(intel_dp, crtc_state);
+        intel_psr_enable_sink(intel_dp);
+        intel_psr_enable_source(intel_dp, crtc_state);
         dev_priv->psr.enabled = intel_dp;
 
         intel_psr_activate(intel_dp);
@@ -677,8 +630,8 @@ unlock:
         mutex_unlock(&dev_priv->psr.lock);
 }
 
-static void hsw_psr_disable(struct intel_dp *intel_dp,
-                            const struct intel_crtc_state *old_crtc_state)
+static void
+intel_psr_disable_source(struct intel_dp *intel_dp)
 {
         struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp);
         struct drm_device *dev = intel_dig_port->base.base.dev;
@@ -717,8 +670,25 @@ static void hsw_psr_disable(struct intel_dp *intel_dp,
                 else
                         WARN_ON(I915_READ(EDP_PSR_CTL) & EDP_PSR_ENABLE);
         }
+}
+
+static void intel_psr_disable_locked(struct intel_dp *intel_dp)
+{
+        struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp);
+        struct drm_device *dev = intel_dig_port->base.base.dev;
+        struct drm_i915_private *dev_priv = to_i915(dev);
 
-        psr_aux_io_power_put(intel_dp);
+        lockdep_assert_held(&dev_priv->psr.lock);
+
+        if (!dev_priv->psr.enabled)
+                return;
+
+        intel_psr_disable_source(intel_dp);
+
+        /* Disable PSR on Sink */
+        drm_dp_dpcd_writeb(&intel_dp->aux, DP_PSR_EN_CFG, 0);
+
+        dev_priv->psr.enabled = NULL;
 }
 
 /**
@@ -742,22 +712,44 @@ void intel_psr_disable(struct intel_dp *intel_dp,
                 return;
 
         mutex_lock(&dev_priv->psr.lock);
-        if (!dev_priv->psr.enabled) {
-                mutex_unlock(&dev_priv->psr.lock);
-                return;
-        }
+        intel_psr_disable_locked(intel_dp);
+        mutex_unlock(&dev_priv->psr.lock);
+        cancel_work_sync(&dev_priv->psr.work);
+}
 
-        dev_priv->psr.disable_source(intel_dp, old_crtc_state);
+int intel_psr_wait_for_idle(struct drm_i915_private *dev_priv)
+{
+        i915_reg_t reg;
+        u32 mask;
 
-        /* Disable PSR on Sink */
-        drm_dp_dpcd_writeb(&intel_dp->aux, DP_PSR_EN_CFG, 0);
+        /*
+         * The sole user right now is intel_pipe_update_start(),
+         * which won't race with psr_enable/disable, which is
+         * where psr2_enabled is written to. So, we don't need
+         * to acquire the psr.lock. More importantly, we want the
+         * latency inside intel_pipe_update_start() to be as low
+         * as possible, so no need to acquire psr.lock when it is
+         * not needed and will induce latencies in the atomic
+         * update path.
+         */
+        if (dev_priv->psr.psr2_enabled) {
+                reg = EDP_PSR2_STATUS;
+                mask = EDP_PSR2_STATUS_STATE_MASK;
+        } else {
+                reg = EDP_PSR_STATUS;
+                mask = EDP_PSR_STATUS_STATE_MASK;
+        }
 
-        dev_priv->psr.enabled = NULL;
-        mutex_unlock(&dev_priv->psr.lock);
-        cancel_work_sync(&dev_priv->psr.work);
+        /*
+         * Max time for PSR to idle = Inverse of the refresh rate +
+         * 6 ms of exit training time + 1.5 ms of aux channel
+         * handshake. 50 msec is defesive enough to cover everything.
+         */
+        return intel_wait_for_register(dev_priv, reg, mask,
+                                       EDP_PSR_STATUS_STATE_IDLE, 50);
 }
 
-static bool psr_wait_for_idle(struct drm_i915_private *dev_priv)
+static bool __psr_wait_for_idle_locked(struct drm_i915_private *dev_priv)
 {
         struct intel_dp *intel_dp;
         i915_reg_t reg;
@@ -803,7 +795,7 @@ static void intel_psr_work(struct work_struct *work)
          * PSR might take some time to get fully disabled
          * and be ready for re-enable.
          */
-        if (!psr_wait_for_idle(dev_priv))
+        if (!__psr_wait_for_idle_locked(dev_priv))
                 goto unlock;
 
         /*
@@ -811,7 +803,7 @@ static void intel_psr_work(struct work_struct *work)
          * recheck. Since psr_flush first clears this and then reschedules we
          * won't ever miss a flush when bailing out here.
          */
-        if (dev_priv->psr.busy_frontbuffer_bits)
+        if (dev_priv->psr.busy_frontbuffer_bits || dev_priv->psr.active)
                 goto unlock;
 
         intel_psr_activate(dev_priv->psr.enabled);
@@ -986,11 +978,58 @@ void intel_psr_init(struct drm_i915_private *dev_priv)
 
         INIT_WORK(&dev_priv->psr.work, intel_psr_work);
         mutex_init(&dev_priv->psr.lock);
+}
 
-        dev_priv->psr.enable_source = hsw_psr_enable_source;
-        dev_priv->psr.disable_source = hsw_psr_disable;
-        dev_priv->psr.enable_sink = hsw_psr_enable_sink;
-        dev_priv->psr.activate = hsw_psr_activate;
-        dev_priv->psr.setup_vsc = hsw_psr_setup_vsc;
+void intel_psr_short_pulse(struct intel_dp *intel_dp)
+{
+        struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp);
+        struct drm_device *dev = intel_dig_port->base.base.dev;
+        struct drm_i915_private *dev_priv = to_i915(dev);
+        struct i915_psr *psr = &dev_priv->psr;
+        u8 val;
+        const u8 errors = DP_PSR_RFB_STORAGE_ERROR |
+                          DP_PSR_VSC_SDP_UNCORRECTABLE_ERROR |
+                          DP_PSR_LINK_CRC_ERROR;
+
+        if (!CAN_PSR(dev_priv) || !intel_dp_is_edp(intel_dp))
+                return;
+
+        mutex_lock(&psr->lock);
+
+        if (psr->enabled != intel_dp)
+                goto exit;
+
+        if (drm_dp_dpcd_readb(&intel_dp->aux, DP_PSR_STATUS, &val) != 1) {
+                DRM_ERROR("PSR_STATUS dpcd read failed\n");
+                goto exit;
+        }
+
+        if ((val & DP_PSR_SINK_STATE_MASK) == DP_PSR_SINK_INTERNAL_ERROR) {
+                DRM_DEBUG_KMS("PSR sink internal error, disabling PSR\n");
+                intel_psr_disable_locked(intel_dp);
+        }
+
+        if (drm_dp_dpcd_readb(&intel_dp->aux, DP_PSR_ERROR_STATUS, &val) != 1) {
+                DRM_ERROR("PSR_ERROR_STATUS dpcd read failed\n");
+                goto exit;
+        }
 
+        if (val & DP_PSR_RFB_STORAGE_ERROR)
+                DRM_DEBUG_KMS("PSR RFB storage error, disabling PSR\n");
+        if (val & DP_PSR_VSC_SDP_UNCORRECTABLE_ERROR)
+                DRM_DEBUG_KMS("PSR VSC SDP uncorrectable error, disabling PSR\n");
+        if (val & DP_PSR_LINK_CRC_ERROR)
+                DRM_ERROR("PSR Link CRC error, disabling PSR\n");
+
+        if (val & ~errors)
+                DRM_ERROR("PSR_ERROR_STATUS unhandled errors %x\n",
+                          val & ~errors);
+        if (val & errors)
+                intel_psr_disable_locked(intel_dp);
+        /* clear status register */
+        drm_dp_dpcd_writeb(&intel_dp->aux, DP_PSR_ERROR_STATUS, val);
+
+        /* TODO: handle PSR2 errors */
+exit:
+        mutex_unlock(&psr->lock);
 }
diff --git a/drivers/gpu/drm/i915/intel_ringbuffer.c b/drivers/gpu/drm/i915/intel_ringbuffer.c
index e0448eff12bd..700f94c371b3 100644
--- a/drivers/gpu/drm/i915/intel_ringbuffer.c
+++ b/drivers/gpu/drm/i915/intel_ringbuffer.c
@@ -1169,8 +1169,11 @@ static void intel_ring_context_destroy(struct intel_context *ce)
 {
         GEM_BUG_ON(ce->pin_count);
 
-        if (ce->state)
-                __i915_gem_object_release_unless_active(ce->state->obj);
+        if (!ce->state)
+                return;
+
+        GEM_BUG_ON(i915_gem_object_is_active(ce->state->obj));
+        i915_gem_object_put(ce->state->obj);
 }
 
 static int __context_pin_ppgtt(struct i915_gem_context *ctx)
diff --git a/drivers/gpu/drm/i915/intel_ringbuffer.h b/drivers/gpu/drm/i915/intel_ringbuffer.h
index a0bc7a8222b4..ce6cc2a6cf7a 100644
--- a/drivers/gpu/drm/i915/intel_ringbuffer.h
+++ b/drivers/gpu/drm/i915/intel_ringbuffer.h
@@ -300,24 +300,44 @@ struct intel_engine_execlists {
         struct rb_node *first;
 
         /**
-         * @fw_domains: forcewake domains for irq tasklet
+         * @csb_read: control register for Context Switch buffer
+         *
+         * Note this register is always in mmio.
          */
-        unsigned int fw_domains;
+        u32 __iomem *csb_read;
 
         /**
-         * @csb_head: context status buffer head
+         * @csb_write: control register for Context Switch buffer
+         *
+         * Note this register may be either mmio or HWSP shadow.
          */
-        unsigned int csb_head;
+        u32 *csb_write;
 
         /**
-         * @csb_use_mmio: access csb through mmio, instead of hwsp
+         * @csb_status: status array for Context Switch buffer
+         *
+         * Note these register may be either mmio or HWSP shadow.
          */
-        bool csb_use_mmio;
+        u32 *csb_status;
 
         /**
          * @preempt_complete_status: expected CSB upon completing preemption
          */
         u32 preempt_complete_status;
+
+        /**
+         * @csb_write_reset: reset value for CSB write pointer
+         *
+         * As the CSB write pointer maybe either in HWSP or as a field
+         * inside an mmio register, we want to reprogram it slightly
+         * differently to avoid later confusion.
+         */
+        u32 csb_write_reset;
+
+        /**
+         * @csb_head: context status buffer head
+         */
+        u8 csb_head;
 };
 
 #define INTEL_ENGINE_CS_MAX_NAME 8
@@ -345,10 +365,8 @@ struct intel_engine_cs {
         struct drm_i915_gem_object *default_state;
         void *pinned_default_state;
 
-        atomic_t irq_count;
         unsigned long irq_posted;
 #define ENGINE_IRQ_BREADCRUMB 0
-#define ENGINE_IRQ_EXECLIST 1
 
         /* Rather than have every client wait upon all user interrupts,
          * with the herd waking after every interrupt and each doing the
@@ -380,6 +398,7 @@ struct intel_engine_cs {
 
                 unsigned int hangcheck_interrupts;
                 unsigned int irq_enabled;
+                unsigned int irq_count;
 
                 bool irq_armed : 1;
                 I915_SELFTEST_DECLARE(bool mock : 1);
@@ -928,11 +947,10 @@ static inline u32 intel_hws_preempt_done_address(struct intel_engine_cs *engine)
 /* intel_breadcrumbs.c -- user interrupt bottom-half for waiters */
 int intel_engine_init_breadcrumbs(struct intel_engine_cs *engine);
 
-static inline void intel_wait_init(struct intel_wait *wait,
-                                   struct i915_request *rq)
+static inline void intel_wait_init(struct intel_wait *wait)
 {
         wait->tsk = current;
-        wait->request = rq;
+        wait->request = NULL;
 }
 
 static inline void intel_wait_init_for_seqno(struct intel_wait *wait, u32 seqno)
diff --git a/drivers/gpu/drm/i915/intel_runtime_pm.c b/drivers/gpu/drm/i915/intel_runtime_pm.c
index de3a81034f77..6b5aa3b074ec 100644
--- a/drivers/gpu/drm/i915/intel_runtime_pm.c
+++ b/drivers/gpu/drm/i915/intel_runtime_pm.c
@@ -134,6 +134,14 @@ intel_display_power_domain_str(enum intel_display_power_domain domain)
                 return "AUX_F";
         case POWER_DOMAIN_AUX_IO_A:
                 return "AUX_IO_A";
+        case POWER_DOMAIN_AUX_TBT1:
+                return "AUX_TBT1";
+        case POWER_DOMAIN_AUX_TBT2:
+                return "AUX_TBT2";
+        case POWER_DOMAIN_AUX_TBT3:
+                return "AUX_TBT3";
+        case POWER_DOMAIN_AUX_TBT4:
+                return "AUX_TBT4";
         case POWER_DOMAIN_GMBUS:
                 return "GMBUS";
         case POWER_DOMAIN_INIT:
@@ -384,7 +392,8 @@ static void hsw_power_well_enable(struct drm_i915_private *dev_priv,
         u32 val;
 
         if (wait_fuses) {
-                pg = SKL_PW_TO_PG(id);
+                pg = INTEL_GEN(dev_priv) >= 11 ? ICL_PW_TO_PG(id) :
+                                                 SKL_PW_TO_PG(id);
                 /*
                  * For PW1 we have to wait both for the PW0/PG0 fuse state
                  * before enabling the power well and PW1/PG1's own fuse
@@ -430,6 +439,43 @@ static void hsw_power_well_disable(struct drm_i915_private *dev_priv,
         hsw_wait_for_power_well_disable(dev_priv, power_well);
 }
 
+#define ICL_AUX_PW_TO_PORT(pw)  ((pw) - ICL_DISP_PW_AUX_A)
+
+static void
+icl_combo_phy_aux_power_well_enable(struct drm_i915_private *dev_priv,
+                                    struct i915_power_well *power_well)
+{
+        enum i915_power_well_id id = power_well->id;
+        enum port port = ICL_AUX_PW_TO_PORT(id);
+        u32 val;
+
+        val = I915_READ(HSW_PWR_WELL_CTL_DRIVER(id));
+        I915_WRITE(HSW_PWR_WELL_CTL_DRIVER(id), val | HSW_PWR_WELL_CTL_REQ(id));
+
+        val = I915_READ(ICL_PORT_CL_DW12(port));
+        I915_WRITE(ICL_PORT_CL_DW12(port), val | ICL_LANE_ENABLE_AUX);
+
+        hsw_wait_for_power_well_enable(dev_priv, power_well);
+}
+
+static void
+icl_combo_phy_aux_power_well_disable(struct drm_i915_private *dev_priv,
+                                     struct i915_power_well *power_well)
+{
+        enum i915_power_well_id id = power_well->id;
+        enum port port = ICL_AUX_PW_TO_PORT(id);
+        u32 val;
+
+        val = I915_READ(ICL_PORT_CL_DW12(port));
+        I915_WRITE(ICL_PORT_CL_DW12(port), val & ~ICL_LANE_ENABLE_AUX);
+
+        val = I915_READ(HSW_PWR_WELL_CTL_DRIVER(id));
+        I915_WRITE(HSW_PWR_WELL_CTL_DRIVER(id),
+                   val & ~HSW_PWR_WELL_CTL_REQ(id));
+
+        hsw_wait_for_power_well_disable(dev_priv, power_well);
+}
+
 /*
  * We should only use the power well if we explicitly asked the hardware to
  * enable it, so check if it's enabled and also check if we've requested it to
@@ -1824,6 +1870,7 @@ void intel_display_power_put(struct drm_i915_private *dev_priv,
         BIT_ULL(POWER_DOMAIN_INIT))
 #define GLK_DISPLAY_AUX_A_POWER_DOMAINS (               \
         BIT_ULL(POWER_DOMAIN_AUX_A) |           \
+        BIT_ULL(POWER_DOMAIN_AUX_IO_A) |                \
         BIT_ULL(POWER_DOMAIN_INIT))
 #define GLK_DISPLAY_AUX_B_POWER_DOMAINS (               \
         BIT_ULL(POWER_DOMAIN_AUX_B) |           \
@@ -1896,6 +1943,105 @@ void intel_display_power_put(struct drm_i915_private *dev_priv,
         BIT_ULL(POWER_DOMAIN_AUX_A) |                   \
         BIT_ULL(POWER_DOMAIN_INIT))
 
+/*
+ * ICL PW_0/PG_0 domains (HW/DMC control):
+ * - PCI
+ * - clocks except port PLL
+ * - central power except FBC
+ * - shared functions except pipe interrupts, pipe MBUS, DBUF registers
+ * ICL PW_1/PG_1 domains (HW/DMC control):
+ * - DBUF function
+ * - PIPE_A and its planes, except VGA
+ * - transcoder EDP + PSR
+ * - transcoder DSI
+ * - DDI_A
+ * - FBC
+ */
+#define ICL_PW_4_POWER_DOMAINS (                        \
+        BIT_ULL(POWER_DOMAIN_PIPE_C) |                  \
+        BIT_ULL(POWER_DOMAIN_PIPE_C_PANEL_FITTER) |     \
+        BIT_ULL(POWER_DOMAIN_INIT))
+        /* VDSC/joining */
+#define ICL_PW_3_POWER_DOMAINS (                        \
+        ICL_PW_4_POWER_DOMAINS |                        \
+        BIT_ULL(POWER_DOMAIN_PIPE_B) |                  \
+        BIT_ULL(POWER_DOMAIN_TRANSCODER_A) |            \
+        BIT_ULL(POWER_DOMAIN_TRANSCODER_B) |            \
+        BIT_ULL(POWER_DOMAIN_TRANSCODER_C) |            \
+        BIT_ULL(POWER_DOMAIN_PIPE_B_PANEL_FITTER) |     \
+        BIT_ULL(POWER_DOMAIN_PORT_DDI_B_LANES) |        \
+        BIT_ULL(POWER_DOMAIN_PORT_DDI_B_IO) |           \
+        BIT_ULL(POWER_DOMAIN_PORT_DDI_C_LANES) |        \
+        BIT_ULL(POWER_DOMAIN_PORT_DDI_C_IO) |           \
+        BIT_ULL(POWER_DOMAIN_PORT_DDI_D_LANES) |        \
+        BIT_ULL(POWER_DOMAIN_PORT_DDI_D_IO) |           \
+        BIT_ULL(POWER_DOMAIN_PORT_DDI_E_LANES) |        \
+        BIT_ULL(POWER_DOMAIN_PORT_DDI_E_IO) |           \
+        BIT_ULL(POWER_DOMAIN_PORT_DDI_F_LANES) |        \
+        BIT_ULL(POWER_DOMAIN_PORT_DDI_F_IO) |           \
+        BIT_ULL(POWER_DOMAIN_AUX_B) |                   \
+        BIT_ULL(POWER_DOMAIN_AUX_C) |                   \
+        BIT_ULL(POWER_DOMAIN_AUX_D) |                   \
+        BIT_ULL(POWER_DOMAIN_AUX_E) |                   \
+        BIT_ULL(POWER_DOMAIN_AUX_F) |                   \
+        BIT_ULL(POWER_DOMAIN_AUX_TBT1) |                \
+        BIT_ULL(POWER_DOMAIN_AUX_TBT2) |                \
+        BIT_ULL(POWER_DOMAIN_AUX_TBT3) |                \
+        BIT_ULL(POWER_DOMAIN_AUX_TBT4) |                \
+        BIT_ULL(POWER_DOMAIN_VGA) |                     \
+        BIT_ULL(POWER_DOMAIN_AUDIO) |                   \
+        BIT_ULL(POWER_DOMAIN_INIT))
+        /*
+         * - transcoder WD
+         * - KVMR (HW control)
+         */
+#define ICL_PW_2_POWER_DOMAINS (                        \
+        ICL_PW_3_POWER_DOMAINS |                        \
+        BIT_ULL(POWER_DOMAIN_INIT))
+        /*
+         * - eDP/DSI VDSC
+         * - KVMR (HW control)
+         */
+#define ICL_DISPLAY_DC_OFF_POWER_DOMAINS (              \
+        ICL_PW_2_POWER_DOMAINS |                        \
+        BIT_ULL(POWER_DOMAIN_MODESET) |                 \
+        BIT_ULL(POWER_DOMAIN_AUX_A) |                   \
+        BIT_ULL(POWER_DOMAIN_INIT))
+
+#define ICL_DDI_IO_A_POWER_DOMAINS (                    \
+        BIT_ULL(POWER_DOMAIN_PORT_DDI_A_IO))
+#define ICL_DDI_IO_B_POWER_DOMAINS (                    \
+        BIT_ULL(POWER_DOMAIN_PORT_DDI_B_IO))
+#define ICL_DDI_IO_C_POWER_DOMAINS (                    \
+        BIT_ULL(POWER_DOMAIN_PORT_DDI_C_IO))
+#define ICL_DDI_IO_D_POWER_DOMAINS (                    \
+        BIT_ULL(POWER_DOMAIN_PORT_DDI_D_IO))
+#define ICL_DDI_IO_E_POWER_DOMAINS (                    \
+        BIT_ULL(POWER_DOMAIN_PORT_DDI_E_IO))
+#define ICL_DDI_IO_F_POWER_DOMAINS (                    \
+        BIT_ULL(POWER_DOMAIN_PORT_DDI_F_IO))
+
+#define ICL_AUX_A_IO_POWER_DOMAINS (                    \
+        BIT_ULL(POWER_DOMAIN_AUX_A))
+#define ICL_AUX_B_IO_POWER_DOMAINS (                    \
+        BIT_ULL(POWER_DOMAIN_AUX_B))
+#define ICL_AUX_C_IO_POWER_DOMAINS (                    \
+        BIT_ULL(POWER_DOMAIN_AUX_C))
+#define ICL_AUX_D_IO_POWER_DOMAINS (                    \
+        BIT_ULL(POWER_DOMAIN_AUX_D))
+#define ICL_AUX_E_IO_POWER_DOMAINS (                    \
+        BIT_ULL(POWER_DOMAIN_AUX_E))
+#define ICL_AUX_F_IO_POWER_DOMAINS (                    \
+        BIT_ULL(POWER_DOMAIN_AUX_F))
+#define ICL_AUX_TBT1_IO_POWER_DOMAINS (                 \
+        BIT_ULL(POWER_DOMAIN_AUX_TBT1))
+#define ICL_AUX_TBT2_IO_POWER_DOMAINS (                 \
+        BIT_ULL(POWER_DOMAIN_AUX_TBT2))
+#define ICL_AUX_TBT3_IO_POWER_DOMAINS (                 \
+        BIT_ULL(POWER_DOMAIN_AUX_TBT3))
+#define ICL_AUX_TBT4_IO_POWER_DOMAINS (                 \
+        BIT_ULL(POWER_DOMAIN_AUX_TBT4))
+
 static const struct i915_power_well_ops i9xx_always_on_power_well_ops = {
         .sync_hw = i9xx_power_well_sync_hw_noop,
         .enable = i9xx_always_on_power_well_noop,
@@ -2453,6 +2599,157 @@ static struct i915_power_well cnl_power_wells[] = {
         },
 };
 
+static const struct i915_power_well_ops icl_combo_phy_aux_power_well_ops = {
+        .sync_hw = hsw_power_well_sync_hw,
+        .enable = icl_combo_phy_aux_power_well_enable,
+        .disable = icl_combo_phy_aux_power_well_disable,
+        .is_enabled = hsw_power_well_enabled,
+};
+
+static struct i915_power_well icl_power_wells[] = {
+        {
+                .name = "always-on",
+                .always_on = 1,
+                .domains = POWER_DOMAIN_MASK,
+                .ops = &i9xx_always_on_power_well_ops,
+                .id = I915_DISP_PW_ALWAYS_ON,
+        },
+        {
+                .name = "power well 1",
+                /* Handled by the DMC firmware */
+                .domains = 0,
+                .ops = &hsw_power_well_ops,
+                .id = ICL_DISP_PW_1,
+                .hsw.has_fuses = true,
+        },
+        {
+                .name = "power well 2",
+                .domains = ICL_PW_2_POWER_DOMAINS,
+                .ops = &hsw_power_well_ops,
+                .id = ICL_DISP_PW_2,
+                .hsw.has_fuses = true,
+        },
+        {
+                .name = "DC off",
+                .domains = ICL_DISPLAY_DC_OFF_POWER_DOMAINS,
+                .ops = &gen9_dc_off_power_well_ops,
+                .id = SKL_DISP_PW_DC_OFF,
+        },
+        {
+                .name = "power well 3",
+                .domains = ICL_PW_3_POWER_DOMAINS,
+                .ops = &hsw_power_well_ops,
+                .id = ICL_DISP_PW_3,
+                .hsw.irq_pipe_mask = BIT(PIPE_B),
+                .hsw.has_vga = true,
+                .hsw.has_fuses = true,
+        },
+        {
+                .name = "DDI A IO",
+                .domains = ICL_DDI_IO_A_POWER_DOMAINS,
+                .ops = &hsw_power_well_ops,
+                .id = ICL_DISP_PW_DDI_A,
+        },
+        {
+                .name = "DDI B IO",
+                .domains = ICL_DDI_IO_B_POWER_DOMAINS,
+                .ops = &hsw_power_well_ops,
+                .id = ICL_DISP_PW_DDI_B,
+        },
+        {
+                .name = "DDI C IO",
+                .domains = ICL_DDI_IO_C_POWER_DOMAINS,
+                .ops = &hsw_power_well_ops,
+                .id = ICL_DISP_PW_DDI_C,
+        },
+        {
+                .name = "DDI D IO",
+                .domains = ICL_DDI_IO_D_POWER_DOMAINS,
+                .ops = &hsw_power_well_ops,
+                .id = ICL_DISP_PW_DDI_D,
+        },
+        {
+                .name = "DDI E IO",
+                .domains = ICL_DDI_IO_E_POWER_DOMAINS,
+                .ops = &hsw_power_well_ops,
+                .id = ICL_DISP_PW_DDI_E,
+        },
+        {
+                .name = "DDI F IO",
+                .domains = ICL_DDI_IO_F_POWER_DOMAINS,
+                .ops = &hsw_power_well_ops,
+                .id = ICL_DISP_PW_DDI_F,
+        },
+        {
+                .name = "AUX A",
+                .domains = ICL_AUX_A_IO_POWER_DOMAINS,
+                .ops = &icl_combo_phy_aux_power_well_ops,
+                .id = ICL_DISP_PW_AUX_A,
+        },
+        {
+                .name = "AUX B",
+                .domains = ICL_AUX_B_IO_POWER_DOMAINS,
+                .ops = &icl_combo_phy_aux_power_well_ops,
+                .id = ICL_DISP_PW_AUX_B,
+        },
+        {
+                .name = "AUX C",
+                .domains = ICL_AUX_C_IO_POWER_DOMAINS,
+                .ops = &hsw_power_well_ops,
+                .id = ICL_DISP_PW_AUX_C,
+        },
+        {
+                .name = "AUX D",
+                .domains = ICL_AUX_D_IO_POWER_DOMAINS,
+                .ops = &hsw_power_well_ops,
+                .id = ICL_DISP_PW_AUX_D,
+        },
+        {
+                .name = "AUX E",
+                .domains = ICL_AUX_E_IO_POWER_DOMAINS,
+                .ops = &hsw_power_well_ops,
+                .id = ICL_DISP_PW_AUX_E,
+        },
+        {
+                .name = "AUX F",
+                .domains = ICL_AUX_F_IO_POWER_DOMAINS,
+                .ops = &hsw_power_well_ops,
+                .id = ICL_DISP_PW_AUX_F,
+        },
+        {
+                .name = "AUX TBT1",
+                .domains = ICL_AUX_TBT1_IO_POWER_DOMAINS,
+                .ops = &hsw_power_well_ops,
+                .id = ICL_DISP_PW_AUX_TBT1,
+        },
+        {
+                .name = "AUX TBT2",
+                .domains = ICL_AUX_TBT2_IO_POWER_DOMAINS,
+                .ops = &hsw_power_well_ops,
+                .id = ICL_DISP_PW_AUX_TBT2,
+        },
+        {
+                .name = "AUX TBT3",
+                .domains = ICL_AUX_TBT3_IO_POWER_DOMAINS,
+                .ops = &hsw_power_well_ops,
+                .id = ICL_DISP_PW_AUX_TBT3,
+        },
+        {
+                .name = "AUX TBT4",
+                .domains = ICL_AUX_TBT4_IO_POWER_DOMAINS,
+                .ops = &hsw_power_well_ops,
+                .id = ICL_DISP_PW_AUX_TBT4,
+        },
+        {
+                .name = "power well 4",
+                .domains = ICL_PW_4_POWER_DOMAINS,
+                .ops = &hsw_power_well_ops,
+                .id = ICL_DISP_PW_4,
+                .hsw.has_fuses = true,
+                .hsw.irq_pipe_mask = BIT(PIPE_C),
+        },
+};
+
 static int
 sanitize_disable_power_well_option(const struct drm_i915_private *dev_priv,
                                    int disable_power_well)
@@ -2470,7 +2767,7 @@ static uint32_t get_allowed_dc_mask(const struct drm_i915_private *dev_priv,
         int requested_dc;
         int max_dc;
 
-        if (IS_GEN9_BC(dev_priv) || IS_CANNONLAKE(dev_priv)) {
+        if (IS_GEN9_BC(dev_priv) || INTEL_INFO(dev_priv)->gen >= 10) {
                 max_dc = 2;
                 mask = 0;
         } else if (IS_GEN9_LP(dev_priv)) {
@@ -2558,7 +2855,9 @@ int intel_power_domains_init(struct drm_i915_private *dev_priv)
          * The enabling order will be from lower to higher indexed wells,
          * the disabling order is reversed.
          */
-        if (IS_HASWELL(dev_priv)) {
+        if (IS_ICELAKE(dev_priv)) {
+                set_power_wells(power_domains, icl_power_wells);
+        } else if (IS_HASWELL(dev_priv)) {
                 set_power_wells(power_domains, hsw_power_wells);
         } else if (IS_BROADWELL(dev_priv)) {
                 set_power_wells(power_domains, bdw_power_wells);
@@ -2913,6 +3212,7 @@ static void cnl_set_procmon_ref_values(struct drm_i915_private *dev_priv,
         switch (val & (PROCESS_INFO_MASK | VOLTAGE_INFO_MASK)) {
         default:
                 MISSING_CASE(val);
+                /* fall through */
         case VOLTAGE_INFO_0_85V | PROCESS_INFO_DOT_0:
                 procmon = &cnl_procmon_values[PROCMON_0_85V_DOT_0];
                 break;
@@ -3025,6 +3325,8 @@ static void cnl_display_core_uninit(struct drm_i915_private *dev_priv)
 static void icl_display_core_init(struct drm_i915_private *dev_priv,
                                   bool resume)
 {
+        struct i915_power_domains *power_domains = &dev_priv->power_domains;
+        struct i915_power_well *well;
         enum port port;
         u32 val;
 
@@ -3053,8 +3355,14 @@ static void icl_display_core_init(struct drm_i915_private *dev_priv,
                 I915_WRITE(ICL_PORT_CL_DW5(port), val);
         }
 
-        /* 4. Enable power well 1 (PG1) and aux IO power. */
-        /* FIXME: ICL power wells code not here yet. */
+        /*
+         * 4. Enable Power Well 1 (PG1).
+         *    The AUX IO power wells will be enabled on demand.
+         */
+        mutex_lock(&power_domains->lock);
+        well = lookup_power_well(dev_priv, ICL_DISP_PW_1);
+        intel_power_well_enable(dev_priv, well);
+        mutex_unlock(&power_domains->lock);
 
         /* 5. Enable CDCLK. */
         icl_init_cdclk(dev_priv);
@@ -3072,6 +3380,8 @@ static void icl_display_core_init(struct drm_i915_private *dev_priv,
 
 static void icl_display_core_uninit(struct drm_i915_private *dev_priv)
 {
+        struct i915_power_domains *power_domains = &dev_priv->power_domains;
+        struct i915_power_well *well;
         enum port port;
         u32 val;
 
@@ -3085,8 +3395,15 @@ static void icl_display_core_uninit(struct drm_i915_private *dev_priv)
         /* 3. Disable CD clock */
         icl_uninit_cdclk(dev_priv);
 
-        /* 4. Disable Power Well 1 (PG1) and Aux IO Power */
-        /* FIXME: ICL power wells code not here yet. */
+        /*
+         * 4. Disable Power Well 1 (PG1).
+         *    The AUX IO power wells are toggled on demand, so they are already
+         *    disabled at this point.
+         */
+        mutex_lock(&power_domains->lock);
+        well = lookup_power_well(dev_priv, ICL_DISP_PW_1);
+        intel_power_well_disable(dev_priv, well);
+        mutex_unlock(&power_domains->lock);
 
         /* 5. Disable Comp */
         for (port = PORT_A; port <= PORT_B; port++) {
diff --git a/drivers/gpu/drm/i915/intel_sdvo.c b/drivers/gpu/drm/i915/intel_sdvo.c
index e6a64b3ecd91..396cb59ca4b8 100644
--- a/drivers/gpu/drm/i915/intel_sdvo.c
+++ b/drivers/gpu/drm/i915/intel_sdvo.c
@@ -1340,6 +1340,7 @@ static void intel_sdvo_pre_enable(struct intel_encoder *intel_encoder,
         switch (crtc_state->pixel_multiplier) {
         default:
                 WARN(1, "unknown pixel multiplier specified\n");
+                /* fall through */
         case 1: rate = SDVO_CLOCK_RATE_MULT_1X; break;
         case 2: rate = SDVO_CLOCK_RATE_MULT_2X; break;
         case 4: rate = SDVO_CLOCK_RATE_MULT_4X; break;
@@ -1400,10 +1401,7 @@ static bool intel_sdvo_connector_get_hw_state(struct intel_connector *connector)
 
         intel_sdvo_get_active_outputs(intel_sdvo, &active_outputs);
 
-        if (active_outputs & intel_sdvo_connector->output_flag)
-                return true;
-        else
-                return false;
+        return active_outputs & intel_sdvo_connector->output_flag;
 }
 
 bool intel_sdvo_port_enabled(struct drm_i915_private *dev_priv,
@@ -2316,14 +2314,19 @@ intel_sdvo_guess_ddc_bus(struct intel_sdvo *sdvo)
         switch (sdvo->controlled_output) {
         case SDVO_OUTPUT_LVDS1:
                 mask |= SDVO_OUTPUT_LVDS1;
+                /* fall through */
         case SDVO_OUTPUT_LVDS0:
                 mask |= SDVO_OUTPUT_LVDS0;
+                /* fall through */
         case SDVO_OUTPUT_TMDS1:
                 mask |= SDVO_OUTPUT_TMDS1;
+                /* fall through */
         case SDVO_OUTPUT_TMDS0:
                 mask |= SDVO_OUTPUT_TMDS0;
+                /* fall through */
         case SDVO_OUTPUT_RGB1:
                 mask |= SDVO_OUTPUT_RGB1;
+                /* fall through */
         case SDVO_OUTPUT_RGB0:
                 mask |= SDVO_OUTPUT_RGB0;
                 break;
diff --git a/drivers/gpu/drm/i915/intel_sprite.c b/drivers/gpu/drm/i915/intel_sprite.c
index 344c0e709b19..4990d6e84ddf 100644
--- a/drivers/gpu/drm/i915/intel_sprite.c
+++ b/drivers/gpu/drm/i915/intel_sprite.c
@@ -107,13 +107,21 @@ void intel_pipe_update_start(const struct intel_crtc_state *new_crtc_state)
                                                       VBLANK_EVASION_TIME_US);
         max = vblank_start - 1;
 
-        local_irq_disable();
-
         if (min <= 0 || max <= 0)
-                return;
+                goto irq_disable;
 
         if (WARN_ON(drm_crtc_vblank_get(&crtc->base)))
-                return;
+                goto irq_disable;
+
+        /*
+         * Wait for psr to idle out after enabling the VBL interrupts
+         * VBL interrupts will start the PSR exit and prevent a PSR
+         * re-entry as well.
+         */
+        if (CAN_PSR(dev_priv) && intel_psr_wait_for_idle(dev_priv))
+                DRM_ERROR("PSR idle timed out, atomic update may fail\n");
+
+        local_irq_disable();
 
         crtc->debug.min_vbl = min;
         crtc->debug.max_vbl = max;
@@ -171,6 +179,10 @@ void intel_pipe_update_start(const struct intel_crtc_state *new_crtc_state)
         crtc->debug.start_vbl_count = intel_crtc_get_vblank_counter(crtc);
 
         trace_i915_pipe_update_vblank_evaded(crtc);
+        return;
+
+irq_disable:
+        local_irq_disable();
 }
 
 /**
diff --git a/drivers/gpu/drm/i915/intel_uc.c b/drivers/gpu/drm/i915/intel_uc.c
index 94e8863bd97c..7c95697e1a35 100644
--- a/drivers/gpu/drm/i915/intel_uc.c
+++ b/drivers/gpu/drm/i915/intel_uc.c
@@ -171,24 +171,11 @@ void intel_uc_init_early(struct drm_i915_private *i915)
         intel_huc_init_early(huc);
 
         sanitize_options_early(i915);
-
-        if (USES_GUC(i915))
-                intel_uc_fw_fetch(i915, &guc->fw);
-
-        if (USES_HUC(i915))
-                intel_uc_fw_fetch(i915, &huc->fw);
 }
 
 void intel_uc_cleanup_early(struct drm_i915_private *i915)
 {
         struct intel_guc *guc = &i915->guc;
-        struct intel_huc *huc = &i915->huc;
-
-        if (USES_HUC(i915))
-                intel_uc_fw_fini(&huc->fw);
-
-        if (USES_GUC(i915))
-                intel_uc_fw_fini(&guc->fw);
 
         guc_free_load_err_log(guc);
 }
@@ -252,28 +239,41 @@ static void guc_disable_communication(struct intel_guc *guc)
 int intel_uc_init_misc(struct drm_i915_private *i915)
 {
         struct intel_guc *guc = &i915->guc;
+        struct intel_huc *huc = &i915->huc;
         int ret;
 
         if (!USES_GUC(i915))
                 return 0;
 
-        intel_guc_init_ggtt_pin_bias(guc);
-
-        ret = intel_guc_init_wq(guc);
+        ret = intel_guc_init_misc(guc);
         if (ret)
                 return ret;
 
+        if (USES_HUC(i915)) {
+                ret = intel_huc_init_misc(huc);
+                if (ret)
+                        goto err_guc;
+        }
+
         return 0;
+
+err_guc:
+        intel_guc_fini_misc(guc);
+        return ret;
 }
 
 void intel_uc_fini_misc(struct drm_i915_private *i915)
 {
         struct intel_guc *guc = &i915->guc;
+        struct intel_huc *huc = &i915->huc;
 
         if (!USES_GUC(i915))
                 return;
 
-        intel_guc_fini_wq(guc);
+        if (USES_HUC(i915))
+                intel_huc_fini_misc(huc);
+
+        intel_guc_fini_misc(guc);
 }
 
 int intel_uc_init(struct drm_i915_private *i915)
diff --git a/drivers/gpu/drm/i915/selftests/huge_pages.c b/drivers/gpu/drm/i915/selftests/huge_pages.c
index fbe4324116d7..d9f439f6219f 100644
--- a/drivers/gpu/drm/i915/selftests/huge_pages.c
+++ b/drivers/gpu/drm/i915/selftests/huge_pages.c
@@ -919,12 +919,12 @@ gpu_write_dw(struct i915_vma *vma, u64 offset, u32 val)
                         *cmd++ = val;
                 } else if (gen >= 4) {
                         *cmd++ = MI_STORE_DWORD_IMM_GEN4 |
-                                (gen < 6 ? 1 << 22 : 0);
+                                (gen < 6 ? MI_USE_GGTT : 0);
                         *cmd++ = 0;
                         *cmd++ = offset;
                         *cmd++ = val;
                 } else {
-                        *cmd++ = MI_STORE_DWORD_IMM | 1 << 22;
+                        *cmd++ = MI_STORE_DWORD_IMM | MI_MEM_VIRTUAL;
                         *cmd++ = offset;
                         *cmd++ = val;
                 }
@@ -985,7 +985,10 @@ static int gpu_write(struct i915_vma *vma,
                 goto err_request;
         }
 
-        i915_vma_move_to_active(batch, rq, 0);
+        err = i915_vma_move_to_active(batch, rq, 0);
+        if (err)
+                goto err_request;
+
         i915_gem_object_set_active_reference(batch->obj);
         i915_vma_unpin(batch);
         i915_vma_close(batch);
@@ -996,11 +999,9 @@ static int gpu_write(struct i915_vma *vma,
         if (err)
                 goto err_request;
 
-        i915_vma_move_to_active(vma, rq, EXEC_OBJECT_WRITE);
-
-        reservation_object_lock(vma->resv, NULL);
-        reservation_object_add_excl_fence(vma->resv, &rq->fence);
-        reservation_object_unlock(vma->resv);
+        err = i915_vma_move_to_active(vma, rq, EXEC_OBJECT_WRITE);
+        if (err)
+                i915_request_skip(rq, err);
 
 err_request:
         i915_request_add(rq);
@@ -1694,7 +1695,7 @@ int i915_gem_huge_page_mock_selftests(void)
         dma_coerce_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(39));
 
         mutex_lock(&dev_priv->drm.struct_mutex);
-        ppgtt = i915_ppgtt_create(dev_priv, ERR_PTR(-ENODEV), "mock");
+        ppgtt = i915_ppgtt_create(dev_priv, ERR_PTR(-ENODEV));
         if (IS_ERR(ppgtt)) {
                 err = PTR_ERR(ppgtt);
                 goto out_unlock;
@@ -1724,7 +1725,7 @@ out_unlock:
 
         i915_modparams.enable_ppgtt = saved_ppgtt;
 
-        drm_dev_unref(&dev_priv->drm);
+        drm_dev_put(&dev_priv->drm);
 
         return err;
 }
@@ -1748,6 +1749,9 @@ int i915_gem_huge_page_live_selftests(struct drm_i915_private *dev_priv)
                 return 0;
         }
 
+        if (i915_terminally_wedged(&dev_priv->gpu_error))
+                return 0;
+
         file = mock_file(dev_priv);
         if (IS_ERR(file))
                 return PTR_ERR(file);
diff --git a/drivers/gpu/drm/i915/selftests/i915_gem_coherency.c b/drivers/gpu/drm/i915/selftests/i915_gem_coherency.c
index a4900091ae3d..3a095c37c120 100644
--- a/drivers/gpu/drm/i915/selftests/i915_gem_coherency.c
+++ b/drivers/gpu/drm/i915/selftests/i915_gem_coherency.c
@@ -42,11 +42,21 @@ static int cpu_set(struct drm_i915_gem_object *obj,
 
         page = i915_gem_object_get_page(obj, offset >> PAGE_SHIFT);
         map = kmap_atomic(page);
-        if (needs_clflush & CLFLUSH_BEFORE)
+
+        if (needs_clflush & CLFLUSH_BEFORE) {
+                mb();
                 clflush(map+offset_in_page(offset) / sizeof(*map));
+                mb();
+        }
+
         map[offset_in_page(offset) / sizeof(*map)] = v;
-        if (needs_clflush & CLFLUSH_AFTER)
+
+        if (needs_clflush & CLFLUSH_AFTER) {
+                mb();
                 clflush(map+offset_in_page(offset) / sizeof(*map));
+                mb();
+        }
+
         kunmap_atomic(map);
 
         i915_gem_obj_finish_shmem_access(obj);
@@ -68,8 +78,13 @@ static int cpu_get(struct drm_i915_gem_object *obj,
 
         page = i915_gem_object_get_page(obj, offset >> PAGE_SHIFT);
         map = kmap_atomic(page);
-        if (needs_clflush & CLFLUSH_BEFORE)
+
+        if (needs_clflush & CLFLUSH_BEFORE) {
+                mb();
                 clflush(map+offset_in_page(offset) / sizeof(*map));
+                mb();
+        }
+
         *v = map[offset_in_page(offset) / sizeof(*map)];
         kunmap_atomic(map);
 
@@ -210,28 +225,24 @@ static int gpu_set(struct drm_i915_gem_object *obj,
                 *cs++ = upper_32_bits(i915_ggtt_offset(vma) + offset);
                 *cs++ = v;
         } else if (INTEL_GEN(i915) >= 4) {
-                *cs++ = MI_STORE_DWORD_IMM_GEN4 | 1 << 22;
+                *cs++ = MI_STORE_DWORD_IMM_GEN4 | MI_USE_GGTT;
                 *cs++ = 0;
                 *cs++ = i915_ggtt_offset(vma) + offset;
                 *cs++ = v;
         } else {
-                *cs++ = MI_STORE_DWORD_IMM | 1 << 22;
+                *cs++ = MI_STORE_DWORD_IMM | MI_MEM_VIRTUAL;
                 *cs++ = i915_ggtt_offset(vma) + offset;
                 *cs++ = v;
                 *cs++ = MI_NOOP;
         }
         intel_ring_advance(rq, cs);
 
-        i915_vma_move_to_active(vma, rq, EXEC_OBJECT_WRITE);
+        err = i915_vma_move_to_active(vma, rq, EXEC_OBJECT_WRITE);
         i915_vma_unpin(vma);
 
-        reservation_object_lock(obj->resv, NULL);
-        reservation_object_add_excl_fence(obj->resv, &rq->fence);
-        reservation_object_unlock(obj->resv);
-
         i915_request_add(rq);
 
-        return 0;
+        return err;
 }
 
 static bool always_valid(struct drm_i915_private *i915)
@@ -239,8 +250,16 @@ static bool always_valid(struct drm_i915_private *i915)
         return true;
 }
 
+static bool needs_fence_registers(struct drm_i915_private *i915)
+{
+        return !i915_terminally_wedged(&i915->gpu_error);
+}
+
 static bool needs_mi_store_dword(struct drm_i915_private *i915)
 {
+        if (i915_terminally_wedged(&i915->gpu_error))
+                return false;
+
         return intel_engine_can_store_dword(i915->engine[RCS]);
 }
 
@@ -251,7 +270,7 @@ static const struct igt_coherency_mode {
         bool (*valid)(struct drm_i915_private *i915);
 } igt_coherency_mode[] = {
         { "cpu", cpu_set, cpu_get, always_valid },
-        { "gtt", gtt_set, gtt_get, always_valid },
+        { "gtt", gtt_set, gtt_get, needs_fence_registers },
         { "wc", wc_set, wc_get, always_valid },
         { "gpu", gpu_set, NULL, needs_mi_store_dword },
         { },
diff --git a/drivers/gpu/drm/i915/selftests/i915_gem_context.c b/drivers/gpu/drm/i915/selftests/i915_gem_context.c
index 90c3c36173ba..ab2590242033 100644
--- a/drivers/gpu/drm/i915/selftests/i915_gem_context.c
+++ b/drivers/gpu/drm/i915/selftests/i915_gem_context.c
@@ -63,12 +63,12 @@ gpu_fill_dw(struct i915_vma *vma, u64 offset, unsigned long count, u32 value)
                         *cmd++ = value;
                 } else if (gen >= 4) {
                         *cmd++ = MI_STORE_DWORD_IMM_GEN4 |
-                                (gen < 6 ? 1 << 22 : 0);
+                                (gen < 6 ? MI_USE_GGTT : 0);
                         *cmd++ = 0;
                         *cmd++ = offset;
                         *cmd++ = value;
                 } else {
-                        *cmd++ = MI_STORE_DWORD_IMM | 1 << 22;
+                        *cmd++ = MI_STORE_DWORD_IMM | MI_MEM_VIRTUAL;
                         *cmd++ = offset;
                         *cmd++ = value;
                 }
@@ -170,22 +170,26 @@ static int gpu_fill(struct drm_i915_gem_object *obj,
         if (err)
                 goto err_request;
 
-        i915_vma_move_to_active(batch, rq, 0);
+        err = i915_vma_move_to_active(batch, rq, 0);
+        if (err)
+                goto skip_request;
+
+        err = i915_vma_move_to_active(vma, rq, EXEC_OBJECT_WRITE);
+        if (err)
+                goto skip_request;
+
         i915_gem_object_set_active_reference(batch->obj);
         i915_vma_unpin(batch);
         i915_vma_close(batch);
 
-        i915_vma_move_to_active(vma, rq, 0);
         i915_vma_unpin(vma);
 
-        reservation_object_lock(obj->resv, NULL);
-        reservation_object_add_excl_fence(obj->resv, &rq->fence);
-        reservation_object_unlock(obj->resv);
-
         i915_request_add(rq);
 
         return 0;
 
+skip_request:
+        i915_request_skip(rq, err);
 err_request:
         i915_request_add(rq);
 err_batch:
@@ -336,11 +340,15 @@ static int igt_ctx_exec(void *arg)
         bool first_shared_gtt = true;
         int err = -ENODEV;
 
-        /* Create a few different contexts (with different mm) and write
+        /*
+         * Create a few different contexts (with different mm) and write
          * through each ctx/mm using the GPU making sure those writes end
          * up in the expected pages of our obj.
          */
 
+        if (!DRIVER_CAPS(i915)->has_logical_contexts)
+                return 0;
+
         file = mock_file(i915);
         if (IS_ERR(file))
                 return PTR_ERR(file);
@@ -367,6 +375,9 @@ static int igt_ctx_exec(void *arg)
                 }
 
                 for_each_engine(engine, i915, id) {
+                        if (!engine->context_size)
+                                continue; /* No logical context support in HW */
+
                         if (!intel_engine_can_store_dword(engine))
                                 continue;
 
@@ -467,7 +478,9 @@ static int __igt_switch_to_kernel_context(struct drm_i915_private *i915,
                 }
         }
 
-        err = i915_gem_wait_for_idle(i915, I915_WAIT_LOCKED);
+        err = i915_gem_wait_for_idle(i915,
+                                     I915_WAIT_LOCKED,
+                                     MAX_SCHEDULE_TIMEOUT);
         if (err)
                 return err;
 
@@ -586,7 +599,7 @@ int i915_gem_context_mock_selftests(void)
 
         err = i915_subtests(tests, i915);
 
-        drm_dev_unref(&i915->drm);
+        drm_dev_put(&i915->drm);
         return err;
 }
 
@@ -599,6 +612,9 @@ int i915_gem_context_live_selftests(struct drm_i915_private *dev_priv)
         bool fake_alias = false;
         int err;
 
+        if (i915_terminally_wedged(&dev_priv->gpu_error))
+                return 0;
+
         /* Install a fake aliasing gtt for exercise */
         if (USES_PPGTT(dev_priv) && !dev_priv->mm.aliasing_ppgtt) {
                 mutex_lock(&dev_priv->drm.struct_mutex);
diff --git a/drivers/gpu/drm/i915/selftests/i915_gem_dmabuf.c b/drivers/gpu/drm/i915/selftests/i915_gem_dmabuf.c
index 89dc25a5a53b..a7055b12e53c 100644
--- a/drivers/gpu/drm/i915/selftests/i915_gem_dmabuf.c
+++ b/drivers/gpu/drm/i915/selftests/i915_gem_dmabuf.c
@@ -389,7 +389,7 @@ int i915_gem_dmabuf_mock_selftests(void)
 
         err = i915_subtests(tests, i915);
 
-        drm_dev_unref(&i915->drm);
+        drm_dev_put(&i915->drm);
         return err;
 }
 
diff --git a/drivers/gpu/drm/i915/selftests/i915_gem_evict.c b/drivers/gpu/drm/i915/selftests/i915_gem_evict.c
index 2dc72a984d45..128ad1cf0647 100644
--- a/drivers/gpu/drm/i915/selftests/i915_gem_evict.c
+++ b/drivers/gpu/drm/i915/selftests/i915_gem_evict.c
@@ -490,7 +490,7 @@ int i915_gem_evict_mock_selftests(void)
         err = i915_subtests(tests, i915);
         mutex_unlock(&i915->drm.struct_mutex);
 
-        drm_dev_unref(&i915->drm);
+        drm_dev_put(&i915->drm);
         return err;
 }
 
@@ -500,5 +500,8 @@ int i915_gem_evict_live_selftests(struct drm_i915_private *i915)
                 SUBTEST(igt_evict_contexts),
         };
 
+        if (i915_terminally_wedged(&i915->gpu_error))
+                return 0;
+
         return i915_subtests(tests, i915);
 }
diff --git a/drivers/gpu/drm/i915/selftests/i915_gem_gtt.c b/drivers/gpu/drm/i915/selftests/i915_gem_gtt.c
index a4060238bef0..600a3bcbd3d6 100644
--- a/drivers/gpu/drm/i915/selftests/i915_gem_gtt.c
+++ b/drivers/gpu/drm/i915/selftests/i915_gem_gtt.c
@@ -32,6 +32,20 @@
 #include "mock_drm.h"
 #include "mock_gem_device.h"
 
+static void cleanup_freed_objects(struct drm_i915_private *i915)
+{
+        /*
+         * As we may hold onto the struct_mutex for inordinate lengths of
+         * time, the NMI khungtaskd detector may fire for the free objects
+         * worker.
+         */
+        mutex_unlock(&i915->drm.struct_mutex);
+
+        i915_gem_drain_freed_objects(i915);
+
+        mutex_lock(&i915->drm.struct_mutex);
+}
+
 static void fake_free_pages(struct drm_i915_gem_object *obj,
                             struct sg_table *pages)
 {
@@ -134,7 +148,7 @@ static int igt_ppgtt_alloc(void *arg)
 {
         struct drm_i915_private *dev_priv = arg;
         struct i915_hw_ppgtt *ppgtt;
-        u64 size, last;
+        u64 size, last, limit;
         int err = 0;
 
         /* Allocate a ppggt and try to fill the entire range */
@@ -142,20 +156,25 @@ static int igt_ppgtt_alloc(void *arg)
         if (!USES_PPGTT(dev_priv))
                 return 0;
 
-        mutex_lock(&dev_priv->drm.struct_mutex);
         ppgtt = __hw_ppgtt_create(dev_priv);
-        if (IS_ERR(ppgtt)) {
-                err = PTR_ERR(ppgtt);
-                goto err_unlock;
-        }
+        if (IS_ERR(ppgtt))
+                return PTR_ERR(ppgtt);
 
         if (!ppgtt->vm.allocate_va_range)
                 goto err_ppgtt_cleanup;
 
+        /*
+         * While we only allocate the page tables here and so we could
+         * address a much larger GTT than we could actually fit into
+         * RAM, a practical limit is the amount of physical pages in the system.
+         * This should ensure that we do not run into the oomkiller during
+         * the test and take down the machine wilfully.
+         */
+        limit = totalram_pages << PAGE_SHIFT;
+        limit = min(ppgtt->vm.total, limit);
+
         /* Check we can allocate the entire range */
-        for (size = 4096;
-             size <= ppgtt->vm.total;
-             size <<= 2) {
+        for (size = 4096; size <= limit; size <<= 2) {
                 err = ppgtt->vm.allocate_va_range(&ppgtt->vm, 0, size);
                 if (err) {
                         if (err == -ENOMEM) {
@@ -166,13 +185,13 @@ static int igt_ppgtt_alloc(void *arg)
                         goto err_ppgtt_cleanup;
                 }
 
+                cond_resched();
+
                 ppgtt->vm.clear_range(&ppgtt->vm, 0, size);
         }
 
         /* Check we can incrementally allocate the entire range */
-        for (last = 0, size = 4096;
-             size <= ppgtt->vm.total;
-             last = size, size <<= 2) {
+        for (last = 0, size = 4096; size <= limit; last = size, size <<= 2) {
                 err = ppgtt->vm.allocate_va_range(&ppgtt->vm,
                                                   last, size - last);
                 if (err) {
@@ -183,12 +202,13 @@ static int igt_ppgtt_alloc(void *arg)
                         }
                         goto err_ppgtt_cleanup;
                 }
+
+                cond_resched();
         }
 
 err_ppgtt_cleanup:
-        ppgtt->vm.cleanup(&ppgtt->vm);
-        kfree(ppgtt);
-err_unlock:
+        mutex_lock(&dev_priv->drm.struct_mutex);
+        i915_ppgtt_put(ppgtt);
         mutex_unlock(&dev_priv->drm.struct_mutex);
         return err;
 }
@@ -291,6 +311,8 @@ static int lowlevel_hole(struct drm_i915_private *i915,
                 i915_gem_object_put(obj);
 
                 kfree(order);
+
+                cleanup_freed_objects(i915);
         }
 
         return 0;
@@ -519,6 +541,7 @@ static int fill_hole(struct drm_i915_private *i915,
                 }
 
                 close_object_list(&objects, vm);
+                cleanup_freed_objects(i915);
         }
 
         return 0;
@@ -605,6 +628,8 @@ err_put:
                 i915_gem_object_put(obj);
                 if (err)
                         return err;
+
+                cleanup_freed_objects(i915);
         }
 
         return 0;
@@ -789,6 +814,8 @@ err_obj:
                 kfree(order);
                 if (err)
                         return err;
+
+                cleanup_freed_objects(i915);
         }
 
         return 0;
@@ -857,6 +884,7 @@ static int __shrink_hole(struct drm_i915_private *i915,
         }
 
         close_object_list(&objects, vm);
+        cleanup_freed_objects(i915);
         return err;
 }
 
@@ -949,6 +977,7 @@ static int shrink_boom(struct drm_i915_private *i915,
                 i915_gem_object_put(explode);
 
                 memset(&vm->fault_attr, 0, sizeof(vm->fault_attr));
+                cleanup_freed_objects(i915);
         }
 
         return 0;
@@ -980,7 +1009,7 @@ static int exercise_ppgtt(struct drm_i915_private *dev_priv,
                 return PTR_ERR(file);
 
         mutex_lock(&dev_priv->drm.struct_mutex);
-        ppgtt = i915_ppgtt_create(dev_priv, file->driver_priv, "mock");
+        ppgtt = i915_ppgtt_create(dev_priv, file->driver_priv);
         if (IS_ERR(ppgtt)) {
                 err = PTR_ERR(ppgtt);
                 goto out_unlock;
@@ -1644,7 +1673,7 @@ int i915_gem_gtt_mock_selftests(void)
         err = i915_subtests(tests, i915);
         mutex_unlock(&i915->drm.struct_mutex);
 
-        drm_dev_unref(&i915->drm);
+        drm_dev_put(&i915->drm);
         return err;
 }
 
diff --git a/drivers/gpu/drm/i915/selftests/i915_gem_object.c b/drivers/gpu/drm/i915/selftests/i915_gem_object.c
index 2b2dde94526f..d77acf4cc439 100644
--- a/drivers/gpu/drm/i915/selftests/i915_gem_object.c
+++ b/drivers/gpu/drm/i915/selftests/i915_gem_object.c
@@ -169,9 +169,16 @@ static u64 tiled_offset(const struct tile *tile, u64 v)
                 v += y * tile->width;
                 v += div64_u64_rem(x, tile->width, &x) << tile->size;
                 v += x;
-        } else {
+        } else if (tile->width == 128) {
                 const unsigned int ytile_span = 16;
-                const unsigned int ytile_height = 32 * ytile_span;
+                const unsigned int ytile_height = 512;
+
+                v += y * ytile_span;
+                v += div64_u64_rem(x, ytile_span, &x) * ytile_height;
+                v += x;
+        } else {
+                const unsigned int ytile_span = 32;
+                const unsigned int ytile_height = 256;
 
                 v += y * ytile_span;
                 v += div64_u64_rem(x, ytile_span, &x) * ytile_height;
@@ -288,6 +295,8 @@ static int check_partial_mapping(struct drm_i915_gem_object *obj,
                 kunmap(p);
                 if (err)
                         return err;
+
+                i915_vma_destroy(vma);
         }
 
         return 0;
@@ -347,6 +356,14 @@ static int igt_partial_tiling(void *arg)
                 unsigned int pitch;
                 struct tile tile;
 
+                if (i915->quirks & QUIRK_PIN_SWIZZLED_PAGES)
+                        /*
+                         * The swizzling pattern is actually unknown as it
+                         * varies based on physical address of each page.
+                         * See i915_gem_detect_bit_6_swizzle().
+                         */
+                        break;
+
                 tile.tiling = tiling;
                 switch (tiling) {
                 case I915_TILING_X:
@@ -357,8 +374,8 @@ static int igt_partial_tiling(void *arg)
                         break;
                 }
 
-                if (tile.swizzle == I915_BIT_6_SWIZZLE_UNKNOWN ||
-                    tile.swizzle == I915_BIT_6_SWIZZLE_9_10_17)
+                GEM_BUG_ON(tile.swizzle == I915_BIT_6_SWIZZLE_UNKNOWN);
+                if (tile.swizzle == I915_BIT_6_SWIZZLE_9_10_17)
                         continue;
 
                 if (INTEL_GEN(i915) <= 2) {
@@ -454,12 +471,14 @@ static int make_obj_busy(struct drm_i915_gem_object *obj)
                 return PTR_ERR(rq);
         }
 
-        i915_vma_move_to_active(vma, rq, 0);
+        err = i915_vma_move_to_active(vma, rq, EXEC_OBJECT_WRITE);
+
         i915_request_add(rq);
 
-        i915_gem_object_set_active_reference(obj);
+        __i915_gem_object_release_unless_active(obj);
         i915_vma_unpin(vma);
-        return 0;
+
+        return err;
 }
 
 static bool assert_mmap_offset(struct drm_i915_private *i915,
@@ -488,6 +507,15 @@ static int igt_mmap_offset_exhaustion(void *arg)
         u64 hole_start, hole_end;
         int loop, err;
 
+        /* Disable background reaper */
+        mutex_lock(&i915->drm.struct_mutex);
+        if (!i915->gt.active_requests++)
+                i915_gem_unpark(i915);
+        mutex_unlock(&i915->drm.struct_mutex);
+        cancel_delayed_work_sync(&i915->gt.retire_work);
+        cancel_delayed_work_sync(&i915->gt.idle_work);
+        GEM_BUG_ON(!i915->gt.awake);
+
         /* Trim the device mmap space to only a page */
         memset(&resv, 0, sizeof(resv));
         drm_mm_for_each_hole(hole, mm, hole_start, hole_end) {
@@ -496,7 +524,7 @@ static int igt_mmap_offset_exhaustion(void *arg)
                 err = drm_mm_reserve_node(mm, &resv);
                 if (err) {
                         pr_err("Failed to trim VMA manager, err=%d\n", err);
-                        return err;
+                        goto out_park;
                 }
                 break;
         }
@@ -538,6 +566,9 @@ static int igt_mmap_offset_exhaustion(void *arg)
 
         /* Now fill with busy dead objects that we expect to reap */
         for (loop = 0; loop < 3; loop++) {
+                if (i915_terminally_wedged(&i915->gpu_error))
+                        break;
+
                 obj = i915_gem_object_create_internal(i915, PAGE_SIZE);
                 if (IS_ERR(obj)) {
                         err = PTR_ERR(obj);
@@ -554,6 +585,7 @@ static int igt_mmap_offset_exhaustion(void *arg)
                         goto err_obj;
                 }
 
+                /* NB we rely on the _active_ reference to access obj now */
                 GEM_BUG_ON(!i915_gem_object_is_active(obj));
                 err = i915_gem_object_create_mmap_offset(obj);
                 if (err) {
@@ -565,6 +597,13 @@ static int igt_mmap_offset_exhaustion(void *arg)
 
 out:
         drm_mm_remove_node(&resv);
+out_park:
+        mutex_lock(&i915->drm.struct_mutex);
+        if (--i915->gt.active_requests)
+                queue_delayed_work(i915->wq, &i915->gt.retire_work, 0);
+        else
+                queue_delayed_work(i915->wq, &i915->gt.idle_work, 0);
+        mutex_unlock(&i915->drm.struct_mutex);
         return err;
 err_obj:
         i915_gem_object_put(obj);
@@ -586,7 +625,7 @@ int i915_gem_object_mock_selftests(void)
 
         err = i915_subtests(tests, i915);
 
-        drm_dev_unref(&i915->drm);
+        drm_dev_put(&i915->drm);
         return err;
 }
 
diff --git a/drivers/gpu/drm/i915/selftests/i915_request.c b/drivers/gpu/drm/i915/selftests/i915_request.c
index 63cd9486cc13..c4aac6141e04 100644
--- a/drivers/gpu/drm/i915/selftests/i915_request.c
+++ b/drivers/gpu/drm/i915/selftests/i915_request.c
@@ -262,7 +262,7 @@ int i915_request_mock_selftests(void)
                 return -ENOMEM;
 
         err = i915_subtests(tests, i915);
-        drm_dev_unref(&i915->drm);
+        drm_dev_put(&i915->drm);
 
         return err;
 }
@@ -286,7 +286,9 @@ static int begin_live_test(struct live_test *t,
         t->func = func;
         t->name = name;
 
-        err = i915_gem_wait_for_idle(i915, I915_WAIT_LOCKED);
+        err = i915_gem_wait_for_idle(i915,
+                                     I915_WAIT_LOCKED,
+                                     MAX_SCHEDULE_TIMEOUT);
         if (err) {
                 pr_err("%s(%s): failed to idle before, with err=%d!",
                        func, name, err);
@@ -594,11 +596,8 @@ static struct i915_vma *recursive_batch(struct drm_i915_private *i915)
         } else if (gen >= 6) {
                 *cmd++ = MI_BATCH_BUFFER_START | 1 << 8;
                 *cmd++ = lower_32_bits(vma->node.start);
-        } else if (gen >= 4) {
-                *cmd++ = MI_BATCH_BUFFER_START | MI_BATCH_GTT;
-                *cmd++ = lower_32_bits(vma->node.start);
         } else {
-                *cmd++ = MI_BATCH_BUFFER_START | MI_BATCH_GTT | 1;
+                *cmd++ = MI_BATCH_BUFFER_START | MI_BATCH_GTT;
                 *cmd++ = lower_32_bits(vma->node.start);
         }
         *cmd++ = MI_BATCH_BUFFER_END; /* terminate early in case of error */
@@ -678,7 +677,9 @@ static int live_all_engines(void *arg)
                         i915_gem_object_set_active_reference(batch->obj);
                 }
 
-                i915_vma_move_to_active(batch, request[id], 0);
+                err = i915_vma_move_to_active(batch, request[id], 0);
+                GEM_BUG_ON(err);
+
                 i915_request_get(request[id]);
                 i915_request_add(request[id]);
         }
@@ -788,7 +789,9 @@ static int live_sequential_engines(void *arg)
                 GEM_BUG_ON(err);
                 request[id]->batch = batch;
 
-                i915_vma_move_to_active(batch, request[id], 0);
+                err = i915_vma_move_to_active(batch, request[id], 0);
+                GEM_BUG_ON(err);
+
                 i915_gem_object_set_active_reference(batch->obj);
                 i915_vma_get(batch);
 
@@ -862,5 +865,9 @@ int i915_request_live_selftests(struct drm_i915_private *i915)
                 SUBTEST(live_sequential_engines),
                 SUBTEST(live_empty_request),
         };
+
+        if (i915_terminally_wedged(&i915->gpu_error))
+                return 0;
+
         return i915_subtests(tests, i915);
 }
diff --git a/drivers/gpu/drm/i915/selftests/i915_vma.c b/drivers/gpu/drm/i915/selftests/i915_vma.c
index 8400a8cc5cf2..ffa74290e054 100644
--- a/drivers/gpu/drm/i915/selftests/i915_vma.c
+++ b/drivers/gpu/drm/i915/selftests/i915_vma.c
@@ -733,7 +733,7 @@ int i915_vma_mock_selftests(void)
         err = i915_subtests(tests, i915);
         mutex_unlock(&i915->drm.struct_mutex);
 
-        drm_dev_unref(&i915->drm);
+        drm_dev_put(&i915->drm);
         return err;
 }
 
diff --git a/drivers/gpu/drm/i915/selftests/igt_flush_test.c b/drivers/gpu/drm/i915/selftests/igt_flush_test.c
index 0d06f559243f..af66e3d4e23a 100644
--- a/drivers/gpu/drm/i915/selftests/igt_flush_test.c
+++ b/drivers/gpu/drm/i915/selftests/igt_flush_test.c
@@ -9,52 +9,8 @@
 #include "../i915_selftest.h"
 #include "igt_flush_test.h"
 
-struct wedge_me {
-        struct delayed_work work;
-        struct drm_i915_private *i915;
-        const void *symbol;
-};
-
-static void wedge_me(struct work_struct *work)
-{
-        struct wedge_me *w = container_of(work, typeof(*w), work.work);
-
-        pr_err("%pS timed out, cancelling all further testing.\n", w->symbol);
-
-        GEM_TRACE("%pS timed out.\n", w->symbol);
-        GEM_TRACE_DUMP();
-
-        i915_gem_set_wedged(w->i915);
-}
-
-static void __init_wedge(struct wedge_me *w,
-                         struct drm_i915_private *i915,
-                         long timeout,
-                         const void *symbol)
-{
-        w->i915 = i915;
-        w->symbol = symbol;
-
-        INIT_DELAYED_WORK_ONSTACK(&w->work, wedge_me);
-        schedule_delayed_work(&w->work, timeout);
-}
-
-static void __fini_wedge(struct wedge_me *w)
-{
-        cancel_delayed_work_sync(&w->work);
-        destroy_delayed_work_on_stack(&w->work);
-        w->i915 = NULL;
-}
-
-#define wedge_on_timeout(W, DEV, TIMEOUT)                               \
-        for (__init_wedge((W), (DEV), (TIMEOUT), __builtin_return_address(0)); \
-             (W)->i915;                                                 \
-             __fini_wedge((W)))
-
 int igt_flush_test(struct drm_i915_private *i915, unsigned int flags)
 {
-        struct wedge_me w;
-
         cond_resched();
 
         if (flags & I915_WAIT_LOCKED &&
@@ -63,8 +19,15 @@ int igt_flush_test(struct drm_i915_private *i915, unsigned int flags)
                 i915_gem_set_wedged(i915);
         }
 
-        wedge_on_timeout(&w, i915, HZ)
-                i915_gem_wait_for_idle(i915, flags);
+        if (i915_gem_wait_for_idle(i915, flags, HZ / 5) == -ETIME) {
+                pr_err("%pS timed out, cancelling all further testing.\n",
+                       __builtin_return_address(0));
+
+                GEM_TRACE("%pS timed out.\n", __builtin_return_address(0));
+                GEM_TRACE_DUMP();
+
+                i915_gem_set_wedged(i915);
+        }
 
         return i915_terminally_wedged(&i915->gpu_error) ? -EIO : 0;
 }
diff --git a/drivers/gpu/drm/i915/selftests/intel_breadcrumbs.c b/drivers/gpu/drm/i915/selftests/intel_breadcrumbs.c
index d6926e7820e5..f03b407fdbe2 100644
--- a/drivers/gpu/drm/i915/selftests/intel_breadcrumbs.c
+++ b/drivers/gpu/drm/i915/selftests/intel_breadcrumbs.c
@@ -464,7 +464,7 @@ int intel_breadcrumbs_mock_selftests(void)
                 return -ENOMEM;
 
         err = i915_subtests(tests, i915->engine[RCS]);
-        drm_dev_unref(&i915->drm);
+        drm_dev_put(&i915->drm);
 
         return err;
 }
diff --git a/drivers/gpu/drm/i915/selftests/intel_hangcheck.c b/drivers/gpu/drm/i915/selftests/intel_hangcheck.c
index fe7d3190ebfe..73462a65a330 100644
--- a/drivers/gpu/drm/i915/selftests/intel_hangcheck.c
+++ b/drivers/gpu/drm/i915/selftests/intel_hangcheck.c
@@ -130,13 +130,19 @@ static int emit_recurse_batch(struct hang *h,
         if (err)
                 goto unpin_vma;
 
-        i915_vma_move_to_active(vma, rq, 0);
+        err = i915_vma_move_to_active(vma, rq, 0);
+        if (err)
+                goto unpin_hws;
+
         if (!i915_gem_object_has_active_reference(vma->obj)) {
                 i915_gem_object_get(vma->obj);
                 i915_gem_object_set_active_reference(vma->obj);
         }
 
-        i915_vma_move_to_active(hws, rq, 0);
+        err = i915_vma_move_to_active(hws, rq, 0);
+        if (err)
+                goto unpin_hws;
+
         if (!i915_gem_object_has_active_reference(hws->obj)) {
                 i915_gem_object_get(hws->obj);
                 i915_gem_object_set_active_reference(hws->obj);
@@ -171,7 +177,7 @@ static int emit_recurse_batch(struct hang *h,
                 *batch++ = MI_BATCH_BUFFER_START | 1 << 8;
                 *batch++ = lower_32_bits(vma->node.start);
         } else if (INTEL_GEN(i915) >= 4) {
-                *batch++ = MI_STORE_DWORD_IMM_GEN4 | 1 << 22;
+                *batch++ = MI_STORE_DWORD_IMM_GEN4 | MI_USE_GGTT;
                 *batch++ = 0;
                 *batch++ = lower_32_bits(hws_address(hws, rq));
                 *batch++ = rq->fence.seqno;
@@ -184,7 +190,7 @@ static int emit_recurse_batch(struct hang *h,
                 *batch++ = MI_BATCH_BUFFER_START | 2 << 6;
                 *batch++ = lower_32_bits(vma->node.start);
         } else {
-                *batch++ = MI_STORE_DWORD_IMM;
+                *batch++ = MI_STORE_DWORD_IMM | MI_MEM_VIRTUAL;
                 *batch++ = lower_32_bits(hws_address(hws, rq));
                 *batch++ = rq->fence.seqno;
                 *batch++ = MI_ARB_CHECK;
@@ -193,7 +199,7 @@ static int emit_recurse_batch(struct hang *h,
                 batch += 1024 / sizeof(*batch);
 
                 *batch++ = MI_ARB_CHECK;
-                *batch++ = MI_BATCH_BUFFER_START | 2 << 6 | 1;
+                *batch++ = MI_BATCH_BUFFER_START | 2 << 6;
                 *batch++ = lower_32_bits(vma->node.start);
         }
         *batch++ = MI_BATCH_BUFFER_END; /* not reached */
@@ -205,6 +211,7 @@ static int emit_recurse_batch(struct hang *h,
 
         err = rq->engine->emit_bb_start(rq, vma->node.start, PAGE_SIZE, flags);
 
+unpin_hws:
         i915_vma_unpin(hws);
 unpin_vma:
         i915_vma_unpin(vma);
@@ -1243,6 +1250,9 @@ int intel_hangcheck_live_selftests(struct drm_i915_private *i915)
         if (!intel_has_gpu_reset(i915))
                 return 0;
 
+        if (i915_terminally_wedged(&i915->gpu_error))
+                return -EIO; /* we're long past hope of a successful reset */
+
         intel_runtime_pm_get(i915);
         saved_hangcheck = fetch_and_zero(&i915_modparams.enable_hangcheck);
 
diff --git a/drivers/gpu/drm/i915/selftests/intel_lrc.c b/drivers/gpu/drm/i915/selftests/intel_lrc.c
index ea27c7cfbf96..636cb68191e3 100644
--- a/drivers/gpu/drm/i915/selftests/intel_lrc.c
+++ b/drivers/gpu/drm/i915/selftests/intel_lrc.c
@@ -104,13 +104,19 @@ static int emit_recurse_batch(struct spinner *spin,
         if (err)
                 goto unpin_vma;
 
-        i915_vma_move_to_active(vma, rq, 0);
+        err = i915_vma_move_to_active(vma, rq, 0);
+        if (err)
+                goto unpin_hws;
+
         if (!i915_gem_object_has_active_reference(vma->obj)) {
                 i915_gem_object_get(vma->obj);
                 i915_gem_object_set_active_reference(vma->obj);
         }
 
-        i915_vma_move_to_active(hws, rq, 0);
+        err = i915_vma_move_to_active(hws, rq, 0);
+        if (err)
+                goto unpin_hws;
+
         if (!i915_gem_object_has_active_reference(hws->obj)) {
                 i915_gem_object_get(hws->obj);
                 i915_gem_object_set_active_reference(hws->obj);
@@ -134,6 +140,7 @@ static int emit_recurse_batch(struct spinner *spin,
 
         err = rq->engine->emit_bb_start(rq, vma->node.start, PAGE_SIZE, 0);
 
+unpin_hws:
         i915_vma_unpin(hws);
 unpin_vma:
         i915_vma_unpin(vma);
@@ -455,5 +462,8 @@ int intel_execlists_live_selftests(struct drm_i915_private *i915)
         if (!HAS_EXECLISTS(i915))
                 return 0;
 
+        if (i915_terminally_wedged(&i915->gpu_error))
+                return 0;
+
         return i915_subtests(tests, i915);
 }
diff --git a/drivers/gpu/drm/i915/selftests/intel_workarounds.c b/drivers/gpu/drm/i915/selftests/intel_workarounds.c
index e1ea2d2bedd2..fafdec3fe83e 100644
--- a/drivers/gpu/drm/i915/selftests/intel_workarounds.c
+++ b/drivers/gpu/drm/i915/selftests/intel_workarounds.c
@@ -49,6 +49,10 @@ read_nonprivs(struct i915_gem_context *ctx, struct intel_engine_cs *engine)
                 goto err_pin;
         }
 
+        err = i915_vma_move_to_active(vma, rq, EXEC_OBJECT_WRITE);
+        if (err)
+                goto err_req;
+
         srm = MI_STORE_REGISTER_MEM | MI_SRM_LRM_GLOBAL_GTT;
         if (INTEL_GEN(ctx->i915) >= 8)
                 srm++;
@@ -67,11 +71,6 @@ read_nonprivs(struct i915_gem_context *ctx, struct intel_engine_cs *engine)
         }
         intel_ring_advance(rq, cs);
 
-        i915_vma_move_to_active(vma, rq, EXEC_OBJECT_WRITE);
-        reservation_object_lock(vma->resv, NULL);
-        reservation_object_add_excl_fence(vma->resv, &rq->fence);
-        reservation_object_unlock(vma->resv);
-
         i915_gem_object_get(result);
         i915_gem_object_set_active_reference(result);
 
@@ -283,6 +282,9 @@ int intel_workarounds_live_selftests(struct drm_i915_private *i915)
         };
         int err;
 
+        if (i915_terminally_wedged(&i915->gpu_error))
+                return 0;
+
         mutex_lock(&i915->drm.struct_mutex);
         err = i915_subtests(tests, i915);
         mutex_unlock(&i915->drm.struct_mutex);
diff --git a/drivers/gpu/drm/i915/selftests/mock_engine.c b/drivers/gpu/drm/i915/selftests/mock_engine.c
index c2a0451336cf..22a73da45ad5 100644
--- a/drivers/gpu/drm/i915/selftests/mock_engine.c
+++ b/drivers/gpu/drm/i915/selftests/mock_engine.c
@@ -200,6 +200,8 @@ struct intel_engine_cs *mock_engine(struct drm_i915_private *i915,
         engine->base.submit_request = mock_submit_request;
 
         i915_timeline_init(i915, &engine->base.timeline, engine->base.name);
+        lockdep_set_subclass(&engine->base.timeline.lock, TIMELINE_ENGINE);
+
         intel_engine_init_breadcrumbs(&engine->base);
         engine->base.breadcrumbs.mock = true; /* prevent touching HW for irqs */
 
diff --git a/drivers/gpu/drm/i915/selftests/mock_gem_device.c b/drivers/gpu/drm/i915/selftests/mock_gem_device.c
index c97075c5ccaf..43ed8b28aeaa 100644
--- a/drivers/gpu/drm/i915/selftests/mock_gem_device.c
+++ b/drivers/gpu/drm/i915/selftests/mock_gem_device.c
@@ -157,7 +157,8 @@ struct drm_i915_private *mock_gem_device(void)
         dev_pm_domain_set(&pdev->dev, &pm_domain);
         pm_runtime_enable(&pdev->dev);
         pm_runtime_dont_use_autosuspend(&pdev->dev);
-        WARN_ON(pm_runtime_get_sync(&pdev->dev));
+        if (pm_runtime_enabled(&pdev->dev))
+                WARN_ON(pm_runtime_get_sync(&pdev->dev));
 
         i915 = (struct drm_i915_private *)(pdev + 1);
         pci_set_drvdata(pdev, i915);
diff --git a/drivers/gpu/drm/i915/selftests/mock_gtt.c b/drivers/gpu/drm/i915/selftests/mock_gtt.c
index 6a7f4da7b523..a140ea5c3a7c 100644
--- a/drivers/gpu/drm/i915/selftests/mock_gtt.c
+++ b/drivers/gpu/drm/i915/selftests/mock_gtt.c
@@ -70,12 +70,7 @@ mock_ppgtt(struct drm_i915_private *i915,
         ppgtt->vm.total = round_down(U64_MAX, PAGE_SIZE);
         ppgtt->vm.file = ERR_PTR(-ENODEV);
 
-        INIT_LIST_HEAD(&ppgtt->vm.active_list);
-        INIT_LIST_HEAD(&ppgtt->vm.inactive_list);
-        INIT_LIST_HEAD(&ppgtt->vm.unbound_list);
-
-        INIT_LIST_HEAD(&ppgtt->vm.global_link);
-        drm_mm_init(&ppgtt->vm.mm, 0, ppgtt->vm.total);
+        i915_address_space_init(&ppgtt->vm, i915);
 
         ppgtt->vm.clear_range = nop_clear_range;
         ppgtt->vm.insert_page = mock_insert_page;
@@ -106,8 +101,6 @@ void mock_init_ggtt(struct drm_i915_private *i915)
 {
         struct i915_ggtt *ggtt = &i915->ggtt;
 
-        INIT_LIST_HEAD(&i915->vm_list);
-
         ggtt->vm.i915 = i915;
 
         ggtt->gmadr = (struct resource) DEFINE_RES_MEM(0, 2048 * PAGE_SIZE);
@@ -124,7 +117,7 @@ void mock_init_ggtt(struct drm_i915_private *i915)
         ggtt->vm.vma_ops.set_pages   = ggtt_set_pages;
         ggtt->vm.vma_ops.clear_pages = clear_pages;
 
-        i915_address_space_init(&ggtt->vm, i915, "global");
+        i915_address_space_init(&ggtt->vm, i915);
 }
 
 void mock_fini_ggtt(struct drm_i915_private *i915)
diff --git a/drivers/gpu/drm/i915/intel_dsi.c b/drivers/gpu/drm/i915/vlv_dsi.c
index 3b7acb5a70b3..435a2c35ee8c 100644
--- a/drivers/gpu/drm/i915/intel_dsi.c
+++ b/drivers/gpu/drm/i915/vlv_dsi.c
@@ -69,7 +69,7 @@ enum mipi_dsi_pixel_format pixel_format_from_register_bits(u32 fmt)
         }
 }
 
-void wait_for_dsi_fifo_empty(struct intel_dsi *intel_dsi, enum port port)
+void vlv_dsi_wait_for_fifo_empty(struct intel_dsi *intel_dsi, enum port port)
 {
         struct drm_encoder *encoder = &intel_dsi->base.base;
         struct drm_device *dev = encoder->dev;
@@ -342,11 +342,15 @@ static bool intel_dsi_compute_config(struct intel_encoder *encoder,
                         pipe_config->cpu_transcoder = TRANSCODER_DSI_C;
                 else
                         pipe_config->cpu_transcoder = TRANSCODER_DSI_A;
-        }
 
-        ret = intel_compute_dsi_pll(encoder, pipe_config);
-        if (ret)
-                return false;
+                ret = bxt_dsi_pll_compute(encoder, pipe_config);
+                if (ret)
+                        return false;
+        } else {
+                ret = vlv_dsi_pll_compute(encoder, pipe_config);
+                if (ret)
+                        return false;
+        }
 
         pipe_config->clock_set = true;
 
@@ -546,12 +550,12 @@ static void intel_dsi_device_ready(struct intel_encoder *encoder)
 {
         struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
 
-        if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
-                vlv_dsi_device_ready(encoder);
-        else if (IS_BROXTON(dev_priv))
-                bxt_dsi_device_ready(encoder);
-        else if (IS_GEMINILAKE(dev_priv))
+        if (IS_GEMINILAKE(dev_priv))
                 glk_dsi_device_ready(encoder);
+        else if (IS_GEN9_LP(dev_priv))
+                bxt_dsi_device_ready(encoder);
+        else
+                vlv_dsi_device_ready(encoder);
 }
 
 static void glk_dsi_enter_low_power_mode(struct intel_encoder *encoder)
@@ -810,8 +814,13 @@ static void intel_dsi_pre_enable(struct intel_encoder *encoder,
          * The BIOS may leave the PLL in a wonky state where it doesn't
          * lock. It needs to be fully powered down to fix it.
          */
-        intel_disable_dsi_pll(encoder);
-        intel_enable_dsi_pll(encoder, pipe_config);
+        if (IS_GEN9_LP(dev_priv)) {
+                bxt_dsi_pll_disable(encoder);
+                bxt_dsi_pll_enable(encoder, pipe_config);
+        } else {
+                vlv_dsi_pll_disable(encoder);
+                vlv_dsi_pll_enable(encoder, pipe_config);
+        }
 
         if (IS_BROXTON(dev_priv)) {
                 /* Add MIPI IO reset programming for modeset */
@@ -929,11 +938,10 @@ static void intel_dsi_clear_device_ready(struct intel_encoder *encoder)
 {
         struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
 
-        if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv) ||
-            IS_BROXTON(dev_priv))
-                vlv_dsi_clear_device_ready(encoder);
-        else if (IS_GEMINILAKE(dev_priv))
+        if (IS_GEMINILAKE(dev_priv))
                 glk_dsi_clear_device_ready(encoder);
+        else
+                vlv_dsi_clear_device_ready(encoder);
 }
 
 static void intel_dsi_post_disable(struct intel_encoder *encoder,
@@ -949,7 +957,7 @@ static void intel_dsi_post_disable(struct intel_encoder *encoder,
 
         if (is_vid_mode(intel_dsi)) {
                 for_each_dsi_port(port, intel_dsi->ports)
-                        wait_for_dsi_fifo_empty(intel_dsi, port);
+                        vlv_dsi_wait_for_fifo_empty(intel_dsi, port);
 
                 intel_dsi_port_disable(encoder);
                 usleep_range(2000, 5000);
@@ -979,11 +987,13 @@ static void intel_dsi_post_disable(struct intel_encoder *encoder,
                                 val & ~MIPIO_RST_CTRL);
         }
 
-        intel_disable_dsi_pll(encoder);
-
-        if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) {
+        if (IS_GEN9_LP(dev_priv)) {
+                bxt_dsi_pll_disable(encoder);
+        } else {
                 u32 val;
 
+                vlv_dsi_pll_disable(encoder);
+
                 val = I915_READ(DSPCLK_GATE_D);
                 val &= ~DPOUNIT_CLOCK_GATE_DISABLE;
                 I915_WRITE(DSPCLK_GATE_D, val);
@@ -1024,7 +1034,7 @@ static bool intel_dsi_get_hw_state(struct intel_encoder *encoder,
          * configuration, otherwise accessing DSI registers will hang the
          * machine. See BSpec North Display Engine registers/MIPI[BXT].
          */
-        if (IS_GEN9_LP(dev_priv) && !intel_dsi_pll_is_enabled(dev_priv))
+        if (IS_GEN9_LP(dev_priv) && !bxt_dsi_pll_is_enabled(dev_priv))
                 goto out_put_power;
 
         /* XXX: this only works for one DSI output */
@@ -1247,16 +1257,19 @@ static void intel_dsi_get_config(struct intel_encoder *encoder,
 
         pipe_config->output_types |= BIT(INTEL_OUTPUT_DSI);
 
-        if (IS_GEN9_LP(dev_priv))
+        if (IS_GEN9_LP(dev_priv)) {
                 bxt_dsi_get_pipe_config(encoder, pipe_config);
+                pclk = bxt_dsi_get_pclk(encoder, pipe_config->pipe_bpp,
+                                        pipe_config);
+        } else {
+                pclk = vlv_dsi_get_pclk(encoder, pipe_config->pipe_bpp,
+                                        pipe_config);
+        }
 
-        pclk = intel_dsi_get_pclk(encoder, pipe_config->pipe_bpp,
-                                  pipe_config);
-        if (!pclk)
-                return;
-
-        pipe_config->base.adjusted_mode.crtc_clock = pclk;
-        pipe_config->port_clock = pclk;
+        if (pclk) {
+                pipe_config->base.adjusted_mode.crtc_clock = pclk;
+                pipe_config->port_clock = pclk;
+        }
 }
 
 static enum drm_mode_status
@@ -1585,20 +1598,24 @@ static void intel_dsi_unprepare(struct intel_encoder *encoder)
         enum port port;
         u32 val;
 
-        if (!IS_GEMINILAKE(dev_priv)) {
-                for_each_dsi_port(port, intel_dsi->ports) {
-                        /* Panel commands can be sent when clock is in LP11 */
-                        I915_WRITE(MIPI_DEVICE_READY(port), 0x0);
+        if (IS_GEMINILAKE(dev_priv))
+                return;
 
-                        intel_dsi_reset_clocks(encoder, port);
-                        I915_WRITE(MIPI_EOT_DISABLE(port), CLOCKSTOP);
+        for_each_dsi_port(port, intel_dsi->ports) {
+                /* Panel commands can be sent when clock is in LP11 */
+                I915_WRITE(MIPI_DEVICE_READY(port), 0x0);
 
-                        val = I915_READ(MIPI_DSI_FUNC_PRG(port));
-                        val &= ~VID_MODE_FORMAT_MASK;
-                        I915_WRITE(MIPI_DSI_FUNC_PRG(port), val);
+                if (IS_GEN9_LP(dev_priv))
+                        bxt_dsi_reset_clocks(encoder, port);
+                else
+                        vlv_dsi_reset_clocks(encoder, port);
+                I915_WRITE(MIPI_EOT_DISABLE(port), CLOCKSTOP);
 
-                        I915_WRITE(MIPI_DEVICE_READY(port), 0x1);
-                }
+                val = I915_READ(MIPI_DSI_FUNC_PRG(port));
+                val &= ~VID_MODE_FORMAT_MASK;
+                I915_WRITE(MIPI_DSI_FUNC_PRG(port), val);
+
+                I915_WRITE(MIPI_DEVICE_READY(port), 0x1);
         }
 }
 
@@ -1713,7 +1730,7 @@ static void intel_dsi_add_properties(struct intel_connector *connector)
         }
 }
 
-void intel_dsi_init(struct drm_i915_private *dev_priv)
+void vlv_dsi_init(struct drm_i915_private *dev_priv)
 {
         struct drm_device *dev = &dev_priv->drm;
         struct intel_dsi *intel_dsi;
@@ -1730,14 +1747,10 @@ void intel_dsi_init(struct drm_i915_private *dev_priv)
         if (!intel_bios_is_dsi_present(dev_priv, &port))
                 return;
 
-        if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) {
-                dev_priv->mipi_mmio_base = VLV_MIPI_BASE;
-        } else if (IS_GEN9_LP(dev_priv)) {
+        if (IS_GEN9_LP(dev_priv))
                 dev_priv->mipi_mmio_base = BXT_MIPI_BASE;
-        } else {
-                DRM_ERROR("Unsupported Mipi device to reg base");
-                return;
-        }
+        else
+                dev_priv->mipi_mmio_base = VLV_MIPI_BASE;
 
         intel_dsi = kzalloc(sizeof(*intel_dsi), GFP_KERNEL);
         if (!intel_dsi)
diff --git a/drivers/gpu/drm/i915/intel_dsi_pll.c b/drivers/gpu/drm/i915/vlv_dsi_pll.c
index 2ff2ee7f3b78..a132a8037ecc 100644
--- a/drivers/gpu/drm/i915/intel_dsi_pll.c
+++ b/drivers/gpu/drm/i915/vlv_dsi_pll.c
@@ -111,8 +111,8 @@ static int dsi_calc_mnp(struct drm_i915_private *dev_priv,
  * XXX: The muxing and gating is hard coded for now. Need to add support for
  * sharing PLLs with two DSI outputs.
  */
-static int vlv_compute_dsi_pll(struct intel_encoder *encoder,
-                               struct intel_crtc_state *config)
+int vlv_dsi_pll_compute(struct intel_encoder *encoder,
+                        struct intel_crtc_state *config)
 {
         struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
         struct intel_dsi *intel_dsi = enc_to_intel_dsi(&encoder->base);
@@ -142,8 +142,8 @@ static int vlv_compute_dsi_pll(struct intel_encoder *encoder,
         return 0;
 }
 
-static void vlv_enable_dsi_pll(struct intel_encoder *encoder,
-                               const struct intel_crtc_state *config)
+void vlv_dsi_pll_enable(struct intel_encoder *encoder,
+                        const struct intel_crtc_state *config)
 {
         struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
 
@@ -175,7 +175,7 @@ static void vlv_enable_dsi_pll(struct intel_encoder *encoder,
         DRM_DEBUG_KMS("DSI PLL locked\n");
 }
 
-static void vlv_disable_dsi_pll(struct intel_encoder *encoder)
+void vlv_dsi_pll_disable(struct intel_encoder *encoder)
 {
         struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
         u32 tmp;
@@ -192,7 +192,7 @@ static void vlv_disable_dsi_pll(struct intel_encoder *encoder)
         mutex_unlock(&dev_priv->sb_lock);
 }
 
-static bool bxt_dsi_pll_is_enabled(struct drm_i915_private *dev_priv)
+bool bxt_dsi_pll_is_enabled(struct drm_i915_private *dev_priv)
 {
         bool enabled;
         u32 val;
@@ -229,7 +229,7 @@ static bool bxt_dsi_pll_is_enabled(struct drm_i915_private *dev_priv)
         return enabled;
 }
 
-static void bxt_disable_dsi_pll(struct intel_encoder *encoder)
+void bxt_dsi_pll_disable(struct intel_encoder *encoder)
 {
         struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
         u32 val;
@@ -261,8 +261,8 @@ static void assert_bpp_mismatch(enum mipi_dsi_pixel_format fmt, int pipe_bpp)
              bpp, pipe_bpp);
 }
 
-static u32 vlv_dsi_get_pclk(struct intel_encoder *encoder, int pipe_bpp,
-                            struct intel_crtc_state *config)
+u32 vlv_dsi_get_pclk(struct intel_encoder *encoder, int pipe_bpp,
+                     struct intel_crtc_state *config)
 {
         struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
         struct intel_dsi *intel_dsi = enc_to_intel_dsi(&encoder->base);
@@ -327,8 +327,8 @@ static u32 vlv_dsi_get_pclk(struct intel_encoder *encoder, int pipe_bpp,
         return pclk;
 }
 
-static u32 bxt_dsi_get_pclk(struct intel_encoder *encoder, int pipe_bpp,
-                            struct intel_crtc_state *config)
+u32 bxt_dsi_get_pclk(struct intel_encoder *encoder, int pipe_bpp,
+                     struct intel_crtc_state *config)
 {
         u32 pclk;
         u32 dsi_clk;
@@ -357,16 +357,7 @@ static u32 bxt_dsi_get_pclk(struct intel_encoder *encoder, int pipe_bpp,
         return pclk;
 }
 
-u32 intel_dsi_get_pclk(struct intel_encoder *encoder, int pipe_bpp,
-                       struct intel_crtc_state *config)
-{
-        if (IS_GEN9_LP(to_i915(encoder->base.dev)))
-                return bxt_dsi_get_pclk(encoder, pipe_bpp, config);
-        else
-                return vlv_dsi_get_pclk(encoder, pipe_bpp, config);
-}
-
-static void vlv_dsi_reset_clocks(struct intel_encoder *encoder, enum port port)
+void vlv_dsi_reset_clocks(struct intel_encoder *encoder, enum port port)
 {
         u32 temp;
         struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
@@ -480,8 +471,8 @@ static void bxt_dsi_program_clocks(struct drm_device *dev, enum port port,
         I915_WRITE(BXT_MIPI_CLOCK_CTL, tmp);
 }
 
-static int gen9lp_compute_dsi_pll(struct intel_encoder *encoder,
-                               struct intel_crtc_state *config)
+int bxt_dsi_pll_compute(struct intel_encoder *encoder,
+                        struct intel_crtc_state *config)
 {
         struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
         struct intel_dsi *intel_dsi = enc_to_intel_dsi(&encoder->base);
@@ -528,8 +519,8 @@ static int gen9lp_compute_dsi_pll(struct intel_encoder *encoder,
         return 0;
 }
 
-static void gen9lp_enable_dsi_pll(struct intel_encoder *encoder,
-                               const struct intel_crtc_state *config)
+void bxt_dsi_pll_enable(struct intel_encoder *encoder,
+                        const struct intel_crtc_state *config)
 {
         struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
         struct intel_dsi *intel_dsi = enc_to_intel_dsi(&encoder->base);
@@ -568,52 +559,7 @@ static void gen9lp_enable_dsi_pll(struct intel_encoder *encoder,
         DRM_DEBUG_KMS("DSI PLL locked\n");
 }
 
-bool intel_dsi_pll_is_enabled(struct drm_i915_private *dev_priv)
-{
-        if (IS_GEN9_LP(dev_priv))
-                return bxt_dsi_pll_is_enabled(dev_priv);
-
-        MISSING_CASE(INTEL_DEVID(dev_priv));
-
-        return false;
-}
-
-int intel_compute_dsi_pll(struct intel_encoder *encoder,
-                          struct intel_crtc_state *config)
-{
-        struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
-
-        if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
-                return vlv_compute_dsi_pll(encoder, config);
-        else if (IS_GEN9_LP(dev_priv))
-                return gen9lp_compute_dsi_pll(encoder, config);
-
-        return -ENODEV;
-}
-
-void intel_enable_dsi_pll(struct intel_encoder *encoder,
-                          const struct intel_crtc_state *config)
-{
-        struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
-
-        if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
-                vlv_enable_dsi_pll(encoder, config);
-        else if (IS_GEN9_LP(dev_priv))
-                gen9lp_enable_dsi_pll(encoder, config);
-}
-
-void intel_disable_dsi_pll(struct intel_encoder *encoder)
-{
-        struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
-
-        if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
-                vlv_disable_dsi_pll(encoder);
-        else if (IS_GEN9_LP(dev_priv))
-                bxt_disable_dsi_pll(encoder);
-}
-
-static void gen9lp_dsi_reset_clocks(struct intel_encoder *encoder,
-                                    enum port port)
+void bxt_dsi_reset_clocks(struct intel_encoder *encoder, enum port port)
 {
         u32 tmp;
         struct drm_device *dev = encoder->base.dev;
@@ -638,13 +584,3 @@ static void gen9lp_dsi_reset_clocks(struct intel_encoder *encoder,
         }
         I915_WRITE(MIPI_EOT_DISABLE(port), CLOCKSTOP);
 }
-
-void intel_dsi_reset_clocks(struct intel_encoder *encoder, enum port port)
-{
-        struct drm_i915_private *dev_priv = to_i915(encoder->base.dev);
-
-        if (IS_GEN9_LP(dev_priv))
-                gen9lp_dsi_reset_clocks(encoder, port);
-        else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
-                vlv_dsi_reset_clocks(encoder, port);
-}