Merge tag 'drm-intel-next-2014-06-20' of git://anongit.freedesktop.org/drm-intel into drm-next

- Accurate frontbuffer tracking and frontbuffer rendering invalidate, flush and
  flip events. This is prep work for proper PSR support and should also be
  useful for DRRS&fbc.
- Runtime suspend hardware on system suspend to support the new SOix sleep
  states, from Jesse.
- PSR updates for broadwell (Rodrigo)
- Universal plane support for cursors (Matt Roper), including core drm patches.
- Prefault gtt mappings (Chris)
- baytrail write-enable pte bit support (Akash Goel)
- mmio based flips (Sourab Gupta) instead of blitter ring flips
- interrupt handling race fixes (Oscar Mateo)

And old, not yet merged features from the previous round:
- rps/turbo support for chv (Deepak)
- some other straggling chv patches (Ville)
- proper universal plane conversion for the primary plane (Matt Roper)
- ppgtt on vlv from Jesse
- pile of cleanups, little fixes for insane corner cases and improved debug
  support all over

* tag 'drm-intel-next-2014-06-20' of git://anongit.freedesktop.org/drm-intel: (99 commits)
  drm/i915: Update DRIVER_DATE to 20140620
  drivers/i915: Fix unnoticed failure of init_ring_common()
  drm/i915: Track frontbuffer invalidation/flushing
  drm/i915: Use new frontbuffer bits to increase pll clock
  drm/i915: don't take runtime PM reference around freeze/thaw
  drm/i915: use runtime irq suspend/resume in freeze/thaw
  drm/i915: Properly track domain of the fbcon fb
  drm/i915: Print obj->frontbuffer_bits in debugfs output
  drm/i915: Introduce accurate frontbuffer tracking
  drm/i915: Drop schedule_back from psr_exit
  drm/i915: Ditch intel_edp_psr_update
  drm/i915: Drop unecessary complexity from psr_inactivate
  drm/i915: Remove ctx->last_ring
  drm/i915/chv: Ack interrupts before handling them (CHV)
  drm/i915/bdw: Ack interrupts before handling them (GEN8)
  drm/i915/vlv: Ack interrupts before handling them (VLV)
  drm/i915: Ack interrupts before handling them (GEN5 - GEN7)
  drm/i915: Don't BUG_ON in i915_gem_obj_offset
  drm/i915: Grab dev->struct_mutex in i915_gem_pageflip_info
  drm/i915: Add some L3 registers to the parser whitelist
  ...

Conflicts:
	drivers/gpu/drm/i915/i915_drv.c

1 files changed, 808 insertions, 95 deletions

diff --git a/drivers/gpu/drm/i915/intel_display.c b/drivers/gpu/drm/i915/intel_display.c
index 1112d9ecc226..927d2476f60a 100644
--- a/drivers/gpu/drm/i915/intel_display.c
+++ b/drivers/gpu/drm/i915/intel_display.c
@@ -39,12 +39,45 @@
 #include "i915_trace.h"
 #include <drm/drm_dp_helper.h>
 #include <drm/drm_crtc_helper.h>
+#include <drm/drm_plane_helper.h>
+#include <drm/drm_rect.h>
 #include <linux/dma_remapping.h>
 
+/* Primary plane formats supported by all gen */
+#define COMMON_PRIMARY_FORMATS \
+        DRM_FORMAT_C8, \
+        DRM_FORMAT_RGB565, \
+        DRM_FORMAT_XRGB8888, \
+        DRM_FORMAT_ARGB8888
+
+/* Primary plane formats for gen <= 3 */
+static const uint32_t intel_primary_formats_gen2[] = {
+        COMMON_PRIMARY_FORMATS,
+        DRM_FORMAT_XRGB1555,
+        DRM_FORMAT_ARGB1555,
+};
+
+/* Primary plane formats for gen >= 4 */
+static const uint32_t intel_primary_formats_gen4[] = {
+        COMMON_PRIMARY_FORMATS, \
+        DRM_FORMAT_XBGR8888,
+        DRM_FORMAT_ABGR8888,
+        DRM_FORMAT_XRGB2101010,
+        DRM_FORMAT_ARGB2101010,
+        DRM_FORMAT_XBGR2101010,
+        DRM_FORMAT_ABGR2101010,
+};
+
+/* Cursor formats */
+static const uint32_t intel_cursor_formats[] = {
+        DRM_FORMAT_ARGB8888,
+};
+
 #define DIV_ROUND_CLOSEST_ULL(ll, d)    \
-        ({ unsigned long long _tmp = (ll)+(d)/2; do_div(_tmp, d); _tmp; })
+({ unsigned long long _tmp = (ll)+(d)/2; do_div(_tmp, d); _tmp; })
 
-static void intel_increase_pllclock(struct drm_crtc *crtc);
+static void intel_increase_pllclock(struct drm_device *dev,
+                                    enum pipe pipe);
 static void intel_crtc_update_cursor(struct drm_crtc *crtc, bool on);
 
 static void i9xx_crtc_clock_get(struct intel_crtc *crtc,
@@ -1712,6 +1745,17 @@ static void chv_disable_pll(struct drm_i915_private *dev_priv, enum pipe pipe)
         val &= ~DPIO_DCLKP_EN;
         vlv_dpio_write(dev_priv, pipe, CHV_CMN_DW14(port), val);
 
+        /* disable left/right clock distribution */
+        if (pipe != PIPE_B) {
+                val = vlv_dpio_read(dev_priv, pipe, _CHV_CMN_DW5_CH0);
+                val &= ~(CHV_BUFLEFTENA1_MASK | CHV_BUFRIGHTENA1_MASK);
+                vlv_dpio_write(dev_priv, pipe, _CHV_CMN_DW5_CH0, val);
+        } else {
+                val = vlv_dpio_read(dev_priv, pipe, _CHV_CMN_DW1_CH1);
+                val &= ~(CHV_BUFLEFTENA2_MASK | CHV_BUFRIGHTENA2_MASK);
+                vlv_dpio_write(dev_priv, pipe, _CHV_CMN_DW1_CH1, val);
+        }
+
         mutex_unlock(&dev_priv->dpio_lock);
 }
 
@@ -1749,6 +1793,9 @@ static void intel_prepare_shared_dpll(struct intel_crtc *crtc)
         struct drm_i915_private *dev_priv = dev->dev_private;
         struct intel_shared_dpll *pll = intel_crtc_to_shared_dpll(crtc);
 
+        if (WARN_ON(pll == NULL))
+                return;
+
         WARN_ON(!pll->refcount);
         if (pll->active == 0) {
                 DRM_DEBUG_DRIVER("setting up %s\n", pll->name);
@@ -2314,6 +2361,7 @@ static bool intel_alloc_plane_obj(struct intel_crtc *crtc,
                 goto out_unref_obj;
         }
 
+        obj->frontbuffer_bits = INTEL_FRONTBUFFER_PRIMARY(crtc->pipe);
         mutex_unlock(&dev->struct_mutex);
 
         DRM_DEBUG_KMS("plane fb obj %p\n", obj);
@@ -2359,6 +2407,7 @@ static void intel_find_plane_obj(struct intel_crtc *intel_crtc,
                 if (i915_gem_obj_ggtt_offset(fb->obj) == plane_config->base) {
                         drm_framebuffer_reference(c->primary->fb);
                         intel_crtc->base.primary->fb = c->primary->fb;
+                        fb->obj->frontbuffer_bits |= INTEL_FRONTBUFFER_PRIMARY(intel_crtc->pipe);
                         break;
                 }
         }
@@ -2546,7 +2595,7 @@ intel_pipe_set_base_atomic(struct drm_crtc *crtc, struct drm_framebuffer *fb,
 
         if (dev_priv->display.disable_fbc)
                 dev_priv->display.disable_fbc(dev);
-        intel_increase_pllclock(crtc);
+        intel_increase_pllclock(dev, to_intel_crtc(crtc)->pipe);
 
         dev_priv->display.update_primary_plane(crtc, fb, x, y);
 
@@ -2647,7 +2696,9 @@ intel_pipe_set_base(struct drm_crtc *crtc, int x, int y,
         struct drm_device *dev = crtc->dev;
         struct drm_i915_private *dev_priv = dev->dev_private;
         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
+        enum pipe pipe = intel_crtc->pipe;
         struct drm_framebuffer *old_fb;
+        struct drm_i915_gem_object *obj = to_intel_framebuffer(fb)->obj;
         int ret;
 
         if (intel_crtc_has_pending_flip(crtc)) {
@@ -2668,10 +2719,13 @@ intel_pipe_set_base(struct drm_crtc *crtc, int x, int y,
                 return -EINVAL;
         }
 
+        old_fb = crtc->primary->fb;
+
         mutex_lock(&dev->struct_mutex);
-        ret = intel_pin_and_fence_fb_obj(dev,
-                                         to_intel_framebuffer(fb)->obj,
-                                         NULL);
+        ret = intel_pin_and_fence_fb_obj(dev, obj, NULL);
+        if (ret == 0)
+                i915_gem_track_fb(to_intel_framebuffer(old_fb)->obj, obj,
+                                  INTEL_FRONTBUFFER_PRIMARY(pipe));
         mutex_unlock(&dev->struct_mutex);
         if (ret != 0) {
                 DRM_ERROR("pin & fence failed\n");
@@ -2711,7 +2765,9 @@ intel_pipe_set_base(struct drm_crtc *crtc, int x, int y,
 
         dev_priv->display.update_primary_plane(crtc, fb, x, y);
 
-        old_fb = crtc->primary->fb;
+        if (intel_crtc->active)
+                intel_frontbuffer_flip(dev, INTEL_FRONTBUFFER_PRIMARY(pipe));
+
         crtc->primary->fb = fb;
         crtc->x = x;
         crtc->y = y;
@@ -2726,7 +2782,6 @@ intel_pipe_set_base(struct drm_crtc *crtc, int x, int y,
 
         mutex_lock(&dev->struct_mutex);
         intel_update_fbc(dev);
-        intel_edp_psr_update(dev);
         mutex_unlock(&dev->struct_mutex);
 
         return 0;
@@ -3892,6 +3947,8 @@ static void intel_crtc_enable_planes(struct drm_crtc *crtc)
         int pipe = intel_crtc->pipe;
         int plane = intel_crtc->plane;
 
+        drm_vblank_on(dev, pipe);
+
         intel_enable_primary_hw_plane(dev_priv, plane, pipe);
         intel_enable_planes(crtc);
         /* The fixup needs to happen before cursor is enabled */
@@ -3904,8 +3961,14 @@ static void intel_crtc_enable_planes(struct drm_crtc *crtc)
 
         mutex_lock(&dev->struct_mutex);
         intel_update_fbc(dev);
-        intel_edp_psr_update(dev);
         mutex_unlock(&dev->struct_mutex);
+
+        /*
+         * FIXME: Once we grow proper nuclear flip support out of this we need
+         * to compute the mask of flip planes precisely. For the time being
+         * consider this a flip from a NULL plane.
+         */
+        intel_frontbuffer_flip(dev, INTEL_FRONTBUFFER_ALL_MASK(pipe));
 }
 
 static void intel_crtc_disable_planes(struct drm_crtc *crtc)
@@ -3917,7 +3980,6 @@ static void intel_crtc_disable_planes(struct drm_crtc *crtc)
         int plane = intel_crtc->plane;
 
         intel_crtc_wait_for_pending_flips(crtc);
-        drm_crtc_vblank_off(crtc);
 
         if (dev_priv->fbc.plane == plane)
                 intel_disable_fbc(dev);
@@ -3928,6 +3990,15 @@ static void intel_crtc_disable_planes(struct drm_crtc *crtc)
         intel_crtc_update_cursor(crtc, false);
         intel_disable_planes(crtc);
         intel_disable_primary_hw_plane(dev_priv, plane, pipe);
+
+        /*
+         * FIXME: Once we grow proper nuclear flip support out of this we need
+         * to compute the mask of flip planes precisely. For the time being
+         * consider this a flip to a NULL plane.
+         */
+        intel_frontbuffer_flip(dev, INTEL_FRONTBUFFER_ALL_MASK(pipe));
+
+        drm_vblank_off(dev, pipe);
 }
 
 static void ironlake_crtc_enable(struct drm_crtc *crtc)
@@ -4006,8 +4077,6 @@ static void ironlake_crtc_enable(struct drm_crtc *crtc)
                 cpt_verify_modeset(dev, intel_crtc->pipe);
 
         intel_crtc_enable_planes(crtc);
-
-        drm_crtc_vblank_on(crtc);
 }
 
 /* IPS only exists on ULT machines and is tied to pipe A. */
@@ -4121,8 +4190,6 @@ static void haswell_crtc_enable(struct drm_crtc *crtc)
          * to change the workaround. */
         haswell_mode_set_planes_workaround(intel_crtc);
         intel_crtc_enable_planes(crtc);
-
-        drm_crtc_vblank_on(crtc);
 }
 
 static void ironlake_pfit_disable(struct intel_crtc *crtc)
@@ -4200,7 +4267,6 @@ static void ironlake_crtc_disable(struct drm_crtc *crtc)
 
         mutex_lock(&dev->struct_mutex);
         intel_update_fbc(dev);
-        intel_edp_psr_update(dev);
         mutex_unlock(&dev->struct_mutex);
 }
 
@@ -4248,7 +4314,6 @@ static void haswell_crtc_disable(struct drm_crtc *crtc)
 
         mutex_lock(&dev->struct_mutex);
         intel_update_fbc(dev);
-        intel_edp_psr_update(dev);
         mutex_unlock(&dev->struct_mutex);
 }
 
@@ -4633,8 +4698,6 @@ static void valleyview_crtc_enable(struct drm_crtc *crtc)
 
         intel_crtc_enable_planes(crtc);
 
-        drm_crtc_vblank_on(crtc);
-
         /* Underruns don't raise interrupts, so check manually. */
         i9xx_check_fifo_underruns(dev);
 }
@@ -4727,8 +4790,6 @@ static void i9xx_crtc_enable(struct drm_crtc *crtc)
         if (IS_GEN2(dev))
                 intel_set_cpu_fifo_underrun_reporting(dev, pipe, true);
 
-        drm_crtc_vblank_on(crtc);
-
         /* Underruns don't raise interrupts, so check manually. */
         i9xx_check_fifo_underruns(dev);
 }
@@ -4805,7 +4866,6 @@ static void i9xx_crtc_disable(struct drm_crtc *crtc)
 
         mutex_lock(&dev->struct_mutex);
         intel_update_fbc(dev);
-        intel_edp_psr_update(dev);
         mutex_unlock(&dev->struct_mutex);
 }
 
@@ -4850,16 +4910,43 @@ void intel_crtc_update_dpms(struct drm_crtc *crtc)
 {
         struct drm_device *dev = crtc->dev;
         struct drm_i915_private *dev_priv = dev->dev_private;
+        struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
         struct intel_encoder *intel_encoder;
+        enum intel_display_power_domain domain;
+        unsigned long domains;
         bool enable = false;
 
         for_each_encoder_on_crtc(dev, crtc, intel_encoder)
                 enable |= intel_encoder->connectors_active;
 
-        if (enable)
-                dev_priv->display.crtc_enable(crtc);
-        else
-                dev_priv->display.crtc_disable(crtc);
+        if (enable) {
+                if (!intel_crtc->active) {
+                        /*
+                         * FIXME: DDI plls and relevant code isn't converted
+                         * yet, so do runtime PM for DPMS only for all other
+                         * platforms for now.
+                         */
+                        if (!HAS_DDI(dev)) {
+                                domains = get_crtc_power_domains(crtc);
+                                for_each_power_domain(domain, domains)
+                                        intel_display_power_get(dev_priv, domain);
+                                intel_crtc->enabled_power_domains = domains;
+                        }
+
+                        dev_priv->display.crtc_enable(crtc);
+                }
+        } else {
+                if (intel_crtc->active) {
+                        dev_priv->display.crtc_disable(crtc);
+
+                        if (!HAS_DDI(dev)) {
+                                domains = intel_crtc->enabled_power_domains;
+                                for_each_power_domain(domain, domains)
+                                        intel_display_power_put(dev_priv, domain);
+                                intel_crtc->enabled_power_domains = 0;
+                        }
+                }
+        }
 
         intel_crtc_update_sarea(crtc, enable);
 }
@@ -4869,6 +4956,8 @@ static void intel_crtc_disable(struct drm_crtc *crtc)
         struct drm_device *dev = crtc->dev;
         struct drm_connector *connector;
         struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_gem_object *old_obj;
+        enum pipe pipe = to_intel_crtc(crtc)->pipe;
 
         /* crtc should still be enabled when we disable it. */
         WARN_ON(!crtc->enabled);
@@ -4878,12 +4967,15 @@ static void intel_crtc_disable(struct drm_crtc *crtc)
         dev_priv->display.off(crtc);
 
         assert_plane_disabled(dev->dev_private, to_intel_crtc(crtc)->plane);
-        assert_cursor_disabled(dev_priv, to_intel_crtc(crtc)->pipe);
-        assert_pipe_disabled(dev->dev_private, to_intel_crtc(crtc)->pipe);
+        assert_cursor_disabled(dev_priv, pipe);
+        assert_pipe_disabled(dev->dev_private, pipe);
 
         if (crtc->primary->fb) {
+                old_obj = to_intel_framebuffer(crtc->primary->fb)->obj;
                 mutex_lock(&dev->struct_mutex);
-                intel_unpin_fb_obj(to_intel_framebuffer(crtc->primary->fb)->obj);
+                intel_unpin_fb_obj(old_obj);
+                i915_gem_track_fb(old_obj, NULL,
+                                  INTEL_FRONTBUFFER_PRIMARY(pipe));
                 mutex_unlock(&dev->struct_mutex);
                 crtc->primary->fb = NULL;
         }
@@ -7991,8 +8083,8 @@ static void intel_crtc_update_cursor(struct drm_crtc *crtc,
         struct drm_i915_private *dev_priv = dev->dev_private;
         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
         int pipe = intel_crtc->pipe;
-        int x = intel_crtc->cursor_x;
-        int y = intel_crtc->cursor_y;
+        int x = crtc->cursor_x;
+        int y = crtc->cursor_y;
         u32 base = 0, pos = 0;
 
         if (on)
@@ -8036,21 +8128,27 @@ static void intel_crtc_update_cursor(struct drm_crtc *crtc,
         intel_crtc->cursor_base = base;
 }
 
-static int intel_crtc_cursor_set(struct drm_crtc *crtc,
-                                 struct drm_file *file,
-                                 uint32_t handle,
-                                 uint32_t width, uint32_t height)
+/*
+ * intel_crtc_cursor_set_obj - Set cursor to specified GEM object
+ *
+ * Note that the object's reference will be consumed if the update fails.  If
+ * the update succeeds, the reference of the old object (if any) will be
+ * consumed.
+ */
+static int intel_crtc_cursor_set_obj(struct drm_crtc *crtc,
+                                     struct drm_i915_gem_object *obj,
+                                     uint32_t width, uint32_t height)
 {
         struct drm_device *dev = crtc->dev;
         struct drm_i915_private *dev_priv = dev->dev_private;
         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
-        struct drm_i915_gem_object *obj;
+        enum pipe pipe = intel_crtc->pipe;
         unsigned old_width;
         uint32_t addr;
         int ret;
 
         /* if we want to turn off the cursor ignore width and height */
-        if (!handle) {
+        if (!obj) {
                 DRM_DEBUG_KMS("cursor off\n");
                 addr = 0;
                 obj = NULL;
@@ -8066,12 +8164,8 @@ static int intel_crtc_cursor_set(struct drm_crtc *crtc,
                 return -EINVAL;
         }
 
-        obj = to_intel_bo(drm_gem_object_lookup(dev, file, handle));
-        if (&obj->base == NULL)
-                return -ENOENT;
-
         if (obj->base.size < width * height * 4) {
-                DRM_DEBUG_KMS("buffer is to small\n");
+                DRM_DEBUG_KMS("buffer is too small\n");
                 ret = -ENOMEM;
                 goto fail;
         }
@@ -8126,9 +8220,10 @@ static int intel_crtc_cursor_set(struct drm_crtc *crtc,
         if (intel_crtc->cursor_bo) {
                 if (!INTEL_INFO(dev)->cursor_needs_physical)
                         i915_gem_object_unpin_from_display_plane(intel_crtc->cursor_bo);
-                drm_gem_object_unreference(&intel_crtc->cursor_bo->base);
         }
 
+        i915_gem_track_fb(intel_crtc->cursor_bo, obj,
+                          INTEL_FRONTBUFFER_CURSOR(pipe));
         mutex_unlock(&dev->struct_mutex);
 
         old_width = intel_crtc->cursor_width;
@@ -8144,6 +8239,8 @@ static int intel_crtc_cursor_set(struct drm_crtc *crtc,
                 intel_crtc_update_cursor(crtc, intel_crtc->cursor_bo != NULL);
         }
 
+        intel_frontbuffer_flip(dev, INTEL_FRONTBUFFER_CURSOR(pipe));
+
         return 0;
 fail_unpin:
         i915_gem_object_unpin_from_display_plane(obj);
@@ -8154,19 +8251,6 @@ fail:
         return ret;
 }
 
-static int intel_crtc_cursor_move(struct drm_crtc *crtc, int x, int y)
-{
-        struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
-
-        intel_crtc->cursor_x = clamp_t(int, x, SHRT_MIN, SHRT_MAX);
-        intel_crtc->cursor_y = clamp_t(int, y, SHRT_MIN, SHRT_MAX);
-
-        if (intel_crtc->active)
-                intel_crtc_update_cursor(crtc, intel_crtc->cursor_bo != NULL);
-
-        return 0;
-}
-
 static void intel_crtc_gamma_set(struct drm_crtc *crtc, u16 *red, u16 *green,
                                  u16 *blue, uint32_t start, uint32_t size)
 {
@@ -8667,12 +8751,10 @@ struct drm_display_mode *intel_crtc_mode_get(struct drm_device *dev,
         return mode;
 }
 
-static void intel_increase_pllclock(struct drm_crtc *crtc)
+static void intel_increase_pllclock(struct drm_device *dev,
+                                    enum pipe pipe)
 {
-        struct drm_device *dev = crtc->dev;
         struct drm_i915_private *dev_priv = dev->dev_private;
-        struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
-        int pipe = intel_crtc->pipe;
         int dpll_reg = DPLL(pipe);
         int dpll;
 
@@ -8773,28 +8855,179 @@ out:
         intel_runtime_pm_put(dev_priv);
 }
 
-void intel_mark_fb_busy(struct drm_i915_gem_object *obj,
-                        struct intel_engine_cs *ring)
+
+/**
+ * intel_mark_fb_busy - mark given planes as busy
+ * @dev: DRM device
+ * @frontbuffer_bits: bits for the affected planes
+ * @ring: optional ring for asynchronous commands
+ *
+ * This function gets called every time the screen contents change. It can be
+ * used to keep e.g. the update rate at the nominal refresh rate with DRRS.
+ */
+static void intel_mark_fb_busy(struct drm_device *dev,
+                               unsigned frontbuffer_bits,
+                               struct intel_engine_cs *ring)
 {
-        struct drm_device *dev = obj->base.dev;
-        struct drm_crtc *crtc;
+        enum pipe pipe;
 
         if (!i915.powersave)
                 return;
 
-        for_each_crtc(dev, crtc) {
-                if (!crtc->primary->fb)
-                        continue;
-
-                if (to_intel_framebuffer(crtc->primary->fb)->obj != obj)
+        for_each_pipe(pipe) {
+                if (!(frontbuffer_bits & INTEL_FRONTBUFFER_ALL_MASK(pipe)))
                         continue;
 
-                intel_increase_pllclock(crtc);
+                intel_increase_pllclock(dev, pipe);
                 if (ring && intel_fbc_enabled(dev))
                         ring->fbc_dirty = true;
         }
 }
 
+/**
+ * intel_fb_obj_invalidate - invalidate frontbuffer object
+ * @obj: GEM object to invalidate
+ * @ring: set for asynchronous rendering
+ *
+ * This function gets called every time rendering on the given object starts and
+ * frontbuffer caching (fbc, low refresh rate for DRRS, panel self refresh) must
+ * be invalidated. If @ring is non-NULL any subsequent invalidation will be delayed
+ * until the rendering completes or a flip on this frontbuffer plane is
+ * scheduled.
+ */
+void intel_fb_obj_invalidate(struct drm_i915_gem_object *obj,
+                             struct intel_engine_cs *ring)
+{
+        struct drm_device *dev = obj->base.dev;
+        struct drm_i915_private *dev_priv = dev->dev_private;
+
+        WARN_ON(!mutex_is_locked(&dev->struct_mutex));
+
+        if (!obj->frontbuffer_bits)
+                return;
+
+        if (ring) {
+                mutex_lock(&dev_priv->fb_tracking.lock);
+                dev_priv->fb_tracking.busy_bits
+                        |= obj->frontbuffer_bits;
+                dev_priv->fb_tracking.flip_bits
+                        &= ~obj->frontbuffer_bits;
+                mutex_unlock(&dev_priv->fb_tracking.lock);
+        }
+
+        intel_mark_fb_busy(dev, obj->frontbuffer_bits, ring);
+
+        intel_edp_psr_exit(dev);
+}
+
+/**
+ * intel_frontbuffer_flush - flush frontbuffer
+ * @dev: DRM device
+ * @frontbuffer_bits: frontbuffer plane tracking bits
+ *
+ * This function gets called every time rendering on the given planes has
+ * completed and frontbuffer caching can be started again. Flushes will get
+ * delayed if they're blocked by some oustanding asynchronous rendering.
+ *
+ * Can be called without any locks held.
+ */
+void intel_frontbuffer_flush(struct drm_device *dev,
+                             unsigned frontbuffer_bits)
+{
+        struct drm_i915_private *dev_priv = dev->dev_private;
+
+        /* Delay flushing when rings are still busy.*/
+        mutex_lock(&dev_priv->fb_tracking.lock);
+        frontbuffer_bits &= ~dev_priv->fb_tracking.busy_bits;
+        mutex_unlock(&dev_priv->fb_tracking.lock);
+
+        intel_mark_fb_busy(dev, frontbuffer_bits, NULL);
+
+        intel_edp_psr_exit(dev);
+}
+
+/**
+ * intel_fb_obj_flush - flush frontbuffer object
+ * @obj: GEM object to flush
+ * @retire: set when retiring asynchronous rendering
+ *
+ * This function gets called every time rendering on the given object has
+ * completed and frontbuffer caching can be started again. If @retire is true
+ * then any delayed flushes will be unblocked.
+ */
+void intel_fb_obj_flush(struct drm_i915_gem_object *obj,
+                        bool retire)
+{
+        struct drm_device *dev = obj->base.dev;
+        struct drm_i915_private *dev_priv = dev->dev_private;
+        unsigned frontbuffer_bits;
+
+        WARN_ON(!mutex_is_locked(&dev->struct_mutex));
+
+        if (!obj->frontbuffer_bits)
+                return;
+
+        frontbuffer_bits = obj->frontbuffer_bits;
+
+        if (retire) {
+                mutex_lock(&dev_priv->fb_tracking.lock);
+                /* Filter out new bits since rendering started. */
+                frontbuffer_bits &= dev_priv->fb_tracking.busy_bits;
+
+                dev_priv->fb_tracking.busy_bits &= ~frontbuffer_bits;
+                mutex_unlock(&dev_priv->fb_tracking.lock);
+        }
+
+        intel_frontbuffer_flush(dev, frontbuffer_bits);
+}
+
+/**
+ * intel_frontbuffer_flip_prepare - prepare asnychronous frontbuffer flip
+ * @dev: DRM device
+ * @frontbuffer_bits: frontbuffer plane tracking bits
+ *
+ * This function gets called after scheduling a flip on @obj. The actual
+ * frontbuffer flushing will be delayed until completion is signalled with
+ * intel_frontbuffer_flip_complete. If an invalidate happens in between this
+ * flush will be cancelled.
+ *
+ * Can be called without any locks held.
+ */
+void intel_frontbuffer_flip_prepare(struct drm_device *dev,
+                                    unsigned frontbuffer_bits)
+{
+        struct drm_i915_private *dev_priv = dev->dev_private;
+
+        mutex_lock(&dev_priv->fb_tracking.lock);
+        dev_priv->fb_tracking.flip_bits
+                |= frontbuffer_bits;
+        mutex_unlock(&dev_priv->fb_tracking.lock);
+}
+
+/**
+ * intel_frontbuffer_flip_complete - complete asynchronous frontbuffer flush
+ * @dev: DRM device
+ * @frontbuffer_bits: frontbuffer plane tracking bits
+ *
+ * This function gets called after the flip has been latched and will complete
+ * on the next vblank. It will execute the fush if it hasn't been cancalled yet.
+ *
+ * Can be called without any locks held.
+ */
+void intel_frontbuffer_flip_complete(struct drm_device *dev,
+                                     unsigned frontbuffer_bits)
+{
+        struct drm_i915_private *dev_priv = dev->dev_private;
+
+        mutex_lock(&dev_priv->fb_tracking.lock);
+        /* Mask any cancelled flips. */
+        frontbuffer_bits &= dev_priv->fb_tracking.flip_bits;
+        dev_priv->fb_tracking.flip_bits &= ~frontbuffer_bits;
+        mutex_unlock(&dev_priv->fb_tracking.lock);
+
+        intel_frontbuffer_flush(dev, frontbuffer_bits);
+}
+
 static void intel_crtc_destroy(struct drm_crtc *crtc)
 {
         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
@@ -8812,8 +9045,6 @@ static void intel_crtc_destroy(struct drm_crtc *crtc)
                 kfree(work);
         }
 
-        intel_crtc_cursor_set(crtc, NULL, 0, 0, 0);
-
         drm_crtc_cleanup(crtc);
 
         kfree(intel_crtc);
@@ -8824,6 +9055,7 @@ static void intel_unpin_work_fn(struct work_struct *__work)
         struct intel_unpin_work *work =
                 container_of(__work, struct intel_unpin_work, work);
         struct drm_device *dev = work->crtc->dev;
+        enum pipe pipe = to_intel_crtc(work->crtc)->pipe;
 
         mutex_lock(&dev->struct_mutex);
         intel_unpin_fb_obj(work->old_fb_obj);
@@ -8833,6 +9065,8 @@ static void intel_unpin_work_fn(struct work_struct *__work)
         intel_update_fbc(dev);
         mutex_unlock(&dev->struct_mutex);
 
+        intel_frontbuffer_flip_complete(dev, INTEL_FRONTBUFFER_PRIMARY(pipe));
+
         BUG_ON(atomic_read(&to_intel_crtc(work->crtc)->unpin_work_count) == 0);
         atomic_dec(&to_intel_crtc(work->crtc)->unpin_work_count);
 
@@ -9202,6 +9436,147 @@ static int intel_gen7_queue_flip(struct drm_device *dev,
         return 0;
 }
 
+static bool use_mmio_flip(struct intel_engine_cs *ring,
+                          struct drm_i915_gem_object *obj)
+{
+        /*
+         * This is not being used for older platforms, because
+         * non-availability of flip done interrupt forces us to use
+         * CS flips. Older platforms derive flip done using some clever
+         * tricks involving the flip_pending status bits and vblank irqs.
+         * So using MMIO flips there would disrupt this mechanism.
+         */
+
+        if (INTEL_INFO(ring->dev)->gen < 5)
+                return false;
+
+        if (i915.use_mmio_flip < 0)
+                return false;
+        else if (i915.use_mmio_flip > 0)
+                return true;
+        else
+                return ring != obj->ring;
+}
+
+static void intel_do_mmio_flip(struct intel_crtc *intel_crtc)
+{
+        struct drm_device *dev = intel_crtc->base.dev;
+        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct intel_framebuffer *intel_fb =
+                to_intel_framebuffer(intel_crtc->base.primary->fb);
+        struct drm_i915_gem_object *obj = intel_fb->obj;
+        u32 dspcntr;
+        u32 reg;
+
+        intel_mark_page_flip_active(intel_crtc);
+
+        reg = DSPCNTR(intel_crtc->plane);
+        dspcntr = I915_READ(reg);
+
+        if (INTEL_INFO(dev)->gen >= 4) {
+                if (obj->tiling_mode != I915_TILING_NONE)
+                        dspcntr |= DISPPLANE_TILED;
+                else
+                        dspcntr &= ~DISPPLANE_TILED;
+        }
+        I915_WRITE(reg, dspcntr);
+
+        I915_WRITE(DSPSURF(intel_crtc->plane),
+                   intel_crtc->unpin_work->gtt_offset);
+        POSTING_READ(DSPSURF(intel_crtc->plane));
+}
+
+static int intel_postpone_flip(struct drm_i915_gem_object *obj)
+{
+        struct intel_engine_cs *ring;
+        int ret;
+
+        lockdep_assert_held(&obj->base.dev->struct_mutex);
+
+        if (!obj->last_write_seqno)
+                return 0;
+
+        ring = obj->ring;
+
+        if (i915_seqno_passed(ring->get_seqno(ring, true),
+                              obj->last_write_seqno))
+                return 0;
+
+        ret = i915_gem_check_olr(ring, obj->last_write_seqno);
+        if (ret)
+                return ret;
+
+        if (WARN_ON(!ring->irq_get(ring)))
+                return 0;
+
+        return 1;
+}
+
+void intel_notify_mmio_flip(struct intel_engine_cs *ring)
+{
+        struct drm_i915_private *dev_priv = to_i915(ring->dev);
+        struct intel_crtc *intel_crtc;
+        unsigned long irq_flags;
+        u32 seqno;
+
+        seqno = ring->get_seqno(ring, false);
+
+        spin_lock_irqsave(&dev_priv->mmio_flip_lock, irq_flags);
+        for_each_intel_crtc(ring->dev, intel_crtc) {
+                struct intel_mmio_flip *mmio_flip;
+
+                mmio_flip = &intel_crtc->mmio_flip;
+                if (mmio_flip->seqno == 0)
+                        continue;
+
+                if (ring->id != mmio_flip->ring_id)
+                        continue;
+
+                if (i915_seqno_passed(seqno, mmio_flip->seqno)) {
+                        intel_do_mmio_flip(intel_crtc);
+                        mmio_flip->seqno = 0;
+                        ring->irq_put(ring);
+                }
+        }
+        spin_unlock_irqrestore(&dev_priv->mmio_flip_lock, irq_flags);
+}
+
+static int intel_queue_mmio_flip(struct drm_device *dev,
+                                 struct drm_crtc *crtc,
+                                 struct drm_framebuffer *fb,
+                                 struct drm_i915_gem_object *obj,
+                                 struct intel_engine_cs *ring,
+                                 uint32_t flags)
+{
+        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
+        unsigned long irq_flags;
+        int ret;
+
+        if (WARN_ON(intel_crtc->mmio_flip.seqno))
+                return -EBUSY;
+
+        ret = intel_postpone_flip(obj);
+        if (ret < 0)
+                return ret;
+        if (ret == 0) {
+                intel_do_mmio_flip(intel_crtc);
+                return 0;
+        }
+
+        spin_lock_irqsave(&dev_priv->mmio_flip_lock, irq_flags);
+        intel_crtc->mmio_flip.seqno = obj->last_write_seqno;
+        intel_crtc->mmio_flip.ring_id = obj->ring->id;
+        spin_unlock_irqrestore(&dev_priv->mmio_flip_lock, irq_flags);
+
+        /*
+         * Double check to catch cases where irq fired before
+         * mmio flip data was ready
+         */
+        intel_notify_mmio_flip(obj->ring);
+        return 0;
+}
+
 static int intel_default_queue_flip(struct drm_device *dev,
                                     struct drm_crtc *crtc,
                                     struct drm_framebuffer *fb,
@@ -9222,6 +9597,7 @@ static int intel_crtc_page_flip(struct drm_crtc *crtc,
         struct drm_framebuffer *old_fb = crtc->primary->fb;
         struct drm_i915_gem_object *obj = to_intel_framebuffer(fb)->obj;
         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
+        enum pipe pipe = intel_crtc->pipe;
         struct intel_unpin_work *work;
         struct intel_engine_cs *ring;
         unsigned long flags;
@@ -9290,7 +9666,7 @@ static int intel_crtc_page_flip(struct drm_crtc *crtc,
         intel_crtc->reset_counter = atomic_read(&dev_priv->gpu_error.reset_counter);
 
         if (INTEL_INFO(dev)->gen >= 5 || IS_G4X(dev))
-                work->flip_count = I915_READ(PIPE_FLIPCOUNT_GM45(intel_crtc->pipe)) + 1;
+                work->flip_count = I915_READ(PIPE_FLIPCOUNT_GM45(pipe)) + 1;
 
         if (IS_VALLEYVIEW(dev)) {
                 ring = &dev_priv->ring[BCS];
@@ -9309,12 +9685,20 @@ static int intel_crtc_page_flip(struct drm_crtc *crtc,
         work->gtt_offset =
                 i915_gem_obj_ggtt_offset(obj) + intel_crtc->dspaddr_offset;
 
-        ret = dev_priv->display.queue_flip(dev, crtc, fb, obj, ring, page_flip_flags);
+        if (use_mmio_flip(ring, obj))
+                ret = intel_queue_mmio_flip(dev, crtc, fb, obj, ring,
+                                            page_flip_flags);
+        else
+                ret = dev_priv->display.queue_flip(dev, crtc, fb, obj, ring,
+                                page_flip_flags);
         if (ret)
                 goto cleanup_unpin;
 
+        i915_gem_track_fb(work->old_fb_obj, obj,
+                          INTEL_FRONTBUFFER_PRIMARY(pipe));
+
         intel_disable_fbc(dev);
-        intel_mark_fb_busy(obj, NULL);
+        intel_frontbuffer_flip_prepare(dev, INTEL_FRONTBUFFER_PRIMARY(pipe));
         mutex_unlock(&dev->struct_mutex);
 
         trace_i915_flip_request(intel_crtc->plane, obj);
@@ -9344,7 +9728,7 @@ out_hang:
                 intel_crtc_wait_for_pending_flips(crtc);
                 ret = intel_pipe_set_base(crtc, crtc->x, crtc->y, fb);
                 if (ret == 0 && event)
-                        drm_send_vblank_event(dev, intel_crtc->pipe, event);
+                        drm_send_vblank_event(dev, pipe, event);
         }
         return ret;
 }
@@ -10379,10 +10763,13 @@ static int __intel_set_mode(struct drm_crtc *crtc,
          */
         for_each_intel_crtc_masked(dev, modeset_pipes, intel_crtc) {
                 struct drm_framebuffer *old_fb;
+                struct drm_i915_gem_object *old_obj = NULL;
+                struct drm_i915_gem_object *obj =
+                        to_intel_framebuffer(fb)->obj;
 
                 mutex_lock(&dev->struct_mutex);
                 ret = intel_pin_and_fence_fb_obj(dev,
-                                                 to_intel_framebuffer(fb)->obj,
+                                                 obj,
                                                  NULL);
                 if (ret != 0) {
                         DRM_ERROR("pin & fence failed\n");
@@ -10390,8 +10777,12 @@ static int __intel_set_mode(struct drm_crtc *crtc,
                         goto done;
                 }
                 old_fb = crtc->primary->fb;
-                if (old_fb)
-                        intel_unpin_fb_obj(to_intel_framebuffer(old_fb)->obj);
+                if (old_fb) {
+                        old_obj = to_intel_framebuffer(old_fb)->obj;
+                        intel_unpin_fb_obj(old_obj);
+                }
+                i915_gem_track_fb(old_obj, obj,
+                                  INTEL_FRONTBUFFER_PRIMARY(intel_crtc->pipe));
                 mutex_unlock(&dev->struct_mutex);
 
                 crtc->primary->fb = fb;
@@ -10563,12 +10954,17 @@ intel_set_config_compute_mode_changes(struct drm_mode_set *set,
         if (is_crtc_connector_off(set)) {
                 config->mode_changed = true;
         } else if (set->crtc->primary->fb != set->fb) {
-                /* If we have no fb then treat it as a full mode set */
+                /*
+                 * If we have no fb, we can only flip as long as the crtc is
+                 * active, otherwise we need a full mode set.  The crtc may
+                 * be active if we've only disabled the primary plane, or
+                 * in fastboot situations.
+                 */
                 if (set->crtc->primary->fb == NULL) {
                         struct intel_crtc *intel_crtc =
                                 to_intel_crtc(set->crtc);
 
-                        if (intel_crtc->active && i915.fastboot) {
+                        if (intel_crtc->active) {
                                 DRM_DEBUG_KMS("crtc has no fb, will flip\n");
                                 config->fb_changed = true;
                         } else {
@@ -10806,10 +11202,24 @@ static int intel_crtc_set_config(struct drm_mode_set *set)
                 ret = intel_set_mode(set->crtc, set->mode,
                                      set->x, set->y, set->fb);
         } else if (config->fb_changed) {
+                struct drm_i915_private *dev_priv = dev->dev_private;
+                struct intel_crtc *intel_crtc = to_intel_crtc(set->crtc);
+
                 intel_crtc_wait_for_pending_flips(set->crtc);
 
                 ret = intel_pipe_set_base(set->crtc,
                                           set->x, set->y, set->fb);
+
+                /*
+                 * We need to make sure the primary plane is re-enabled if it
+                 * has previously been turned off.
+                 */
+                if (!intel_crtc->primary_enabled && ret == 0) {
+                        WARN_ON(!intel_crtc->active);
+                        intel_enable_primary_hw_plane(dev_priv, intel_crtc->plane,
+                                                      intel_crtc->pipe);
+                }
+
                 /*
                  * In the fastboot case this may be our only check of the
                  * state after boot.  It would be better to only do it on
@@ -10850,8 +11260,6 @@ out_config:
 }
 
 static const struct drm_crtc_funcs intel_crtc_funcs = {
-        .cursor_set = intel_crtc_cursor_set,
-        .cursor_move = intel_crtc_cursor_move,
         .gamma_set = intel_crtc_gamma_set,
         .set_config = intel_crtc_set_config,
         .destroy = intel_crtc_destroy,
@@ -10959,17 +11367,318 @@ static void intel_shared_dpll_init(struct drm_device *dev)
         BUG_ON(dev_priv->num_shared_dpll > I915_NUM_PLLS);
 }
 
+static int
+intel_primary_plane_disable(struct drm_plane *plane)
+{
+        struct drm_device *dev = plane->dev;
+        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct intel_plane *intel_plane = to_intel_plane(plane);
+        struct intel_crtc *intel_crtc;
+
+        if (!plane->fb)
+                return 0;
+
+        BUG_ON(!plane->crtc);
+
+        intel_crtc = to_intel_crtc(plane->crtc);
+
+        /*
+         * Even though we checked plane->fb above, it's still possible that
+         * the primary plane has been implicitly disabled because the crtc
+         * coordinates given weren't visible, or because we detected
+         * that it was 100% covered by a sprite plane.  Or, the CRTC may be
+         * off and we've set a fb, but haven't actually turned on the CRTC yet.
+         * In either case, we need to unpin the FB and let the fb pointer get
+         * updated, but otherwise we don't need to touch the hardware.
+         */
+        if (!intel_crtc->primary_enabled)
+                goto disable_unpin;
+
+        intel_crtc_wait_for_pending_flips(plane->crtc);
+        intel_disable_primary_hw_plane(dev_priv, intel_plane->plane,
+                                       intel_plane->pipe);
+disable_unpin:
+        i915_gem_track_fb(to_intel_framebuffer(plane->fb)->obj, NULL,
+                          INTEL_FRONTBUFFER_PRIMARY(intel_crtc->pipe));
+        intel_unpin_fb_obj(to_intel_framebuffer(plane->fb)->obj);
+        plane->fb = NULL;
+
+        return 0;
+}
+
+static int
+intel_primary_plane_setplane(struct drm_plane *plane, struct drm_crtc *crtc,
+                             struct drm_framebuffer *fb, int crtc_x, int crtc_y,
+                             unsigned int crtc_w, unsigned int crtc_h,
+                             uint32_t src_x, uint32_t src_y,
+                             uint32_t src_w, uint32_t src_h)
+{
+        struct drm_device *dev = crtc->dev;
+        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
+        struct intel_plane *intel_plane = to_intel_plane(plane);
+        struct drm_i915_gem_object *obj, *old_obj = NULL;
+        struct drm_rect dest = {
+                /* integer pixels */
+                .x1 = crtc_x,
+                .y1 = crtc_y,
+                .x2 = crtc_x + crtc_w,
+                .y2 = crtc_y + crtc_h,
+        };
+        struct drm_rect src = {
+                /* 16.16 fixed point */
+                .x1 = src_x,
+                .y1 = src_y,
+                .x2 = src_x + src_w,
+                .y2 = src_y + src_h,
+        };
+        const struct drm_rect clip = {
+                /* integer pixels */
+                .x2 = intel_crtc->active ? intel_crtc->config.pipe_src_w : 0,
+                .y2 = intel_crtc->active ? intel_crtc->config.pipe_src_h : 0,
+        };
+        bool visible;
+        int ret;
+
+        ret = drm_plane_helper_check_update(plane, crtc, fb,
+                                            &src, &dest, &clip,
+                                            DRM_PLANE_HELPER_NO_SCALING,
+                                            DRM_PLANE_HELPER_NO_SCALING,
+                                            false, true, &visible);
+
+        if (ret)
+                return ret;
+
+        if (plane->fb)
+                old_obj = to_intel_framebuffer(plane->fb)->obj;
+        obj = to_intel_framebuffer(fb)->obj;
+
+        /*
+         * If the CRTC isn't enabled, we're just pinning the framebuffer,
+         * updating the fb pointer, and returning without touching the
+         * hardware.  This allows us to later do a drmModeSetCrtc with fb=-1 to
+         * turn on the display with all planes setup as desired.
+         */
+        if (!crtc->enabled) {
+                /*
+                 * If we already called setplane while the crtc was disabled,
+                 * we may have an fb pinned; unpin it.
+                 */
+                if (plane->fb)
+                        intel_unpin_fb_obj(old_obj);
+
+                i915_gem_track_fb(old_obj, obj,
+                                  INTEL_FRONTBUFFER_PRIMARY(intel_crtc->pipe));
+
+                /* Pin and return without programming hardware */
+                return intel_pin_and_fence_fb_obj(dev, obj, NULL);
+        }
+
+        intel_crtc_wait_for_pending_flips(crtc);
+
+        /*
+         * If clipping results in a non-visible primary plane, we'll disable
+         * the primary plane.  Note that this is a bit different than what
+         * happens if userspace explicitly disables the plane by passing fb=0
+         * because plane->fb still gets set and pinned.
+         */
+        if (!visible) {
+                /*
+                 * Try to pin the new fb first so that we can bail out if we
+                 * fail.
+                 */
+                if (plane->fb != fb) {
+                        ret = intel_pin_and_fence_fb_obj(dev, obj, NULL);
+                        if (ret)
+                                return ret;
+                }
+
+                i915_gem_track_fb(old_obj, obj,
+                                  INTEL_FRONTBUFFER_PRIMARY(intel_crtc->pipe));
+
+                if (intel_crtc->primary_enabled)
+                        intel_disable_primary_hw_plane(dev_priv,
+                                                       intel_plane->plane,
+                                                       intel_plane->pipe);
+
+
+                if (plane->fb != fb)
+                        if (plane->fb)
+                                intel_unpin_fb_obj(old_obj);
+
+                return 0;
+        }
+
+        ret = intel_pipe_set_base(crtc, src.x1, src.y1, fb);
+        if (ret)
+                return ret;
+
+        if (!intel_crtc->primary_enabled)
+                intel_enable_primary_hw_plane(dev_priv, intel_crtc->plane,
+                                              intel_crtc->pipe);
+
+        return 0;
+}
+
+/* Common destruction function for both primary and cursor planes */
+static void intel_plane_destroy(struct drm_plane *plane)
+{
+        struct intel_plane *intel_plane = to_intel_plane(plane);
+        drm_plane_cleanup(plane);
+        kfree(intel_plane);
+}
+
+static const struct drm_plane_funcs intel_primary_plane_funcs = {
+        .update_plane = intel_primary_plane_setplane,
+        .disable_plane = intel_primary_plane_disable,
+        .destroy = intel_plane_destroy,
+};
+
+static struct drm_plane *intel_primary_plane_create(struct drm_device *dev,
+                                                    int pipe)
+{
+        struct intel_plane *primary;
+        const uint32_t *intel_primary_formats;
+        int num_formats;
+
+        primary = kzalloc(sizeof(*primary), GFP_KERNEL);
+        if (primary == NULL)
+                return NULL;
+
+        primary->can_scale = false;
+        primary->max_downscale = 1;
+        primary->pipe = pipe;
+        primary->plane = pipe;
+        if (HAS_FBC(dev) && INTEL_INFO(dev)->gen < 4)
+                primary->plane = !pipe;
+
+        if (INTEL_INFO(dev)->gen <= 3) {
+                intel_primary_formats = intel_primary_formats_gen2;
+                num_formats = ARRAY_SIZE(intel_primary_formats_gen2);
+        } else {
+                intel_primary_formats = intel_primary_formats_gen4;
+                num_formats = ARRAY_SIZE(intel_primary_formats_gen4);
+        }
+
+        drm_universal_plane_init(dev, &primary->base, 0,
+                                 &intel_primary_plane_funcs,
+                                 intel_primary_formats, num_formats,
+                                 DRM_PLANE_TYPE_PRIMARY);
+        return &primary->base;
+}
+
+static int
+intel_cursor_plane_disable(struct drm_plane *plane)
+{
+        if (!plane->fb)
+                return 0;
+
+        BUG_ON(!plane->crtc);
+
+        return intel_crtc_cursor_set_obj(plane->crtc, NULL, 0, 0);
+}
+
+static int
+intel_cursor_plane_update(struct drm_plane *plane, struct drm_crtc *crtc,
+                          struct drm_framebuffer *fb, int crtc_x, int crtc_y,
+                          unsigned int crtc_w, unsigned int crtc_h,
+                          uint32_t src_x, uint32_t src_y,
+                          uint32_t src_w, uint32_t src_h)
+{
+        struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
+        struct intel_framebuffer *intel_fb = to_intel_framebuffer(fb);
+        struct drm_i915_gem_object *obj = intel_fb->obj;
+        struct drm_rect dest = {
+                /* integer pixels */
+                .x1 = crtc_x,
+                .y1 = crtc_y,
+                .x2 = crtc_x + crtc_w,
+                .y2 = crtc_y + crtc_h,
+        };
+        struct drm_rect src = {
+                /* 16.16 fixed point */
+                .x1 = src_x,
+                .y1 = src_y,
+                .x2 = src_x + src_w,
+                .y2 = src_y + src_h,
+        };
+        const struct drm_rect clip = {
+                /* integer pixels */
+                .x2 = intel_crtc->config.pipe_src_w,
+                .y2 = intel_crtc->config.pipe_src_h,
+        };
+        bool visible;
+        int ret;
+
+        ret = drm_plane_helper_check_update(plane, crtc, fb,
+                                            &src, &dest, &clip,
+                                            DRM_PLANE_HELPER_NO_SCALING,
+                                            DRM_PLANE_HELPER_NO_SCALING,
+                                            true, true, &visible);
+        if (ret)
+                return ret;
+
+        crtc->cursor_x = crtc_x;
+        crtc->cursor_y = crtc_y;
+        if (fb != crtc->cursor->fb) {
+                return intel_crtc_cursor_set_obj(crtc, obj, crtc_w, crtc_h);
+        } else {
+                intel_crtc_update_cursor(crtc, visible);
+                return 0;
+        }
+}
+static const struct drm_plane_funcs intel_cursor_plane_funcs = {
+        .update_plane = intel_cursor_plane_update,
+        .disable_plane = intel_cursor_plane_disable,
+        .destroy = intel_plane_destroy,
+};
+
+static struct drm_plane *intel_cursor_plane_create(struct drm_device *dev,
+                                                   int pipe)
+{
+        struct intel_plane *cursor;
+
+        cursor = kzalloc(sizeof(*cursor), GFP_KERNEL);
+        if (cursor == NULL)
+                return NULL;
+
+        cursor->can_scale = false;
+        cursor->max_downscale = 1;
+        cursor->pipe = pipe;
+        cursor->plane = pipe;
+
+        drm_universal_plane_init(dev, &cursor->base, 0,
+                                 &intel_cursor_plane_funcs,
+                                 intel_cursor_formats,
+                                 ARRAY_SIZE(intel_cursor_formats),
+                                 DRM_PLANE_TYPE_CURSOR);
+        return &cursor->base;
+}
+
 static void intel_crtc_init(struct drm_device *dev, int pipe)
 {
         struct drm_i915_private *dev_priv = dev->dev_private;
         struct intel_crtc *intel_crtc;
-        int i;
+        struct drm_plane *primary = NULL;
+        struct drm_plane *cursor = NULL;
+        int i, ret;
 
         intel_crtc = kzalloc(sizeof(*intel_crtc), GFP_KERNEL);
         if (intel_crtc == NULL)
                 return;
 
-        drm_crtc_init(dev, &intel_crtc->base, &intel_crtc_funcs);
+        primary = intel_primary_plane_create(dev, pipe);
+        if (!primary)
+                goto fail;
+
+        cursor = intel_cursor_plane_create(dev, pipe);
+        if (!cursor)
+                goto fail;
+
+        ret = drm_crtc_init_with_planes(dev, &intel_crtc->base, primary,
+                                        cursor, &intel_crtc_funcs);
+        if (ret)
+                goto fail;
 
         drm_mode_crtc_set_gamma_size(&intel_crtc->base, 256);
         for (i = 0; i < 256; i++) {
@@ -10980,7 +11689,7 @@ static void intel_crtc_init(struct drm_device *dev, int pipe)
 
         /*
          * On gen2/3 only plane A can do fbc, but the panel fitter and lvds port
-         * is hooked to plane B. Hence we want plane A feeding pipe B.
+         * is hooked to pipe B. Hence we want plane A feeding pipe B.
          */
         intel_crtc->pipe = pipe;
         intel_crtc->plane = pipe;
@@ -11002,6 +11711,14 @@ static void intel_crtc_init(struct drm_device *dev, int pipe)
         drm_crtc_helper_add(&intel_crtc->base, &intel_helper_funcs);
 
         WARN_ON(drm_crtc_index(&intel_crtc->base) != intel_crtc->pipe);
+        return;
+
+fail:
+        if (primary)
+                drm_plane_cleanup(primary);
+        if (cursor)
+                drm_plane_cleanup(cursor);
+        kfree(intel_crtc);
 }
 
 enum pipe intel_get_pipe_from_connector(struct intel_connector *connector)
@@ -11236,6 +11953,8 @@ static void intel_setup_outputs(struct drm_device *dev)
         if (SUPPORTS_TV(dev))
                 intel_tv_init(dev);
 
+        intel_edp_psr_init(dev);
+
         list_for_each_entry(encoder, &dev->mode_config.encoder_list, base.head) {
                 encoder->base.possible_crtcs = encoder->crtc_mask;
                 encoder->base.possible_clones =
@@ -11249,11 +11968,14 @@ static void intel_setup_outputs(struct drm_device *dev)
 
 static void intel_user_framebuffer_destroy(struct drm_framebuffer *fb)
 {
+        struct drm_device *dev = fb->dev;
         struct intel_framebuffer *intel_fb = to_intel_framebuffer(fb);
 
         drm_framebuffer_cleanup(fb);
+        mutex_lock(&dev->struct_mutex);
         WARN_ON(!intel_fb->obj->framebuffer_references--);
-        drm_gem_object_unreference_unlocked(&intel_fb->obj->base);
+        drm_gem_object_unreference(&intel_fb->obj->base);
+        mutex_unlock(&dev->struct_mutex);
         kfree(intel_fb);
 }
 
@@ -12266,7 +12988,6 @@ void intel_connector_unregister(struct intel_connector *intel_connector)
 void intel_modeset_cleanup(struct drm_device *dev)
 {
         struct drm_i915_private *dev_priv = dev->dev_private;
-        struct drm_crtc *crtc;
         struct drm_connector *connector;
 
         /*
@@ -12286,14 +13007,6 @@ void intel_modeset_cleanup(struct drm_device *dev)
 
         intel_unregister_dsm_handler();
 
-        for_each_crtc(dev, crtc) {
-                /* Skip inactive CRTCs */
-                if (!crtc->primary->fb)
-                        continue;
-
-                intel_increase_pllclock(crtc);
-        }
-
         intel_disable_fbc(dev);
 
         intel_disable_gt_powersave(dev);