Merge branch 'drm-next' of git://people.freedesktop.org/~airlied/linux

Pull drm updates from Dave Airlie:
 "This is the main git pull for the drm,

  I pretty much froze major pulls at -rc5/6 time, and haven't had much
  fallout, so will probably continue doing that.

  Lots of changes all over, big internal header cleanup to make it clear
  drm features are legacy things and what are things that modern KMS
  drivers should be using.  Also big move to use the new generic fences
  in all the TTM drivers.

  core:
        atomic prep work,
        vblank rework changes, allows immediate vblank disables
        major header reworking and cleanups to better delinate legacy
        interfaces from what KMS drivers should be using.
        cursor planes locking fixes

  ttm:
        move to generic fences (affects all TTM drivers)
        ppc64 caching fixes

  radeon:
        userptr support,
        uvd for old asics,
        reset rework for fence changes
        better buffer placement changes,
        dpm feature enablement
        hdmi audio support fixes

  intel:
        Cherryview work,
        180 degree rotation,
        skylake prep work,
        execlist command submission
        full ppgtt prep work
        cursor improvements
        edid caching,
        vdd handling improvements

  nouveau:
        fence reworking
        kepler memory clock work
        gt21x clock work
        fan control improvements
        hdmi infoframe fixes
        DP audio

  ast:
        ppc64 fixes
        caching fix

  rcar:
        rcar-du DT support

  ipuv3:
        prep work for capture support

  msm:
        LVDS support for mdp4, new panel, gpu refactoring

  exynos:
        exynos3250 SoC support, drop bad mmap interface,
        mipi dsi changes, and component match support"

* 'drm-next' of git://people.freedesktop.org/~airlied/linux: (640 commits)
  drm/mst: rework payload table allocation to conform better.
  drm/ast: Fix HW cursor image
  drm/radeon/kv: add uvd/vce info to dpm debugfs output
  drm/radeon/ci: add uvd/vce info to dpm debugfs output
  drm/radeon: export reservation_object from dmabuf to ttm
  drm/radeon: cope with foreign fences inside the reservation object
  drm/radeon: cope with foreign fences inside display
  drm/core: use helper to check driver features
  drm/radeon/cik: write gfx ucode version to ucode addr reg
  drm/radeon/si: print full CS when we hit a packet 0
  drm/radeon: remove unecessary includes
  drm/radeon/combios: declare legacy_connector_convert as static
  drm/radeon/atombios: declare connector convert tables as static
  drm/radeon: drop btc_get_max_clock_from_voltage_dependency_table
  drm/radeon/dpm: drop clk/voltage dependency filters for BTC
  drm/radeon/dpm: drop clk/voltage dependency filters for CI
  drm/radeon/dpm: drop clk/voltage dependency filters for SI
  drm/radeon/dpm: drop clk/voltage dependency filters for NI
  drm/radeon: disable audio when we disable hdmi (v2)
  drm/radeon: split audio enable between eg and r600 (v2)
  ...

47 files changed, 7172 insertions, 2717 deletions

diff --git a/drivers/gpu/drm/i915/Makefile b/drivers/gpu/drm/i915/Makefile
index 91bd167e1cb7..c1dd485aeb6c 100644
--- a/drivers/gpu/drm/i915/Makefile
+++ b/drivers/gpu/drm/i915/Makefile
@@ -31,6 +31,7 @@ i915-y += i915_cmd_parser.o \
           i915_gpu_error.o \
           i915_irq.o \
           i915_trace_points.o \
+          intel_lrc.o \
           intel_ringbuffer.o \
           intel_uncore.o
 
diff --git a/drivers/gpu/drm/i915/dvo_ns2501.c b/drivers/gpu/drm/i915/dvo_ns2501.c
index 74f2af7c2d3e..441630434d34 100644
--- a/drivers/gpu/drm/i915/dvo_ns2501.c
+++ b/drivers/gpu/drm/i915/dvo_ns2501.c
@@ -60,16 +60,297 @@
 
 #define NS2501_REGC 0x0c
 
+enum {
+        MODE_640x480,
+        MODE_800x600,
+        MODE_1024x768,
+};
+
+struct ns2501_reg {
+         uint8_t offset;
+         uint8_t value;
+};
+
+/*
+ * Magic values based on what the BIOS on
+ * Fujitsu-Siemens Lifebook S6010 programs (1024x768 panel).
+ */
+static const struct ns2501_reg regs_1024x768[][86] = {
+        [MODE_640x480] = {
+                [0] = { .offset = 0x0a, .value = 0x81, },
+                [1] = { .offset = 0x18, .value = 0x07, },
+                [2] = { .offset = 0x19, .value = 0x00, },
+                [3] = { .offset = 0x1a, .value = 0x00, },
+                [4] = { .offset = 0x1b, .value = 0x11, },
+                [5] = { .offset = 0x1c, .value = 0x54, },
+                [6] = { .offset = 0x1d, .value = 0x03, },
+                [7] = { .offset = 0x1e, .value = 0x02, },
+                [8] = { .offset = 0xf3, .value = 0x90, },
+                [9] = { .offset = 0xf9, .value = 0x00, },
+                [10] = { .offset = 0xc1, .value = 0x90, },
+                [11] = { .offset = 0xc2, .value = 0x00, },
+                [12] = { .offset = 0xc3, .value = 0x0f, },
+                [13] = { .offset = 0xc4, .value = 0x03, },
+                [14] = { .offset = 0xc5, .value = 0x16, },
+                [15] = { .offset = 0xc6, .value = 0x00, },
+                [16] = { .offset = 0xc7, .value = 0x02, },
+                [17] = { .offset = 0xc8, .value = 0x02, },
+                [18] = { .offset = 0xf4, .value = 0x00, },
+                [19] = { .offset = 0x80, .value = 0xff, },
+                [20] = { .offset = 0x81, .value = 0x07, },
+                [21] = { .offset = 0x82, .value = 0x3d, },
+                [22] = { .offset = 0x83, .value = 0x05, },
+                [23] = { .offset = 0x94, .value = 0x00, },
+                [24] = { .offset = 0x95, .value = 0x00, },
+                [25] = { .offset = 0x96, .value = 0x05, },
+                [26] = { .offset = 0x97, .value = 0x00, },
+                [27] = { .offset = 0x9a, .value = 0x88, },
+                [28] = { .offset = 0x9b, .value = 0x00, },
+                [29] = { .offset = 0x98, .value = 0x00, },
+                [30] = { .offset = 0x99, .value = 0x00, },
+                [31] = { .offset = 0xf7, .value = 0x88, },
+                [32] = { .offset = 0xf8, .value = 0x0a, },
+                [33] = { .offset = 0x9c, .value = 0x24, },
+                [34] = { .offset = 0x9d, .value = 0x00, },
+                [35] = { .offset = 0x9e, .value = 0x25, },
+                [36] = { .offset = 0x9f, .value = 0x03, },
+                [37] = { .offset = 0xa0, .value = 0x28, },
+                [38] = { .offset = 0xa1, .value = 0x01, },
+                [39] = { .offset = 0xa2, .value = 0x28, },
+                [40] = { .offset = 0xa3, .value = 0x05, },
+                [41] = { .offset = 0xb6, .value = 0x09, },
+                [42] = { .offset = 0xb8, .value = 0x00, },
+                [43] = { .offset = 0xb9, .value = 0xa0, },
+                [44] = { .offset = 0xba, .value = 0x00, },
+                [45] = { .offset = 0xbb, .value = 0x20, },
+                [46] = { .offset = 0x10, .value = 0x00, },
+                [47] = { .offset = 0x11, .value = 0xa0, },
+                [48] = { .offset = 0x12, .value = 0x02, },
+                [49] = { .offset = 0x20, .value = 0x00, },
+                [50] = { .offset = 0x22, .value = 0x00, },
+                [51] = { .offset = 0x23, .value = 0x00, },
+                [52] = { .offset = 0x24, .value = 0x00, },
+                [53] = { .offset = 0x25, .value = 0x00, },
+                [54] = { .offset = 0x8c, .value = 0x10, },
+                [55] = { .offset = 0x8d, .value = 0x02, },
+                [56] = { .offset = 0x8e, .value = 0x10, },
+                [57] = { .offset = 0x8f, .value = 0x00, },
+                [58] = { .offset = 0x90, .value = 0xff, },
+                [59] = { .offset = 0x91, .value = 0x07, },
+                [60] = { .offset = 0x92, .value = 0xa0, },
+                [61] = { .offset = 0x93, .value = 0x02, },
+                [62] = { .offset = 0xa5, .value = 0x00, },
+                [63] = { .offset = 0xa6, .value = 0x00, },
+                [64] = { .offset = 0xa7, .value = 0x00, },
+                [65] = { .offset = 0xa8, .value = 0x00, },
+                [66] = { .offset = 0xa9, .value = 0x04, },
+                [67] = { .offset = 0xaa, .value = 0x70, },
+                [68] = { .offset = 0xab, .value = 0x4f, },
+                [69] = { .offset = 0xac, .value = 0x00, },
+                [70] = { .offset = 0xa4, .value = 0x84, },
+                [71] = { .offset = 0x7e, .value = 0x18, },
+                [72] = { .offset = 0x84, .value = 0x00, },
+                [73] = { .offset = 0x85, .value = 0x00, },
+                [74] = { .offset = 0x86, .value = 0x00, },
+                [75] = { .offset = 0x87, .value = 0x00, },
+                [76] = { .offset = 0x88, .value = 0x00, },
+                [77] = { .offset = 0x89, .value = 0x00, },
+                [78] = { .offset = 0x8a, .value = 0x00, },
+                [79] = { .offset = 0x8b, .value = 0x00, },
+                [80] = { .offset = 0x26, .value = 0x00, },
+                [81] = { .offset = 0x27, .value = 0x00, },
+                [82] = { .offset = 0xad, .value = 0x00, },
+                [83] = { .offset = 0x08, .value = 0x30, }, /* 0x31 */
+                [84] = { .offset = 0x41, .value = 0x00, },
+                [85] = { .offset = 0xc0, .value = 0x05, },
+        },
+        [MODE_800x600] = {
+                [0] = { .offset = 0x0a, .value = 0x81, },
+                [1] = { .offset = 0x18, .value = 0x07, },
+                [2] = { .offset = 0x19, .value = 0x00, },
+                [3] = { .offset = 0x1a, .value = 0x00, },
+                [4] = { .offset = 0x1b, .value = 0x19, },
+                [5] = { .offset = 0x1c, .value = 0x64, },
+                [6] = { .offset = 0x1d, .value = 0x02, },
+                [7] = { .offset = 0x1e, .value = 0x02, },
+                [8] = { .offset = 0xf3, .value = 0x90, },
+                [9] = { .offset = 0xf9, .value = 0x00, },
+                [10] = { .offset = 0xc1, .value = 0xd7, },
+                [11] = { .offset = 0xc2, .value = 0x00, },
+                [12] = { .offset = 0xc3, .value = 0xf8, },
+                [13] = { .offset = 0xc4, .value = 0x03, },
+                [14] = { .offset = 0xc5, .value = 0x1a, },
+                [15] = { .offset = 0xc6, .value = 0x00, },
+                [16] = { .offset = 0xc7, .value = 0x73, },
+                [17] = { .offset = 0xc8, .value = 0x02, },
+                [18] = { .offset = 0xf4, .value = 0x00, },
+                [19] = { .offset = 0x80, .value = 0x27, },
+                [20] = { .offset = 0x81, .value = 0x03, },
+                [21] = { .offset = 0x82, .value = 0x41, },
+                [22] = { .offset = 0x83, .value = 0x05, },
+                [23] = { .offset = 0x94, .value = 0x00, },
+                [24] = { .offset = 0x95, .value = 0x00, },
+                [25] = { .offset = 0x96, .value = 0x05, },
+                [26] = { .offset = 0x97, .value = 0x00, },
+                [27] = { .offset = 0x9a, .value = 0x88, },
+                [28] = { .offset = 0x9b, .value = 0x00, },
+                [29] = { .offset = 0x98, .value = 0x00, },
+                [30] = { .offset = 0x99, .value = 0x00, },
+                [31] = { .offset = 0xf7, .value = 0x88, },
+                [32] = { .offset = 0xf8, .value = 0x06, },
+                [33] = { .offset = 0x9c, .value = 0x23, },
+                [34] = { .offset = 0x9d, .value = 0x00, },
+                [35] = { .offset = 0x9e, .value = 0x25, },
+                [36] = { .offset = 0x9f, .value = 0x03, },
+                [37] = { .offset = 0xa0, .value = 0x28, },
+                [38] = { .offset = 0xa1, .value = 0x01, },
+                [39] = { .offset = 0xa2, .value = 0x28, },
+                [40] = { .offset = 0xa3, .value = 0x05, },
+                [41] = { .offset = 0xb6, .value = 0x09, },
+                [42] = { .offset = 0xb8, .value = 0x30, },
+                [43] = { .offset = 0xb9, .value = 0xc8, },
+                [44] = { .offset = 0xba, .value = 0x00, },
+                [45] = { .offset = 0xbb, .value = 0x20, },
+                [46] = { .offset = 0x10, .value = 0x20, },
+                [47] = { .offset = 0x11, .value = 0xc8, },
+                [48] = { .offset = 0x12, .value = 0x02, },
+                [49] = { .offset = 0x20, .value = 0x00, },
+                [50] = { .offset = 0x22, .value = 0x00, },
+                [51] = { .offset = 0x23, .value = 0x00, },
+                [52] = { .offset = 0x24, .value = 0x00, },
+                [53] = { .offset = 0x25, .value = 0x00, },
+                [54] = { .offset = 0x8c, .value = 0x10, },
+                [55] = { .offset = 0x8d, .value = 0x02, },
+                [56] = { .offset = 0x8e, .value = 0x04, },
+                [57] = { .offset = 0x8f, .value = 0x00, },
+                [58] = { .offset = 0x90, .value = 0xff, },
+                [59] = { .offset = 0x91, .value = 0x07, },
+                [60] = { .offset = 0x92, .value = 0xa0, },
+                [61] = { .offset = 0x93, .value = 0x02, },
+                [62] = { .offset = 0xa5, .value = 0x00, },
+                [63] = { .offset = 0xa6, .value = 0x00, },
+                [64] = { .offset = 0xa7, .value = 0x00, },
+                [65] = { .offset = 0xa8, .value = 0x00, },
+                [66] = { .offset = 0xa9, .value = 0x83, },
+                [67] = { .offset = 0xaa, .value = 0x40, },
+                [68] = { .offset = 0xab, .value = 0x32, },
+                [69] = { .offset = 0xac, .value = 0x00, },
+                [70] = { .offset = 0xa4, .value = 0x80, },
+                [71] = { .offset = 0x7e, .value = 0x18, },
+                [72] = { .offset = 0x84, .value = 0x00, },
+                [73] = { .offset = 0x85, .value = 0x00, },
+                [74] = { .offset = 0x86, .value = 0x00, },
+                [75] = { .offset = 0x87, .value = 0x00, },
+                [76] = { .offset = 0x88, .value = 0x00, },
+                [77] = { .offset = 0x89, .value = 0x00, },
+                [78] = { .offset = 0x8a, .value = 0x00, },
+                [79] = { .offset = 0x8b, .value = 0x00, },
+                [80] = { .offset = 0x26, .value = 0x00, },
+                [81] = { .offset = 0x27, .value = 0x00, },
+                [82] = { .offset = 0xad, .value = 0x00, },
+                [83] = { .offset = 0x08, .value = 0x30, }, /* 0x31 */
+                [84] = { .offset = 0x41, .value = 0x00, },
+                [85] = { .offset = 0xc0, .value = 0x07, },
+        },
+        [MODE_1024x768] = {
+                [0] = { .offset = 0x0a, .value = 0x81, },
+                [1] = { .offset = 0x18, .value = 0x07, },
+                [2] = { .offset = 0x19, .value = 0x00, },
+                [3] = { .offset = 0x1a, .value = 0x00, },
+                [4] = { .offset = 0x1b, .value = 0x11, },
+                [5] = { .offset = 0x1c, .value = 0x54, },
+                [6] = { .offset = 0x1d, .value = 0x03, },
+                [7] = { .offset = 0x1e, .value = 0x02, },
+                [8] = { .offset = 0xf3, .value = 0x90, },
+                [9] = { .offset = 0xf9, .value = 0x00, },
+                [10] = { .offset = 0xc1, .value = 0x90, },
+                [11] = { .offset = 0xc2, .value = 0x00, },
+                [12] = { .offset = 0xc3, .value = 0x0f, },
+                [13] = { .offset = 0xc4, .value = 0x03, },
+                [14] = { .offset = 0xc5, .value = 0x16, },
+                [15] = { .offset = 0xc6, .value = 0x00, },
+                [16] = { .offset = 0xc7, .value = 0x02, },
+                [17] = { .offset = 0xc8, .value = 0x02, },
+                [18] = { .offset = 0xf4, .value = 0x00, },
+                [19] = { .offset = 0x80, .value = 0xff, },
+                [20] = { .offset = 0x81, .value = 0x07, },
+                [21] = { .offset = 0x82, .value = 0x3d, },
+                [22] = { .offset = 0x83, .value = 0x05, },
+                [23] = { .offset = 0x94, .value = 0x00, },
+                [24] = { .offset = 0x95, .value = 0x00, },
+                [25] = { .offset = 0x96, .value = 0x05, },
+                [26] = { .offset = 0x97, .value = 0x00, },
+                [27] = { .offset = 0x9a, .value = 0x88, },
+                [28] = { .offset = 0x9b, .value = 0x00, },
+                [29] = { .offset = 0x98, .value = 0x00, },
+                [30] = { .offset = 0x99, .value = 0x00, },
+                [31] = { .offset = 0xf7, .value = 0x88, },
+                [32] = { .offset = 0xf8, .value = 0x0a, },
+                [33] = { .offset = 0x9c, .value = 0x24, },
+                [34] = { .offset = 0x9d, .value = 0x00, },
+                [35] = { .offset = 0x9e, .value = 0x25, },
+                [36] = { .offset = 0x9f, .value = 0x03, },
+                [37] = { .offset = 0xa0, .value = 0x28, },
+                [38] = { .offset = 0xa1, .value = 0x01, },
+                [39] = { .offset = 0xa2, .value = 0x28, },
+                [40] = { .offset = 0xa3, .value = 0x05, },
+                [41] = { .offset = 0xb6, .value = 0x09, },
+                [42] = { .offset = 0xb8, .value = 0x00, },
+                [43] = { .offset = 0xb9, .value = 0xa0, },
+                [44] = { .offset = 0xba, .value = 0x00, },
+                [45] = { .offset = 0xbb, .value = 0x20, },
+                [46] = { .offset = 0x10, .value = 0x00, },
+                [47] = { .offset = 0x11, .value = 0xa0, },
+                [48] = { .offset = 0x12, .value = 0x02, },
+                [49] = { .offset = 0x20, .value = 0x00, },
+                [50] = { .offset = 0x22, .value = 0x00, },
+                [51] = { .offset = 0x23, .value = 0x00, },
+                [52] = { .offset = 0x24, .value = 0x00, },
+                [53] = { .offset = 0x25, .value = 0x00, },
+                [54] = { .offset = 0x8c, .value = 0x10, },
+                [55] = { .offset = 0x8d, .value = 0x02, },
+                [56] = { .offset = 0x8e, .value = 0x10, },
+                [57] = { .offset = 0x8f, .value = 0x00, },
+                [58] = { .offset = 0x90, .value = 0xff, },
+                [59] = { .offset = 0x91, .value = 0x07, },
+                [60] = { .offset = 0x92, .value = 0xa0, },
+                [61] = { .offset = 0x93, .value = 0x02, },
+                [62] = { .offset = 0xa5, .value = 0x00, },
+                [63] = { .offset = 0xa6, .value = 0x00, },
+                [64] = { .offset = 0xa7, .value = 0x00, },
+                [65] = { .offset = 0xa8, .value = 0x00, },
+                [66] = { .offset = 0xa9, .value = 0x04, },
+                [67] = { .offset = 0xaa, .value = 0x70, },
+                [68] = { .offset = 0xab, .value = 0x4f, },
+                [69] = { .offset = 0xac, .value = 0x00, },
+                [70] = { .offset = 0xa4, .value = 0x84, },
+                [71] = { .offset = 0x7e, .value = 0x18, },
+                [72] = { .offset = 0x84, .value = 0x00, },
+                [73] = { .offset = 0x85, .value = 0x00, },
+                [74] = { .offset = 0x86, .value = 0x00, },
+                [75] = { .offset = 0x87, .value = 0x00, },
+                [76] = { .offset = 0x88, .value = 0x00, },
+                [77] = { .offset = 0x89, .value = 0x00, },
+                [78] = { .offset = 0x8a, .value = 0x00, },
+                [79] = { .offset = 0x8b, .value = 0x00, },
+                [80] = { .offset = 0x26, .value = 0x00, },
+                [81] = { .offset = 0x27, .value = 0x00, },
+                [82] = { .offset = 0xad, .value = 0x00, },
+                [83] = { .offset = 0x08, .value = 0x34, }, /* 0x35 */
+                [84] = { .offset = 0x41, .value = 0x00, },
+                [85] = { .offset = 0xc0, .value = 0x01, },
+        },
+};
+
+static const struct ns2501_reg regs_init[] = {
+        [0] = { .offset = 0x35, .value = 0xff, },
+        [1] = { .offset = 0x34, .value = 0x00, },
+        [2] = { .offset = 0x08, .value = 0x30, },
+};
+
 struct ns2501_priv {
-        //I2CDevRec d;
         bool quiet;
-        int reg_8_shadow;
-        int reg_8_set;
-        // Shadow registers for i915
-        int dvoc;
-        int pll_a;
-        int srcdim;
-        int fw_blc;
+        const struct ns2501_reg *regs;
 };
 
 #define NSPTR(d) ((NS2501Ptr)(d->DriverPrivate.ptr))
@@ -205,11 +486,9 @@ static bool ns2501_init(struct intel_dvo_device *dvo,
                 goto out;
         }
         ns->quiet = false;
-        ns->reg_8_set = 0;
-        ns->reg_8_shadow =
-            NS2501_8_PD | NS2501_8_BPAS | NS2501_8_VEN | NS2501_8_HEN;
 
         DRM_DEBUG_KMS("init ns2501 dvo controller successfully!\n");
+
         return true;
 
 out:
@@ -242,9 +521,9 @@ static enum drm_mode_status ns2501_mode_valid(struct intel_dvo_device *dvo,
          * of the panel in here so we could always accept it
          * by disabling the scaler.
          */
-        if ((mode->hdisplay == 800 && mode->vdisplay == 600) ||
-            (mode->hdisplay == 640 && mode->vdisplay == 480) ||
-            (mode->hdisplay == 1024 && mode->vdisplay == 768)) {
+        if ((mode->hdisplay == 640 && mode->vdisplay == 480 && mode->clock == 25175) ||
+            (mode->hdisplay == 800 && mode->vdisplay == 600 && mode->clock == 40000) ||
+            (mode->hdisplay == 1024 && mode->vdisplay == 768 && mode->clock == 65000)) {
                 return MODE_OK;
         } else {
                 return MODE_ONE_SIZE;   /* Is this a reasonable error? */
@@ -255,180 +534,30 @@ static void ns2501_mode_set(struct intel_dvo_device *dvo,
                             struct drm_display_mode *mode,
                             struct drm_display_mode *adjusted_mode)
 {
-        bool ok;
-        int retries = 10;
         struct ns2501_priv *ns = (struct ns2501_priv *)(dvo->dev_priv);
+        int mode_idx, i;
 
         DRM_DEBUG_KMS
             ("set mode (hdisplay=%d,htotal=%d,vdisplay=%d,vtotal=%d).\n",
              mode->hdisplay, mode->htotal, mode->vdisplay, mode->vtotal);
 
-        /*
-         * Where do I find the native resolution for which scaling is not required???
-         *
-         * First trigger the DVO on as otherwise the chip does not appear on the i2c
-         * bus.
-         */
-        do {
-                ok = true;
-
-                if (mode->hdisplay == 800 && mode->vdisplay == 600) {
-                        /* mode 277 */
-                        ns->reg_8_shadow &= ~NS2501_8_BPAS;
-                        DRM_DEBUG_KMS("switching to 800x600\n");
-
-                        /*
-                         * No, I do not know where this data comes from.
-                         * It is just what the video bios left in the DVO, so
-                         * I'm just copying it here over.
-                         * This also means that I cannot support any other modes
-                         * except the ones supported by the bios.
-                         */
-                        ok &= ns2501_writeb(dvo, 0x11, 0xc8);   // 0xc7 also works.
-                        ok &= ns2501_writeb(dvo, 0x1b, 0x19);
-                        ok &= ns2501_writeb(dvo, 0x1c, 0x62);   // VBIOS left 0x64 here, but 0x62 works nicer
-                        ok &= ns2501_writeb(dvo, 0x1d, 0x02);
-
-                        ok &= ns2501_writeb(dvo, 0x34, 0x03);
-                        ok &= ns2501_writeb(dvo, 0x35, 0xff);
+        if (mode->hdisplay == 640 && mode->vdisplay == 480)
+                mode_idx = MODE_640x480;
+        else if (mode->hdisplay == 800 && mode->vdisplay == 600)
+                mode_idx = MODE_800x600;
+        else if (mode->hdisplay == 1024 && mode->vdisplay == 768)
+                mode_idx = MODE_1024x768;
+        else
+                return;
 
-                        ok &= ns2501_writeb(dvo, 0x80, 0x27);
-                        ok &= ns2501_writeb(dvo, 0x81, 0x03);
-                        ok &= ns2501_writeb(dvo, 0x82, 0x41);
-                        ok &= ns2501_writeb(dvo, 0x83, 0x05);
+        /* Hopefully doing it every time won't hurt... */
+        for (i = 0; i < ARRAY_SIZE(regs_init); i++)
+                ns2501_writeb(dvo, regs_init[i].offset, regs_init[i].value);
 
-                        ok &= ns2501_writeb(dvo, 0x8d, 0x02);
-                        ok &= ns2501_writeb(dvo, 0x8e, 0x04);
-                        ok &= ns2501_writeb(dvo, 0x8f, 0x00);
+        ns->regs = regs_1024x768[mode_idx];
 
-                        ok &= ns2501_writeb(dvo, 0x90, 0xfe);   /* vertical. VBIOS left 0xff here, but 0xfe works better */
-                        ok &= ns2501_writeb(dvo, 0x91, 0x07);
-                        ok &= ns2501_writeb(dvo, 0x94, 0x00);
-                        ok &= ns2501_writeb(dvo, 0x95, 0x00);
-
-                        ok &= ns2501_writeb(dvo, 0x96, 0x00);
-
-                        ok &= ns2501_writeb(dvo, 0x99, 0x00);
-                        ok &= ns2501_writeb(dvo, 0x9a, 0x88);
-
-                        ok &= ns2501_writeb(dvo, 0x9c, 0x23);   /* Looks like first and last line of the image. */
-                        ok &= ns2501_writeb(dvo, 0x9d, 0x00);
-                        ok &= ns2501_writeb(dvo, 0x9e, 0x25);
-                        ok &= ns2501_writeb(dvo, 0x9f, 0x03);
-
-                        ok &= ns2501_writeb(dvo, 0xa4, 0x80);
-
-                        ok &= ns2501_writeb(dvo, 0xb6, 0x00);
-
-                        ok &= ns2501_writeb(dvo, 0xb9, 0xc8);   /* horizontal? */
-                        ok &= ns2501_writeb(dvo, 0xba, 0x00);   /* horizontal? */
-
-                        ok &= ns2501_writeb(dvo, 0xc0, 0x05);   /* horizontal? */
-                        ok &= ns2501_writeb(dvo, 0xc1, 0xd7);
-
-                        ok &= ns2501_writeb(dvo, 0xc2, 0x00);
-                        ok &= ns2501_writeb(dvo, 0xc3, 0xf8);
-
-                        ok &= ns2501_writeb(dvo, 0xc4, 0x03);
-                        ok &= ns2501_writeb(dvo, 0xc5, 0x1a);
-
-                        ok &= ns2501_writeb(dvo, 0xc6, 0x00);
-                        ok &= ns2501_writeb(dvo, 0xc7, 0x73);
-                        ok &= ns2501_writeb(dvo, 0xc8, 0x02);
-
-                } else if (mode->hdisplay == 640 && mode->vdisplay == 480) {
-                        /* mode 274 */
-                        DRM_DEBUG_KMS("switching to 640x480\n");
-                        /*
-                         * No, I do not know where this data comes from.
-                         * It is just what the video bios left in the DVO, so
-                         * I'm just copying it here over.
-                         * This also means that I cannot support any other modes
-                         * except the ones supported by the bios.
-                         */
-                        ns->reg_8_shadow &= ~NS2501_8_BPAS;
-
-                        ok &= ns2501_writeb(dvo, 0x11, 0xa0);
-                        ok &= ns2501_writeb(dvo, 0x1b, 0x11);
-                        ok &= ns2501_writeb(dvo, 0x1c, 0x54);
-                        ok &= ns2501_writeb(dvo, 0x1d, 0x03);
-
-                        ok &= ns2501_writeb(dvo, 0x34, 0x03);
-                        ok &= ns2501_writeb(dvo, 0x35, 0xff);
-
-                        ok &= ns2501_writeb(dvo, 0x80, 0xff);
-                        ok &= ns2501_writeb(dvo, 0x81, 0x07);
-                        ok &= ns2501_writeb(dvo, 0x82, 0x3d);
-                        ok &= ns2501_writeb(dvo, 0x83, 0x05);
-
-                        ok &= ns2501_writeb(dvo, 0x8d, 0x02);
-                        ok &= ns2501_writeb(dvo, 0x8e, 0x10);
-                        ok &= ns2501_writeb(dvo, 0x8f, 0x00);
-
-                        ok &= ns2501_writeb(dvo, 0x90, 0xff);   /* vertical */
-                        ok &= ns2501_writeb(dvo, 0x91, 0x07);
-                        ok &= ns2501_writeb(dvo, 0x94, 0x00);
-                        ok &= ns2501_writeb(dvo, 0x95, 0x00);
-
-                        ok &= ns2501_writeb(dvo, 0x96, 0x05);
-
-                        ok &= ns2501_writeb(dvo, 0x99, 0x00);
-                        ok &= ns2501_writeb(dvo, 0x9a, 0x88);
-
-                        ok &= ns2501_writeb(dvo, 0x9c, 0x24);
-                        ok &= ns2501_writeb(dvo, 0x9d, 0x00);
-                        ok &= ns2501_writeb(dvo, 0x9e, 0x25);
-                        ok &= ns2501_writeb(dvo, 0x9f, 0x03);
-
-                        ok &= ns2501_writeb(dvo, 0xa4, 0x84);
-
-                        ok &= ns2501_writeb(dvo, 0xb6, 0x09);
-
-                        ok &= ns2501_writeb(dvo, 0xb9, 0xa0);   /* horizontal? */
-                        ok &= ns2501_writeb(dvo, 0xba, 0x00);   /* horizontal? */
-
-                        ok &= ns2501_writeb(dvo, 0xc0, 0x05);   /* horizontal? */
-                        ok &= ns2501_writeb(dvo, 0xc1, 0x90);
-
-                        ok &= ns2501_writeb(dvo, 0xc2, 0x00);
-                        ok &= ns2501_writeb(dvo, 0xc3, 0x0f);
-
-                        ok &= ns2501_writeb(dvo, 0xc4, 0x03);
-                        ok &= ns2501_writeb(dvo, 0xc5, 0x16);
-
-                        ok &= ns2501_writeb(dvo, 0xc6, 0x00);
-                        ok &= ns2501_writeb(dvo, 0xc7, 0x02);
-                        ok &= ns2501_writeb(dvo, 0xc8, 0x02);
-
-                } else if (mode->hdisplay == 1024 && mode->vdisplay == 768) {
-                        /* mode 280 */
-                        DRM_DEBUG_KMS("switching to 1024x768\n");
-                        /*
-                         * This might or might not work, actually. I'm silently
-                         * assuming here that the native panel resolution is
-                         * 1024x768. If not, then this leaves the scaler disabled
-                         * generating a picture that is likely not the expected.
-                         *
-                         * Problem is that I do not know where to take the panel
-                         * dimensions from.
-                         *
-                         * Enable the bypass, scaling not required.
-                         *
-                         * The scaler registers are irrelevant here....
-                         *
-                         */
-                        ns->reg_8_shadow |= NS2501_8_BPAS;
-                        ok &= ns2501_writeb(dvo, 0x37, 0x44);
-                } else {
-                        /*
-                         * Data not known. Bummer!
-                         * Hopefully, the code should not go here
-                         * as mode_OK delivered no other modes.
-                         */
-                        ns->reg_8_shadow |= NS2501_8_BPAS;
-                }
-                ok &= ns2501_writeb(dvo, NS2501_REG8, ns->reg_8_shadow);
-        } while (!ok && retries--);
+        for (i = 0; i < 84; i++)
+                ns2501_writeb(dvo, ns->regs[i].offset, ns->regs[i].value);
 }
 
 /* set the NS2501 power state */
@@ -439,60 +568,46 @@ static bool ns2501_get_hw_state(struct intel_dvo_device *dvo)
         if (!ns2501_readb(dvo, NS2501_REG8, &ch))
                 return false;
 
-        if (ch & NS2501_8_PD)
-                return true;
-        else
-                return false;
+        return ch & NS2501_8_PD;
 }
 
 /* set the NS2501 power state */
 static void ns2501_dpms(struct intel_dvo_device *dvo, bool enable)
 {
-        bool ok;
-        int retries = 10;
         struct ns2501_priv *ns = (struct ns2501_priv *)(dvo->dev_priv);
-        unsigned char ch;
 
         DRM_DEBUG_KMS("Trying set the dpms of the DVO to %i\n", enable);
 
-        ch = ns->reg_8_shadow;
+        if (enable) {
+                if (WARN_ON(ns->regs[83].offset != 0x08 ||
+                            ns->regs[84].offset != 0x41 ||
+                            ns->regs[85].offset != 0xc0))
+                        return;
 
-        if (enable)
-                ch |= NS2501_8_PD;
-        else
-                ch &= ~NS2501_8_PD;
-
-        if (ns->reg_8_set == 0 || ns->reg_8_shadow != ch) {
-                ns->reg_8_set = 1;
-                ns->reg_8_shadow = ch;
-
-                do {
-                        ok = true;
-                        ok &= ns2501_writeb(dvo, NS2501_REG8, ch);
-                        ok &=
-                            ns2501_writeb(dvo, 0x34,
-                                          enable ? 0x03 : 0x00);
-                        ok &=
-                            ns2501_writeb(dvo, 0x35,
-                                          enable ? 0xff : 0x00);
-                } while (!ok && retries--);
-        }
-}
+                ns2501_writeb(dvo, 0xc0, ns->regs[85].value | 0x08);
 
-static void ns2501_dump_regs(struct intel_dvo_device *dvo)
-{
-        uint8_t val;
-
-        ns2501_readb(dvo, NS2501_FREQ_LO, &val);
-        DRM_DEBUG_KMS("NS2501_FREQ_LO: 0x%02x\n", val);
-        ns2501_readb(dvo, NS2501_FREQ_HI, &val);
-        DRM_DEBUG_KMS("NS2501_FREQ_HI: 0x%02x\n", val);
-        ns2501_readb(dvo, NS2501_REG8, &val);
-        DRM_DEBUG_KMS("NS2501_REG8: 0x%02x\n", val);
-        ns2501_readb(dvo, NS2501_REG9, &val);
-        DRM_DEBUG_KMS("NS2501_REG9: 0x%02x\n", val);
-        ns2501_readb(dvo, NS2501_REGC, &val);
-        DRM_DEBUG_KMS("NS2501_REGC: 0x%02x\n", val);
+                ns2501_writeb(dvo, 0x41, ns->regs[84].value);
+
+                ns2501_writeb(dvo, 0x34, 0x01);
+                msleep(15);
+
+                ns2501_writeb(dvo, 0x08, 0x35);
+                if (!(ns->regs[83].value & NS2501_8_BPAS))
+                        ns2501_writeb(dvo, 0x08, 0x31);
+                msleep(200);
+
+                ns2501_writeb(dvo, 0x34, 0x03);
+
+                ns2501_writeb(dvo, 0xc0, ns->regs[85].value);
+        } else {
+                ns2501_writeb(dvo, 0x34, 0x01);
+                msleep(200);
+
+                ns2501_writeb(dvo, 0x08, 0x34);
+                msleep(15);
+
+                ns2501_writeb(dvo, 0x34, 0x00);
+        }
 }
 
 static void ns2501_destroy(struct intel_dvo_device *dvo)
@@ -512,6 +627,5 @@ struct intel_dvo_dev_ops ns2501_ops = {
         .mode_set = ns2501_mode_set,
         .dpms = ns2501_dpms,
         .get_hw_state = ns2501_get_hw_state,
-        .dump_regs = ns2501_dump_regs,
         .destroy = ns2501_destroy,
 };
diff --git a/drivers/gpu/drm/i915/i915_cmd_parser.c b/drivers/gpu/drm/i915/i915_cmd_parser.c
index 4b7ed5289217..593b657d3e59 100644
--- a/drivers/gpu/drm/i915/i915_cmd_parser.c
+++ b/drivers/gpu/drm/i915/i915_cmd_parser.c
@@ -844,8 +844,6 @@ finish:
  */
 bool i915_needs_cmd_parser(struct intel_engine_cs *ring)
 {
-        struct drm_i915_private *dev_priv = ring->dev->dev_private;
-
         if (!ring->needs_cmd_parser)
                 return false;
 
@@ -854,7 +852,7 @@ bool i915_needs_cmd_parser(struct intel_engine_cs *ring)
          * disabled. That will cause all of the parser's PPGTT checks to
          * fail. For now, disable parsing when PPGTT is off.
          */
-        if (!dev_priv->mm.aliasing_ppgtt)
+        if (USES_PPGTT(ring->dev))
                 return false;
 
         return (i915.enable_cmd_parser == 1);
diff --git a/drivers/gpu/drm/i915/i915_debugfs.c b/drivers/gpu/drm/i915/i915_debugfs.c
index 9e737b771c40..063b44817e08 100644
--- a/drivers/gpu/drm/i915/i915_debugfs.c
+++ b/drivers/gpu/drm/i915/i915_debugfs.c
@@ -136,7 +136,7 @@ describe_obj(struct seq_file *m, struct drm_i915_gem_object *obj)
                    obj->last_read_seqno,
                    obj->last_write_seqno,
                    obj->last_fenced_seqno,
-                   i915_cache_level_str(obj->cache_level),
+                   i915_cache_level_str(to_i915(obj->base.dev), obj->cache_level),
                    obj->dirty ? " dirty" : "",
                    obj->madv == I915_MADV_DONTNEED ? " purgeable" : "");
         if (obj->base.name)
@@ -333,7 +333,7 @@ static int per_file_stats(int id, void *ptr, void *data)
                         }
 
                         ppgtt = container_of(vma->vm, struct i915_hw_ppgtt, base);
-                        if (ppgtt->ctx && ppgtt->ctx->file_priv != stats->file_priv)
+                        if (ppgtt->file_priv != stats->file_priv)
                                 continue;
 
                         if (obj->ring) /* XXX per-vma statistic */
@@ -515,6 +515,7 @@ static int i915_gem_pageflip_info(struct seq_file *m, void *data)
 {
         struct drm_info_node *node = m->private;
         struct drm_device *dev = node->minor->dev;
+        struct drm_i915_private *dev_priv = dev->dev_private;
         unsigned long flags;
         struct intel_crtc *crtc;
         int ret;
@@ -534,6 +535,8 @@ static int i915_gem_pageflip_info(struct seq_file *m, void *data)
                         seq_printf(m, "No flip due on pipe %c (plane %c)\n",
                                    pipe, plane);
                 } else {
+                        u32 addr;
+
                         if (atomic_read(&work->pending) < INTEL_FLIP_COMPLETE) {
                                 seq_printf(m, "Flip queued on pipe %c (plane %c)\n",
                                            pipe, plane);
@@ -541,23 +544,35 @@ static int i915_gem_pageflip_info(struct seq_file *m, void *data)
                                 seq_printf(m, "Flip pending (waiting for vsync) on pipe %c (plane %c)\n",
                                            pipe, plane);
                         }
+                        if (work->flip_queued_ring) {
+                                seq_printf(m, "Flip queued on %s at seqno %u, next seqno %u [current breadcrumb %u], completed? %d\n",
+                                           work->flip_queued_ring->name,
+                                           work->flip_queued_seqno,
+                                           dev_priv->next_seqno,
+                                           work->flip_queued_ring->get_seqno(work->flip_queued_ring, true),
+                                           i915_seqno_passed(work->flip_queued_ring->get_seqno(work->flip_queued_ring, true),
+                                                             work->flip_queued_seqno));
+                        } else
+                                seq_printf(m, "Flip not associated with any ring\n");
+                        seq_printf(m, "Flip queued on frame %d, (was ready on frame %d), now %d\n",
+                                   work->flip_queued_vblank,
+                                   work->flip_ready_vblank,
+                                   drm_vblank_count(dev, crtc->pipe));
                         if (work->enable_stall_check)
                                 seq_puts(m, "Stall check enabled, ");
                         else
                                 seq_puts(m, "Stall check waiting for page flip ioctl, ");
                         seq_printf(m, "%d prepares\n", atomic_read(&work->pending));
 
-                        if (work->old_fb_obj) {
-                                struct drm_i915_gem_object *obj = work->old_fb_obj;
-                                if (obj)
-                                        seq_printf(m, "Old framebuffer gtt_offset 0x%08lx\n",
-                                                   i915_gem_obj_ggtt_offset(obj));
-                        }
+                        if (INTEL_INFO(dev)->gen >= 4)
+                                addr = I915_HI_DISPBASE(I915_READ(DSPSURF(crtc->plane)));
+                        else
+                                addr = I915_READ(DSPADDR(crtc->plane));
+                        seq_printf(m, "Current scanout address 0x%08x\n", addr);
+
                         if (work->pending_flip_obj) {
-                                struct drm_i915_gem_object *obj = work->pending_flip_obj;
-                                if (obj)
-                                        seq_printf(m, "New framebuffer gtt_offset 0x%08lx\n",
-                                                   i915_gem_obj_ggtt_offset(obj));
+                                seq_printf(m, "New framebuffer address 0x%08lx\n", (long)work->gtt_offset);
+                                seq_printf(m, "MMIO update completed? %d\n",  addr == work->gtt_offset);
                         }
                 }
                 spin_unlock_irqrestore(&dev->event_lock, flags);
@@ -650,7 +665,6 @@ static int i915_interrupt_info(struct seq_file *m, void *data)
         intel_runtime_pm_get(dev_priv);
 
         if (IS_CHERRYVIEW(dev)) {
-                int i;
                 seq_printf(m, "Master Interrupt Control:\t%08x\n",
                            I915_READ(GEN8_MASTER_IRQ));
 
@@ -662,7 +676,7 @@ static int i915_interrupt_info(struct seq_file *m, void *data)
                            I915_READ(VLV_IIR_RW));
                 seq_printf(m, "Display IMR:\t%08x\n",
                            I915_READ(VLV_IMR));
-                for_each_pipe(pipe)
+                for_each_pipe(dev_priv, pipe)
                         seq_printf(m, "Pipe %c stat:\t%08x\n",
                                    pipe_name(pipe),
                                    I915_READ(PIPESTAT(pipe)));
@@ -702,7 +716,13 @@ static int i915_interrupt_info(struct seq_file *m, void *data)
                                    i, I915_READ(GEN8_GT_IER(i)));
                 }
 
-                for_each_pipe(pipe) {
+                for_each_pipe(dev_priv, pipe) {
+                        if (!intel_display_power_enabled(dev_priv,
+                                                POWER_DOMAIN_PIPE(pipe))) {
+                                seq_printf(m, "Pipe %c power disabled\n",
+                                           pipe_name(pipe));
+                                continue;
+                        }
                         seq_printf(m, "Pipe %c IMR:\t%08x\n",
                                    pipe_name(pipe),
                                    I915_READ(GEN8_DE_PIPE_IMR(pipe)));
@@ -743,7 +763,7 @@ static int i915_interrupt_info(struct seq_file *m, void *data)
                            I915_READ(VLV_IIR_RW));
                 seq_printf(m, "Display IMR:\t%08x\n",
                            I915_READ(VLV_IMR));
-                for_each_pipe(pipe)
+                for_each_pipe(dev_priv, pipe)
                         seq_printf(m, "Pipe %c stat:\t%08x\n",
                                    pipe_name(pipe),
                                    I915_READ(PIPESTAT(pipe)));
@@ -779,7 +799,7 @@ static int i915_interrupt_info(struct seq_file *m, void *data)
                            I915_READ(IIR));
                 seq_printf(m, "Interrupt mask:      %08x\n",
                            I915_READ(IMR));
-                for_each_pipe(pipe)
+                for_each_pipe(dev_priv, pipe)
                         seq_printf(m, "Pipe %c stat:         %08x\n",
                                    pipe_name(pipe),
                                    I915_READ(PIPESTAT(pipe)));
@@ -927,7 +947,7 @@ static ssize_t i915_error_state_read(struct file *file, char __user *userbuf,
         ssize_t ret_count = 0;
         int ret;
 
-        ret = i915_error_state_buf_init(&error_str, count, *pos);
+        ret = i915_error_state_buf_init(&error_str, to_i915(error_priv->dev), count, *pos);
         if (ret)
                 return ret;
 
@@ -1024,6 +1044,7 @@ static int i915_frequency_info(struct seq_file *m, void *unused)
                 u32 rpstat, cagf, reqf;
                 u32 rpupei, rpcurup, rpprevup;
                 u32 rpdownei, rpcurdown, rpprevdown;
+                u32 pm_ier, pm_imr, pm_isr, pm_iir, pm_mask;
                 int max_freq;
 
                 /* RPSTAT1 is in the GT power well */
@@ -1061,12 +1082,21 @@ static int i915_frequency_info(struct seq_file *m, void *unused)
                 gen6_gt_force_wake_put(dev_priv, FORCEWAKE_ALL);
                 mutex_unlock(&dev->struct_mutex);
 
+                if (IS_GEN6(dev) || IS_GEN7(dev)) {
+                        pm_ier = I915_READ(GEN6_PMIER);
+                        pm_imr = I915_READ(GEN6_PMIMR);
+                        pm_isr = I915_READ(GEN6_PMISR);
+                        pm_iir = I915_READ(GEN6_PMIIR);
+                        pm_mask = I915_READ(GEN6_PMINTRMSK);
+                } else {
+                        pm_ier = I915_READ(GEN8_GT_IER(2));
+                        pm_imr = I915_READ(GEN8_GT_IMR(2));
+                        pm_isr = I915_READ(GEN8_GT_ISR(2));
+                        pm_iir = I915_READ(GEN8_GT_IIR(2));
+                        pm_mask = I915_READ(GEN6_PMINTRMSK);
+                }
                 seq_printf(m, "PM IER=0x%08x IMR=0x%08x ISR=0x%08x IIR=0x%08x, MASK=0x%08x\n",
-                           I915_READ(GEN6_PMIER),
-                           I915_READ(GEN6_PMIMR),
-                           I915_READ(GEN6_PMISR),
-                           I915_READ(GEN6_PMIIR),
-                           I915_READ(GEN6_PMINTRMSK));
+                           pm_ier, pm_imr, pm_isr, pm_iir, pm_mask);
                 seq_printf(m, "GT_PERF_STATUS: 0x%08x\n", gt_perf_status);
                 seq_printf(m, "Render p-state ratio: %d\n",
                            (gt_perf_status & 0xff00) >> 8);
@@ -1365,7 +1395,7 @@ static int i915_drpc_info(struct seq_file *m, void *unused)
 
         if (IS_VALLEYVIEW(dev))
                 return vlv_drpc_info(m);
-        else if (IS_GEN6(dev) || IS_GEN7(dev))
+        else if (INTEL_INFO(dev)->gen >= 6)
                 return gen6_drpc_info(m);
         else
                 return ironlake_drpc_info(m);
@@ -1433,6 +1463,47 @@ static int i915_fbc_status(struct seq_file *m, void *unused)
         return 0;
 }
 
+static int i915_fbc_fc_get(void *data, u64 *val)
+{
+        struct drm_device *dev = data;
+        struct drm_i915_private *dev_priv = dev->dev_private;
+
+        if (INTEL_INFO(dev)->gen < 7 || !HAS_FBC(dev))
+                return -ENODEV;
+
+        drm_modeset_lock_all(dev);
+        *val = dev_priv->fbc.false_color;
+        drm_modeset_unlock_all(dev);
+
+        return 0;
+}
+
+static int i915_fbc_fc_set(void *data, u64 val)
+{
+        struct drm_device *dev = data;
+        struct drm_i915_private *dev_priv = dev->dev_private;
+        u32 reg;
+
+        if (INTEL_INFO(dev)->gen < 7 || !HAS_FBC(dev))
+                return -ENODEV;
+
+        drm_modeset_lock_all(dev);
+
+        reg = I915_READ(ILK_DPFC_CONTROL);
+        dev_priv->fbc.false_color = val;
+
+        I915_WRITE(ILK_DPFC_CONTROL, val ?
+                   (reg | FBC_CTL_FALSE_COLOR) :
+                   (reg & ~FBC_CTL_FALSE_COLOR));
+
+        drm_modeset_unlock_all(dev);
+        return 0;
+}
+
+DEFINE_SIMPLE_ATTRIBUTE(i915_fbc_fc_fops,
+                        i915_fbc_fc_get, i915_fbc_fc_set,
+                        "%llu\n");
+
 static int i915_ips_status(struct seq_file *m, void *unused)
 {
         struct drm_info_node *node = m->private;
@@ -1630,6 +1701,14 @@ static int i915_gem_framebuffer_info(struct seq_file *m, void *data)
         return 0;
 }
 
+static void describe_ctx_ringbuf(struct seq_file *m,
+                                 struct intel_ringbuffer *ringbuf)
+{
+        seq_printf(m, " (ringbuffer, space: %d, head: %u, tail: %u, last head: %d)",
+                   ringbuf->space, ringbuf->head, ringbuf->tail,
+                   ringbuf->last_retired_head);
+}
+
 static int i915_context_status(struct seq_file *m, void *unused)
 {
         struct drm_info_node *node = m->private;
@@ -1656,16 +1735,168 @@ static int i915_context_status(struct seq_file *m, void *unused)
         }
 
         list_for_each_entry(ctx, &dev_priv->context_list, link) {
-                if (ctx->legacy_hw_ctx.rcs_state == NULL)
+                if (!i915.enable_execlists &&
+                    ctx->legacy_hw_ctx.rcs_state == NULL)
                         continue;
 
                 seq_puts(m, "HW context ");
                 describe_ctx(m, ctx);
-                for_each_ring(ring, dev_priv, i)
+                for_each_ring(ring, dev_priv, i) {
+                        if (ring->default_context == ctx)
+                                seq_printf(m, "(default context %s) ",
+                                           ring->name);
+                }
+
+                if (i915.enable_execlists) {
+                        seq_putc(m, '\n');
+                        for_each_ring(ring, dev_priv, i) {
+                                struct drm_i915_gem_object *ctx_obj =
+                                        ctx->engine[i].state;
+                                struct intel_ringbuffer *ringbuf =
+                                        ctx->engine[i].ringbuf;
+
+                                seq_printf(m, "%s: ", ring->name);
+                                if (ctx_obj)
+                                        describe_obj(m, ctx_obj);
+                                if (ringbuf)
+                                        describe_ctx_ringbuf(m, ringbuf);
+                                seq_putc(m, '\n');
+                        }
+                } else {
+                        describe_obj(m, ctx->legacy_hw_ctx.rcs_state);
+                }
+
+                seq_putc(m, '\n');
+        }
+
+        mutex_unlock(&dev->struct_mutex);
+
+        return 0;
+}
+
+static int i915_dump_lrc(struct seq_file *m, void *unused)
+{
+        struct drm_info_node *node = (struct drm_info_node *) m->private;
+        struct drm_device *dev = node->minor->dev;
+        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct intel_engine_cs *ring;
+        struct intel_context *ctx;
+        int ret, i;
+
+        if (!i915.enable_execlists) {
+                seq_printf(m, "Logical Ring Contexts are disabled\n");
+                return 0;
+        }
+
+        ret = mutex_lock_interruptible(&dev->struct_mutex);
+        if (ret)
+                return ret;
+
+        list_for_each_entry(ctx, &dev_priv->context_list, link) {
+                for_each_ring(ring, dev_priv, i) {
+                        struct drm_i915_gem_object *ctx_obj = ctx->engine[i].state;
+
                         if (ring->default_context == ctx)
-                                seq_printf(m, "(default context %s) ", ring->name);
+                                continue;
+
+                        if (ctx_obj) {
+                                struct page *page = i915_gem_object_get_page(ctx_obj, 1);
+                                uint32_t *reg_state = kmap_atomic(page);
+                                int j;
+
+                                seq_printf(m, "CONTEXT: %s %u\n", ring->name,
+                                                intel_execlists_ctx_id(ctx_obj));
+
+                                for (j = 0; j < 0x600 / sizeof(u32) / 4; j += 4) {
+                                        seq_printf(m, "\t[0x%08lx] 0x%08x 0x%08x 0x%08x 0x%08x\n",
+                                        i915_gem_obj_ggtt_offset(ctx_obj) + 4096 + (j * 4),
+                                        reg_state[j], reg_state[j + 1],
+                                        reg_state[j + 2], reg_state[j + 3]);
+                                }
+                                kunmap_atomic(reg_state);
+
+                                seq_putc(m, '\n');
+                        }
+                }
+        }
+
+        mutex_unlock(&dev->struct_mutex);
+
+        return 0;
+}
+
+static int i915_execlists(struct seq_file *m, void *data)
+{
+        struct drm_info_node *node = (struct drm_info_node *)m->private;
+        struct drm_device *dev = node->minor->dev;
+        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct intel_engine_cs *ring;
+        u32 status_pointer;
+        u8 read_pointer;
+        u8 write_pointer;
+        u32 status;
+        u32 ctx_id;
+        struct list_head *cursor;
+        int ring_id, i;
+        int ret;
+
+        if (!i915.enable_execlists) {
+                seq_puts(m, "Logical Ring Contexts are disabled\n");
+                return 0;
+        }
+
+        ret = mutex_lock_interruptible(&dev->struct_mutex);
+        if (ret)
+                return ret;
+
+        for_each_ring(ring, dev_priv, ring_id) {
+                struct intel_ctx_submit_request *head_req = NULL;
+                int count = 0;
+                unsigned long flags;
+
+                seq_printf(m, "%s\n", ring->name);
+
+                status = I915_READ(RING_EXECLIST_STATUS(ring));
+                ctx_id = I915_READ(RING_EXECLIST_STATUS(ring) + 4);
+                seq_printf(m, "\tExeclist status: 0x%08X, context: %u\n",
+                           status, ctx_id);
+
+                status_pointer = I915_READ(RING_CONTEXT_STATUS_PTR(ring));
+                seq_printf(m, "\tStatus pointer: 0x%08X\n", status_pointer);
+
+                read_pointer = ring->next_context_status_buffer;
+                write_pointer = status_pointer & 0x07;
+                if (read_pointer > write_pointer)
+                        write_pointer += 6;
+                seq_printf(m, "\tRead pointer: 0x%08X, write pointer 0x%08X\n",
+                           read_pointer, write_pointer);
+
+                for (i = 0; i < 6; i++) {
+                        status = I915_READ(RING_CONTEXT_STATUS_BUF(ring) + 8*i);
+                        ctx_id = I915_READ(RING_CONTEXT_STATUS_BUF(ring) + 8*i + 4);
+
+                        seq_printf(m, "\tStatus buffer %d: 0x%08X, context: %u\n",
+                                   i, status, ctx_id);
+                }
+
+                spin_lock_irqsave(&ring->execlist_lock, flags);
+                list_for_each(cursor, &ring->execlist_queue)
+                        count++;
+                head_req = list_first_entry_or_null(&ring->execlist_queue,
+                                struct intel_ctx_submit_request, execlist_link);
+                spin_unlock_irqrestore(&ring->execlist_lock, flags);
+
+                seq_printf(m, "\t%d requests in queue\n", count);
+                if (head_req) {
+                        struct drm_i915_gem_object *ctx_obj;
+
+                        ctx_obj = head_req->ctx->engine[ring_id].state;
+                        seq_printf(m, "\tHead request id: %u\n",
+                                   intel_execlists_ctx_id(ctx_obj));
+                        seq_printf(m, "\tHead request tail: %u\n",
+                                   head_req->tail);
+                }
 
-                describe_obj(m, ctx->legacy_hw_ctx.rcs_state);
                 seq_putc(m, '\n');
         }
 
@@ -1774,7 +2005,13 @@ static int per_file_ctx(int id, void *ptr, void *data)
 {
         struct intel_context *ctx = ptr;
         struct seq_file *m = data;
-        struct i915_hw_ppgtt *ppgtt = ctx_to_ppgtt(ctx);
+        struct i915_hw_ppgtt *ppgtt = ctx->ppgtt;
+
+        if (!ppgtt) {
+                seq_printf(m, "  no ppgtt for context %d\n",
+                           ctx->user_handle);
+                return 0;
+        }
 
         if (i915_gem_context_is_default(ctx))
                 seq_puts(m, "  default context:\n");
@@ -1834,8 +2071,7 @@ static void gen6_ppgtt_info(struct seq_file *m, struct drm_device *dev)
                 seq_printf(m, "pd gtt offset: 0x%08x\n", ppgtt->pd_offset);
 
                 ppgtt->debug_dump(ppgtt, m);
-        } else
-                return;
+        }
 
         list_for_each_entry_reverse(file, &dev->filelist, lhead) {
                 struct drm_i915_file_private *file_priv = file->driver_priv;
@@ -2406,6 +2642,40 @@ static int i915_shared_dplls_info(struct seq_file *m, void *unused)
         return 0;
 }
 
+static int i915_wa_registers(struct seq_file *m, void *unused)
+{
+        int i;
+        int ret;
+        struct drm_info_node *node = (struct drm_info_node *) m->private;
+        struct drm_device *dev = node->minor->dev;
+        struct drm_i915_private *dev_priv = dev->dev_private;
+
+        ret = mutex_lock_interruptible(&dev->struct_mutex);
+        if (ret)
+                return ret;
+
+        intel_runtime_pm_get(dev_priv);
+
+        seq_printf(m, "Workarounds applied: %d\n", dev_priv->num_wa_regs);
+        for (i = 0; i < dev_priv->num_wa_regs; ++i) {
+                u32 addr, mask;
+
+                addr = dev_priv->intel_wa_regs[i].addr;
+                mask = dev_priv->intel_wa_regs[i].mask;
+                dev_priv->intel_wa_regs[i].value = I915_READ(addr) | mask;
+                if (dev_priv->intel_wa_regs[i].addr)
+                        seq_printf(m, "0x%X: 0x%08X, mask: 0x%08X\n",
+                                   dev_priv->intel_wa_regs[i].addr,
+                                   dev_priv->intel_wa_regs[i].value,
+                                   dev_priv->intel_wa_regs[i].mask);
+        }
+
+        intel_runtime_pm_put(dev_priv);
+        mutex_unlock(&dev->struct_mutex);
+
+        return 0;
+}
+
 struct pipe_crc_info {
         const char *name;
         struct drm_device *dev;
@@ -2667,8 +2937,7 @@ static int i9xx_pipe_crc_auto_source(struct drm_device *dev, enum pipe pipe,
         *source = INTEL_PIPE_CRC_SOURCE_PIPE;
 
         drm_modeset_lock_all(dev);
-        list_for_each_entry(encoder, &dev->mode_config.encoder_list,
-                            base.head) {
+        for_each_intel_encoder(dev, encoder) {
                 if (!encoder->base.crtc)
                         continue;
 
@@ -3557,9 +3826,6 @@ i915_drop_caches_set(void *data, u64 val)
 {
         struct drm_device *dev = data;
         struct drm_i915_private *dev_priv = dev->dev_private;
-        struct drm_i915_gem_object *obj, *next;
-        struct i915_address_space *vm;
-        struct i915_vma *vma, *x;
         int ret;
 
         DRM_DEBUG("Dropping caches: 0x%08llx\n", val);
@@ -3579,29 +3845,11 @@ i915_drop_caches_set(void *data, u64 val)
         if (val & (DROP_RETIRE | DROP_ACTIVE))
                 i915_gem_retire_requests(dev);
 
-        if (val & DROP_BOUND) {
-                list_for_each_entry(vm, &dev_priv->vm_list, global_link) {
-                        list_for_each_entry_safe(vma, x, &vm->inactive_list,
-                                                 mm_list) {
-                                if (vma->pin_count)
-                                        continue;
+        if (val & DROP_BOUND)
+                i915_gem_shrink(dev_priv, LONG_MAX, I915_SHRINK_BOUND);
 
-                                ret = i915_vma_unbind(vma);
-                                if (ret)
-                                        goto unlock;
-                        }
-                }
-        }
-
-        if (val & DROP_UNBOUND) {
-                list_for_each_entry_safe(obj, next, &dev_priv->mm.unbound_list,
-                                         global_list)
-                        if (obj->pages_pin_count == 0) {
-                                ret = i915_gem_object_put_pages(obj);
-                                if (ret)
-                                        goto unlock;
-                        }
-        }
+        if (val & DROP_UNBOUND)
+                i915_gem_shrink(dev_priv, LONG_MAX, I915_SHRINK_UNBOUND);
 
 unlock:
         mutex_unlock(&dev->struct_mutex);
@@ -3923,6 +4171,8 @@ static const struct drm_info_list i915_debugfs_list[] = {
         {"i915_opregion", i915_opregion, 0},
         {"i915_gem_framebuffer", i915_gem_framebuffer_info, 0},
         {"i915_context_status", i915_context_status, 0},
+        {"i915_dump_lrc", i915_dump_lrc, 0},
+        {"i915_execlists", i915_execlists, 0},
         {"i915_gen6_forcewake_count", i915_gen6_forcewake_count_info, 0},
         {"i915_swizzle_info", i915_swizzle_info, 0},
         {"i915_ppgtt_info", i915_ppgtt_info, 0},
@@ -3936,6 +4186,7 @@ static const struct drm_info_list i915_debugfs_list[] = {
         {"i915_semaphore_status", i915_semaphore_status, 0},
         {"i915_shared_dplls_info", i915_shared_dplls_info, 0},
         {"i915_dp_mst_info", i915_dp_mst_info, 0},
+        {"i915_wa_registers", i915_wa_registers, 0},
 };
 #define I915_DEBUGFS_ENTRIES ARRAY_SIZE(i915_debugfs_list)
 
@@ -3957,6 +4208,7 @@ static const struct i915_debugfs_files {
         {"i915_pri_wm_latency", &i915_pri_wm_latency_fops},
         {"i915_spr_wm_latency", &i915_spr_wm_latency_fops},
         {"i915_cur_wm_latency", &i915_cur_wm_latency_fops},
+        {"i915_fbc_false_color", &i915_fbc_fc_fops},
 };
 
 void intel_display_crc_init(struct drm_device *dev)
@@ -3964,7 +4216,7 @@ void intel_display_crc_init(struct drm_device *dev)
         struct drm_i915_private *dev_priv = dev->dev_private;
         enum pipe pipe;
 
-        for_each_pipe(pipe) {
+        for_each_pipe(dev_priv, pipe) {
                 struct intel_pipe_crc *pipe_crc = &dev_priv->pipe_crc[pipe];
 
                 pipe_crc->opened = false;
diff --git a/drivers/gpu/drm/i915/i915_dma.c b/drivers/gpu/drm/i915/i915_dma.c
index 9933c26017ed..1403b01e8216 100644
--- a/drivers/gpu/drm/i915/i915_dma.c
+++ b/drivers/gpu/drm/i915/i915_dma.c
@@ -28,9 +28,11 @@
 
 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
 
+#include <linux/async.h>
 #include <drm/drmP.h>
 #include <drm/drm_crtc_helper.h>
 #include <drm/drm_fb_helper.h>
+#include <drm/drm_legacy.h>
 #include "intel_drv.h"
 #include <drm/i915_drm.h>
 #include "i915_drv.h"
@@ -196,7 +198,7 @@ static int i915_initialize(struct drm_device *dev, drm_i915_init_t *init)
         struct drm_i915_master_private *master_priv = dev->primary->master->driver_priv;
         int ret;
 
-        master_priv->sarea = drm_getsarea(dev);
+        master_priv->sarea = drm_legacy_getsarea(dev);
         if (master_priv->sarea) {
                 master_priv->sarea_priv = (drm_i915_sarea_t *)
                         ((u8 *)master_priv->sarea->handle + init->sarea_priv_offset);
@@ -999,7 +1001,7 @@ static int i915_getparam(struct drm_device *dev, void *data,
                 value = HAS_WT(dev);
                 break;
         case I915_PARAM_HAS_ALIASING_PPGTT:
-                value = dev_priv->mm.aliasing_ppgtt || USES_FULL_PPGTT(dev);
+                value = USES_PPGTT(dev);
                 break;
         case I915_PARAM_HAS_WAIT_TIMEOUT:
                 value = 1;
@@ -1355,8 +1357,6 @@ static int i915_load_modeset_init(struct drm_device *dev)
         if (ret)
                 goto cleanup_irq;
 
-        INIT_WORK(&dev_priv->console_resume_work, intel_console_resume);
-
         intel_modeset_gem_init(dev);
 
         /* Always safe in the mode setting case. */
@@ -1382,7 +1382,7 @@ static int i915_load_modeset_init(struct drm_device *dev)
          * scanning against hotplug events. Hence do this first and ignore the
          * tiny window where we will loose hotplug notifactions.
          */
-        intel_fbdev_initial_config(dev);
+        async_schedule(intel_fbdev_initial_config, dev_priv);
 
         drm_kms_helper_poll_init(dev);
 
@@ -1393,7 +1393,6 @@ cleanup_gem:
         i915_gem_cleanup_ringbuffer(dev);
         i915_gem_context_fini(dev);
         mutex_unlock(&dev->struct_mutex);
-        WARN_ON(dev_priv->mm.aliasing_ppgtt);
 cleanup_irq:
         drm_irq_uninstall(dev);
 cleanup_gem_stolen:
@@ -1536,10 +1535,10 @@ static void intel_device_info_runtime_init(struct drm_device *dev)
         info = (struct intel_device_info *)&dev_priv->info;
 
         if (IS_VALLEYVIEW(dev))
-                for_each_pipe(pipe)
+                for_each_pipe(dev_priv, pipe)
                         info->num_sprites[pipe] = 2;
         else
-                for_each_pipe(pipe)
+                for_each_pipe(dev_priv, pipe)
                         info->num_sprites[pipe] = 1;
 
         if (i915.disable_display) {
@@ -1608,9 +1607,10 @@ int i915_driver_load(struct drm_device *dev, unsigned long flags)
         dev->dev_private = dev_priv;
         dev_priv->dev = dev;
 
-        /* copy initial configuration to dev_priv->info */
+        /* Setup the write-once "constant" device info */
         device_info = (struct intel_device_info *)&dev_priv->info;
-        *device_info = *info;
+        memcpy(device_info, info, sizeof(dev_priv->info));
+        device_info->device_id = dev->pdev->device;
 
         spin_lock_init(&dev_priv->irq_lock);
         spin_lock_init(&dev_priv->gpu_error.lock);
@@ -1822,7 +1822,7 @@ out_mtrrfree:
         arch_phys_wc_del(dev_priv->gtt.mtrr);
         io_mapping_free(dev_priv->gtt.mappable);
 out_gtt:
-        dev_priv->gtt.base.cleanup(&dev_priv->gtt.base);
+        i915_global_gtt_cleanup(dev);
 out_regs:
         intel_uncore_fini(dev);
         pci_iounmap(dev->pdev, dev_priv->regs);
@@ -1869,7 +1869,6 @@ int i915_driver_unload(struct drm_device *dev)
         if (drm_core_check_feature(dev, DRIVER_MODESET)) {
                 intel_fbdev_fini(dev);
                 intel_modeset_cleanup(dev);
-                cancel_work_sync(&dev_priv->console_resume_work);
 
                 /*
                  * free the memory space allocated for the child device
@@ -1902,7 +1901,6 @@ int i915_driver_unload(struct drm_device *dev)
                 mutex_lock(&dev->struct_mutex);
                 i915_gem_cleanup_ringbuffer(dev);
                 i915_gem_context_fini(dev);
-                WARN_ON(dev_priv->mm.aliasing_ppgtt);
                 mutex_unlock(&dev->struct_mutex);
                 i915_gem_cleanup_stolen(dev);
 
@@ -1910,8 +1908,6 @@ int i915_driver_unload(struct drm_device *dev)
                         i915_free_hws(dev);
         }
 
-        WARN_ON(!list_empty(&dev_priv->vm_list));
-
         drm_vblank_cleanup(dev);
 
         intel_teardown_gmbus(dev);
@@ -1921,7 +1917,7 @@ int i915_driver_unload(struct drm_device *dev)
         destroy_workqueue(dev_priv->wq);
         pm_qos_remove_request(&dev_priv->pm_qos);
 
-        dev_priv->gtt.base.cleanup(&dev_priv->gtt.base);
+        i915_global_gtt_cleanup(dev);
 
         intel_uncore_fini(dev);
         if (dev_priv->regs != NULL)
@@ -1986,6 +1982,9 @@ void i915_driver_preclose(struct drm_device *dev, struct drm_file *file)
         i915_gem_context_close(dev, file);
         i915_gem_release(dev, file);
         mutex_unlock(&dev->struct_mutex);
+
+        if (drm_core_check_feature(dev, DRIVER_MODESET))
+                intel_modeset_preclose(dev, file);
 }
 
 void i915_driver_postclose(struct drm_device *dev, struct drm_file *file)
diff --git a/drivers/gpu/drm/i915/i915_drv.c b/drivers/gpu/drm/i915/i915_drv.c
index e27cdbe9d524..055d5e7fbf12 100644
--- a/drivers/gpu/drm/i915/i915_drv.c
+++ b/drivers/gpu/drm/i915/i915_drv.c
@@ -481,6 +481,10 @@ bool i915_semaphore_is_enabled(struct drm_device *dev)
         if (i915.semaphores >= 0)
                 return i915.semaphores;
 
+        /* TODO: make semaphores and Execlists play nicely together */
+        if (i915.enable_execlists)
+                return false;
+
         /* Until we get further testing... */
         if (IS_GEN8(dev))
                 return false;
@@ -524,6 +528,10 @@ static void intel_suspend_encoders(struct drm_i915_private *dev_priv)
         drm_modeset_unlock_all(dev);
 }
 
+static int intel_suspend_complete(struct drm_i915_private *dev_priv);
+static int intel_resume_prepare(struct drm_i915_private *dev_priv,
+                                bool rpm_resume);
+
 static int i915_drm_freeze(struct drm_device *dev)
 {
         struct drm_i915_private *dev_priv = dev->dev_private;
@@ -591,9 +599,7 @@ static int i915_drm_freeze(struct drm_device *dev)
         intel_uncore_forcewake_reset(dev, false);
         intel_opregion_fini(dev);
 
-        console_lock();
-        intel_fbdev_set_suspend(dev, FBINFO_STATE_SUSPENDED);
-        console_unlock();
+        intel_fbdev_set_suspend(dev, FBINFO_STATE_SUSPENDED, true);
 
         dev_priv->suspend_count++;
 
@@ -632,30 +638,20 @@ int i915_suspend(struct drm_device *dev, pm_message_t state)
         return 0;
 }
 
-void intel_console_resume(struct work_struct *work)
-{
-        struct drm_i915_private *dev_priv =
-                container_of(work, struct drm_i915_private,
-                             console_resume_work);
-        struct drm_device *dev = dev_priv->dev;
-
-        console_lock();
-        intel_fbdev_set_suspend(dev, FBINFO_STATE_RUNNING);
-        console_unlock();
-}
-
 static int i915_drm_thaw_early(struct drm_device *dev)
 {
         struct drm_i915_private *dev_priv = dev->dev_private;
+        int ret;
 
-        if (IS_HASWELL(dev) || IS_BROADWELL(dev))
-                hsw_disable_pc8(dev_priv);
+        ret = intel_resume_prepare(dev_priv, false);
+        if (ret)
+                DRM_ERROR("Resume prepare failed: %d,Continuing resume\n", ret);
 
         intel_uncore_early_sanitize(dev, true);
         intel_uncore_sanitize(dev);
         intel_power_domains_init_hw(dev_priv);
 
-        return 0;
+        return ret;
 }
 
 static int __i915_drm_thaw(struct drm_device *dev, bool restore_gtt_mappings)
@@ -714,17 +710,7 @@ static int __i915_drm_thaw(struct drm_device *dev, bool restore_gtt_mappings)
 
         intel_opregion_init(dev);
 
-        /*
-         * The console lock can be pretty contented on resume due
-         * to all the printk activity.  Try to keep it out of the hot
-         * path of resume if possible.
-         */
-        if (console_trylock()) {
-                intel_fbdev_set_suspend(dev, FBINFO_STATE_RUNNING);
-                console_unlock();
-        } else {
-                schedule_work(&dev_priv->console_resume_work);
-        }
+        intel_fbdev_set_suspend(dev, FBINFO_STATE_RUNNING, false);
 
         mutex_lock(&dev_priv->modeset_restore_lock);
         dev_priv->modeset_restore = MODESET_DONE;
@@ -858,7 +844,13 @@ int i915_reset(struct drm_device *dev)
                         !dev_priv->ums.mm_suspended) {
                 dev_priv->ums.mm_suspended = 0;
 
+                /* Used to prevent gem_check_wedged returning -EAGAIN during gpu reset */
+                dev_priv->gpu_error.reload_in_reset = true;
+
                 ret = i915_gem_init_hw(dev);
+
+                dev_priv->gpu_error.reload_in_reset = false;
+
                 mutex_unlock(&dev->struct_mutex);
                 if (ret) {
                         DRM_ERROR("Failed hw init on reset %d\n", ret);
@@ -879,8 +871,6 @@ int i915_reset(struct drm_device *dev)
                  */
                 if (INTEL_INFO(dev)->gen > 5)
                         intel_reset_gt_powersave(dev);
-
-                intel_hpd_init(dev);
         } else {
                 mutex_unlock(&dev->struct_mutex);
         }
@@ -941,6 +931,7 @@ static int i915_pm_suspend_late(struct device *dev)
         struct pci_dev *pdev = to_pci_dev(dev);
         struct drm_device *drm_dev = pci_get_drvdata(pdev);
         struct drm_i915_private *dev_priv = drm_dev->dev_private;
+        int ret;
 
         /*
          * We have a suspedn ordering issue with the snd-hda driver also
@@ -954,13 +945,16 @@ static int i915_pm_suspend_late(struct device *dev)
         if (drm_dev->switch_power_state == DRM_SWITCH_POWER_OFF)
                 return 0;
 
-        if (IS_HASWELL(drm_dev) || IS_BROADWELL(drm_dev))
-                hsw_enable_pc8(dev_priv);
+        ret = intel_suspend_complete(dev_priv);
 
-        pci_disable_device(pdev);
-        pci_set_power_state(pdev, PCI_D3hot);
+        if (ret)
+                DRM_ERROR("Suspend complete failed: %d\n", ret);
+        else {
+                pci_disable_device(pdev);
+                pci_set_power_state(pdev, PCI_D3hot);
+        }
 
-        return 0;
+        return ret;
 }
 
 static int i915_pm_resume_early(struct device *dev)
@@ -1016,23 +1010,26 @@ static int i915_pm_poweroff(struct device *dev)
         return i915_drm_freeze(drm_dev);
 }
 
-static int hsw_runtime_suspend(struct drm_i915_private *dev_priv)
+static int hsw_suspend_complete(struct drm_i915_private *dev_priv)
 {
         hsw_enable_pc8(dev_priv);
 
         return 0;
 }
 
-static int snb_runtime_resume(struct drm_i915_private *dev_priv)
+static int snb_resume_prepare(struct drm_i915_private *dev_priv,
+                                bool rpm_resume)
 {
         struct drm_device *dev = dev_priv->dev;
 
-        intel_init_pch_refclk(dev);
+        if (rpm_resume)
+                intel_init_pch_refclk(dev);
 
         return 0;
 }
 
-static int hsw_runtime_resume(struct drm_i915_private *dev_priv)
+static int hsw_resume_prepare(struct drm_i915_private *dev_priv,
+                                bool rpm_resume)
 {
         hsw_disable_pc8(dev_priv);
 
@@ -1328,7 +1325,7 @@ static void vlv_check_no_gt_access(struct drm_i915_private *dev_priv)
         I915_WRITE(VLV_GTLC_PW_STATUS, VLV_GTLC_ALLOWWAKEERR);
 }
 
-static int vlv_runtime_suspend(struct drm_i915_private *dev_priv)
+static int vlv_suspend_complete(struct drm_i915_private *dev_priv)
 {
         u32 mask;
         int err;
@@ -1368,7 +1365,8 @@ err1:
         return err;
 }
 
-static int vlv_runtime_resume(struct drm_i915_private *dev_priv)
+static int vlv_resume_prepare(struct drm_i915_private *dev_priv,
+                                bool rpm_resume)
 {
         struct drm_device *dev = dev_priv->dev;
         int err;
@@ -1393,8 +1391,10 @@ static int vlv_runtime_resume(struct drm_i915_private *dev_priv)
 
         vlv_check_no_gt_access(dev_priv);
 
-        intel_init_clock_gating(dev);
-        i915_gem_restore_fences(dev);
+        if (rpm_resume) {
+                intel_init_clock_gating(dev);
+                i915_gem_restore_fences(dev);
+        }
 
         return ret;
 }
@@ -1409,7 +1409,9 @@ static int intel_runtime_suspend(struct device *device)
         if (WARN_ON_ONCE(!(dev_priv->rps.enabled && intel_enable_rc6(dev))))
                 return -ENODEV;
 
-        WARN_ON(!HAS_RUNTIME_PM(dev));
+        if (WARN_ON_ONCE(!HAS_RUNTIME_PM(dev)))
+                return -ENODEV;
+
         assert_force_wake_inactive(dev_priv);
 
         DRM_DEBUG_KMS("Suspending device\n");
@@ -1446,17 +1448,7 @@ static int intel_runtime_suspend(struct device *device)
         cancel_work_sync(&dev_priv->rps.work);
         intel_runtime_pm_disable_interrupts(dev);
 
-        if (IS_GEN6(dev)) {
-                ret = 0;
-        } else if (IS_HASWELL(dev) || IS_BROADWELL(dev)) {
-                ret = hsw_runtime_suspend(dev_priv);
-        } else if (IS_VALLEYVIEW(dev)) {
-                ret = vlv_runtime_suspend(dev_priv);
-        } else {
-                ret = -ENODEV;
-                WARN_ON(1);
-        }
-
+        ret = intel_suspend_complete(dev_priv);
         if (ret) {
                 DRM_ERROR("Runtime suspend failed, disabling it (%d)\n", ret);
                 intel_runtime_pm_restore_interrupts(dev);
@@ -1468,13 +1460,29 @@ static int intel_runtime_suspend(struct device *device)
         dev_priv->pm.suspended = true;
 
         /*
-         * current versions of firmware which depend on this opregion
-         * notification have repurposed the D1 definition to mean
-         * "runtime suspended" vs. what you would normally expect (D3)
-         * to distinguish it from notifications that might be sent
-         * via the suspend path.
+         * FIXME: We really should find a document that references the arguments
+         * used below!
          */
-        intel_opregion_notify_adapter(dev, PCI_D1);
+        if (IS_HASWELL(dev)) {
+                /*
+                 * current versions of firmware which depend on this opregion
+                 * notification have repurposed the D1 definition to mean
+                 * "runtime suspended" vs. what you would normally expect (D3)
+                 * to distinguish it from notifications that might be sent via
+                 * the suspend path.
+                 */
+                intel_opregion_notify_adapter(dev, PCI_D1);
+        } else {
+                /*
+                 * On Broadwell, if we use PCI_D1 the PCH DDI ports will stop
+                 * being detected, and the call we do at intel_runtime_resume()
+                 * won't be able to restore them. Since PCI_D3hot matches the
+                 * actual specification and appears to be working, use it. Let's
+                 * assume the other non-Haswell platforms will stay the same as
+                 * Broadwell.
+                 */
+                intel_opregion_notify_adapter(dev, PCI_D3hot);
+        }
 
         DRM_DEBUG_KMS("Device suspended\n");
         return 0;
@@ -1487,24 +1495,15 @@ static int intel_runtime_resume(struct device *device)
         struct drm_i915_private *dev_priv = dev->dev_private;
         int ret;
 
-        WARN_ON(!HAS_RUNTIME_PM(dev));
+        if (WARN_ON_ONCE(!HAS_RUNTIME_PM(dev)))
+                return -ENODEV;
 
         DRM_DEBUG_KMS("Resuming device\n");
 
         intel_opregion_notify_adapter(dev, PCI_D0);
         dev_priv->pm.suspended = false;
 
-        if (IS_GEN6(dev)) {
-                ret = snb_runtime_resume(dev_priv);
-        } else if (IS_HASWELL(dev) || IS_BROADWELL(dev)) {
-                ret = hsw_runtime_resume(dev_priv);
-        } else if (IS_VALLEYVIEW(dev)) {
-                ret = vlv_runtime_resume(dev_priv);
-        } else {
-                WARN_ON(1);
-                ret = -ENODEV;
-        }
-
+        ret = intel_resume_prepare(dev_priv, true);
         /*
          * No point of rolling back things in case of an error, as the best
          * we can do is to hope that things will still work (and disable RPM).
@@ -1523,6 +1522,48 @@ static int intel_runtime_resume(struct device *device)
         return ret;
 }
 
+/*
+ * This function implements common functionality of runtime and system
+ * suspend sequence.
+ */
+static int intel_suspend_complete(struct drm_i915_private *dev_priv)
+{
+        struct drm_device *dev = dev_priv->dev;
+        int ret;
+
+        if (IS_HASWELL(dev) || IS_BROADWELL(dev))
+                ret = hsw_suspend_complete(dev_priv);
+        else if (IS_VALLEYVIEW(dev))
+                ret = vlv_suspend_complete(dev_priv);
+        else
+                ret = 0;
+
+        return ret;
+}
+
+/*
+ * This function implements common functionality of runtime and system
+ * resume sequence. Variable rpm_resume used for implementing different
+ * code paths.
+ */
+static int intel_resume_prepare(struct drm_i915_private *dev_priv,
+                                bool rpm_resume)
+{
+        struct drm_device *dev = dev_priv->dev;
+        int ret;
+
+        if (IS_GEN6(dev))
+                ret = snb_resume_prepare(dev_priv, rpm_resume);
+        else if (IS_HASWELL(dev) || IS_BROADWELL(dev))
+                ret = hsw_resume_prepare(dev_priv, rpm_resume);
+        else if (IS_VALLEYVIEW(dev))
+                ret = vlv_resume_prepare(dev_priv, rpm_resume);
+        else
+                ret = 0;
+
+        return ret;
+}
+
 static const struct dev_pm_ops i915_pm_ops = {
         .suspend = i915_pm_suspend,
         .suspend_late = i915_pm_suspend_late,
@@ -1572,6 +1613,7 @@ static struct drm_driver driver = {
         .lastclose = i915_driver_lastclose,
         .preclose = i915_driver_preclose,
         .postclose = i915_driver_postclose,
+        .set_busid = drm_pci_set_busid,
 
         /* Used in place of i915_pm_ops for non-DRIVER_MODESET */
         .suspend = i915_suspend,
@@ -1663,6 +1705,8 @@ static void __exit i915_exit(void)
 module_init(i915_init);
 module_exit(i915_exit);
 
-MODULE_AUTHOR(DRIVER_AUTHOR);
+MODULE_AUTHOR("Tungsten Graphics, Inc.");
+MODULE_AUTHOR("Intel Corporation");
+
 MODULE_DESCRIPTION(DRIVER_DESC);
 MODULE_LICENSE("GPL and additional rights");
diff --git a/drivers/gpu/drm/i915/i915_drv.h b/drivers/gpu/drm/i915/i915_drv.h
index 3524306d8cfb..16a6f6d187a1 100644
--- a/drivers/gpu/drm/i915/i915_drv.h
+++ b/drivers/gpu/drm/i915/i915_drv.h
@@ -35,11 +35,15 @@
 #include "i915_reg.h"
 #include "intel_bios.h"
 #include "intel_ringbuffer.h"
+#include "intel_lrc.h"
 #include "i915_gem_gtt.h"
+#include "i915_gem_render_state.h"
 #include <linux/io-mapping.h>
 #include <linux/i2c.h>
 #include <linux/i2c-algo-bit.h>
 #include <drm/intel-gtt.h>
+#include <drm/drm_legacy.h> /* for struct drm_dma_handle */
+#include <drm/drm_gem.h>
 #include <linux/backlight.h>
 #include <linux/hashtable.h>
 #include <linux/intel-iommu.h>
@@ -49,11 +53,9 @@
 /* General customization:
  */
 
-#define DRIVER_AUTHOR           "Tungsten Graphics, Inc."
-
 #define DRIVER_NAME             "i915"
 #define DRIVER_DESC             "Intel Graphics"
-#define DRIVER_DATE             "20140725"
+#define DRIVER_DATE             "20140905"
 
 enum pipe {
         INVALID_PIPE = -1,
@@ -162,7 +164,10 @@ enum hpd_pin {
          I915_GEM_DOMAIN_INSTRUCTION | \
          I915_GEM_DOMAIN_VERTEX)
 
-#define for_each_pipe(p) for ((p) = 0; (p) < INTEL_INFO(dev)->num_pipes; (p)++)
+#define for_each_pipe(__dev_priv, __p) \
+        for ((__p) = 0; (__p) < INTEL_INFO(__dev_priv)->num_pipes; (__p)++)
+#define for_each_plane(pipe, p) \
+        for ((p) = 0; (p) < INTEL_INFO(dev)->num_sprites[(pipe)] + 1; (p)++)
 #define for_each_sprite(p, s) for ((s) = 0; (s) < INTEL_INFO(dev)->num_sprites[(p)]; (s)++)
 
 #define for_each_crtc(dev, crtc) \
@@ -171,6 +176,11 @@ enum hpd_pin {
 #define for_each_intel_crtc(dev, intel_crtc) \
         list_for_each_entry(intel_crtc, &dev->mode_config.crtc_list, base.head)
 
+#define for_each_intel_encoder(dev, intel_encoder)              \
+        list_for_each_entry(intel_encoder,                      \
+                            &(dev)->mode_config.encoder_list,   \
+                            base.head)
+
 #define for_each_encoder_on_crtc(dev, __crtc, intel_encoder) \
         list_for_each_entry((intel_encoder), &(dev)->mode_config.encoder_list, base.head) \
                 if ((intel_encoder)->base.crtc == (__crtc))
@@ -198,10 +208,13 @@ enum intel_dpll_id {
 #define I915_NUM_PLLS 2
 
 struct intel_dpll_hw_state {
+        /* i9xx, pch plls */
         uint32_t dpll;
         uint32_t dpll_md;
         uint32_t fp0;
         uint32_t fp1;
+
+        /* hsw, bdw */
         uint32_t wrpll;
 };
 
@@ -277,8 +290,10 @@ struct intel_opregion {
 struct intel_overlay;
 struct intel_overlay_error_state;
 
+struct drm_local_map;
+
 struct drm_i915_master_private {
-        drm_local_map_t *sarea;
+        struct drm_local_map *sarea;
         struct _drm_i915_sarea *sarea_priv;
 };
 #define I915_FENCE_REG_NONE -1
@@ -388,6 +403,7 @@ struct drm_i915_error_state {
                 pid_t pid;
                 char comm[TASK_COMM_LEN];
         } ring[I915_NUM_RINGS];
+
         struct drm_i915_error_buffer {
                 u32 size;
                 u32 name;
@@ -406,6 +422,7 @@ struct drm_i915_error_state {
         } **active_bo, **pinned_bo;
 
         u32 *active_bo_count, *pinned_bo_count;
+        u32 vm_count;
 };
 
 struct intel_connector;
@@ -551,6 +568,7 @@ struct intel_uncore {
 
 struct intel_device_info {
         u32 display_mmio_offset;
+        u16 device_id;
         u8 num_pipes:3;
         u8 num_sprites[I915_MAX_PIPES];
         u8 gen;
@@ -615,13 +633,21 @@ struct intel_context {
         uint8_t remap_slice;
         struct drm_i915_file_private *file_priv;
         struct i915_ctx_hang_stats hang_stats;
-        struct i915_address_space *vm;
+        struct i915_hw_ppgtt *ppgtt;
 
+        /* Legacy ring buffer submission */
         struct {
                 struct drm_i915_gem_object *rcs_state;
                 bool initialized;
         } legacy_hw_ctx;
 
+        /* Execlists */
+        bool rcs_initialized;
+        struct {
+                struct drm_i915_gem_object *state;
+                struct intel_ringbuffer *ringbuf;
+        } engine[I915_NUM_RINGS];
+
         struct list_head link;
 };
 
@@ -635,6 +661,8 @@ struct i915_fbc {
         struct drm_mm_node compressed_fb;
         struct drm_mm_node *compressed_llb;
 
+        bool false_color;
+
         struct intel_fbc_work {
                 struct delayed_work work;
                 struct drm_crtc *crtc;
@@ -688,6 +716,7 @@ enum intel_sbi_destination {
 #define QUIRK_LVDS_SSC_DISABLE (1<<1)
 #define QUIRK_INVERT_BRIGHTNESS (1<<2)
 #define QUIRK_BACKLIGHT_PRESENT (1<<3)
+#define QUIRK_PIPEB_FORCE (1<<4)
 
 struct intel_fbdev;
 struct intel_fbc_work;
@@ -1147,6 +1176,7 @@ struct i915_gem_mm {
 };
 
 struct drm_i915_error_state_buf {
+        struct drm_i915_private *i915;
         unsigned bytes;
         unsigned size;
         int err;
@@ -1219,6 +1249,9 @@ struct i915_gpu_error {
 
         /* For missed irq/seqno simulation. */
         unsigned int test_irq_rings;
+
+        /* Used to prevent gem_check_wedged returning -EAGAIN during gpu reset   */
+        bool reload_in_reset;
 };
 
 enum modeset_restore {
@@ -1228,6 +1261,12 @@ enum modeset_restore {
 };
 
 struct ddi_vbt_port_info {
+        /*
+         * This is an index in the HDMI/DVI DDI buffer translation table.
+         * The special value HDMI_LEVEL_SHIFT_UNKNOWN means the VBT didn't
+         * populate this field.
+         */
+#define HDMI_LEVEL_SHIFT_UNKNOWN        0xff
         uint8_t hdmi_level_shift;
 
         uint8_t supports_dvi:1;
@@ -1421,7 +1460,7 @@ struct drm_i915_private {
         struct drm_i915_gem_object *semaphore_obj;
         uint32_t last_seqno, next_seqno;
 
-        drm_dma_handle_t *status_page_dmah;
+        struct drm_dma_handle *status_page_dmah;
         struct resource mch_res;
 
         /* protects the irq masks */
@@ -1475,6 +1514,9 @@ struct drm_i915_private {
         /* LVDS info */
         bool no_aux_handshake;
 
+        /* protects panel power sequencer state */
+        struct mutex pps_mutex;
+
         struct drm_i915_fence_reg fence_regs[I915_MAX_NUM_FENCES]; /* assume 965 */
         int fence_reg_start; /* 4 if userland hasn't ioctl'd us yet */
         int num_fence_regs; /* 8 on pre-965, 16 otherwise */
@@ -1526,6 +1568,20 @@ struct drm_i915_private {
         struct intel_shared_dpll shared_dplls[I915_NUM_PLLS];
         int dpio_phy_iosf_port[I915_NUM_PHYS_VLV];
 
+        /*
+         * workarounds are currently applied at different places and
+         * changes are being done to consolidate them so exact count is
+         * not clear at this point, use a max value for now.
+         */
+#define I915_MAX_WA_REGS  16
+        struct {
+                u32 addr;
+                u32 value;
+                /* bitmask representing WA bits */
+                u32 mask;
+        } intel_wa_regs[I915_MAX_WA_REGS];
+        u32 num_wa_regs;
+
         /* Reclocking support */
         bool render_reclock_avail;
         bool lvds_downclock_avail;
@@ -1561,14 +1617,9 @@ struct drm_i915_private {
 #ifdef CONFIG_DRM_I915_FBDEV
         /* list of fbdev register on this device */
         struct intel_fbdev *fbdev;
+        struct work_struct fbdev_suspend_work;
 #endif
 
-        /*
-         * The console may be contended at resume, but we don't
-         * want it to block on it.
-         */
-        struct work_struct console_resume_work;
-
         struct drm_property *broadcast_rgb_property;
         struct drm_property *force_audio_property;
 
@@ -1614,12 +1665,28 @@ struct drm_i915_private {
          */
         struct workqueue_struct *dp_wq;
 
+        uint32_t bios_vgacntr;
+
         /* Old dri1 support infrastructure, beware the dragons ya fools entering
          * here! */
         struct i915_dri1_state dri1;
         /* Old ums support infrastructure, same warning applies. */
         struct i915_ums_state ums;
 
+        /* Abstract the submission mechanism (legacy ringbuffer or execlists) away */
+        struct {
+                int (*do_execbuf)(struct drm_device *dev, struct drm_file *file,
+                                  struct intel_engine_cs *ring,
+                                  struct intel_context *ctx,
+                                  struct drm_i915_gem_execbuffer2 *args,
+                                  struct list_head *vmas,
+                                  struct drm_i915_gem_object *batch_obj,
+                                  u64 exec_start, u32 flags);
+                int (*init_rings)(struct drm_device *dev);
+                void (*cleanup_ring)(struct intel_engine_cs *ring);
+                void (*stop_ring)(struct intel_engine_cs *ring);
+        } gt;
+
         /*
          * NOTE: This is the dri1/ums dungeon, don't add stuff here. Your patch
          * will be rejected. Instead look for a better place.
@@ -1761,13 +1828,6 @@ struct drm_i915_gem_object {
          * Only honoured if hardware has relevant pte bit
          */
         unsigned long gt_ro:1;
-
-        /*
-         * Is the GPU currently using a fence to access this buffer,
-         */
-        unsigned int pending_fenced_gpu_access:1;
-        unsigned int fenced_gpu_access:1;
-
         unsigned int cache_level:3;
 
         unsigned int has_aliasing_ppgtt_mapping:1;
@@ -1805,7 +1865,7 @@ struct drm_i915_gem_object {
         struct drm_file *pin_filp;
 
         /** for phy allocated objects */
-        drm_dma_handle_t *phys_handle;
+        struct drm_dma_handle *phys_handle;
 
         union {
                 struct i915_gem_userptr {
@@ -1971,51 +2031,63 @@ struct drm_i915_cmd_table {
         int count;
 };
 
-#define INTEL_INFO(dev) (&to_i915(dev)->info)
-
-#define IS_I830(dev)            ((dev)->pdev->device == 0x3577)
-#define IS_845G(dev)            ((dev)->pdev->device == 0x2562)
+/* Note that the (struct drm_i915_private *) cast is just to shut up gcc. */
+#define __I915__(p) ({ \
+        struct drm_i915_private *__p; \
+        if (__builtin_types_compatible_p(typeof(*p), struct drm_i915_private)) \
+                __p = (struct drm_i915_private *)p; \
+        else if (__builtin_types_compatible_p(typeof(*p), struct drm_device)) \
+                __p = to_i915((struct drm_device *)p); \
+        else \
+                BUILD_BUG(); \
+        __p; \
+})
+#define INTEL_INFO(p)   (&__I915__(p)->info)
+#define INTEL_DEVID(p)  (INTEL_INFO(p)->device_id)
+
+#define IS_I830(dev)            (INTEL_DEVID(dev) == 0x3577)
+#define IS_845G(dev)            (INTEL_DEVID(dev) == 0x2562)
 #define IS_I85X(dev)            (INTEL_INFO(dev)->is_i85x)
-#define IS_I865G(dev)           ((dev)->pdev->device == 0x2572)
+#define IS_I865G(dev)           (INTEL_DEVID(dev) == 0x2572)
 #define IS_I915G(dev)           (INTEL_INFO(dev)->is_i915g)
-#define IS_I915GM(dev)          ((dev)->pdev->device == 0x2592)
-#define IS_I945G(dev)           ((dev)->pdev->device == 0x2772)
+#define IS_I915GM(dev)          (INTEL_DEVID(dev) == 0x2592)
+#define IS_I945G(dev)           (INTEL_DEVID(dev) == 0x2772)
 #define IS_I945GM(dev)          (INTEL_INFO(dev)->is_i945gm)
 #define IS_BROADWATER(dev)      (INTEL_INFO(dev)->is_broadwater)
 #define IS_CRESTLINE(dev)       (INTEL_INFO(dev)->is_crestline)
-#define IS_GM45(dev)            ((dev)->pdev->device == 0x2A42)
+#define IS_GM45(dev)            (INTEL_DEVID(dev) == 0x2A42)
 #define IS_G4X(dev)             (INTEL_INFO(dev)->is_g4x)
-#define IS_PINEVIEW_G(dev)      ((dev)->pdev->device == 0xa001)
-#define IS_PINEVIEW_M(dev)      ((dev)->pdev->device == 0xa011)
+#define IS_PINEVIEW_G(dev)      (INTEL_DEVID(dev) == 0xa001)
+#define IS_PINEVIEW_M(dev)      (INTEL_DEVID(dev) == 0xa011)
 #define IS_PINEVIEW(dev)        (INTEL_INFO(dev)->is_pineview)
 #define IS_G33(dev)             (INTEL_INFO(dev)->is_g33)
-#define IS_IRONLAKE_M(dev)      ((dev)->pdev->device == 0x0046)
+#define IS_IRONLAKE_M(dev)      (INTEL_DEVID(dev) == 0x0046)
 #define IS_IVYBRIDGE(dev)       (INTEL_INFO(dev)->is_ivybridge)
-#define IS_IVB_GT1(dev)         ((dev)->pdev->device == 0x0156 || \
-                                 (dev)->pdev->device == 0x0152 || \
-                                 (dev)->pdev->device == 0x015a)
-#define IS_SNB_GT1(dev)         ((dev)->pdev->device == 0x0102 || \
-                                 (dev)->pdev->device == 0x0106 || \
-                                 (dev)->pdev->device == 0x010A)
+#define IS_IVB_GT1(dev)         (INTEL_DEVID(dev) == 0x0156 || \
+                                 INTEL_DEVID(dev) == 0x0152 || \
+                                 INTEL_DEVID(dev) == 0x015a)
+#define IS_SNB_GT1(dev)         (INTEL_DEVID(dev) == 0x0102 || \
+                                 INTEL_DEVID(dev) == 0x0106 || \
+                                 INTEL_DEVID(dev) == 0x010A)
 #define IS_VALLEYVIEW(dev)      (INTEL_INFO(dev)->is_valleyview)
 #define IS_CHERRYVIEW(dev)      (INTEL_INFO(dev)->is_valleyview && IS_GEN8(dev))
 #define IS_HASWELL(dev) (INTEL_INFO(dev)->is_haswell)
 #define IS_BROADWELL(dev)       (!INTEL_INFO(dev)->is_valleyview && IS_GEN8(dev))
 #define IS_MOBILE(dev)          (INTEL_INFO(dev)->is_mobile)
 #define IS_HSW_EARLY_SDV(dev)   (IS_HASWELL(dev) && \
-                                 ((dev)->pdev->device & 0xFF00) == 0x0C00)
+                                 (INTEL_DEVID(dev) & 0xFF00) == 0x0C00)
 #define IS_BDW_ULT(dev)         (IS_BROADWELL(dev) && \
-                                 (((dev)->pdev->device & 0xf) == 0x2  || \
-                                 ((dev)->pdev->device & 0xf) == 0x6 || \
-                                 ((dev)->pdev->device & 0xf) == 0xe))
+                                 ((INTEL_DEVID(dev) & 0xf) == 0x2  || \
+                                 (INTEL_DEVID(dev) & 0xf) == 0x6 || \
+                                 (INTEL_DEVID(dev) & 0xf) == 0xe))
 #define IS_HSW_ULT(dev)         (IS_HASWELL(dev) && \
-                                 ((dev)->pdev->device & 0xFF00) == 0x0A00)
+                                 (INTEL_DEVID(dev) & 0xFF00) == 0x0A00)
 #define IS_ULT(dev)             (IS_HSW_ULT(dev) || IS_BDW_ULT(dev))
 #define IS_HSW_GT3(dev)         (IS_HASWELL(dev) && \
-                                 ((dev)->pdev->device & 0x00F0) == 0x0020)
+                                 (INTEL_DEVID(dev) & 0x00F0) == 0x0020)
 /* ULX machines are also considered ULT. */
-#define IS_HSW_ULX(dev)         ((dev)->pdev->device == 0x0A0E || \
-                                 (dev)->pdev->device == 0x0A1E)
+#define IS_HSW_ULX(dev)         (INTEL_DEVID(dev) == 0x0A0E || \
+                                 INTEL_DEVID(dev) == 0x0A1E)
 #define IS_PRELIMINARY_HW(intel_info) ((intel_info)->is_preliminary)
 
 /*
@@ -2047,10 +2119,11 @@ struct drm_i915_cmd_table {
 #define I915_NEED_GFX_HWS(dev)  (INTEL_INFO(dev)->need_gfx_hws)
 
 #define HAS_HW_CONTEXTS(dev)    (INTEL_INFO(dev)->gen >= 6)
+#define HAS_LOGICAL_RING_CONTEXTS(dev)  (INTEL_INFO(dev)->gen >= 8)
 #define HAS_ALIASING_PPGTT(dev) (INTEL_INFO(dev)->gen >= 6)
 #define HAS_PPGTT(dev)          (INTEL_INFO(dev)->gen >= 7 && !IS_GEN8(dev))
-#define USES_PPGTT(dev)         intel_enable_ppgtt(dev, false)
-#define USES_FULL_PPGTT(dev)    intel_enable_ppgtt(dev, true)
+#define USES_PPGTT(dev)         (i915.enable_ppgtt)
+#define USES_FULL_PPGTT(dev)    (i915.enable_ppgtt == 2)
 
 #define HAS_OVERLAY(dev)                (INTEL_INFO(dev)->has_overlay)
 #define OVERLAY_NEEDS_PHYSICAL(dev)     (INTEL_INFO(dev)->overlay_needs_physical)
@@ -2134,6 +2207,7 @@ struct i915_params {
         int enable_rc6;
         int enable_fbc;
         int enable_ppgtt;
+        int enable_execlists;
         int enable_psr;
         unsigned int preliminary_hw_support;
         int disable_power_well;
@@ -2180,8 +2254,6 @@ extern void i915_update_gfx_val(struct drm_i915_private *dev_priv);
 int vlv_force_gfx_clock(struct drm_i915_private *dev_priv, bool on);
 void intel_hpd_cancel_work(struct drm_i915_private *dev_priv);
 
-extern void intel_console_resume(struct work_struct *work);
-
 /* i915_irq.c */
 void i915_queue_hangcheck(struct drm_device *dev);
 __printf(3, 4)
@@ -2229,6 +2301,20 @@ int i915_gem_set_domain_ioctl(struct drm_device *dev, void *data,
                               struct drm_file *file_priv);
 int i915_gem_sw_finish_ioctl(struct drm_device *dev, void *data,
                              struct drm_file *file_priv);
+void i915_gem_execbuffer_move_to_active(struct list_head *vmas,
+                                        struct intel_engine_cs *ring);
+void i915_gem_execbuffer_retire_commands(struct drm_device *dev,
+                                         struct drm_file *file,
+                                         struct intel_engine_cs *ring,
+                                         struct drm_i915_gem_object *obj);
+int i915_gem_ringbuffer_submission(struct drm_device *dev,
+                                   struct drm_file *file,
+                                   struct intel_engine_cs *ring,
+                                   struct intel_context *ctx,
+                                   struct drm_i915_gem_execbuffer2 *args,
+                                   struct list_head *vmas,
+                                   struct drm_i915_gem_object *batch_obj,
+                                   u64 exec_start, u32 flags);
 int i915_gem_execbuffer(struct drm_device *dev, void *data,
                         struct drm_file *file_priv);
 int i915_gem_execbuffer2(struct drm_device *dev, void *data,
@@ -2263,6 +2349,12 @@ int i915_gem_get_aperture_ioctl(struct drm_device *dev, void *data,
 int i915_gem_wait_ioctl(struct drm_device *dev, void *data,
                         struct drm_file *file_priv);
 void i915_gem_load(struct drm_device *dev);
+unsigned long i915_gem_shrink(struct drm_i915_private *dev_priv,
+                              long target,
+                              unsigned flags);
+#define I915_SHRINK_PURGEABLE 0x1
+#define I915_SHRINK_UNBOUND 0x2
+#define I915_SHRINK_BOUND 0x4
 void *i915_gem_object_alloc(struct drm_device *dev);
 void i915_gem_object_free(struct drm_i915_gem_object *obj);
 void i915_gem_object_init(struct drm_i915_gem_object *obj,
@@ -2381,6 +2473,7 @@ void i915_gem_reset(struct drm_device *dev);
 bool i915_gem_clflush_object(struct drm_i915_gem_object *obj, bool force);
 int __must_check i915_gem_object_finish_gpu(struct drm_i915_gem_object *obj);
 int __must_check i915_gem_init(struct drm_device *dev);
+int i915_gem_init_rings(struct drm_device *dev);
 int __must_check i915_gem_init_hw(struct drm_device *dev);
 int i915_gem_l3_remap(struct intel_engine_cs *ring, int slice);
 void i915_gem_init_swizzling(struct drm_device *dev);
@@ -2451,7 +2544,7 @@ static inline bool i915_gem_obj_is_pinned(struct drm_i915_gem_object *obj) {
 }
 
 /* Some GGTT VM helpers */
-#define obj_to_ggtt(obj) \
+#define i915_obj_to_ggtt(obj) \
         (&((struct drm_i915_private *)(obj)->base.dev->dev_private)->gtt.base)
 static inline bool i915_is_ggtt(struct i915_address_space *vm)
 {
@@ -2460,21 +2553,30 @@ static inline bool i915_is_ggtt(struct i915_address_space *vm)
         return vm == ggtt;
 }
 
+static inline struct i915_hw_ppgtt *
+i915_vm_to_ppgtt(struct i915_address_space *vm)
+{
+        WARN_ON(i915_is_ggtt(vm));
+
+        return container_of(vm, struct i915_hw_ppgtt, base);
+}
+
+
 static inline bool i915_gem_obj_ggtt_bound(struct drm_i915_gem_object *obj)
 {
-        return i915_gem_obj_bound(obj, obj_to_ggtt(obj));
+        return i915_gem_obj_bound(obj, i915_obj_to_ggtt(obj));
 }
 
 static inline unsigned long
 i915_gem_obj_ggtt_offset(struct drm_i915_gem_object *obj)
 {
-        return i915_gem_obj_offset(obj, obj_to_ggtt(obj));
+        return i915_gem_obj_offset(obj, i915_obj_to_ggtt(obj));
 }
 
 static inline unsigned long
 i915_gem_obj_ggtt_size(struct drm_i915_gem_object *obj)
 {
-        return i915_gem_obj_size(obj, obj_to_ggtt(obj));
+        return i915_gem_obj_size(obj, i915_obj_to_ggtt(obj));
 }
 
 static inline int __must_check
@@ -2482,7 +2584,8 @@ i915_gem_obj_ggtt_pin(struct drm_i915_gem_object *obj,
                       uint32_t alignment,
                       unsigned flags)
 {
-        return i915_gem_object_pin(obj, obj_to_ggtt(obj), alignment, flags | PIN_GLOBAL);
+        return i915_gem_object_pin(obj, i915_obj_to_ggtt(obj),
+                                   alignment, flags | PIN_GLOBAL);
 }
 
 static inline int
@@ -2494,7 +2597,6 @@ i915_gem_object_ggtt_unbind(struct drm_i915_gem_object *obj)
 void i915_gem_object_ggtt_unpin(struct drm_i915_gem_object *obj);
 
 /* i915_gem_context.c */
-#define ctx_to_ppgtt(ctx) container_of((ctx)->vm, struct i915_hw_ppgtt, base)
 int __must_check i915_gem_context_init(struct drm_device *dev);
 void i915_gem_context_fini(struct drm_device *dev);
 void i915_gem_context_reset(struct drm_device *dev);
@@ -2506,6 +2608,8 @@ int i915_switch_context(struct intel_engine_cs *ring,
 struct intel_context *
 i915_gem_context_get(struct drm_i915_file_private *file_priv, u32 id);
 void i915_gem_context_free(struct kref *ctx_ref);
+struct drm_i915_gem_object *
+i915_gem_alloc_context_obj(struct drm_device *dev, size_t size);
 static inline void i915_gem_context_reference(struct intel_context *ctx)
 {
         kref_get(&ctx->ref);
@@ -2526,8 +2630,6 @@ int i915_gem_context_create_ioctl(struct drm_device *dev, void *data,
 int i915_gem_context_destroy_ioctl(struct drm_device *dev, void *data,
                                    struct drm_file *file);
 
-/* i915_gem_render_state.c */
-int i915_gem_render_state_init(struct intel_engine_cs *ring);
 /* i915_gem_evict.c */
 int __must_check i915_gem_evict_something(struct drm_device *dev,
                                           struct i915_address_space *vm,
@@ -2595,6 +2697,7 @@ void i915_error_printf(struct drm_i915_error_state_buf *e, const char *f, ...);
 int i915_error_state_to_str(struct drm_i915_error_state_buf *estr,
                             const struct i915_error_state_file_priv *error);
 int i915_error_state_buf_init(struct drm_i915_error_state_buf *eb,
+                              struct drm_i915_private *i915,
                               size_t count, loff_t pos);
 static inline void i915_error_state_buf_release(
         struct drm_i915_error_state_buf *eb)
@@ -2609,7 +2712,7 @@ void i915_error_state_put(struct i915_error_state_file_priv *error_priv);
 void i915_destroy_error_state(struct drm_device *dev);
 
 void i915_get_extra_instdone(struct drm_device *dev, uint32_t *instdone);
-const char *i915_cache_level_str(int type);
+const char *i915_cache_level_str(struct drm_i915_private *i915, int type);
 
 /* i915_cmd_parser.c */
 int i915_cmd_parser_get_version(void);
@@ -2701,6 +2804,7 @@ extern void intel_modeset_setup_hw_state(struct drm_device *dev,
 extern void i915_redisable_vga(struct drm_device *dev);
 extern void i915_redisable_vga_power_on(struct drm_device *dev);
 extern bool intel_fbc_enabled(struct drm_device *dev);
+extern void gen8_fbc_sw_flush(struct drm_device *dev, u32 value);
 extern void intel_disable_fbc(struct drm_device *dev);
 extern bool ironlake_set_drps(struct drm_device *dev, u8 val);
 extern void intel_init_pch_refclk(struct drm_device *dev);
diff --git a/drivers/gpu/drm/i915/i915_gem.c b/drivers/gpu/drm/i915/i915_gem.c
index ad55b06a3cb1..28f91df2604d 100644
--- a/drivers/gpu/drm/i915/i915_gem.c
+++ b/drivers/gpu/drm/i915/i915_gem.c
@@ -60,7 +60,6 @@ static unsigned long i915_gem_shrinker_scan(struct shrinker *shrinker,
 static int i915_gem_shrinker_oom(struct notifier_block *nb,
                                  unsigned long event,
                                  void *ptr);
-static unsigned long i915_gem_purge(struct drm_i915_private *dev_priv, long target);
 static unsigned long i915_gem_shrink_all(struct drm_i915_private *dev_priv);
 
 static bool cpu_cache_is_coherent(struct drm_device *dev,
@@ -1085,7 +1084,13 @@ i915_gem_check_wedge(struct i915_gpu_error *error,
                 if (i915_terminally_wedged(error))
                         return -EIO;
 
-                return -EAGAIN;
+                /*
+                 * Check if GPU Reset is in progress - we need intel_ring_begin
+                 * to work properly to reinit the hw state while the gpu is
+                 * still marked as reset-in-progress. Handle this with a flag.
+                 */
+                if (!error->reload_in_reset)
+                        return -EAGAIN;
         }
 
         return 0;
@@ -1735,7 +1740,11 @@ static int i915_gem_object_create_mmap_offset(struct drm_i915_gem_object *obj)
          * offsets on purgeable objects by truncating it and marking it purged,
          * which prevents userspace from ever using that object again.
          */
-        i915_gem_purge(dev_priv, obj->base.size >> PAGE_SHIFT);
+        i915_gem_shrink(dev_priv,
+                        obj->base.size >> PAGE_SHIFT,
+                        I915_SHRINK_BOUND |
+                        I915_SHRINK_UNBOUND |
+                        I915_SHRINK_PURGEABLE);
         ret = drm_gem_create_mmap_offset(&obj->base);
         if (ret != -ENOSPC)
                 goto out;
@@ -1932,12 +1941,11 @@ i915_gem_object_put_pages(struct drm_i915_gem_object *obj)
         return 0;
 }
 
-static unsigned long
-__i915_gem_shrink(struct drm_i915_private *dev_priv, long target,
-                  bool purgeable_only)
+unsigned long
+i915_gem_shrink(struct drm_i915_private *dev_priv,
+                long target, unsigned flags)
 {
-        struct list_head still_in_list;
-        struct drm_i915_gem_object *obj;
+        const bool purgeable_only = flags & I915_SHRINK_PURGEABLE;
         unsigned long count = 0;
 
         /*
@@ -1959,62 +1967,68 @@ __i915_gem_shrink(struct drm_i915_private *dev_priv, long target,
          * dev->struct_mutex and so we won't ever be able to observe an
          * object on the bound_list with a reference count equals 0.
          */
-        INIT_LIST_HEAD(&still_in_list);
-        while (count < target && !list_empty(&dev_priv->mm.unbound_list)) {
-                obj = list_first_entry(&dev_priv->mm.unbound_list,
-                                       typeof(*obj), global_list);
-                list_move_tail(&obj->global_list, &still_in_list);
+        if (flags & I915_SHRINK_UNBOUND) {
+                struct list_head still_in_list;
 
-                if (!i915_gem_object_is_purgeable(obj) && purgeable_only)
-                        continue;
+                INIT_LIST_HEAD(&still_in_list);
+                while (count < target && !list_empty(&dev_priv->mm.unbound_list)) {
+                        struct drm_i915_gem_object *obj;
 
-                drm_gem_object_reference(&obj->base);
+                        obj = list_first_entry(&dev_priv->mm.unbound_list,
+                                               typeof(*obj), global_list);
+                        list_move_tail(&obj->global_list, &still_in_list);
 
-                if (i915_gem_object_put_pages(obj) == 0)
-                        count += obj->base.size >> PAGE_SHIFT;
+                        if (!i915_gem_object_is_purgeable(obj) && purgeable_only)
+                                continue;
 
-                drm_gem_object_unreference(&obj->base);
+                        drm_gem_object_reference(&obj->base);
+
+                        if (i915_gem_object_put_pages(obj) == 0)
+                                count += obj->base.size >> PAGE_SHIFT;
+
+                        drm_gem_object_unreference(&obj->base);
+                }
+                list_splice(&still_in_list, &dev_priv->mm.unbound_list);
         }
-        list_splice(&still_in_list, &dev_priv->mm.unbound_list);
 
-        INIT_LIST_HEAD(&still_in_list);
-        while (count < target && !list_empty(&dev_priv->mm.bound_list)) {
-                struct i915_vma *vma, *v;
+        if (flags & I915_SHRINK_BOUND) {
+                struct list_head still_in_list;
 
-                obj = list_first_entry(&dev_priv->mm.bound_list,
-                                       typeof(*obj), global_list);
-                list_move_tail(&obj->global_list, &still_in_list);
+                INIT_LIST_HEAD(&still_in_list);
+                while (count < target && !list_empty(&dev_priv->mm.bound_list)) {
+                        struct drm_i915_gem_object *obj;
+                        struct i915_vma *vma, *v;
 
-                if (!i915_gem_object_is_purgeable(obj) && purgeable_only)
-                        continue;
+                        obj = list_first_entry(&dev_priv->mm.bound_list,
+                                               typeof(*obj), global_list);
+                        list_move_tail(&obj->global_list, &still_in_list);
 
-                drm_gem_object_reference(&obj->base);
+                        if (!i915_gem_object_is_purgeable(obj) && purgeable_only)
+                                continue;
 
-                list_for_each_entry_safe(vma, v, &obj->vma_list, vma_link)
-                        if (i915_vma_unbind(vma))
-                                break;
+                        drm_gem_object_reference(&obj->base);
 
-                if (i915_gem_object_put_pages(obj) == 0)
-                        count += obj->base.size >> PAGE_SHIFT;
+                        list_for_each_entry_safe(vma, v, &obj->vma_list, vma_link)
+                                if (i915_vma_unbind(vma))
+                                        break;
 
-                drm_gem_object_unreference(&obj->base);
+                        if (i915_gem_object_put_pages(obj) == 0)
+                                count += obj->base.size >> PAGE_SHIFT;
+
+                        drm_gem_object_unreference(&obj->base);
+                }
+                list_splice(&still_in_list, &dev_priv->mm.bound_list);
         }
-        list_splice(&still_in_list, &dev_priv->mm.bound_list);
 
         return count;
 }
 
 static unsigned long
-i915_gem_purge(struct drm_i915_private *dev_priv, long target)
-{
-        return __i915_gem_shrink(dev_priv, target, true);
-}
-
-static unsigned long
 i915_gem_shrink_all(struct drm_i915_private *dev_priv)
 {
         i915_gem_evict_everything(dev_priv->dev);
-        return __i915_gem_shrink(dev_priv, LONG_MAX, false);
+        return i915_gem_shrink(dev_priv, LONG_MAX,
+                               I915_SHRINK_BOUND | I915_SHRINK_UNBOUND);
 }
 
 static int
@@ -2061,7 +2075,11 @@ i915_gem_object_get_pages_gtt(struct drm_i915_gem_object *obj)
         for (i = 0; i < page_count; i++) {
                 page = shmem_read_mapping_page_gfp(mapping, i, gfp);
                 if (IS_ERR(page)) {
-                        i915_gem_purge(dev_priv, page_count);
+                        i915_gem_shrink(dev_priv,
+                                        page_count,
+                                        I915_SHRINK_BOUND |
+                                        I915_SHRINK_UNBOUND |
+                                        I915_SHRINK_PURGEABLE);
                         page = shmem_read_mapping_page_gfp(mapping, i, gfp);
                 }
                 if (IS_ERR(page)) {
@@ -2163,8 +2181,6 @@ static void
 i915_gem_object_move_to_active(struct drm_i915_gem_object *obj,
                                struct intel_engine_cs *ring)
 {
-        struct drm_device *dev = obj->base.dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
         u32 seqno = intel_ring_get_seqno(ring);
 
         BUG_ON(ring == NULL);
@@ -2183,19 +2199,6 @@ i915_gem_object_move_to_active(struct drm_i915_gem_object *obj,
         list_move_tail(&obj->ring_list, &ring->active_list);
 
         obj->last_read_seqno = seqno;
-
-        if (obj->fenced_gpu_access) {
-                obj->last_fenced_seqno = seqno;
-
-                /* Bump MRU to take account of the delayed flush */
-                if (obj->fence_reg != I915_FENCE_REG_NONE) {
-                        struct drm_i915_fence_reg *reg;
-
-                        reg = &dev_priv->fence_regs[obj->fence_reg];
-                        list_move_tail(&reg->lru_list,
-                                       &dev_priv->mm.fence_list);
-                }
-        }
 }
 
 void i915_vma_move_to_active(struct i915_vma *vma,
@@ -2231,7 +2234,6 @@ i915_gem_object_move_to_inactive(struct drm_i915_gem_object *obj)
         obj->base.write_domain = 0;
 
         obj->last_fenced_seqno = 0;
-        obj->fenced_gpu_access = false;
 
         obj->active = 0;
         drm_gem_object_unreference(&obj->base);
@@ -2329,10 +2331,21 @@ int __i915_add_request(struct intel_engine_cs *ring,
 {
         struct drm_i915_private *dev_priv = ring->dev->dev_private;
         struct drm_i915_gem_request *request;
+        struct intel_ringbuffer *ringbuf;
         u32 request_ring_position, request_start;
         int ret;
 
-        request_start = intel_ring_get_tail(ring->buffer);
+        request = ring->preallocated_lazy_request;
+        if (WARN_ON(request == NULL))
+                return -ENOMEM;
+
+        if (i915.enable_execlists) {
+                struct intel_context *ctx = request->ctx;
+                ringbuf = ctx->engine[ring->id].ringbuf;
+        } else
+                ringbuf = ring->buffer;
+
+        request_start = intel_ring_get_tail(ringbuf);
         /*
          * Emit any outstanding flushes - execbuf can fail to emit the flush
          * after having emitted the batchbuffer command. Hence we need to fix
@@ -2340,24 +2353,32 @@ int __i915_add_request(struct intel_engine_cs *ring,
          * is that the flush _must_ happen before the next request, no matter
          * what.
          */
-        ret = intel_ring_flush_all_caches(ring);
-        if (ret)
-                return ret;
-
-        request = ring->preallocated_lazy_request;
-        if (WARN_ON(request == NULL))
-                return -ENOMEM;
+        if (i915.enable_execlists) {
+                ret = logical_ring_flush_all_caches(ringbuf);
+                if (ret)
+                        return ret;
+        } else {
+                ret = intel_ring_flush_all_caches(ring);
+                if (ret)
+                        return ret;
+        }
 
         /* Record the position of the start of the request so that
          * should we detect the updated seqno part-way through the
          * GPU processing the request, we never over-estimate the
          * position of the head.
          */
-        request_ring_position = intel_ring_get_tail(ring->buffer);
+        request_ring_position = intel_ring_get_tail(ringbuf);
 
-        ret = ring->add_request(ring);
-        if (ret)
-                return ret;
+        if (i915.enable_execlists) {
+                ret = ring->emit_request(ringbuf);
+                if (ret)
+                        return ret;
+        } else {
+                ret = ring->add_request(ring);
+                if (ret)
+                        return ret;
+        }
 
         request->seqno = intel_ring_get_seqno(ring);
         request->ring = ring;
@@ -2372,12 +2393,14 @@ int __i915_add_request(struct intel_engine_cs *ring,
          */
         request->batch_obj = obj;
 
-        /* Hold a reference to the current context so that we can inspect
-         * it later in case a hangcheck error event fires.
-         */
-        request->ctx = ring->last_context;
-        if (request->ctx)
-                i915_gem_context_reference(request->ctx);
+        if (!i915.enable_execlists) {
+                /* Hold a reference to the current context so that we can inspect
+                 * it later in case a hangcheck error event fires.
+                 */
+                request->ctx = ring->last_context;
+                if (request->ctx)
+                        i915_gem_context_reference(request->ctx);
+        }
 
         request->emitted_jiffies = jiffies;
         list_add_tail(&request->list, &ring->request_list);
@@ -2548,6 +2571,18 @@ static void i915_gem_reset_ring_cleanup(struct drm_i915_private *dev_priv,
                 i915_gem_free_request(request);
         }
 
+        while (!list_empty(&ring->execlist_queue)) {
+                struct intel_ctx_submit_request *submit_req;
+
+                submit_req = list_first_entry(&ring->execlist_queue,
+                                struct intel_ctx_submit_request,
+                                execlist_link);
+                list_del(&submit_req->execlist_link);
+                intel_runtime_pm_put(dev_priv);
+                i915_gem_context_unreference(submit_req->ctx);
+                kfree(submit_req);
+        }
+
         /* These may not have been flush before the reset, do so now */
         kfree(ring->preallocated_lazy_request);
         ring->preallocated_lazy_request = NULL;
@@ -2632,6 +2667,7 @@ i915_gem_retire_requests_ring(struct intel_engine_cs *ring)
 
         while (!list_empty(&ring->request_list)) {
                 struct drm_i915_gem_request *request;
+                struct intel_ringbuffer *ringbuf;
 
                 request = list_first_entry(&ring->request_list,
                                            struct drm_i915_gem_request,
@@ -2641,12 +2677,24 @@ i915_gem_retire_requests_ring(struct intel_engine_cs *ring)
                         break;
 
                 trace_i915_gem_request_retire(ring, request->seqno);
+
+                /* This is one of the few common intersection points
+                 * between legacy ringbuffer submission and execlists:
+                 * we need to tell them apart in order to find the correct
+                 * ringbuffer to which the request belongs to.
+                 */
+                if (i915.enable_execlists) {
+                        struct intel_context *ctx = request->ctx;
+                        ringbuf = ctx->engine[ring->id].ringbuf;
+                } else
+                        ringbuf = ring->buffer;
+
                 /* We know the GPU must have read the request to have
                  * sent us the seqno + interrupt, so use the position
                  * of tail of the request to update the last known position
                  * of the GPU head.
                  */
-                ring->buffer->last_retired_head = request->tail;
+                ringbuf->last_retired_head = request->tail;
 
                 i915_gem_free_request(request);
         }
@@ -2908,6 +2956,9 @@ int i915_vma_unbind(struct i915_vma *vma)
          * cause memory corruption through use-after-free.
          */
 
+        /* Throw away the active reference before moving to the unbound list */
+        i915_gem_object_retire(obj);
+
         if (i915_is_ggtt(vma->vm)) {
                 i915_gem_object_finish_gtt(obj);
 
@@ -2922,9 +2973,8 @@ int i915_vma_unbind(struct i915_vma *vma)
         vma->unbind_vma(vma);
 
         list_del_init(&vma->mm_list);
-        /* Avoid an unnecessary call to unbind on rebind. */
         if (i915_is_ggtt(vma->vm))
-                obj->map_and_fenceable = true;
+                obj->map_and_fenceable = false;
 
         drm_mm_remove_node(&vma->node);
         i915_gem_vma_destroy(vma);
@@ -2953,9 +3003,11 @@ int i915_gpu_idle(struct drm_device *dev)
 
         /* Flush everything onto the inactive list. */
         for_each_ring(ring, dev_priv, i) {
-                ret = i915_switch_context(ring, ring->default_context);
-                if (ret)
-                        return ret;
+                if (!i915.enable_execlists) {
+                        ret = i915_switch_context(ring, ring->default_context);
+                        if (ret)
+                                return ret;
+                }
 
                 ret = intel_ring_idle(ring);
                 if (ret)
@@ -3169,7 +3221,6 @@ i915_gem_object_wait_fence(struct drm_i915_gem_object *obj)
                 obj->last_fenced_seqno = 0;
         }
 
-        obj->fenced_gpu_access = false;
         return 0;
 }
 
@@ -3276,6 +3327,9 @@ i915_gem_object_get_fence(struct drm_i915_gem_object *obj)
                         return 0;
                 }
         } else if (enable) {
+                if (WARN_ON(!obj->map_and_fenceable))
+                        return -EINVAL;
+
                 reg = i915_find_fence_reg(dev);
                 if (IS_ERR(reg))
                         return PTR_ERR(reg);
@@ -3297,17 +3351,20 @@ i915_gem_object_get_fence(struct drm_i915_gem_object *obj)
         return 0;
 }
 
-static bool i915_gem_valid_gtt_space(struct drm_device *dev,
-                                     struct drm_mm_node *gtt_space,
+static bool i915_gem_valid_gtt_space(struct i915_vma *vma,
                                      unsigned long cache_level)
 {
+        struct drm_mm_node *gtt_space = &vma->node;
         struct drm_mm_node *other;
 
-        /* On non-LLC machines we have to be careful when putting differing
-         * types of snoopable memory together to avoid the prefetcher
-         * crossing memory domains and dying.
+        /*
+         * On some machines we have to be careful when putting differing types
+         * of snoopable memory together to avoid the prefetcher crossing memory
+         * domains and dying. During vm initialisation, we decide whether or not
+         * these constraints apply and set the drm_mm.color_adjust
+         * appropriately.
          */
-        if (HAS_LLC(dev))
+        if (vma->vm->mm.color_adjust == NULL)
                 return true;
 
         if (!drm_mm_node_allocated(gtt_space))
@@ -3445,8 +3502,7 @@ search_free:
 
                 goto err_free_vma;
         }
-        if (WARN_ON(!i915_gem_valid_gtt_space(dev, &vma->node,
-                                              obj->cache_level))) {
+        if (WARN_ON(!i915_gem_valid_gtt_space(vma, obj->cache_level))) {
                 ret = -EINVAL;
                 goto err_remove_node;
         }
@@ -3586,11 +3642,12 @@ int
 i915_gem_object_set_to_gtt_domain(struct drm_i915_gem_object *obj, bool write)
 {
         struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
+        struct i915_vma *vma = i915_gem_obj_to_ggtt(obj);
         uint32_t old_write_domain, old_read_domains;
         int ret;
 
         /* Not valid to be called on unbound objects. */
-        if (!i915_gem_obj_bound_any(obj))
+        if (vma == NULL)
                 return -EINVAL;
 
         if (obj->base.write_domain == I915_GEM_DOMAIN_GTT)
@@ -3632,13 +3689,9 @@ i915_gem_object_set_to_gtt_domain(struct drm_i915_gem_object *obj, bool write)
                                             old_write_domain);
 
         /* And bump the LRU for this access */
-        if (i915_gem_object_is_inactive(obj)) {
-                struct i915_vma *vma = i915_gem_obj_to_ggtt(obj);
-                if (vma)
-                        list_move_tail(&vma->mm_list,
-                                       &dev_priv->gtt.base.inactive_list);
-
-        }
+        if (i915_gem_object_is_inactive(obj))
+                list_move_tail(&vma->mm_list,
+                               &dev_priv->gtt.base.inactive_list);
 
         return 0;
 }
@@ -3659,7 +3712,7 @@ int i915_gem_object_set_cache_level(struct drm_i915_gem_object *obj,
         }
 
         list_for_each_entry_safe(vma, next, &obj->vma_list, vma_link) {
-                if (!i915_gem_valid_gtt_space(dev, &vma->node, cache_level)) {
+                if (!i915_gem_valid_gtt_space(vma, cache_level)) {
                         ret = i915_vma_unbind(vma);
                         if (ret)
                                 return ret;
@@ -3802,9 +3855,6 @@ static bool is_pin_display(struct drm_i915_gem_object *obj)
 {
         struct i915_vma *vma;
 
-        if (list_empty(&obj->vma_list))
-                return false;
-
         vma = i915_gem_obj_to_ggtt(obj);
         if (!vma)
                 return false;
@@ -4331,8 +4381,6 @@ void i915_gem_object_init(struct drm_i915_gem_object *obj,
 
         obj->fence_reg = I915_FENCE_REG_NONE;
         obj->madv = I915_MADV_WILLNEED;
-        /* Avoid an unnecessary call to unbind on the first bind. */
-        obj->map_and_fenceable = true;
 
         i915_gem_info_add_obj(obj->base.dev->dev_private, obj->base.size);
 }
@@ -4493,12 +4541,18 @@ struct i915_vma *i915_gem_obj_to_vma(struct drm_i915_gem_object *obj,
 
 void i915_gem_vma_destroy(struct i915_vma *vma)
 {
+        struct i915_address_space *vm = NULL;
         WARN_ON(vma->node.allocated);
 
         /* Keep the vma as a placeholder in the execbuffer reservation lists */
         if (!list_empty(&vma->exec_list))
                 return;
 
+        vm = vma->vm;
+
+        if (!i915_is_ggtt(vm))
+                i915_ppgtt_put(i915_vm_to_ppgtt(vm));
+
         list_del(&vma->vma_link);
 
         kfree(vma);
@@ -4512,7 +4566,7 @@ i915_gem_stop_ringbuffers(struct drm_device *dev)
         int i;
 
         for_each_ring(ring, dev_priv, i)
-                intel_stop_ring_buffer(ring);
+                dev_priv->gt.stop_ring(ring);
 }
 
 int
@@ -4629,11 +4683,46 @@ intel_enable_blt(struct drm_device *dev)
         return true;
 }
 
-static int i915_gem_init_rings(struct drm_device *dev)
+static void init_unused_ring(struct drm_device *dev, u32 base)
+{
+        struct drm_i915_private *dev_priv = dev->dev_private;
+
+        I915_WRITE(RING_CTL(base), 0);
+        I915_WRITE(RING_HEAD(base), 0);
+        I915_WRITE(RING_TAIL(base), 0);
+        I915_WRITE(RING_START(base), 0);
+}
+
+static void init_unused_rings(struct drm_device *dev)
+{
+        if (IS_I830(dev)) {
+                init_unused_ring(dev, PRB1_BASE);
+                init_unused_ring(dev, SRB0_BASE);
+                init_unused_ring(dev, SRB1_BASE);
+                init_unused_ring(dev, SRB2_BASE);
+                init_unused_ring(dev, SRB3_BASE);
+        } else if (IS_GEN2(dev)) {
+                init_unused_ring(dev, SRB0_BASE);
+                init_unused_ring(dev, SRB1_BASE);
+        } else if (IS_GEN3(dev)) {
+                init_unused_ring(dev, PRB1_BASE);
+                init_unused_ring(dev, PRB2_BASE);
+        }
+}
+
+int i915_gem_init_rings(struct drm_device *dev)
 {
         struct drm_i915_private *dev_priv = dev->dev_private;
         int ret;
 
+        /*
+         * At least 830 can leave some of the unused rings
+         * "active" (ie. head != tail) after resume which
+         * will prevent c3 entry. Makes sure all unused rings
+         * are totally idle.
+         */
+        init_unused_rings(dev);
+
         ret = intel_init_render_ring_buffer(dev);
         if (ret)
                 return ret;
@@ -4712,7 +4801,7 @@ i915_gem_init_hw(struct drm_device *dev)
 
         i915_gem_init_swizzling(dev);
 
-        ret = i915_gem_init_rings(dev);
+        ret = dev_priv->gt.init_rings(dev);
         if (ret)
                 return ret;
 
@@ -4730,6 +4819,14 @@ i915_gem_init_hw(struct drm_device *dev)
         if (ret && ret != -EIO) {
                 DRM_ERROR("Context enable failed %d\n", ret);
                 i915_gem_cleanup_ringbuffer(dev);
+
+                return ret;
+        }
+
+        ret = i915_ppgtt_init_hw(dev);
+        if (ret && ret != -EIO) {
+                DRM_ERROR("PPGTT enable failed %d\n", ret);
+                i915_gem_cleanup_ringbuffer(dev);
         }
 
         return ret;
@@ -4740,6 +4837,9 @@ int i915_gem_init(struct drm_device *dev)
         struct drm_i915_private *dev_priv = dev->dev_private;
         int ret;
 
+        i915.enable_execlists = intel_sanitize_enable_execlists(dev,
+                        i915.enable_execlists);
+
         mutex_lock(&dev->struct_mutex);
 
         if (IS_VALLEYVIEW(dev)) {
@@ -4750,7 +4850,24 @@ int i915_gem_init(struct drm_device *dev)
                         DRM_DEBUG_DRIVER("allow wake ack timed out\n");
         }
 
-        i915_gem_init_userptr(dev);
+        if (!i915.enable_execlists) {
+                dev_priv->gt.do_execbuf = i915_gem_ringbuffer_submission;
+                dev_priv->gt.init_rings = i915_gem_init_rings;
+                dev_priv->gt.cleanup_ring = intel_cleanup_ring_buffer;
+                dev_priv->gt.stop_ring = intel_stop_ring_buffer;
+        } else {
+                dev_priv->gt.do_execbuf = intel_execlists_submission;
+                dev_priv->gt.init_rings = intel_logical_rings_init;
+                dev_priv->gt.cleanup_ring = intel_logical_ring_cleanup;
+                dev_priv->gt.stop_ring = intel_logical_ring_stop;
+        }
+
+        ret = i915_gem_init_userptr(dev);
+        if (ret) {
+                mutex_unlock(&dev->struct_mutex);
+                return ret;
+        }
+
         i915_gem_init_global_gtt(dev);
 
         ret = i915_gem_context_init(dev);
@@ -4785,7 +4902,7 @@ i915_gem_cleanup_ringbuffer(struct drm_device *dev)
         int i;
 
         for_each_ring(ring, dev_priv, i)
-                intel_cleanup_ring_buffer(ring);
+                dev_priv->gt.cleanup_ring(ring);
 }
 
 int
@@ -5097,9 +5214,7 @@ unsigned long i915_gem_obj_offset(struct drm_i915_gem_object *o,
         struct drm_i915_private *dev_priv = o->base.dev->dev_private;
         struct i915_vma *vma;
 
-        if (!dev_priv->mm.aliasing_ppgtt ||
-            vm == &dev_priv->mm.aliasing_ppgtt->base)
-                vm = &dev_priv->gtt.base;
+        WARN_ON(vm == &dev_priv->mm.aliasing_ppgtt->base);
 
         list_for_each_entry(vma, &o->vma_list, vma_link) {
                 if (vma->vm == vm)
@@ -5140,9 +5255,7 @@ unsigned long i915_gem_obj_size(struct drm_i915_gem_object *o,
         struct drm_i915_private *dev_priv = o->base.dev->dev_private;
         struct i915_vma *vma;
 
-        if (!dev_priv->mm.aliasing_ppgtt ||
-            vm == &dev_priv->mm.aliasing_ppgtt->base)
-                vm = &dev_priv->gtt.base;
+        WARN_ON(vm == &dev_priv->mm.aliasing_ppgtt->base);
 
         BUG_ON(list_empty(&o->vma_list));
 
@@ -5165,11 +5278,16 @@ i915_gem_shrinker_scan(struct shrinker *shrinker, struct shrink_control *sc)
         if (!i915_gem_shrinker_lock(dev, &unlock))
                 return SHRINK_STOP;
 
-        freed = i915_gem_purge(dev_priv, sc->nr_to_scan);
+        freed = i915_gem_shrink(dev_priv,
+                                sc->nr_to_scan,
+                                I915_SHRINK_BOUND |
+                                I915_SHRINK_UNBOUND |
+                                I915_SHRINK_PURGEABLE);
         if (freed < sc->nr_to_scan)
-                freed += __i915_gem_shrink(dev_priv,
-                                           sc->nr_to_scan - freed,
-                                           false);
+                freed += i915_gem_shrink(dev_priv,
+                                         sc->nr_to_scan - freed,
+                                         I915_SHRINK_BOUND |
+                                         I915_SHRINK_UNBOUND);
         if (unlock)
                 mutex_unlock(&dev->struct_mutex);
 
@@ -5247,14 +5365,8 @@ struct i915_vma *i915_gem_obj_to_ggtt(struct drm_i915_gem_object *obj)
 {
         struct i915_vma *vma;
 
-        /* This WARN has probably outlived its usefulness (callers already
-         * WARN if they don't find the GGTT vma they expect). When removing,
-         * remember to remove the pre-check in is_pin_display() as well */
-        if (WARN_ON(list_empty(&obj->vma_list)))
-                return NULL;
-
         vma = list_first_entry(&obj->vma_list, typeof(*vma), vma_link);
-        if (vma->vm != obj_to_ggtt(obj))
+        if (vma->vm != i915_obj_to_ggtt(obj))
                 return NULL;
 
         return vma;
diff --git a/drivers/gpu/drm/i915/i915_gem_context.c b/drivers/gpu/drm/i915/i915_gem_context.c
index 3b99390e467a..a5221d8f1580 100644
--- a/drivers/gpu/drm/i915/i915_gem_context.c
+++ b/drivers/gpu/drm/i915/i915_gem_context.c
@@ -96,50 +96,6 @@
 #define GEN6_CONTEXT_ALIGN (64<<10)
 #define GEN7_CONTEXT_ALIGN 4096
 
-static void do_ppgtt_cleanup(struct i915_hw_ppgtt *ppgtt)
-{
-        struct drm_device *dev = ppgtt->base.dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
-        struct i915_address_space *vm = &ppgtt->base;
-
-        if (ppgtt == dev_priv->mm.aliasing_ppgtt ||
-            (list_empty(&vm->active_list) && list_empty(&vm->inactive_list))) {
-                ppgtt->base.cleanup(&ppgtt->base);
-                return;
-        }
-
-        /*
-         * Make sure vmas are unbound before we take down the drm_mm
-         *
-         * FIXME: Proper refcounting should take care of this, this shouldn't be
-         * needed at all.
-         */
-        if (!list_empty(&vm->active_list)) {
-                struct i915_vma *vma;
-
-                list_for_each_entry(vma, &vm->active_list, mm_list)
-                        if (WARN_ON(list_empty(&vma->vma_link) ||
-                                    list_is_singular(&vma->vma_link)))
-                                break;
-
-                i915_gem_evict_vm(&ppgtt->base, true);
-        } else {
-                i915_gem_retire_requests(dev);
-                i915_gem_evict_vm(&ppgtt->base, false);
-        }
-
-        ppgtt->base.cleanup(&ppgtt->base);
-}
-
-static void ppgtt_release(struct kref *kref)
-{
-        struct i915_hw_ppgtt *ppgtt =
-                container_of(kref, struct i915_hw_ppgtt, ref);
-
-        do_ppgtt_cleanup(ppgtt);
-        kfree(ppgtt);
-}
-
 static size_t get_context_alignment(struct drm_device *dev)
 {
         if (IS_GEN6(dev))
@@ -179,24 +135,20 @@ static int get_context_size(struct drm_device *dev)
 void i915_gem_context_free(struct kref *ctx_ref)
 {
         struct intel_context *ctx = container_of(ctx_ref,
-                                                   typeof(*ctx), ref);
-        struct i915_hw_ppgtt *ppgtt = NULL;
+                                                 typeof(*ctx), ref);
 
-        if (ctx->legacy_hw_ctx.rcs_state) {
-                /* We refcount even the aliasing PPGTT to keep the code symmetric */
-                if (USES_PPGTT(ctx->legacy_hw_ctx.rcs_state->base.dev))
-                        ppgtt = ctx_to_ppgtt(ctx);
-        }
+        if (i915.enable_execlists)
+                intel_lr_context_free(ctx);
+
+        i915_ppgtt_put(ctx->ppgtt);
 
-        if (ppgtt)
-                kref_put(&ppgtt->ref, ppgtt_release);
         if (ctx->legacy_hw_ctx.rcs_state)
                 drm_gem_object_unreference(&ctx->legacy_hw_ctx.rcs_state->base);
         list_del(&ctx->link);
         kfree(ctx);
 }
 
-static struct drm_i915_gem_object *
+struct drm_i915_gem_object *
 i915_gem_alloc_context_obj(struct drm_device *dev, size_t size)
 {
         struct drm_i915_gem_object *obj;
@@ -226,29 +178,9 @@ i915_gem_alloc_context_obj(struct drm_device *dev, size_t size)
         return obj;
 }
 
-static struct i915_hw_ppgtt *
-create_vm_for_ctx(struct drm_device *dev, struct intel_context *ctx)
-{
-        struct i915_hw_ppgtt *ppgtt;
-        int ret;
-
-        ppgtt = kzalloc(sizeof(*ppgtt), GFP_KERNEL);
-        if (!ppgtt)
-                return ERR_PTR(-ENOMEM);
-
-        ret = i915_gem_init_ppgtt(dev, ppgtt);
-        if (ret) {
-                kfree(ppgtt);
-                return ERR_PTR(ret);
-        }
-
-        ppgtt->ctx = ctx;
-        return ppgtt;
-}
-
 static struct intel_context *
 __create_hw_context(struct drm_device *dev,
-                  struct drm_i915_file_private *file_priv)
+                    struct drm_i915_file_private *file_priv)
 {
         struct drm_i915_private *dev_priv = dev->dev_private;
         struct intel_context *ctx;
@@ -301,11 +233,9 @@ err_out:
  */
 static struct intel_context *
 i915_gem_create_context(struct drm_device *dev,
-                        struct drm_i915_file_private *file_priv,
-                        bool create_vm)
+                        struct drm_i915_file_private *file_priv)
 {
         const bool is_global_default_ctx = file_priv == NULL;
-        struct drm_i915_private *dev_priv = dev->dev_private;
         struct intel_context *ctx;
         int ret = 0;
 
@@ -331,34 +261,18 @@ i915_gem_create_context(struct drm_device *dev,
                 }
         }
 
-        if (create_vm) {
-                struct i915_hw_ppgtt *ppgtt = create_vm_for_ctx(dev, ctx);
+        if (USES_FULL_PPGTT(dev)) {
+                struct i915_hw_ppgtt *ppgtt = i915_ppgtt_create(dev, file_priv);
 
                 if (IS_ERR_OR_NULL(ppgtt)) {
                         DRM_DEBUG_DRIVER("PPGTT setup failed (%ld)\n",
                                          PTR_ERR(ppgtt));
                         ret = PTR_ERR(ppgtt);
                         goto err_unpin;
-                } else
-                        ctx->vm = &ppgtt->base;
-
-                /* This case is reserved for the global default context and
-                 * should only happen once. */
-                if (is_global_default_ctx) {
-                        if (WARN_ON(dev_priv->mm.aliasing_ppgtt)) {
-                                ret = -EEXIST;
-                                goto err_unpin;
-                        }
-
-                        dev_priv->mm.aliasing_ppgtt = ppgtt;
                 }
-        } else if (USES_PPGTT(dev)) {
-                /* For platforms which only have aliasing PPGTT, we fake the
-                 * address space and refcounting. */
-                ctx->vm = &dev_priv->mm.aliasing_ppgtt->base;
-                kref_get(&dev_priv->mm.aliasing_ppgtt->ref);
-        } else
-                ctx->vm = &dev_priv->gtt.base;
+
+                ctx->ppgtt = ppgtt;
+        }
 
         return ctx;
 
@@ -375,34 +289,23 @@ void i915_gem_context_reset(struct drm_device *dev)
         struct drm_i915_private *dev_priv = dev->dev_private;
         int i;
 
-        /* Prevent the hardware from restoring the last context (which hung) on
-         * the next switch */
+        /* In execlists mode we will unreference the context when the execlist
+         * queue is cleared and the requests destroyed.
+         */
+        if (i915.enable_execlists)
+                return;
+
         for (i = 0; i < I915_NUM_RINGS; i++) {
                 struct intel_engine_cs *ring = &dev_priv->ring[i];
-                struct intel_context *dctx = ring->default_context;
                 struct intel_context *lctx = ring->last_context;
 
-                /* Do a fake switch to the default context */
-                if (lctx == dctx)
-                        continue;
-
-                if (!lctx)
-                        continue;
+                if (lctx) {
+                        if (lctx->legacy_hw_ctx.rcs_state && i == RCS)
+                                i915_gem_object_ggtt_unpin(lctx->legacy_hw_ctx.rcs_state);
 
-                if (dctx->legacy_hw_ctx.rcs_state && i == RCS) {
-                        WARN_ON(i915_gem_obj_ggtt_pin(dctx->legacy_hw_ctx.rcs_state,
-                                                      get_context_alignment(dev), 0));
-                        /* Fake a finish/inactive */
-                        dctx->legacy_hw_ctx.rcs_state->base.write_domain = 0;
-                        dctx->legacy_hw_ctx.rcs_state->active = 0;
+                        i915_gem_context_unreference(lctx);
+                        ring->last_context = NULL;
                 }
-
-                if (lctx->legacy_hw_ctx.rcs_state && i == RCS)
-                        i915_gem_object_ggtt_unpin(lctx->legacy_hw_ctx.rcs_state);
-
-                i915_gem_context_unreference(lctx);
-                i915_gem_context_reference(dctx);
-                ring->last_context = dctx;
         }
 }
 
@@ -417,7 +320,11 @@ int i915_gem_context_init(struct drm_device *dev)
         if (WARN_ON(dev_priv->ring[RCS].default_context))
                 return 0;
 
-        if (HAS_HW_CONTEXTS(dev)) {
+        if (i915.enable_execlists) {
+                /* NB: intentionally left blank. We will allocate our own
+                 * backing objects as we need them, thank you very much */
+                dev_priv->hw_context_size = 0;
+        } else if (HAS_HW_CONTEXTS(dev)) {
                 dev_priv->hw_context_size = round_up(get_context_size(dev), 4096);
                 if (dev_priv->hw_context_size > (1<<20)) {
                         DRM_DEBUG_DRIVER("Disabling HW Contexts; invalid size %d\n",
@@ -426,18 +333,23 @@ int i915_gem_context_init(struct drm_device *dev)
                 }
         }
 
-        ctx = i915_gem_create_context(dev, NULL, USES_PPGTT(dev));
+        ctx = i915_gem_create_context(dev, NULL);
         if (IS_ERR(ctx)) {
                 DRM_ERROR("Failed to create default global context (error %ld)\n",
                           PTR_ERR(ctx));
                 return PTR_ERR(ctx);
         }
 
-        /* NB: RCS will hold a ref for all rings */
-        for (i = 0; i < I915_NUM_RINGS; i++)
-                dev_priv->ring[i].default_context = ctx;
+        for (i = 0; i < I915_NUM_RINGS; i++) {
+                struct intel_engine_cs *ring = &dev_priv->ring[i];
 
-        DRM_DEBUG_DRIVER("%s context support initialized\n", dev_priv->hw_context_size ? "HW" : "fake");
+                /* NB: RCS will hold a ref for all rings */
+                ring->default_context = ctx;
+        }
+
+        DRM_DEBUG_DRIVER("%s context support initialized\n",
+                        i915.enable_execlists ? "LR" :
+                        dev_priv->hw_context_size ? "HW" : "fake");
         return 0;
 }
 
@@ -489,19 +401,11 @@ int i915_gem_context_enable(struct drm_i915_private *dev_priv)
         struct intel_engine_cs *ring;
         int ret, i;
 
-        /* This is the only place the aliasing PPGTT gets enabled, which means
-         * it has to happen before we bail on reset */
-        if (dev_priv->mm.aliasing_ppgtt) {
-                struct i915_hw_ppgtt *ppgtt = dev_priv->mm.aliasing_ppgtt;
-                ppgtt->enable(ppgtt);
-        }
+        BUG_ON(!dev_priv->ring[RCS].default_context);
 
-        /* FIXME: We should make this work, even in reset */
-        if (i915_reset_in_progress(&dev_priv->gpu_error))
+        if (i915.enable_execlists)
                 return 0;
 
-        BUG_ON(!dev_priv->ring[RCS].default_context);
-
         for_each_ring(ring, dev_priv, i) {
                 ret = i915_switch_context(ring, ring->default_context);
                 if (ret)
@@ -527,7 +431,7 @@ int i915_gem_context_open(struct drm_device *dev, struct drm_file *file)
         idr_init(&file_priv->context_idr);
 
         mutex_lock(&dev->struct_mutex);
-        ctx = i915_gem_create_context(dev, file_priv, USES_FULL_PPGTT(dev));
+        ctx = i915_gem_create_context(dev, file_priv);
         mutex_unlock(&dev->struct_mutex);
 
         if (IS_ERR(ctx)) {
@@ -563,6 +467,7 @@ mi_set_context(struct intel_engine_cs *ring,
                struct intel_context *new_context,
                u32 hw_flags)
 {
+        u32 flags = hw_flags | MI_MM_SPACE_GTT;
         int ret;
 
         /* w/a: If Flush TLB Invalidation Mode is enabled, driver must do a TLB
@@ -576,6 +481,10 @@ mi_set_context(struct intel_engine_cs *ring,
                         return ret;
         }
 
+        /* These flags are for resource streamer on HSW+ */
+        if (!IS_HASWELL(ring->dev) && INTEL_INFO(ring->dev)->gen < 8)
+                flags |= (MI_SAVE_EXT_STATE_EN | MI_RESTORE_EXT_STATE_EN);
+
         ret = intel_ring_begin(ring, 6);
         if (ret)
                 return ret;
@@ -589,10 +498,7 @@ mi_set_context(struct intel_engine_cs *ring,
         intel_ring_emit(ring, MI_NOOP);
         intel_ring_emit(ring, MI_SET_CONTEXT);
         intel_ring_emit(ring, i915_gem_obj_ggtt_offset(new_context->legacy_hw_ctx.rcs_state) |
-                        MI_MM_SPACE_GTT |
-                        MI_SAVE_EXT_STATE_EN |
-                        MI_RESTORE_EXT_STATE_EN |
-                        hw_flags);
+                        flags);
         /*
          * w/a: MI_SET_CONTEXT must always be followed by MI_NOOP
          * WaMiSetContext_Hang:snb,ivb,vlv
@@ -614,7 +520,6 @@ static int do_switch(struct intel_engine_cs *ring,
 {
         struct drm_i915_private *dev_priv = ring->dev->dev_private;
         struct intel_context *from = ring->last_context;
-        struct i915_hw_ppgtt *ppgtt = ctx_to_ppgtt(to);
         u32 hw_flags = 0;
         bool uninitialized = false;
         int ret, i;
@@ -642,8 +547,8 @@ static int do_switch(struct intel_engine_cs *ring,
          */
         from = ring->last_context;
 
-        if (USES_FULL_PPGTT(ring->dev)) {
-                ret = ppgtt->switch_mm(ppgtt, ring, false);
+        if (to->ppgtt) {
+                ret = to->ppgtt->switch_mm(to->ppgtt, ring);
                 if (ret)
                         goto unpin_out;
         }
@@ -723,6 +628,12 @@ done:
         ring->last_context = to;
 
         if (uninitialized) {
+                if (ring->init_context) {
+                        ret = ring->init_context(ring);
+                        if (ret)
+                                DRM_ERROR("ring init context: %d\n", ret);
+                }
+
                 ret = i915_gem_render_state_init(ring);
                 if (ret)
                         DRM_ERROR("init render state: %d\n", ret);
@@ -743,14 +654,19 @@ unpin_out:
  *
  * The context life cycle is simple. The context refcount is incremented and
  * decremented by 1 and create and destroy. If the context is in use by the GPU,
- * it will have a refoucnt > 1. This allows us to destroy the context abstract
+ * it will have a refcount > 1. This allows us to destroy the context abstract
  * object while letting the normal object tracking destroy the backing BO.
+ *
+ * This function should not be used in execlists mode.  Instead the context is
+ * switched by writing to the ELSP and requests keep a reference to their
+ * context.
  */
 int i915_switch_context(struct intel_engine_cs *ring,
                         struct intel_context *to)
 {
         struct drm_i915_private *dev_priv = ring->dev->dev_private;
 
+        WARN_ON(i915.enable_execlists);
         WARN_ON(!mutex_is_locked(&dev_priv->dev->struct_mutex));
 
         if (to->legacy_hw_ctx.rcs_state == NULL) { /* We have the fake context */
@@ -766,9 +682,9 @@ int i915_switch_context(struct intel_engine_cs *ring,
         return do_switch(ring, to);
 }
 
-static bool hw_context_enabled(struct drm_device *dev)
+static bool contexts_enabled(struct drm_device *dev)
 {
-        return to_i915(dev)->hw_context_size;
+        return i915.enable_execlists || to_i915(dev)->hw_context_size;
 }
 
 int i915_gem_context_create_ioctl(struct drm_device *dev, void *data,
@@ -779,14 +695,14 @@ int i915_gem_context_create_ioctl(struct drm_device *dev, void *data,
         struct intel_context *ctx;
         int ret;
 
-        if (!hw_context_enabled(dev))
+        if (!contexts_enabled(dev))
                 return -ENODEV;
 
         ret = i915_mutex_lock_interruptible(dev);
         if (ret)
                 return ret;
 
-        ctx = i915_gem_create_context(dev, file_priv, USES_FULL_PPGTT(dev));
+        ctx = i915_gem_create_context(dev, file_priv);
         mutex_unlock(&dev->struct_mutex);
         if (IS_ERR(ctx))
                 return PTR_ERR(ctx);
diff --git a/drivers/gpu/drm/i915/i915_gem_evict.c b/drivers/gpu/drm/i915/i915_gem_evict.c
index bbf4b12d842e..886ff2ee7a28 100644
--- a/drivers/gpu/drm/i915/i915_gem_evict.c
+++ b/drivers/gpu/drm/i915/i915_gem_evict.c
@@ -243,7 +243,7 @@ int
 i915_gem_evict_everything(struct drm_device *dev)
 {
         struct drm_i915_private *dev_priv = dev->dev_private;
-        struct i915_address_space *vm;
+        struct i915_address_space *vm, *v;
         bool lists_empty = true;
         int ret;
 
@@ -270,7 +270,7 @@ i915_gem_evict_everything(struct drm_device *dev)
         i915_gem_retire_requests(dev);
 
         /* Having flushed everything, unbind() should never raise an error */
-        list_for_each_entry(vm, &dev_priv->vm_list, global_link)
+        list_for_each_entry_safe(vm, v, &dev_priv->vm_list, global_link)
                 WARN_ON(i915_gem_evict_vm(vm, false));
 
         return 0;
diff --git a/drivers/gpu/drm/i915/i915_gem_execbuffer.c b/drivers/gpu/drm/i915/i915_gem_execbuffer.c
index 60998fc4e5b2..1a0611bb576b 100644
--- a/drivers/gpu/drm/i915/i915_gem_execbuffer.c
+++ b/drivers/gpu/drm/i915/i915_gem_execbuffer.c
@@ -35,6 +35,7 @@
 
 #define  __EXEC_OBJECT_HAS_PIN (1<<31)
 #define  __EXEC_OBJECT_HAS_FENCE (1<<30)
+#define  __EXEC_OBJECT_NEEDS_MAP (1<<29)
 #define  __EXEC_OBJECT_NEEDS_BIAS (1<<28)
 
 #define BATCH_OFFSET_BIAS (256*1024)
@@ -94,7 +95,6 @@ eb_lookup_vmas(struct eb_vmas *eb,
                struct i915_address_space *vm,
                struct drm_file *file)
 {
-        struct drm_i915_private *dev_priv = vm->dev->dev_private;
         struct drm_i915_gem_object *obj;
         struct list_head objects;
         int i, ret;
@@ -129,20 +129,6 @@ eb_lookup_vmas(struct eb_vmas *eb,
         i = 0;
         while (!list_empty(&objects)) {
                 struct i915_vma *vma;
-                struct i915_address_space *bind_vm = vm;
-
-                if (exec[i].flags & EXEC_OBJECT_NEEDS_GTT &&
-                    USES_FULL_PPGTT(vm->dev)) {
-                        ret = -EINVAL;
-                        goto err;
-                }
-
-                /* If we have secure dispatch, or the userspace assures us that
-                 * they know what they're doing, use the GGTT VM.
-                 */
-                if (((args->flags & I915_EXEC_SECURE) &&
-                    (i == (args->buffer_count - 1))))
-                        bind_vm = &dev_priv->gtt.base;
 
                 obj = list_first_entry(&objects,
                                        struct drm_i915_gem_object,
@@ -156,7 +142,7 @@ eb_lookup_vmas(struct eb_vmas *eb,
                  * from the (obj, vm) we don't run the risk of creating
                  * duplicated vmas for the same vm.
                  */
-                vma = i915_gem_obj_lookup_or_create_vma(obj, bind_vm);
+                vma = i915_gem_obj_lookup_or_create_vma(obj, vm);
                 if (IS_ERR(vma)) {
                         DRM_DEBUG("Failed to lookup VMA\n");
                         ret = PTR_ERR(vma);
@@ -307,7 +293,7 @@ relocate_entry_gtt(struct drm_i915_gem_object *obj,
         struct drm_device *dev = obj->base.dev;
         struct drm_i915_private *dev_priv = dev->dev_private;
         uint64_t delta = reloc->delta + target_offset;
-        uint32_t __iomem *reloc_entry;
+        uint64_t offset;
         void __iomem *reloc_page;
         int ret;
 
@@ -320,25 +306,24 @@ relocate_entry_gtt(struct drm_i915_gem_object *obj,
                 return ret;
 
         /* Map the page containing the relocation we're going to perform.  */
-        reloc->offset += i915_gem_obj_ggtt_offset(obj);
+        offset = i915_gem_obj_ggtt_offset(obj);
+        offset += reloc->offset;
         reloc_page = io_mapping_map_atomic_wc(dev_priv->gtt.mappable,
-                        reloc->offset & PAGE_MASK);
-        reloc_entry = (uint32_t __iomem *)
-                (reloc_page + offset_in_page(reloc->offset));
-        iowrite32(lower_32_bits(delta), reloc_entry);
+                                              offset & PAGE_MASK);
+        iowrite32(lower_32_bits(delta), reloc_page + offset_in_page(offset));
 
         if (INTEL_INFO(dev)->gen >= 8) {
-                reloc_entry += 1;
+                offset += sizeof(uint32_t);
 
-                if (offset_in_page(reloc->offset + sizeof(uint32_t)) == 0) {
+                if (offset_in_page(offset) == 0) {
                         io_mapping_unmap_atomic(reloc_page);
-                        reloc_page = io_mapping_map_atomic_wc(
-                                        dev_priv->gtt.mappable,
-                                        reloc->offset + sizeof(uint32_t));
-                        reloc_entry = reloc_page;
+                        reloc_page =
+                                io_mapping_map_atomic_wc(dev_priv->gtt.mappable,
+                                                         offset);
                 }
 
-                iowrite32(upper_32_bits(delta), reloc_entry);
+                iowrite32(upper_32_bits(delta),
+                          reloc_page + offset_in_page(offset));
         }
 
         io_mapping_unmap_atomic(reloc_page);
@@ -535,34 +520,18 @@ i915_gem_execbuffer_relocate(struct eb_vmas *eb)
 }
 
 static int
-need_reloc_mappable(struct i915_vma *vma)
-{
-        struct drm_i915_gem_exec_object2 *entry = vma->exec_entry;
-        return entry->relocation_count && !use_cpu_reloc(vma->obj) &&
-                i915_is_ggtt(vma->vm);
-}
-
-static int
 i915_gem_execbuffer_reserve_vma(struct i915_vma *vma,
                                 struct intel_engine_cs *ring,
                                 bool *need_reloc)
 {
         struct drm_i915_gem_object *obj = vma->obj;
         struct drm_i915_gem_exec_object2 *entry = vma->exec_entry;
-        bool has_fenced_gpu_access = INTEL_INFO(ring->dev)->gen < 4;
-        bool need_fence;
         uint64_t flags;
         int ret;
 
         flags = 0;
-
-        need_fence =
-                has_fenced_gpu_access &&
-                entry->flags & EXEC_OBJECT_NEEDS_FENCE &&
-                obj->tiling_mode != I915_TILING_NONE;
-        if (need_fence || need_reloc_mappable(vma))
+        if (entry->flags & __EXEC_OBJECT_NEEDS_MAP)
                 flags |= PIN_MAPPABLE;
-
         if (entry->flags & EXEC_OBJECT_NEEDS_GTT)
                 flags |= PIN_GLOBAL;
         if (entry->flags & __EXEC_OBJECT_NEEDS_BIAS)
@@ -574,17 +543,13 @@ i915_gem_execbuffer_reserve_vma(struct i915_vma *vma,
 
         entry->flags |= __EXEC_OBJECT_HAS_PIN;
 
-        if (has_fenced_gpu_access) {
-                if (entry->flags & EXEC_OBJECT_NEEDS_FENCE) {
-                        ret = i915_gem_object_get_fence(obj);
-                        if (ret)
-                                return ret;
-
-                        if (i915_gem_object_pin_fence(obj))
-                                entry->flags |= __EXEC_OBJECT_HAS_FENCE;
+        if (entry->flags & EXEC_OBJECT_NEEDS_FENCE) {
+                ret = i915_gem_object_get_fence(obj);
+                if (ret)
+                        return ret;
 
-                        obj->pending_fenced_gpu_access = true;
-                }
+                if (i915_gem_object_pin_fence(obj))
+                        entry->flags |= __EXEC_OBJECT_HAS_FENCE;
         }
 
         if (entry->offset != vma->node.start) {
@@ -601,26 +566,40 @@ i915_gem_execbuffer_reserve_vma(struct i915_vma *vma,
 }
 
 static bool
-eb_vma_misplaced(struct i915_vma *vma, bool has_fenced_gpu_access)
+need_reloc_mappable(struct i915_vma *vma)
 {
         struct drm_i915_gem_exec_object2 *entry = vma->exec_entry;
-        struct drm_i915_gem_object *obj = vma->obj;
-        bool need_fence, need_mappable;
 
-        need_fence =
-                has_fenced_gpu_access &&
-                entry->flags & EXEC_OBJECT_NEEDS_FENCE &&
-                obj->tiling_mode != I915_TILING_NONE;
-        need_mappable = need_fence || need_reloc_mappable(vma);
+        if (entry->relocation_count == 0)
+                return false;
+
+        if (!i915_is_ggtt(vma->vm))
+                return false;
+
+        /* See also use_cpu_reloc() */
+        if (HAS_LLC(vma->obj->base.dev))
+                return false;
 
-        WARN_ON((need_mappable || need_fence) &&
+        if (vma->obj->base.write_domain == I915_GEM_DOMAIN_CPU)
+                return false;
+
+        return true;
+}
+
+static bool
+eb_vma_misplaced(struct i915_vma *vma)
+{
+        struct drm_i915_gem_exec_object2 *entry = vma->exec_entry;
+        struct drm_i915_gem_object *obj = vma->obj;
+
+        WARN_ON(entry->flags & __EXEC_OBJECT_NEEDS_MAP &&
                !i915_is_ggtt(vma->vm));
 
         if (entry->alignment &&
             vma->node.start & (entry->alignment - 1))
                 return true;
 
-        if (need_mappable && !obj->map_and_fenceable)
+        if (entry->flags & __EXEC_OBJECT_NEEDS_MAP && !obj->map_and_fenceable)
                 return true;
 
         if (entry->flags & __EXEC_OBJECT_NEEDS_BIAS &&
@@ -642,9 +621,6 @@ i915_gem_execbuffer_reserve(struct intel_engine_cs *ring,
         bool has_fenced_gpu_access = INTEL_INFO(ring->dev)->gen < 4;
         int retry;
 
-        if (list_empty(vmas))
-                return 0;
-
         i915_gem_retire_requests_ring(ring);
 
         vm = list_first_entry(vmas, struct i915_vma, exec_list)->vm;
@@ -658,20 +634,21 @@ i915_gem_execbuffer_reserve(struct intel_engine_cs *ring,
                 obj = vma->obj;
                 entry = vma->exec_entry;
 
+                if (!has_fenced_gpu_access)
+                        entry->flags &= ~EXEC_OBJECT_NEEDS_FENCE;
                 need_fence =
-                        has_fenced_gpu_access &&
                         entry->flags & EXEC_OBJECT_NEEDS_FENCE &&
                         obj->tiling_mode != I915_TILING_NONE;
                 need_mappable = need_fence || need_reloc_mappable(vma);
 
-                if (need_mappable)
+                if (need_mappable) {
+                        entry->flags |= __EXEC_OBJECT_NEEDS_MAP;
                         list_move(&vma->exec_list, &ordered_vmas);
-                else
+                } else
                         list_move_tail(&vma->exec_list, &ordered_vmas);
 
                 obj->base.pending_read_domains = I915_GEM_GPU_DOMAINS & ~I915_GEM_DOMAIN_COMMAND;
                 obj->base.pending_write_domain = 0;
-                obj->pending_fenced_gpu_access = false;
         }
         list_splice(&ordered_vmas, vmas);
 
@@ -696,7 +673,7 @@ i915_gem_execbuffer_reserve(struct intel_engine_cs *ring,
                         if (!drm_mm_node_allocated(&vma->node))
                                 continue;
 
-                        if (eb_vma_misplaced(vma, has_fenced_gpu_access))
+                        if (eb_vma_misplaced(vma))
                                 ret = i915_vma_unbind(vma);
                         else
                                 ret = i915_gem_execbuffer_reserve_vma(vma, ring, need_relocs);
@@ -744,9 +721,6 @@ i915_gem_execbuffer_relocate_slow(struct drm_device *dev,
         int i, total, ret;
         unsigned count = args->buffer_count;
 
-        if (WARN_ON(list_empty(&eb->vmas)))
-                return 0;
-
         vm = list_first_entry(&eb->vmas, struct i915_vma, exec_list)->vm;
 
         /* We may process another execbuffer during the unlock... */
@@ -890,18 +864,24 @@ i915_gem_check_execbuffer(struct drm_i915_gem_execbuffer2 *exec)
 }
 
 static int
-validate_exec_list(struct drm_i915_gem_exec_object2 *exec,
+validate_exec_list(struct drm_device *dev,
+                   struct drm_i915_gem_exec_object2 *exec,
                    int count)
 {
-        int i;
         unsigned relocs_total = 0;
         unsigned relocs_max = UINT_MAX / sizeof(struct drm_i915_gem_relocation_entry);
+        unsigned invalid_flags;
+        int i;
+
+        invalid_flags = __EXEC_OBJECT_UNKNOWN_FLAGS;
+        if (USES_FULL_PPGTT(dev))
+                invalid_flags |= EXEC_OBJECT_NEEDS_GTT;
 
         for (i = 0; i < count; i++) {
                 char __user *ptr = to_user_ptr(exec[i].relocs_ptr);
                 int length; /* limited by fault_in_pages_readable() */
 
-                if (exec[i].flags & __EXEC_OBJECT_UNKNOWN_FLAGS)
+                if (exec[i].flags & invalid_flags)
                         return -EINVAL;
 
                 /* First check for malicious input causing overflow in
@@ -951,16 +931,26 @@ i915_gem_validate_context(struct drm_device *dev, struct drm_file *file,
                 return ERR_PTR(-EIO);
         }
 
+        if (i915.enable_execlists && !ctx->engine[ring->id].state) {
+                int ret = intel_lr_context_deferred_create(ctx, ring);
+                if (ret) {
+                        DRM_DEBUG("Could not create LRC %u: %d\n", ctx_id, ret);
+                        return ERR_PTR(ret);
+                }
+        }
+
         return ctx;
 }
 
-static void
+void
 i915_gem_execbuffer_move_to_active(struct list_head *vmas,
                                    struct intel_engine_cs *ring)
 {
+        u32 seqno = intel_ring_get_seqno(ring);
         struct i915_vma *vma;
 
         list_for_each_entry(vma, vmas, exec_list) {
+                struct drm_i915_gem_exec_object2 *entry = vma->exec_entry;
                 struct drm_i915_gem_object *obj = vma->obj;
                 u32 old_read = obj->base.read_domains;
                 u32 old_write = obj->base.write_domain;
@@ -969,24 +959,31 @@ i915_gem_execbuffer_move_to_active(struct list_head *vmas,
                 if (obj->base.write_domain == 0)
                         obj->base.pending_read_domains |= obj->base.read_domains;
                 obj->base.read_domains = obj->base.pending_read_domains;
-                obj->fenced_gpu_access = obj->pending_fenced_gpu_access;
 
                 i915_vma_move_to_active(vma, ring);
                 if (obj->base.write_domain) {
                         obj->dirty = 1;
-                        obj->last_write_seqno = intel_ring_get_seqno(ring);
+                        obj->last_write_seqno = seqno;
 
                         intel_fb_obj_invalidate(obj, ring);
 
                         /* update for the implicit flush after a batch */
                         obj->base.write_domain &= ~I915_GEM_GPU_DOMAINS;
                 }
+                if (entry->flags & EXEC_OBJECT_NEEDS_FENCE) {
+                        obj->last_fenced_seqno = seqno;
+                        if (entry->flags & __EXEC_OBJECT_HAS_FENCE) {
+                                struct drm_i915_private *dev_priv = to_i915(ring->dev);
+                                list_move_tail(&dev_priv->fence_regs[obj->fence_reg].lru_list,
+                                               &dev_priv->mm.fence_list);
+                        }
+                }
 
                 trace_i915_gem_object_change_domain(obj, old_read, old_write);
         }
 }
 
-static void
+void
 i915_gem_execbuffer_retire_commands(struct drm_device *dev,
                                     struct drm_file *file,
                                     struct intel_engine_cs *ring,
@@ -1026,14 +1023,14 @@ i915_reset_gen7_sol_offsets(struct drm_device *dev,
         return 0;
 }
 
-static int
-legacy_ringbuffer_submission(struct drm_device *dev, struct drm_file *file,
-                             struct intel_engine_cs *ring,
-                             struct intel_context *ctx,
-                             struct drm_i915_gem_execbuffer2 *args,
-                             struct list_head *vmas,
-                             struct drm_i915_gem_object *batch_obj,
-                             u64 exec_start, u32 flags)
+int
+i915_gem_ringbuffer_submission(struct drm_device *dev, struct drm_file *file,
+                               struct intel_engine_cs *ring,
+                               struct intel_context *ctx,
+                               struct drm_i915_gem_execbuffer2 *args,
+                               struct list_head *vmas,
+                               struct drm_i915_gem_object *batch_obj,
+                               u64 exec_start, u32 flags)
 {
         struct drm_clip_rect *cliprects = NULL;
         struct drm_i915_private *dev_priv = dev->dev_private;
@@ -1254,7 +1251,7 @@ i915_gem_do_execbuffer(struct drm_device *dev, void *data,
         if (!i915_gem_check_execbuffer(args))
                 return -EINVAL;
 
-        ret = validate_exec_list(exec, args->buffer_count);
+        ret = validate_exec_list(dev, exec, args->buffer_count);
         if (ret)
                 return ret;
 
@@ -1318,8 +1315,9 @@ i915_gem_do_execbuffer(struct drm_device *dev, void *data,
 
         i915_gem_context_reference(ctx);
 
-        vm = ctx->vm;
-        if (!USES_FULL_PPGTT(dev))
+        if (ctx->ppgtt)
+                vm = &ctx->ppgtt->base;
+        else
                 vm = &dev_priv->gtt.base;
 
         eb = eb_create(args);
@@ -1386,25 +1384,36 @@ i915_gem_do_execbuffer(struct drm_device *dev, void *data,
         /* snb/ivb/vlv conflate the "batch in ppgtt" bit with the "non-secure
          * batch" bit. Hence we need to pin secure batches into the global gtt.
          * hsw should have this fixed, but bdw mucks it up again. */
-        if (flags & I915_DISPATCH_SECURE &&
-            !batch_obj->has_global_gtt_mapping) {
-                /* When we have multiple VMs, we'll need to make sure that we
-                 * allocate space first */
-                struct i915_vma *vma = i915_gem_obj_to_ggtt(batch_obj);
-                BUG_ON(!vma);
-                vma->bind_vma(vma, batch_obj->cache_level, GLOBAL_BIND);
-        }
+        if (flags & I915_DISPATCH_SECURE) {
+                /*
+                 * So on first glance it looks freaky that we pin the batch here
+                 * outside of the reservation loop. But:
+                 * - The batch is already pinned into the relevant ppgtt, so we
+                 *   already have the backing storage fully allocated.
+                 * - No other BO uses the global gtt (well contexts, but meh),
+                 *   so we don't really have issues with mutliple objects not
+                 *   fitting due to fragmentation.
+                 * So this is actually safe.
+                 */
+                ret = i915_gem_obj_ggtt_pin(batch_obj, 0, 0);
+                if (ret)
+                        goto err;
 
-        if (flags & I915_DISPATCH_SECURE)
                 exec_start += i915_gem_obj_ggtt_offset(batch_obj);
-        else
+        } else
                 exec_start += i915_gem_obj_offset(batch_obj, vm);
 
-        ret = legacy_ringbuffer_submission(dev, file, ring, ctx,
-                        args, &eb->vmas, batch_obj, exec_start, flags);
-        if (ret)
-                goto err;
+        ret = dev_priv->gt.do_execbuf(dev, file, ring, ctx, args,
+                                      &eb->vmas, batch_obj, exec_start, flags);
 
+        /*
+         * FIXME: We crucially rely upon the active tracking for the (ppgtt)
+         * batch vma for correctness. For less ugly and less fragility this
+         * needs to be adjusted to also track the ggtt batch vma properly as
+         * active.
+         */
+        if (flags & I915_DISPATCH_SECURE)
+                i915_gem_object_ggtt_unpin(batch_obj);
 err:
         /* the request owns the ref now */
         i915_gem_context_unreference(ctx);
diff --git a/drivers/gpu/drm/i915/i915_gem_gtt.c b/drivers/gpu/drm/i915/i915_gem_gtt.c
index e42925f76b4b..b672b843fd5e 100644
--- a/drivers/gpu/drm/i915/i915_gem_gtt.c
+++ b/drivers/gpu/drm/i915/i915_gem_gtt.c
@@ -33,17 +33,6 @@
 static void bdw_setup_private_ppat(struct drm_i915_private *dev_priv);
 static void chv_setup_private_ppat(struct drm_i915_private *dev_priv);
 
-bool intel_enable_ppgtt(struct drm_device *dev, bool full)
-{
-        if (i915.enable_ppgtt == 0)
-                return false;
-
-        if (i915.enable_ppgtt == 1 && full)
-                return false;
-
-        return true;
-}
-
 static int sanitize_enable_ppgtt(struct drm_device *dev, int enable_ppgtt)
 {
         if (enable_ppgtt == 0 || !HAS_ALIASING_PPGTT(dev))
@@ -78,7 +67,6 @@ static void ppgtt_bind_vma(struct i915_vma *vma,
                            enum i915_cache_level cache_level,
                            u32 flags);
 static void ppgtt_unbind_vma(struct i915_vma *vma);
-static int gen8_ppgtt_enable(struct i915_hw_ppgtt *ppgtt);
 
 static inline gen8_gtt_pte_t gen8_pte_encode(dma_addr_t addr,
                                              enum i915_cache_level level,
@@ -216,19 +204,12 @@ static gen6_gtt_pte_t iris_pte_encode(dma_addr_t addr,
 
 /* Broadwell Page Directory Pointer Descriptors */
 static int gen8_write_pdp(struct intel_engine_cs *ring, unsigned entry,
-                           uint64_t val, bool synchronous)
+                           uint64_t val)
 {
-        struct drm_i915_private *dev_priv = ring->dev->dev_private;
         int ret;
 
         BUG_ON(entry >= 4);
 
-        if (synchronous) {
-                I915_WRITE(GEN8_RING_PDP_UDW(ring, entry), val >> 32);
-                I915_WRITE(GEN8_RING_PDP_LDW(ring, entry), (u32)val);
-                return 0;
-        }
-
         ret = intel_ring_begin(ring, 6);
         if (ret)
                 return ret;
@@ -245,8 +226,7 @@ static int gen8_write_pdp(struct intel_engine_cs *ring, unsigned entry,
 }
 
 static int gen8_mm_switch(struct i915_hw_ppgtt *ppgtt,
-                          struct intel_engine_cs *ring,
-                          bool synchronous)
+                          struct intel_engine_cs *ring)
 {
         int i, ret;
 
@@ -255,7 +235,7 @@ static int gen8_mm_switch(struct i915_hw_ppgtt *ppgtt,
 
         for (i = used_pd - 1; i >= 0; i--) {
                 dma_addr_t addr = ppgtt->pd_dma_addr[i];
-                ret = gen8_write_pdp(ring, i, addr, synchronous);
+                ret = gen8_write_pdp(ring, i, addr);
                 if (ret)
                         return ret;
         }
@@ -403,9 +383,6 @@ static void gen8_ppgtt_cleanup(struct i915_address_space *vm)
         struct i915_hw_ppgtt *ppgtt =
                 container_of(vm, struct i915_hw_ppgtt, base);
 
-        list_del(&vm->global_link);
-        drm_mm_takedown(&vm->mm);
-
         gen8_ppgtt_unmap_pages(ppgtt);
         gen8_ppgtt_free(ppgtt);
 }
@@ -615,7 +592,6 @@ static int gen8_ppgtt_init(struct i915_hw_ppgtt *ppgtt, uint64_t size)
                 kunmap_atomic(pd_vaddr);
         }
 
-        ppgtt->enable = gen8_ppgtt_enable;
         ppgtt->switch_mm = gen8_mm_switch;
         ppgtt->base.clear_range = gen8_ppgtt_clear_range;
         ppgtt->base.insert_entries = gen8_ppgtt_insert_entries;
@@ -724,29 +700,10 @@ static uint32_t get_pd_offset(struct i915_hw_ppgtt *ppgtt)
 }
 
 static int hsw_mm_switch(struct i915_hw_ppgtt *ppgtt,
-                         struct intel_engine_cs *ring,
-                         bool synchronous)
+                         struct intel_engine_cs *ring)
 {
-        struct drm_device *dev = ppgtt->base.dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
         int ret;
 
-        /* If we're in reset, we can assume the GPU is sufficiently idle to
-         * manually frob these bits. Ideally we could use the ring functions,
-         * except our error handling makes it quite difficult (can't use
-         * intel_ring_begin, ring->flush, or intel_ring_advance)
-         *
-         * FIXME: We should try not to special case reset
-         */
-        if (synchronous ||
-            i915_reset_in_progress(&dev_priv->gpu_error)) {
-                WARN_ON(ppgtt != dev_priv->mm.aliasing_ppgtt);
-                I915_WRITE(RING_PP_DIR_DCLV(ring), PP_DIR_DCLV_2G);
-                I915_WRITE(RING_PP_DIR_BASE(ring), get_pd_offset(ppgtt));
-                POSTING_READ(RING_PP_DIR_BASE(ring));
-                return 0;
-        }
-
         /* NB: TLBs must be flushed and invalidated before a switch */
         ret = ring->flush(ring, I915_GEM_GPU_DOMAINS, I915_GEM_GPU_DOMAINS);
         if (ret)
@@ -768,29 +725,10 @@ static int hsw_mm_switch(struct i915_hw_ppgtt *ppgtt,
 }
 
 static int gen7_mm_switch(struct i915_hw_ppgtt *ppgtt,
-                          struct intel_engine_cs *ring,
-                          bool synchronous)
+                          struct intel_engine_cs *ring)
 {
-        struct drm_device *dev = ppgtt->base.dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
         int ret;
 
-        /* If we're in reset, we can assume the GPU is sufficiently idle to
-         * manually frob these bits. Ideally we could use the ring functions,
-         * except our error handling makes it quite difficult (can't use
-         * intel_ring_begin, ring->flush, or intel_ring_advance)
-         *
-         * FIXME: We should try not to special case reset
-         */
-        if (synchronous ||
-            i915_reset_in_progress(&dev_priv->gpu_error)) {
-                WARN_ON(ppgtt != dev_priv->mm.aliasing_ppgtt);
-                I915_WRITE(RING_PP_DIR_DCLV(ring), PP_DIR_DCLV_2G);
-                I915_WRITE(RING_PP_DIR_BASE(ring), get_pd_offset(ppgtt));
-                POSTING_READ(RING_PP_DIR_BASE(ring));
-                return 0;
-        }
-
         /* NB: TLBs must be flushed and invalidated before a switch */
         ret = ring->flush(ring, I915_GEM_GPU_DOMAINS, I915_GEM_GPU_DOMAINS);
         if (ret)
@@ -819,14 +757,11 @@ static int gen7_mm_switch(struct i915_hw_ppgtt *ppgtt,
 }
 
 static int gen6_mm_switch(struct i915_hw_ppgtt *ppgtt,
-                          struct intel_engine_cs *ring,
-                          bool synchronous)
+                          struct intel_engine_cs *ring)
 {
         struct drm_device *dev = ppgtt->base.dev;
         struct drm_i915_private *dev_priv = dev->dev_private;
 
-        if (!synchronous)
-                return 0;
 
         I915_WRITE(RING_PP_DIR_DCLV(ring), PP_DIR_DCLV_2G);
         I915_WRITE(RING_PP_DIR_BASE(ring), get_pd_offset(ppgtt));
@@ -836,39 +771,20 @@ static int gen6_mm_switch(struct i915_hw_ppgtt *ppgtt,
         return 0;
 }
 
-static int gen8_ppgtt_enable(struct i915_hw_ppgtt *ppgtt)
+static void gen8_ppgtt_enable(struct drm_device *dev)
 {
-        struct drm_device *dev = ppgtt->base.dev;
         struct drm_i915_private *dev_priv = dev->dev_private;
         struct intel_engine_cs *ring;
-        int j, ret;
+        int j;
 
         for_each_ring(ring, dev_priv, j) {
                 I915_WRITE(RING_MODE_GEN7(ring),
                            _MASKED_BIT_ENABLE(GFX_PPGTT_ENABLE));
-
-                /* We promise to do a switch later with FULL PPGTT. If this is
-                 * aliasing, this is the one and only switch we'll do */
-                if (USES_FULL_PPGTT(dev))
-                        continue;
-
-                ret = ppgtt->switch_mm(ppgtt, ring, true);
-                if (ret)
-                        goto err_out;
         }
-
-        return 0;
-
-err_out:
-        for_each_ring(ring, dev_priv, j)
-                I915_WRITE(RING_MODE_GEN7(ring),
-                           _MASKED_BIT_DISABLE(GFX_PPGTT_ENABLE));
-        return ret;
 }
 
-static int gen7_ppgtt_enable(struct i915_hw_ppgtt *ppgtt)
+static void gen7_ppgtt_enable(struct drm_device *dev)
 {
-        struct drm_device *dev = ppgtt->base.dev;
         struct drm_i915_private *dev_priv = dev->dev_private;
         struct intel_engine_cs *ring;
         uint32_t ecochk, ecobits;
@@ -887,31 +803,16 @@ static int gen7_ppgtt_enable(struct i915_hw_ppgtt *ppgtt)
         I915_WRITE(GAM_ECOCHK, ecochk);
 
         for_each_ring(ring, dev_priv, i) {
-                int ret;
                 /* GFX_MODE is per-ring on gen7+ */
                 I915_WRITE(RING_MODE_GEN7(ring),
                            _MASKED_BIT_ENABLE(GFX_PPGTT_ENABLE));
-
-                /* We promise to do a switch later with FULL PPGTT. If this is
-                 * aliasing, this is the one and only switch we'll do */
-                if (USES_FULL_PPGTT(dev))
-                        continue;
-
-                ret = ppgtt->switch_mm(ppgtt, ring, true);
-                if (ret)
-                        return ret;
         }
-
-        return 0;
 }
 
-static int gen6_ppgtt_enable(struct i915_hw_ppgtt *ppgtt)
+static void gen6_ppgtt_enable(struct drm_device *dev)
 {
-        struct drm_device *dev = ppgtt->base.dev;
         struct drm_i915_private *dev_priv = dev->dev_private;
-        struct intel_engine_cs *ring;
         uint32_t ecochk, gab_ctl, ecobits;
-        int i;
 
         ecobits = I915_READ(GAC_ECO_BITS);
         I915_WRITE(GAC_ECO_BITS, ecobits | ECOBITS_SNB_BIT |
@@ -924,14 +825,6 @@ static int gen6_ppgtt_enable(struct i915_hw_ppgtt *ppgtt)
         I915_WRITE(GAM_ECOCHK, ecochk | ECOCHK_SNB_BIT | ECOCHK_PPGTT_CACHE64B);
 
         I915_WRITE(GFX_MODE, _MASKED_BIT_ENABLE(GFX_PPGTT_ENABLE));
-
-        for_each_ring(ring, dev_priv, i) {
-                int ret = ppgtt->switch_mm(ppgtt, ring, true);
-                if (ret)
-                        return ret;
-        }
-
-        return 0;
 }
 
 /* PPGTT support for Sandybdrige/Gen6 and later */
@@ -1029,8 +922,6 @@ static void gen6_ppgtt_cleanup(struct i915_address_space *vm)
         struct i915_hw_ppgtt *ppgtt =
                 container_of(vm, struct i915_hw_ppgtt, base);
 
-        list_del(&vm->global_link);
-        drm_mm_takedown(&ppgtt->base.mm);
         drm_mm_remove_node(&ppgtt->node);
 
         gen6_ppgtt_unmap_pages(ppgtt);
@@ -1151,13 +1042,10 @@ static int gen6_ppgtt_init(struct i915_hw_ppgtt *ppgtt)
 
         ppgtt->base.pte_encode = dev_priv->gtt.base.pte_encode;
         if (IS_GEN6(dev)) {
-                ppgtt->enable = gen6_ppgtt_enable;
                 ppgtt->switch_mm = gen6_mm_switch;
         } else if (IS_HASWELL(dev)) {
-                ppgtt->enable = gen7_ppgtt_enable;
                 ppgtt->switch_mm = hsw_mm_switch;
         } else if (IS_GEN7(dev)) {
-                ppgtt->enable = gen7_ppgtt_enable;
                 ppgtt->switch_mm = gen7_mm_switch;
         } else
                 BUG();
@@ -1188,39 +1076,114 @@ static int gen6_ppgtt_init(struct i915_hw_ppgtt *ppgtt)
                          ppgtt->node.size >> 20,
                          ppgtt->node.start / PAGE_SIZE);
 
+        gen6_write_pdes(ppgtt);
+        DRM_DEBUG("Adding PPGTT at offset %x\n",
+                  ppgtt->pd_offset << 10);
+
         return 0;
 }
 
-int i915_gem_init_ppgtt(struct drm_device *dev, struct i915_hw_ppgtt *ppgtt)
+static int __hw_ppgtt_init(struct drm_device *dev, struct i915_hw_ppgtt *ppgtt)
 {
         struct drm_i915_private *dev_priv = dev->dev_private;
-        int ret = 0;
 
         ppgtt->base.dev = dev;
         ppgtt->base.scratch = dev_priv->gtt.base.scratch;
 
         if (INTEL_INFO(dev)->gen < 8)
-                ret = gen6_ppgtt_init(ppgtt);
+                return gen6_ppgtt_init(ppgtt);
         else if (IS_GEN8(dev))
-                ret = gen8_ppgtt_init(ppgtt, dev_priv->gtt.base.total);
+                return gen8_ppgtt_init(ppgtt, dev_priv->gtt.base.total);
         else
                 BUG();
+}
+int i915_ppgtt_init(struct drm_device *dev, struct i915_hw_ppgtt *ppgtt)
+{
+        struct drm_i915_private *dev_priv = dev->dev_private;
+        int ret = 0;
 
-        if (!ret) {
-                struct drm_i915_private *dev_priv = dev->dev_private;
+        ret = __hw_ppgtt_init(dev, ppgtt);
+        if (ret == 0) {
                 kref_init(&ppgtt->ref);
                 drm_mm_init(&ppgtt->base.mm, ppgtt->base.start,
                             ppgtt->base.total);
                 i915_init_vm(dev_priv, &ppgtt->base);
-                if (INTEL_INFO(dev)->gen < 8) {
-                        gen6_write_pdes(ppgtt);
-                        DRM_DEBUG("Adding PPGTT at offset %x\n",
-                                  ppgtt->pd_offset << 10);
+        }
+
+        return ret;
+}
+
+int i915_ppgtt_init_hw(struct drm_device *dev)
+{
+        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct intel_engine_cs *ring;
+        struct i915_hw_ppgtt *ppgtt = dev_priv->mm.aliasing_ppgtt;
+        int i, ret = 0;
+
+        /* In the case of execlists, PPGTT is enabled by the context descriptor
+         * and the PDPs are contained within the context itself.  We don't
+         * need to do anything here. */
+        if (i915.enable_execlists)
+                return 0;
+
+        if (!USES_PPGTT(dev))
+                return 0;
+
+        if (IS_GEN6(dev))
+                gen6_ppgtt_enable(dev);
+        else if (IS_GEN7(dev))
+                gen7_ppgtt_enable(dev);
+        else if (INTEL_INFO(dev)->gen >= 8)
+                gen8_ppgtt_enable(dev);
+        else
+                WARN_ON(1);
+
+        if (ppgtt) {
+                for_each_ring(ring, dev_priv, i) {
+                        ret = ppgtt->switch_mm(ppgtt, ring);
+                        if (ret != 0)
+                                return ret;
                 }
         }
 
         return ret;
 }
+struct i915_hw_ppgtt *
+i915_ppgtt_create(struct drm_device *dev, struct drm_i915_file_private *fpriv)
+{
+        struct i915_hw_ppgtt *ppgtt;
+        int ret;
+
+        ppgtt = kzalloc(sizeof(*ppgtt), GFP_KERNEL);
+        if (!ppgtt)
+                return ERR_PTR(-ENOMEM);
+
+        ret = i915_ppgtt_init(dev, ppgtt);
+        if (ret) {
+                kfree(ppgtt);
+                return ERR_PTR(ret);
+        }
+
+        ppgtt->file_priv = fpriv;
+
+        return ppgtt;
+}
+
+void  i915_ppgtt_release(struct kref *kref)
+{
+        struct i915_hw_ppgtt *ppgtt =
+                container_of(kref, struct i915_hw_ppgtt, ref);
+
+        /* vmas should already be unbound */
+        WARN_ON(!list_empty(&ppgtt->base.active_list));
+        WARN_ON(!list_empty(&ppgtt->base.inactive_list));
+
+        list_del(&ppgtt->base.global_link);
+        drm_mm_takedown(&ppgtt->base.mm);
+
+        ppgtt->base.cleanup(&ppgtt->base);
+        kfree(ppgtt);
+}
 
 static void
 ppgtt_bind_vma(struct i915_vma *vma,
@@ -1687,10 +1650,10 @@ static void i915_gtt_color_adjust(struct drm_mm_node *node,
         }
 }
 
-void i915_gem_setup_global_gtt(struct drm_device *dev,
-                               unsigned long start,
-                               unsigned long mappable_end,
-                               unsigned long end)
+int i915_gem_setup_global_gtt(struct drm_device *dev,
+                              unsigned long start,
+                              unsigned long mappable_end,
+                              unsigned long end)
 {
         /* Let GEM Manage all of the aperture.
          *
@@ -1706,6 +1669,7 @@ void i915_gem_setup_global_gtt(struct drm_device *dev,
         struct drm_mm_node *entry;
         struct drm_i915_gem_object *obj;
         unsigned long hole_start, hole_end;
+        int ret;
 
         BUG_ON(mappable_end > end);
 
@@ -1717,14 +1681,16 @@ void i915_gem_setup_global_gtt(struct drm_device *dev,
         /* Mark any preallocated objects as occupied */
         list_for_each_entry(obj, &dev_priv->mm.bound_list, global_list) {
                 struct i915_vma *vma = i915_gem_obj_to_vma(obj, ggtt_vm);
-                int ret;
+
                 DRM_DEBUG_KMS("reserving preallocated space: %lx + %zx\n",
                               i915_gem_obj_ggtt_offset(obj), obj->base.size);
 
                 WARN_ON(i915_gem_obj_ggtt_bound(obj));
                 ret = drm_mm_reserve_node(&ggtt_vm->mm, &vma->node);
-                if (ret)
-                        DRM_DEBUG_KMS("Reservation failed\n");
+                if (ret) {
+                        DRM_DEBUG_KMS("Reservation failed: %i\n", ret);
+                        return ret;
+                }
                 obj->has_global_gtt_mapping = 1;
         }
 
@@ -1741,6 +1707,22 @@ void i915_gem_setup_global_gtt(struct drm_device *dev,
 
         /* And finally clear the reserved guard page */
         ggtt_vm->clear_range(ggtt_vm, end - PAGE_SIZE, PAGE_SIZE, true);
+
+        if (USES_PPGTT(dev) && !USES_FULL_PPGTT(dev)) {
+                struct i915_hw_ppgtt *ppgtt;
+
+                ppgtt = kzalloc(sizeof(*ppgtt), GFP_KERNEL);
+                if (!ppgtt)
+                        return -ENOMEM;
+
+                ret = __hw_ppgtt_init(dev, ppgtt);
+                if (ret != 0)
+                        return ret;
+
+                dev_priv->mm.aliasing_ppgtt = ppgtt;
+        }
+
+        return 0;
 }
 
 void i915_gem_init_global_gtt(struct drm_device *dev)
@@ -1754,6 +1736,25 @@ void i915_gem_init_global_gtt(struct drm_device *dev)
         i915_gem_setup_global_gtt(dev, 0, mappable_size, gtt_size);
 }
 
+void i915_global_gtt_cleanup(struct drm_device *dev)
+{
+        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct i915_address_space *vm = &dev_priv->gtt.base;
+
+        if (dev_priv->mm.aliasing_ppgtt) {
+                struct i915_hw_ppgtt *ppgtt = dev_priv->mm.aliasing_ppgtt;
+
+                ppgtt->base.cleanup(&ppgtt->base);
+        }
+
+        if (drm_mm_initialized(&vm->mm)) {
+                drm_mm_takedown(&vm->mm);
+                list_del(&vm->global_link);
+        }
+
+        vm->cleanup(vm);
+}
+
 static int setup_scratch_page(struct drm_device *dev)
 {
         struct drm_i915_private *dev_priv = dev->dev_private;
@@ -2022,10 +2023,6 @@ static void gen6_gmch_remove(struct i915_address_space *vm)
 
         struct i915_gtt *gtt = container_of(vm, struct i915_gtt, base);
 
-        if (drm_mm_initialized(&vm->mm)) {
-                drm_mm_takedown(&vm->mm);
-                list_del(&vm->global_link);
-        }
         iounmap(gtt->gsm);
         teardown_scratch_page(vm->dev);
 }
@@ -2058,10 +2055,6 @@ static int i915_gmch_probe(struct drm_device *dev,
 
 static void i915_gmch_remove(struct i915_address_space *vm)
 {
-        if (drm_mm_initialized(&vm->mm)) {
-                drm_mm_takedown(&vm->mm);
-                list_del(&vm->global_link);
-        }
         intel_gmch_remove();
 }
 
@@ -2160,8 +2153,10 @@ static struct i915_vma *__i915_gem_vma_create(struct drm_i915_gem_object *obj,
         /* Keep GGTT vmas first to make debug easier */
         if (i915_is_ggtt(vm))
                 list_add(&vma->vma_link, &obj->vma_list);
-        else
+        else {
                 list_add_tail(&vma->vma_link, &obj->vma_list);
+                i915_ppgtt_get(i915_vm_to_ppgtt(vm));
+        }
 
         return vma;
 }
diff --git a/drivers/gpu/drm/i915/i915_gem_gtt.h b/drivers/gpu/drm/i915/i915_gem_gtt.h
index 8d6f7c18c404..d5c14af51e99 100644
--- a/drivers/gpu/drm/i915/i915_gem_gtt.h
+++ b/drivers/gpu/drm/i915/i915_gem_gtt.h
@@ -34,6 +34,8 @@
 #ifndef __I915_GEM_GTT_H__
 #define __I915_GEM_GTT_H__
 
+struct drm_i915_file_private;
+
 typedef uint32_t gen6_gtt_pte_t;
 typedef uint64_t gen8_gtt_pte_t;
 typedef gen8_gtt_pte_t gen8_ppgtt_pde_t;
@@ -258,22 +260,36 @@ struct i915_hw_ppgtt {
                 dma_addr_t *gen8_pt_dma_addr[4];
         };
 
-        struct intel_context *ctx;
+        struct drm_i915_file_private *file_priv;
 
         int (*enable)(struct i915_hw_ppgtt *ppgtt);
         int (*switch_mm)(struct i915_hw_ppgtt *ppgtt,
-                         struct intel_engine_cs *ring,
-                         bool synchronous);
+                         struct intel_engine_cs *ring);
         void (*debug_dump)(struct i915_hw_ppgtt *ppgtt, struct seq_file *m);
 };
 
 int i915_gem_gtt_init(struct drm_device *dev);
 void i915_gem_init_global_gtt(struct drm_device *dev);
-void i915_gem_setup_global_gtt(struct drm_device *dev, unsigned long start,
-                               unsigned long mappable_end, unsigned long end);
-
-bool intel_enable_ppgtt(struct drm_device *dev, bool full);
-int i915_gem_init_ppgtt(struct drm_device *dev, struct i915_hw_ppgtt *ppgtt);
+int i915_gem_setup_global_gtt(struct drm_device *dev, unsigned long start,
+                              unsigned long mappable_end, unsigned long end);
+void i915_global_gtt_cleanup(struct drm_device *dev);
+
+
+int i915_ppgtt_init(struct drm_device *dev, struct i915_hw_ppgtt *ppgtt);
+int i915_ppgtt_init_hw(struct drm_device *dev);
+void i915_ppgtt_release(struct kref *kref);
+struct i915_hw_ppgtt *i915_ppgtt_create(struct drm_device *dev,
+                                        struct drm_i915_file_private *fpriv);
+static inline void i915_ppgtt_get(struct i915_hw_ppgtt *ppgtt)
+{
+        if (ppgtt)
+                kref_get(&ppgtt->ref);
+}
+static inline void i915_ppgtt_put(struct i915_hw_ppgtt *ppgtt)
+{
+        if (ppgtt)
+                kref_put(&ppgtt->ref, i915_ppgtt_release);
+}
 
 void i915_check_and_clear_faults(struct drm_device *dev);
 void i915_gem_suspend_gtt_mappings(struct drm_device *dev);
diff --git a/drivers/gpu/drm/i915/i915_gem_render_state.c b/drivers/gpu/drm/i915/i915_gem_render_state.c
index e60be3f552a6..a9a62d75aa57 100644
--- a/drivers/gpu/drm/i915/i915_gem_render_state.c
+++ b/drivers/gpu/drm/i915/i915_gem_render_state.c
@@ -28,13 +28,6 @@
 #include "i915_drv.h"
 #include "intel_renderstate.h"
 
-struct render_state {
-        const struct intel_renderstate_rodata *rodata;
-        struct drm_i915_gem_object *obj;
-        u64 ggtt_offset;
-        int gen;
-};
-
 static const struct intel_renderstate_rodata *
 render_state_get_rodata(struct drm_device *dev, const int gen)
 {
@@ -127,30 +120,47 @@ static int render_state_setup(struct render_state *so)
         return 0;
 }
 
-static void render_state_fini(struct render_state *so)
+void i915_gem_render_state_fini(struct render_state *so)
 {
         i915_gem_object_ggtt_unpin(so->obj);
         drm_gem_object_unreference(&so->obj->base);
 }
 
-int i915_gem_render_state_init(struct intel_engine_cs *ring)
+int i915_gem_render_state_prepare(struct intel_engine_cs *ring,
+                                  struct render_state *so)
 {
-        struct render_state so;
         int ret;
 
         if (WARN_ON(ring->id != RCS))
                 return -ENOENT;
 
-        ret = render_state_init(&so, ring->dev);
+        ret = render_state_init(so, ring->dev);
         if (ret)
                 return ret;
 
-        if (so.rodata == NULL)
+        if (so->rodata == NULL)
                 return 0;
 
-        ret = render_state_setup(&so);
+        ret = render_state_setup(so);
+        if (ret) {
+                i915_gem_render_state_fini(so);
+                return ret;
+        }
+
+        return 0;
+}
+
+int i915_gem_render_state_init(struct intel_engine_cs *ring)
+{
+        struct render_state so;
+        int ret;
+
+        ret = i915_gem_render_state_prepare(ring, &so);
         if (ret)
-                goto out;
+                return ret;
+
+        if (so.rodata == NULL)
+                return 0;
 
         ret = ring->dispatch_execbuffer(ring,
                                         so.ggtt_offset,
@@ -164,6 +174,6 @@ int i915_gem_render_state_init(struct intel_engine_cs *ring)
         ret = __i915_add_request(ring, NULL, so.obj, NULL);
         /* __i915_add_request moves object to inactive if it fails */
 out:
-        render_state_fini(&so);
+        i915_gem_render_state_fini(&so);
         return ret;
 }
diff --git a/drivers/gpu/drm/i915/i915_gem_render_state.h b/drivers/gpu/drm/i915/i915_gem_render_state.h
new file mode 100644
index 000000000000..c44961ed3fad
--- /dev/null
+++ b/drivers/gpu/drm/i915/i915_gem_render_state.h
@@ -0,0 +1,47 @@
+/*
+ * Copyright © 2014 Intel Corporation
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef _I915_GEM_RENDER_STATE_H_
+#define _I915_GEM_RENDER_STATE_H_
+
+#include <linux/types.h>
+
+struct intel_renderstate_rodata {
+        const u32 *reloc;
+        const u32 *batch;
+        const u32 batch_items;
+};
+
+struct render_state {
+        const struct intel_renderstate_rodata *rodata;
+        struct drm_i915_gem_object *obj;
+        u64 ggtt_offset;
+        int gen;
+};
+
+int i915_gem_render_state_init(struct intel_engine_cs *ring);
+void i915_gem_render_state_fini(struct render_state *so);
+int i915_gem_render_state_prepare(struct intel_engine_cs *ring,
+                                  struct render_state *so);
+
+#endif /* _I915_GEM_RENDER_STATE_H_ */
diff --git a/drivers/gpu/drm/i915/i915_gem_stolen.c b/drivers/gpu/drm/i915/i915_gem_stolen.c
index 21c025a209c0..85fda6b803e4 100644
--- a/drivers/gpu/drm/i915/i915_gem_stolen.c
+++ b/drivers/gpu/drm/i915/i915_gem_stolen.c
@@ -289,6 +289,7 @@ void i915_gem_cleanup_stolen(struct drm_device *dev)
 int i915_gem_init_stolen(struct drm_device *dev)
 {
         struct drm_i915_private *dev_priv = dev->dev_private;
+        u32 tmp;
         int bios_reserved = 0;
 
 #ifdef CONFIG_INTEL_IOMMU
@@ -308,8 +309,16 @@ int i915_gem_init_stolen(struct drm_device *dev)
         DRM_DEBUG_KMS("found %zd bytes of stolen memory at %08lx\n",
                       dev_priv->gtt.stolen_size, dev_priv->mm.stolen_base);
 
-        if (IS_VALLEYVIEW(dev))
-                bios_reserved = 1024*1024; /* top 1M on VLV/BYT */
+        if (INTEL_INFO(dev)->gen >= 8) {
+                tmp = I915_READ(GEN7_BIOS_RESERVED);
+                tmp >>= GEN8_BIOS_RESERVED_SHIFT;
+                tmp &= GEN8_BIOS_RESERVED_MASK;
+                bios_reserved = (1024*1024) << tmp;
+        } else if (IS_GEN7(dev)) {
+                tmp = I915_READ(GEN7_BIOS_RESERVED);
+                bios_reserved = tmp & GEN7_BIOS_RESERVED_256K ?
+                        256*1024 : 1024*1024;
+        }
 
         if (WARN_ON(bios_reserved > dev_priv->gtt.stolen_size))
                 return 0;
diff --git a/drivers/gpu/drm/i915/i915_gem_tiling.c b/drivers/gpu/drm/i915/i915_gem_tiling.c
index cb150e8b4336..2cefb597df6d 100644
--- a/drivers/gpu/drm/i915/i915_gem_tiling.c
+++ b/drivers/gpu/drm/i915/i915_gem_tiling.c
@@ -91,7 +91,14 @@ i915_gem_detect_bit_6_swizzle(struct drm_device *dev)
         uint32_t swizzle_x = I915_BIT_6_SWIZZLE_UNKNOWN;
         uint32_t swizzle_y = I915_BIT_6_SWIZZLE_UNKNOWN;
 
-        if (IS_VALLEYVIEW(dev)) {
+        if (INTEL_INFO(dev)->gen >= 8 || IS_VALLEYVIEW(dev)) {
+                /*
+                 * On BDW+, swizzling is not used. We leave the CPU memory
+                 * controller in charge of optimizing memory accesses without
+                 * the extra address manipulation GPU side.
+                 *
+                 * VLV and CHV don't have GPU swizzling.
+                 */
                 swizzle_x = I915_BIT_6_SWIZZLE_NONE;
                 swizzle_y = I915_BIT_6_SWIZZLE_NONE;
         } else if (INTEL_INFO(dev)->gen >= 6) {
@@ -376,7 +383,7 @@ i915_gem_set_tiling(struct drm_device *dev, void *data,
 
                 if (ret == 0) {
                         obj->fence_dirty =
-                                obj->fenced_gpu_access ||
+                                obj->last_fenced_seqno ||
                                 obj->fence_reg != I915_FENCE_REG_NONE;
 
                         obj->tiling_mode = args->tiling_mode;
diff --git a/drivers/gpu/drm/i915/i915_gem_userptr.c b/drivers/gpu/drm/i915/i915_gem_userptr.c
index d38413997379..d182058383a9 100644
--- a/drivers/gpu/drm/i915/i915_gem_userptr.c
+++ b/drivers/gpu/drm/i915/i915_gem_userptr.c
@@ -293,15 +293,23 @@ i915_gem_userptr_release__mmu_notifier(struct drm_i915_gem_object *obj)
 static struct i915_mmu_notifier *
 i915_mmu_notifier_find(struct i915_mm_struct *mm)
 {
-        if (mm->mn == NULL) {
-                down_write(&mm->mm->mmap_sem);
-                mutex_lock(&to_i915(mm->dev)->mm_lock);
-                if (mm->mn == NULL)
-                        mm->mn = i915_mmu_notifier_create(mm->mm);
-                mutex_unlock(&to_i915(mm->dev)->mm_lock);
-                up_write(&mm->mm->mmap_sem);
+        struct i915_mmu_notifier *mn = mm->mn;
+
+        mn = mm->mn;
+        if (mn)
+                return mn;
+
+        down_write(&mm->mm->mmap_sem);
+        mutex_lock(&to_i915(mm->dev)->mm_lock);
+        if ((mn = mm->mn) == NULL) {
+                mn = i915_mmu_notifier_create(mm->mm);
+                if (!IS_ERR(mn))
+                        mm->mn = mn;
         }
-        return mm->mn;
+        mutex_unlock(&to_i915(mm->dev)->mm_lock);
+        up_write(&mm->mm->mmap_sem);
+
+        return mn;
 }
 
 static int
@@ -681,16 +689,15 @@ i915_gem_userptr_get_pages(struct drm_i915_gem_object *obj)
 static void
 i915_gem_userptr_put_pages(struct drm_i915_gem_object *obj)
 {
-        struct scatterlist *sg;
-        int i;
+        struct sg_page_iter sg_iter;
 
         BUG_ON(obj->userptr.work != NULL);
 
         if (obj->madv != I915_MADV_WILLNEED)
                 obj->dirty = 0;
 
-        for_each_sg(obj->pages->sgl, sg, obj->pages->nents, i) {
-                struct page *page = sg_page(sg);
+        for_each_sg_page(obj->pages->sgl, &sg_iter, obj->pages->nents, 0) {
+                struct page *page = sg_page_iter_page(&sg_iter);
 
                 if (obj->dirty)
                         set_page_dirty(page);
diff --git a/drivers/gpu/drm/i915/i915_gpu_error.c b/drivers/gpu/drm/i915/i915_gpu_error.c
index eab41f9390f8..2c87a797213f 100644
--- a/drivers/gpu/drm/i915/i915_gpu_error.c
+++ b/drivers/gpu/drm/i915/i915_gpu_error.c
@@ -192,10 +192,10 @@ static void print_error_buffers(struct drm_i915_error_state_buf *m,
                                 struct drm_i915_error_buffer *err,
                                 int count)
 {
-        err_printf(m, "%s [%d]:\n", name, count);
+        err_printf(m, "  %s [%d]:\n", name, count);
 
         while (count--) {
-                err_printf(m, "  %08x %8u %02x %02x %x %x",
+                err_printf(m, "    %08x %8u %02x %02x %x %x",
                            err->gtt_offset,
                            err->size,
                            err->read_domains,
@@ -208,7 +208,7 @@ static void print_error_buffers(struct drm_i915_error_state_buf *m,
                 err_puts(m, err->userptr ? " userptr" : "");
                 err_puts(m, err->ring != -1 ? " " : "");
                 err_puts(m, ring_str(err->ring));
-                err_puts(m, i915_cache_level_str(err->cache_level));
+                err_puts(m, i915_cache_level_str(m->i915, err->cache_level));
 
                 if (err->name)
                         err_printf(m, " (name: %d)", err->name);
@@ -393,15 +393,17 @@ int i915_error_state_to_str(struct drm_i915_error_state_buf *m,
                 i915_ring_error_state(m, dev, &error->ring[i]);
         }
 
-        if (error->active_bo)
+        for (i = 0; i < error->vm_count; i++) {
+                err_printf(m, "vm[%d]\n", i);
+
                 print_error_buffers(m, "Active",
-                                    error->active_bo[0],
-                                    error->active_bo_count[0]);
+                                    error->active_bo[i],
+                                    error->active_bo_count[i]);
 
-        if (error->pinned_bo)
                 print_error_buffers(m, "Pinned",
-                                    error->pinned_bo[0],
-                                    error->pinned_bo_count[0]);
+                                    error->pinned_bo[i],
+                                    error->pinned_bo_count[i]);
+        }
 
         for (i = 0; i < ARRAY_SIZE(error->ring); i++) {
                 obj = error->ring[i].batchbuffer;
@@ -492,9 +494,11 @@ out:
 }
 
 int i915_error_state_buf_init(struct drm_i915_error_state_buf *ebuf,
+                              struct drm_i915_private *i915,
                               size_t count, loff_t pos)
 {
         memset(ebuf, 0, sizeof(*ebuf));
+        ebuf->i915 = i915;
 
         /* We need to have enough room to store any i915_error_state printf
          * so that we can move it to start position.
@@ -556,24 +560,54 @@ static void i915_error_state_free(struct kref *error_ref)
 }
 
 static struct drm_i915_error_object *
-i915_error_object_create_sized(struct drm_i915_private *dev_priv,
-                               struct drm_i915_gem_object *src,
-                               struct i915_address_space *vm,
-                               const int num_pages)
+i915_error_object_create(struct drm_i915_private *dev_priv,
+                         struct drm_i915_gem_object *src,
+                         struct i915_address_space *vm)
 {
         struct drm_i915_error_object *dst;
-        int i;
+        int num_pages;
+        bool use_ggtt;
+        int i = 0;
         u32 reloc_offset;
 
         if (src == NULL || src->pages == NULL)
                 return NULL;
 
+        num_pages = src->base.size >> PAGE_SHIFT;
+
         dst = kmalloc(sizeof(*dst) + num_pages * sizeof(u32 *), GFP_ATOMIC);
         if (dst == NULL)
                 return NULL;
 
-        reloc_offset = dst->gtt_offset = i915_gem_obj_offset(src, vm);
-        for (i = 0; i < num_pages; i++) {
+        if (i915_gem_obj_bound(src, vm))
+                dst->gtt_offset = i915_gem_obj_offset(src, vm);
+        else
+                dst->gtt_offset = -1;
+
+        reloc_offset = dst->gtt_offset;
+        use_ggtt = (src->cache_level == I915_CACHE_NONE &&
+                    i915_is_ggtt(vm) &&
+                    src->has_global_gtt_mapping &&
+                    reloc_offset + num_pages * PAGE_SIZE <= dev_priv->gtt.mappable_end);
+
+        /* Cannot access stolen address directly, try to use the aperture */
+        if (src->stolen) {
+                use_ggtt = true;
+
+                if (!src->has_global_gtt_mapping)
+                        goto unwind;
+
+                reloc_offset = i915_gem_obj_ggtt_offset(src);
+                if (reloc_offset + num_pages * PAGE_SIZE > dev_priv->gtt.mappable_end)
+                        goto unwind;
+        }
+
+        /* Cannot access snooped pages through the aperture */
+        if (use_ggtt && src->cache_level != I915_CACHE_NONE && !HAS_LLC(dev_priv->dev))
+                goto unwind;
+
+        dst->page_count = num_pages;
+        while (num_pages--) {
                 unsigned long flags;
                 void *d;
 
@@ -582,10 +616,7 @@ i915_error_object_create_sized(struct drm_i915_private *dev_priv,
                         goto unwind;
 
                 local_irq_save(flags);
-                if (src->cache_level == I915_CACHE_NONE &&
-                    reloc_offset < dev_priv->gtt.mappable_end &&
-                    src->has_global_gtt_mapping &&
-                    i915_is_ggtt(vm)) {
+                if (use_ggtt) {
                         void __iomem *s;
 
                         /* Simply ignore tiling or any overlapping fence.
@@ -597,14 +628,6 @@ i915_error_object_create_sized(struct drm_i915_private *dev_priv,
                                                      reloc_offset);
                         memcpy_fromio(d, s, PAGE_SIZE);
                         io_mapping_unmap_atomic(s);
-                } else if (src->stolen) {
-                        unsigned long offset;
-
-                        offset = dev_priv->mm.stolen_base;
-                        offset += src->stolen->start;
-                        offset += i << PAGE_SHIFT;
-
-                        memcpy_fromio(d, (void __iomem *) offset, PAGE_SIZE);
                 } else {
                         struct page *page;
                         void *s;
@@ -621,11 +644,9 @@ i915_error_object_create_sized(struct drm_i915_private *dev_priv,
                 }
                 local_irq_restore(flags);
 
-                dst->pages[i] = d;
-
+                dst->pages[i++] = d;
                 reloc_offset += PAGE_SIZE;
         }
-        dst->page_count = num_pages;
 
         return dst;
 
@@ -635,22 +656,19 @@ unwind:
         kfree(dst);
         return NULL;
 }
-#define i915_error_object_create(dev_priv, src, vm) \
-        i915_error_object_create_sized((dev_priv), (src), (vm), \
-                                       (src)->base.size>>PAGE_SHIFT)
-
 #define i915_error_ggtt_object_create(dev_priv, src) \
-        i915_error_object_create_sized((dev_priv), (src), &(dev_priv)->gtt.base, \
-                                       (src)->base.size>>PAGE_SHIFT)
+        i915_error_object_create((dev_priv), (src), &(dev_priv)->gtt.base)
 
 static void capture_bo(struct drm_i915_error_buffer *err,
-                       struct drm_i915_gem_object *obj)
+                       struct i915_vma *vma)
 {
+        struct drm_i915_gem_object *obj = vma->obj;
+
         err->size = obj->base.size;
         err->name = obj->base.name;
         err->rseqno = obj->last_read_seqno;
         err->wseqno = obj->last_write_seqno;
-        err->gtt_offset = i915_gem_obj_ggtt_offset(obj);
+        err->gtt_offset = vma->node.start;
         err->read_domains = obj->base.read_domains;
         err->write_domain = obj->base.write_domain;
         err->fence_reg = obj->fence_reg;
@@ -674,7 +692,7 @@ static u32 capture_active_bo(struct drm_i915_error_buffer *err,
         int i = 0;
 
         list_for_each_entry(vma, head, mm_list) {
-                capture_bo(err++, vma->obj);
+                capture_bo(err++, vma);
                 if (++i == count)
                         break;
         }
@@ -683,21 +701,27 @@ static u32 capture_active_bo(struct drm_i915_error_buffer *err,
 }
 
 static u32 capture_pinned_bo(struct drm_i915_error_buffer *err,
-                             int count, struct list_head *head)
+                             int count, struct list_head *head,
+                             struct i915_address_space *vm)
 {
         struct drm_i915_gem_object *obj;
-        int i = 0;
+        struct drm_i915_error_buffer * const first = err;
+        struct drm_i915_error_buffer * const last = err + count;
 
         list_for_each_entry(obj, head, global_list) {
-                if (!i915_gem_obj_is_pinned(obj))
-                        continue;
+                struct i915_vma *vma;
 
-                capture_bo(err++, obj);
-                if (++i == count)
+                if (err == last)
                         break;
+
+                list_for_each_entry(vma, &obj->vma_list, vma_link)
+                        if (vma->vm == vm && vma->pin_count > 0) {
+                                capture_bo(err++, vma);
+                                break;
+                        }
         }
 
-        return i;
+        return err - first;
 }
 
 /* Generate a semi-unique error code. The code is not meant to have meaning, The
@@ -890,9 +914,6 @@ static void i915_record_ring_state(struct drm_device *dev,
                 ering->hws = I915_READ(mmio);
         }
 
-        ering->cpu_ring_head = ring->buffer->head;
-        ering->cpu_ring_tail = ring->buffer->tail;
-
         ering->hangcheck_score = ring->hangcheck.score;
         ering->hangcheck_action = ring->hangcheck.action;
 
@@ -955,6 +976,7 @@ static void i915_gem_record_rings(struct drm_device *dev,
 
         for (i = 0; i < I915_NUM_RINGS; i++) {
                 struct intel_engine_cs *ring = &dev_priv->ring[i];
+                struct intel_ringbuffer *rbuf;
 
                 error->ring[i].pid = -1;
 
@@ -967,6 +989,12 @@ static void i915_gem_record_rings(struct drm_device *dev,
 
                 request = i915_gem_find_active_request(ring);
                 if (request) {
+                        struct i915_address_space *vm;
+
+                        vm = request->ctx && request->ctx->ppgtt ?
+                                &request->ctx->ppgtt->base :
+                                &dev_priv->gtt.base;
+
                         /* We need to copy these to an anonymous buffer
                          * as the simplest method to avoid being overwritten
                          * by userspace.
@@ -974,12 +1002,9 @@ static void i915_gem_record_rings(struct drm_device *dev,
                         error->ring[i].batchbuffer =
                                 i915_error_object_create(dev_priv,
                                                          request->batch_obj,
-                                                         request->ctx ?
-                                                         request->ctx->vm :
-                                                         &dev_priv->gtt.base);
+                                                         vm);
 
-                        if (HAS_BROKEN_CS_TLB(dev_priv->dev) &&
-                            ring->scratch.obj)
+                        if (HAS_BROKEN_CS_TLB(dev_priv->dev))
                                 error->ring[i].wa_batchbuffer =
                                         i915_error_ggtt_object_create(dev_priv,
                                                              ring->scratch.obj);
@@ -998,12 +1023,27 @@ static void i915_gem_record_rings(struct drm_device *dev,
                         }
                 }
 
+                if (i915.enable_execlists) {
+                        /* TODO: This is only a small fix to keep basic error
+                         * capture working, but we need to add more information
+                         * for it to be useful (e.g. dump the context being
+                         * executed).
+                         */
+                        if (request)
+                                rbuf = request->ctx->engine[ring->id].ringbuf;
+                        else
+                                rbuf = ring->default_context->engine[ring->id].ringbuf;
+                } else
+                        rbuf = ring->buffer;
+
+                error->ring[i].cpu_ring_head = rbuf->head;
+                error->ring[i].cpu_ring_tail = rbuf->tail;
+
                 error->ring[i].ringbuffer =
-                        i915_error_ggtt_object_create(dev_priv, ring->buffer->obj);
+                        i915_error_ggtt_object_create(dev_priv, rbuf->obj);
 
-                if (ring->status_page.obj)
-                        error->ring[i].hws_page =
-                                i915_error_ggtt_object_create(dev_priv, ring->status_page.obj);
+                error->ring[i].hws_page =
+                        i915_error_ggtt_object_create(dev_priv, ring->status_page.obj);
 
                 i915_gem_record_active_context(ring, error, &error->ring[i]);
 
@@ -1049,9 +1089,14 @@ static void i915_gem_capture_vm(struct drm_i915_private *dev_priv,
         list_for_each_entry(vma, &vm->active_list, mm_list)
                 i++;
         error->active_bo_count[ndx] = i;
-        list_for_each_entry(obj, &dev_priv->mm.bound_list, global_list)
-                if (i915_gem_obj_is_pinned(obj))
-                        i++;
+
+        list_for_each_entry(obj, &dev_priv->mm.bound_list, global_list) {
+                list_for_each_entry(vma, &obj->vma_list, vma_link)
+                        if (vma->vm == vm && vma->pin_count > 0) {
+                                i++;
+                                break;
+                        }
+        }
         error->pinned_bo_count[ndx] = i - error->active_bo_count[ndx];
 
         if (i) {
@@ -1070,7 +1115,7 @@ static void i915_gem_capture_vm(struct drm_i915_private *dev_priv,
                 error->pinned_bo_count[ndx] =
                         capture_pinned_bo(pinned_bo,
                                           error->pinned_bo_count[ndx],
-                                          &dev_priv->mm.bound_list);
+                                          &dev_priv->mm.bound_list, vm);
         error->active_bo[ndx] = active_bo;
         error->pinned_bo[ndx] = pinned_bo;
 }
@@ -1091,8 +1136,25 @@ static void i915_gem_capture_buffers(struct drm_i915_private *dev_priv,
         error->pinned_bo_count = kcalloc(cnt, sizeof(*error->pinned_bo_count),
                                          GFP_ATOMIC);
 
-        list_for_each_entry(vm, &dev_priv->vm_list, global_link)
-                i915_gem_capture_vm(dev_priv, error, vm, i++);
+        if (error->active_bo == NULL ||
+            error->pinned_bo == NULL ||
+            error->active_bo_count == NULL ||
+            error->pinned_bo_count == NULL) {
+                kfree(error->active_bo);
+                kfree(error->active_bo_count);
+                kfree(error->pinned_bo);
+                kfree(error->pinned_bo_count);
+
+                error->active_bo = NULL;
+                error->active_bo_count = NULL;
+                error->pinned_bo = NULL;
+                error->pinned_bo_count = NULL;
+        } else {
+                list_for_each_entry(vm, &dev_priv->vm_list, global_link)
+                        i915_gem_capture_vm(dev_priv, error, vm, i++);
+
+                error->vm_count = cnt;
+        }
 }
 
 /* Capture all registers which don't fit into another category. */
@@ -1295,11 +1357,11 @@ void i915_destroy_error_state(struct drm_device *dev)
                 kref_put(&error->ref, i915_error_state_free);
 }
 
-const char *i915_cache_level_str(int type)
+const char *i915_cache_level_str(struct drm_i915_private *i915, int type)
 {
         switch (type) {
         case I915_CACHE_NONE: return " uncached";
-        case I915_CACHE_LLC: return " snooped or LLC";
+        case I915_CACHE_LLC: return HAS_LLC(i915) ? " LLC" : " snooped";
         case I915_CACHE_L3_LLC: return " L3+LLC";
         case I915_CACHE_WT: return " WT";
         default: return "";
diff --git a/drivers/gpu/drm/i915/i915_irq.c b/drivers/gpu/drm/i915/i915_irq.c
index 0050ee9470f1..3201986bf25e 100644
--- a/drivers/gpu/drm/i915/i915_irq.c
+++ b/drivers/gpu/drm/i915/i915_irq.c
@@ -151,7 +151,7 @@ ironlake_disable_display_irq(struct drm_i915_private *dev_priv, u32 mask)
 {
         assert_spin_locked(&dev_priv->irq_lock);
 
-        if (!intel_irqs_enabled(dev_priv))
+        if (WARN_ON(!intel_irqs_enabled(dev_priv)))
                 return;
 
         if ((dev_priv->irq_mask & mask) != mask) {
@@ -238,7 +238,7 @@ static bool ivb_can_enable_err_int(struct drm_device *dev)
 
         assert_spin_locked(&dev_priv->irq_lock);
 
-        for_each_pipe(pipe) {
+        for_each_pipe(dev_priv, pipe) {
                 crtc = to_intel_crtc(dev_priv->pipe_to_crtc_mapping[pipe]);
 
                 if (crtc->cpu_fifo_underrun_disabled)
@@ -296,7 +296,7 @@ static bool cpt_can_enable_serr_int(struct drm_device *dev)
 
         assert_spin_locked(&dev_priv->irq_lock);
 
-        for_each_pipe(pipe) {
+        for_each_pipe(dev_priv, pipe) {
                 crtc = to_intel_crtc(dev_priv->pipe_to_crtc_mapping[pipe]);
 
                 if (crtc->pch_fifo_underrun_disabled)
@@ -497,7 +497,7 @@ static bool __intel_set_cpu_fifo_underrun_reporting(struct drm_device *dev,
         old = !intel_crtc->cpu_fifo_underrun_disabled;
         intel_crtc->cpu_fifo_underrun_disabled = !enable;
 
-        if (INTEL_INFO(dev)->gen < 5 || IS_VALLEYVIEW(dev))
+        if (HAS_GMCH_DISPLAY(dev))
                 i9xx_set_fifo_underrun_reporting(dev, pipe, enable, old);
         else if (IS_GEN5(dev) || IS_GEN6(dev))
                 ironlake_set_fifo_underrun_reporting(dev, pipe, enable);
@@ -1020,7 +1020,7 @@ static int i915_get_crtc_scanoutpos(struct drm_device *dev, int pipe,
 
         /* In vblank? */
         if (in_vbl)
-                ret |= DRM_SCANOUTPOS_INVBL;
+                ret |= DRM_SCANOUTPOS_IN_VBLANK;
 
         return ret;
 }
@@ -1322,10 +1322,10 @@ static u32 vlv_c0_residency(struct drm_i915_private *dev_priv,
  * @dev_priv: DRM device private
  *
  */
-static u32 vlv_calc_delay_from_C0_counters(struct drm_i915_private *dev_priv)
+static int vlv_calc_delay_from_C0_counters(struct drm_i915_private *dev_priv)
 {
         u32 residency_C0_up = 0, residency_C0_down = 0;
-        u8 new_delay, adj;
+        int new_delay, adj;
 
         dev_priv->rps.ei_interrupt_count++;
 
@@ -1627,6 +1627,7 @@ static irqreturn_t gen8_gt_irq_handler(struct drm_device *dev,
                                        struct drm_i915_private *dev_priv,
                                        u32 master_ctl)
 {
+        struct intel_engine_cs *ring;
         u32 rcs, bcs, vcs;
         uint32_t tmp = 0;
         irqreturn_t ret = IRQ_NONE;
@@ -1636,12 +1637,20 @@ static irqreturn_t gen8_gt_irq_handler(struct drm_device *dev,
                 if (tmp) {
                         I915_WRITE(GEN8_GT_IIR(0), tmp);
                         ret = IRQ_HANDLED;
+
                         rcs = tmp >> GEN8_RCS_IRQ_SHIFT;
-                        bcs = tmp >> GEN8_BCS_IRQ_SHIFT;
+                        ring = &dev_priv->ring[RCS];
                         if (rcs & GT_RENDER_USER_INTERRUPT)
-                                notify_ring(dev, &dev_priv->ring[RCS]);
+                                notify_ring(dev, ring);
+                        if (rcs & GT_CONTEXT_SWITCH_INTERRUPT)
+                                intel_execlists_handle_ctx_events(ring);
+
+                        bcs = tmp >> GEN8_BCS_IRQ_SHIFT;
+                        ring = &dev_priv->ring[BCS];
                         if (bcs & GT_RENDER_USER_INTERRUPT)
-                                notify_ring(dev, &dev_priv->ring[BCS]);
+                                notify_ring(dev, ring);
+                        if (bcs & GT_CONTEXT_SWITCH_INTERRUPT)
+                                intel_execlists_handle_ctx_events(ring);
                 } else
                         DRM_ERROR("The master control interrupt lied (GT0)!\n");
         }
@@ -1651,12 +1660,20 @@ static irqreturn_t gen8_gt_irq_handler(struct drm_device *dev,
                 if (tmp) {
                         I915_WRITE(GEN8_GT_IIR(1), tmp);
                         ret = IRQ_HANDLED;
+
                         vcs = tmp >> GEN8_VCS1_IRQ_SHIFT;
+                        ring = &dev_priv->ring[VCS];
                         if (vcs & GT_RENDER_USER_INTERRUPT)
-                                notify_ring(dev, &dev_priv->ring[VCS]);
+                                notify_ring(dev, ring);
+                        if (vcs & GT_CONTEXT_SWITCH_INTERRUPT)
+                                intel_execlists_handle_ctx_events(ring);
+
                         vcs = tmp >> GEN8_VCS2_IRQ_SHIFT;
+                        ring = &dev_priv->ring[VCS2];
                         if (vcs & GT_RENDER_USER_INTERRUPT)
-                                notify_ring(dev, &dev_priv->ring[VCS2]);
+                                notify_ring(dev, ring);
+                        if (vcs & GT_CONTEXT_SWITCH_INTERRUPT)
+                                intel_execlists_handle_ctx_events(ring);
                 } else
                         DRM_ERROR("The master control interrupt lied (GT1)!\n");
         }
@@ -1677,9 +1694,13 @@ static irqreturn_t gen8_gt_irq_handler(struct drm_device *dev,
                 if (tmp) {
                         I915_WRITE(GEN8_GT_IIR(3), tmp);
                         ret = IRQ_HANDLED;
+
                         vcs = tmp >> GEN8_VECS_IRQ_SHIFT;
+                        ring = &dev_priv->ring[VECS];
                         if (vcs & GT_RENDER_USER_INTERRUPT)
-                                notify_ring(dev, &dev_priv->ring[VECS]);
+                                notify_ring(dev, ring);
+                        if (vcs & GT_CONTEXT_SWITCH_INTERRUPT)
+                                intel_execlists_handle_ctx_events(ring);
                 } else
                         DRM_ERROR("The master control interrupt lied (GT3)!\n");
         }
@@ -1772,7 +1793,9 @@ static inline void intel_hpd_irq_handler(struct drm_device *dev,
                                 long_hpd = (dig_hotplug_reg >> dig_shift) & PORTB_HOTPLUG_LONG_DETECT;
                         }
 
-                        DRM_DEBUG_DRIVER("digital hpd port %d %d\n", port, long_hpd);
+                        DRM_DEBUG_DRIVER("digital hpd port %c - %s\n",
+                                         port_name(port),
+                                         long_hpd ? "long" : "short");
                         /* for long HPD pulses we want to have the digital queue happen,
                            but we still want HPD storm detection to function. */
                         if (long_hpd) {
@@ -1984,14 +2007,9 @@ static void gen6_rps_irq_handler(struct drm_i915_private *dev_priv, u32 pm_iir)
 
 static bool intel_pipe_handle_vblank(struct drm_device *dev, enum pipe pipe)
 {
-        struct intel_crtc *crtc;
-
         if (!drm_handle_vblank(dev, pipe))
                 return false;
 
-        crtc = to_intel_crtc(intel_get_crtc_for_pipe(dev, pipe));
-        wake_up(&crtc->vbl_wait);
-
         return true;
 }
 
@@ -2002,7 +2020,7 @@ static void valleyview_pipestat_irq_handler(struct drm_device *dev, u32 iir)
         int pipe;
 
         spin_lock(&dev_priv->irq_lock);
-        for_each_pipe(pipe) {
+        for_each_pipe(dev_priv, pipe) {
                 int reg;
                 u32 mask, iir_bit = 0;
 
@@ -2047,9 +2065,10 @@ static void valleyview_pipestat_irq_handler(struct drm_device *dev, u32 iir)
         }
         spin_unlock(&dev_priv->irq_lock);
 
-        for_each_pipe(pipe) {
-                if (pipe_stats[pipe] & PIPE_START_VBLANK_INTERRUPT_STATUS)
-                        intel_pipe_handle_vblank(dev, pipe);
+        for_each_pipe(dev_priv, pipe) {
+                if (pipe_stats[pipe] & PIPE_START_VBLANK_INTERRUPT_STATUS &&
+                    intel_pipe_handle_vblank(dev, pipe))
+                        intel_check_page_flip(dev, pipe);
 
                 if (pipe_stats[pipe] & PLANE_FLIP_DONE_INT_STATUS_VLV) {
                         intel_prepare_page_flip(dev, pipe);
@@ -2216,7 +2235,7 @@ static void ibx_irq_handler(struct drm_device *dev, u32 pch_iir)
                 DRM_ERROR("PCH poison interrupt\n");
 
         if (pch_iir & SDE_FDI_MASK)
-                for_each_pipe(pipe)
+                for_each_pipe(dev_priv, pipe)
                         DRM_DEBUG_DRIVER("  pipe %c FDI IIR: 0x%08x\n",
                                          pipe_name(pipe),
                                          I915_READ(FDI_RX_IIR(pipe)));
@@ -2247,7 +2266,7 @@ static void ivb_err_int_handler(struct drm_device *dev)
         if (err_int & ERR_INT_POISON)
                 DRM_ERROR("Poison interrupt\n");
 
-        for_each_pipe(pipe) {
+        for_each_pipe(dev_priv, pipe) {
                 if (err_int & ERR_INT_FIFO_UNDERRUN(pipe)) {
                         if (intel_set_cpu_fifo_underrun_reporting(dev, pipe,
                                                                   false))
@@ -2324,7 +2343,7 @@ static void cpt_irq_handler(struct drm_device *dev, u32 pch_iir)
                 DRM_DEBUG_DRIVER("Audio CP change interrupt\n");
 
         if (pch_iir & SDE_FDI_MASK_CPT)
-                for_each_pipe(pipe)
+                for_each_pipe(dev_priv, pipe)
                         DRM_DEBUG_DRIVER("  pipe %c FDI IIR: 0x%08x\n",
                                          pipe_name(pipe),
                                          I915_READ(FDI_RX_IIR(pipe)));
@@ -2347,9 +2366,10 @@ static void ilk_display_irq_handler(struct drm_device *dev, u32 de_iir)
         if (de_iir & DE_POISON)
                 DRM_ERROR("Poison interrupt\n");
 
-        for_each_pipe(pipe) {
-                if (de_iir & DE_PIPE_VBLANK(pipe))
-                        intel_pipe_handle_vblank(dev, pipe);
+        for_each_pipe(dev_priv, pipe) {
+                if (de_iir & DE_PIPE_VBLANK(pipe) &&
+                    intel_pipe_handle_vblank(dev, pipe))
+                        intel_check_page_flip(dev, pipe);
 
                 if (de_iir & DE_PIPE_FIFO_UNDERRUN(pipe))
                         if (intel_set_cpu_fifo_underrun_reporting(dev, pipe, false))
@@ -2397,9 +2417,10 @@ static void ivb_display_irq_handler(struct drm_device *dev, u32 de_iir)
         if (de_iir & DE_GSE_IVB)
                 intel_opregion_asle_intr(dev);
 
-        for_each_pipe(pipe) {
-                if (de_iir & (DE_PIPE_VBLANK_IVB(pipe)))
-                        intel_pipe_handle_vblank(dev, pipe);
+        for_each_pipe(dev_priv, pipe) {
+                if (de_iir & (DE_PIPE_VBLANK_IVB(pipe)) &&
+                    intel_pipe_handle_vblank(dev, pipe))
+                        intel_check_page_flip(dev, pipe);
 
                 /* plane/pipes map 1:1 on ilk+ */
                 if (de_iir & DE_PLANE_FLIP_DONE_IVB(pipe)) {
@@ -2544,7 +2565,7 @@ static irqreturn_t gen8_irq_handler(int irq, void *arg)
                         DRM_ERROR("The master control interrupt lied (DE PORT)!\n");
         }
 
-        for_each_pipe(pipe) {
+        for_each_pipe(dev_priv, pipe) {
                 uint32_t pipe_iir;
 
                 if (!(master_ctl & GEN8_DE_PIPE_IRQ(pipe)))
@@ -2554,8 +2575,9 @@ static irqreturn_t gen8_irq_handler(int irq, void *arg)
                 if (pipe_iir) {
                         ret = IRQ_HANDLED;
                         I915_WRITE(GEN8_DE_PIPE_IIR(pipe), pipe_iir);
-                        if (pipe_iir & GEN8_PIPE_VBLANK)
-                                intel_pipe_handle_vblank(dev, pipe);
+                        if (pipe_iir & GEN8_PIPE_VBLANK &&
+                            intel_pipe_handle_vblank(dev, pipe))
+                                intel_check_page_flip(dev, pipe);
 
                         if (pipe_iir & GEN8_PIPE_PRIMARY_FLIP_DONE) {
                                 intel_prepare_page_flip(dev, pipe);
@@ -2763,7 +2785,7 @@ static void i915_report_and_clear_eir(struct drm_device *dev)
 
         if (eir & I915_ERROR_MEMORY_REFRESH) {
                 pr_err("memory refresh error:\n");
-                for_each_pipe(pipe)
+                for_each_pipe(dev_priv, pipe)
                         pr_err("pipe %c stat: 0x%08x\n",
                                pipe_name(pipe), I915_READ(PIPESTAT(pipe)));
                 /* pipestat has already been acked */
@@ -2860,52 +2882,6 @@ void i915_handle_error(struct drm_device *dev, bool wedged,
         schedule_work(&dev_priv->gpu_error.work);
 }
 
-static void __always_unused i915_pageflip_stall_check(struct drm_device *dev, int pipe)
-{
-        struct drm_i915_private *dev_priv = dev->dev_private;
-        struct drm_crtc *crtc = dev_priv->pipe_to_crtc_mapping[pipe];
-        struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
-        struct drm_i915_gem_object *obj;
-        struct intel_unpin_work *work;
-        unsigned long flags;
-        bool stall_detected;
-
-        /* Ignore early vblank irqs */
-        if (intel_crtc == NULL)
-                return;
-
-        spin_lock_irqsave(&dev->event_lock, flags);
-        work = intel_crtc->unpin_work;
-
-        if (work == NULL ||
-            atomic_read(&work->pending) >= INTEL_FLIP_COMPLETE ||
-            !work->enable_stall_check) {
-                /* Either the pending flip IRQ arrived, or we're too early. Don't check */
-                spin_unlock_irqrestore(&dev->event_lock, flags);
-                return;
-        }
-
-        /* Potential stall - if we see that the flip has happened, assume a missed interrupt */
-        obj = work->pending_flip_obj;
-        if (INTEL_INFO(dev)->gen >= 4) {
-                int dspsurf = DSPSURF(intel_crtc->plane);
-                stall_detected = I915_HI_DISPBASE(I915_READ(dspsurf)) ==
-                                        i915_gem_obj_ggtt_offset(obj);
-        } else {
-                int dspaddr = DSPADDR(intel_crtc->plane);
-                stall_detected = I915_READ(dspaddr) == (i915_gem_obj_ggtt_offset(obj) +
-                                                        crtc->y * crtc->primary->fb->pitches[0] +
-                                                        crtc->x * crtc->primary->fb->bits_per_pixel/8);
-        }
-
-        spin_unlock_irqrestore(&dev->event_lock, flags);
-
-        if (stall_detected) {
-                DRM_DEBUG_DRIVER("Pageflip stall detected\n");
-                intel_prepare_page_flip(dev, intel_crtc->plane);
-        }
-}
-
 /* Called from drm generic code, passed 'crtc' which
  * we use as a pipe index
  */
@@ -3441,7 +3417,7 @@ static void valleyview_irq_preinstall(struct drm_device *dev)
 
         I915_WRITE(PORT_HOTPLUG_EN, 0);
         I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT));
-        for_each_pipe(pipe)
+        for_each_pipe(dev_priv, pipe)
                 I915_WRITE(PIPESTAT(pipe), 0xffff);
         I915_WRITE(VLV_IIR, 0xffffffff);
         I915_WRITE(VLV_IMR, 0xffffffff);
@@ -3467,7 +3443,7 @@ static void gen8_irq_reset(struct drm_device *dev)
 
         gen8_gt_irq_reset(dev_priv);
 
-        for_each_pipe(pipe)
+        for_each_pipe(dev_priv, pipe)
                 if (intel_display_power_enabled(dev_priv,
                                                 POWER_DOMAIN_PIPE(pipe)))
                         GEN8_IRQ_RESET_NDX(DE_PIPE, pipe);
@@ -3510,7 +3486,7 @@ static void cherryview_irq_preinstall(struct drm_device *dev)
         I915_WRITE(PORT_HOTPLUG_EN, 0);
         I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT));
 
-        for_each_pipe(pipe)
+        for_each_pipe(dev_priv, pipe)
                 I915_WRITE(PIPESTAT(pipe), 0xffff);
 
         I915_WRITE(VLV_IMR, 0xffffffff);
@@ -3522,18 +3498,17 @@ static void cherryview_irq_preinstall(struct drm_device *dev)
 static void ibx_hpd_irq_setup(struct drm_device *dev)
 {
         struct drm_i915_private *dev_priv = dev->dev_private;
-        struct drm_mode_config *mode_config = &dev->mode_config;
         struct intel_encoder *intel_encoder;
         u32 hotplug_irqs, hotplug, enabled_irqs = 0;
 
         if (HAS_PCH_IBX(dev)) {
                 hotplug_irqs = SDE_HOTPLUG_MASK;
-                list_for_each_entry(intel_encoder, &mode_config->encoder_list, base.head)
+                for_each_intel_encoder(dev, intel_encoder)
                         if (dev_priv->hpd_stats[intel_encoder->hpd_pin].hpd_mark == HPD_ENABLED)
                                 enabled_irqs |= hpd_ibx[intel_encoder->hpd_pin];
         } else {
                 hotplug_irqs = SDE_HOTPLUG_MASK_CPT;
-                list_for_each_entry(intel_encoder, &mode_config->encoder_list, base.head)
+                for_each_intel_encoder(dev, intel_encoder)
                         if (dev_priv->hpd_stats[intel_encoder->hpd_pin].hpd_mark == HPD_ENABLED)
                                 enabled_irqs |= hpd_cpt[intel_encoder->hpd_pin];
         }
@@ -3782,28 +3757,31 @@ static int valleyview_irq_postinstall(struct drm_device *dev)
 
 static void gen8_gt_irq_postinstall(struct drm_i915_private *dev_priv)
 {
-        int i;
-
         /* These are interrupts we'll toggle with the ring mask register */
         uint32_t gt_interrupts[] = {
                 GT_RENDER_USER_INTERRUPT << GEN8_RCS_IRQ_SHIFT |
+                        GT_CONTEXT_SWITCH_INTERRUPT << GEN8_RCS_IRQ_SHIFT |
                         GT_RENDER_L3_PARITY_ERROR_INTERRUPT |
-                        GT_RENDER_USER_INTERRUPT << GEN8_BCS_IRQ_SHIFT,
+                        GT_RENDER_USER_INTERRUPT << GEN8_BCS_IRQ_SHIFT |
+                        GT_CONTEXT_SWITCH_INTERRUPT << GEN8_BCS_IRQ_SHIFT,
                 GT_RENDER_USER_INTERRUPT << GEN8_VCS1_IRQ_SHIFT |
-                        GT_RENDER_USER_INTERRUPT << GEN8_VCS2_IRQ_SHIFT,
+                        GT_CONTEXT_SWITCH_INTERRUPT << GEN8_VCS1_IRQ_SHIFT |
+                        GT_RENDER_USER_INTERRUPT << GEN8_VCS2_IRQ_SHIFT |
+                        GT_CONTEXT_SWITCH_INTERRUPT << GEN8_VCS2_IRQ_SHIFT,
                 0,
-                GT_RENDER_USER_INTERRUPT << GEN8_VECS_IRQ_SHIFT
+                GT_RENDER_USER_INTERRUPT << GEN8_VECS_IRQ_SHIFT |
+                        GT_CONTEXT_SWITCH_INTERRUPT << GEN8_VECS_IRQ_SHIFT
                 };
 
-        for (i = 0; i < ARRAY_SIZE(gt_interrupts); i++)
-                GEN8_IRQ_INIT_NDX(GT, i, ~gt_interrupts[i], gt_interrupts[i]);
-
         dev_priv->pm_irq_mask = 0xffffffff;
+        GEN8_IRQ_INIT_NDX(GT, 0, ~gt_interrupts[0], gt_interrupts[0]);
+        GEN8_IRQ_INIT_NDX(GT, 1, ~gt_interrupts[1], gt_interrupts[1]);
+        GEN8_IRQ_INIT_NDX(GT, 2, dev_priv->pm_irq_mask, dev_priv->pm_rps_events);
+        GEN8_IRQ_INIT_NDX(GT, 3, ~gt_interrupts[3], gt_interrupts[3]);
 }
 
 static void gen8_de_irq_postinstall(struct drm_i915_private *dev_priv)
 {
-        struct drm_device *dev = dev_priv->dev;
         uint32_t de_pipe_masked = GEN8_PIPE_PRIMARY_FLIP_DONE |
                 GEN8_PIPE_CDCLK_CRC_DONE |
                 GEN8_DE_PIPE_IRQ_FAULT_ERRORS;
@@ -3814,7 +3792,7 @@ static void gen8_de_irq_postinstall(struct drm_i915_private *dev_priv)
         dev_priv->de_irq_mask[PIPE_B] = ~de_pipe_masked;
         dev_priv->de_irq_mask[PIPE_C] = ~de_pipe_masked;
 
-        for_each_pipe(pipe)
+        for_each_pipe(dev_priv, pipe)
                 if (intel_display_power_enabled(dev_priv,
                                 POWER_DOMAIN_PIPE(pipe)))
                         GEN8_IRQ_INIT_NDX(DE_PIPE, pipe,
@@ -3859,12 +3837,12 @@ static int cherryview_irq_postinstall(struct drm_device *dev)
          */
         dev_priv->irq_mask = ~enable_mask;
 
-        for_each_pipe(pipe)
+        for_each_pipe(dev_priv, pipe)
                 I915_WRITE(PIPESTAT(pipe), 0xffff);
 
         spin_lock_irqsave(&dev_priv->irq_lock, irqflags);
         i915_enable_pipestat(dev_priv, PIPE_A, PIPE_GMBUS_INTERRUPT_STATUS);
-        for_each_pipe(pipe)
+        for_each_pipe(dev_priv, pipe)
                 i915_enable_pipestat(dev_priv, pipe, pipestat_enable);
         spin_unlock_irqrestore(&dev_priv->irq_lock, irqflags);
 
@@ -3901,7 +3879,7 @@ static void valleyview_irq_uninstall(struct drm_device *dev)
 
         I915_WRITE(VLV_MASTER_IER, 0);
 
-        for_each_pipe(pipe)
+        for_each_pipe(dev_priv, pipe)
                 I915_WRITE(PIPESTAT(pipe), 0xffff);
 
         I915_WRITE(HWSTAM, 0xffffffff);
@@ -3963,7 +3941,7 @@ do {							\
         I915_WRITE(PORT_HOTPLUG_EN, 0);
         I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT));
 
-        for_each_pipe(pipe)
+        for_each_pipe(dev_priv, pipe)
                 I915_WRITE(PIPESTAT(pipe), 0xffff);
 
         I915_WRITE(VLV_IMR, 0xffffffff);
@@ -3987,7 +3965,7 @@ static void i8xx_irq_preinstall(struct drm_device * dev)
         struct drm_i915_private *dev_priv = dev->dev_private;
         int pipe;
 
-        for_each_pipe(pipe)
+        for_each_pipe(dev_priv, pipe)
                 I915_WRITE(PIPESTAT(pipe), 0);
         I915_WRITE16(IMR, 0xffff);
         I915_WRITE16(IER, 0x0);
@@ -4041,7 +4019,7 @@ static bool i8xx_handle_vblank(struct drm_device *dev,
                 return false;
 
         if ((iir & flip_pending) == 0)
-                return false;
+                goto check_page_flip;
 
         intel_prepare_page_flip(dev, plane);
 
@@ -4052,11 +4030,14 @@ static bool i8xx_handle_vblank(struct drm_device *dev,
          * an interrupt per se, we watch for the change at vblank.
          */
         if (I915_READ16(ISR) & flip_pending)
-                return false;
+                goto check_page_flip;
 
         intel_finish_page_flip(dev, pipe);
-
         return true;
+
+check_page_flip:
+        intel_check_page_flip(dev, pipe);
+        return false;
 }
 
 static irqreturn_t i8xx_irq_handler(int irq, void *arg)
@@ -4087,7 +4068,7 @@ static irqreturn_t i8xx_irq_handler(int irq, void *arg)
                                           "Command parser error, iir 0x%08x",
                                           iir);
 
-                for_each_pipe(pipe) {
+                for_each_pipe(dev_priv, pipe) {
                         int reg = PIPESTAT(pipe);
                         pipe_stats[pipe] = I915_READ(reg);
 
@@ -4107,7 +4088,7 @@ static irqreturn_t i8xx_irq_handler(int irq, void *arg)
                 if (iir & I915_USER_INTERRUPT)
                         notify_ring(dev, &dev_priv->ring[RCS]);
 
-                for_each_pipe(pipe) {
+                for_each_pipe(dev_priv, pipe) {
                         int plane = pipe;
                         if (HAS_FBC(dev))
                                 plane = !plane;
@@ -4135,7 +4116,7 @@ static void i8xx_irq_uninstall(struct drm_device * dev)
         struct drm_i915_private *dev_priv = dev->dev_private;
         int pipe;
 
-        for_each_pipe(pipe) {
+        for_each_pipe(dev_priv, pipe) {
                 /* Clear enable bits; then clear status bits */
                 I915_WRITE(PIPESTAT(pipe), 0);
                 I915_WRITE(PIPESTAT(pipe), I915_READ(PIPESTAT(pipe)));
@@ -4156,7 +4137,7 @@ static void i915_irq_preinstall(struct drm_device * dev)
         }
 
         I915_WRITE16(HWSTAM, 0xeffe);
-        for_each_pipe(pipe)
+        for_each_pipe(dev_priv, pipe)
                 I915_WRITE(PIPESTAT(pipe), 0);
         I915_WRITE(IMR, 0xffffffff);
         I915_WRITE(IER, 0x0);
@@ -4226,7 +4207,7 @@ static bool i915_handle_vblank(struct drm_device *dev,
                 return false;
 
         if ((iir & flip_pending) == 0)
-                return false;
+                goto check_page_flip;
 
         intel_prepare_page_flip(dev, plane);
 
@@ -4237,11 +4218,14 @@ static bool i915_handle_vblank(struct drm_device *dev,
          * an interrupt per se, we watch for the change at vblank.
          */
         if (I915_READ(ISR) & flip_pending)
-                return false;
+                goto check_page_flip;
 
         intel_finish_page_flip(dev, pipe);
-
         return true;
+
+check_page_flip:
+        intel_check_page_flip(dev, pipe);
+        return false;
 }
 
 static irqreturn_t i915_irq_handler(int irq, void *arg)
@@ -4271,7 +4255,7 @@ static irqreturn_t i915_irq_handler(int irq, void *arg)
                                           "Command parser error, iir 0x%08x",
                                           iir);
 
-                for_each_pipe(pipe) {
+                for_each_pipe(dev_priv, pipe) {
                         int reg = PIPESTAT(pipe);
                         pipe_stats[pipe] = I915_READ(reg);
 
@@ -4297,7 +4281,7 @@ static irqreturn_t i915_irq_handler(int irq, void *arg)
                 if (iir & I915_USER_INTERRUPT)
                         notify_ring(dev, &dev_priv->ring[RCS]);
 
-                for_each_pipe(pipe) {
+                for_each_pipe(dev_priv, pipe) {
                         int plane = pipe;
                         if (HAS_FBC(dev))
                                 plane = !plane;
@@ -4355,7 +4339,7 @@ static void i915_irq_uninstall(struct drm_device * dev)
         }
 
         I915_WRITE16(HWSTAM, 0xffff);
-        for_each_pipe(pipe) {
+        for_each_pipe(dev_priv, pipe) {
                 /* Clear enable bits; then clear status bits */
                 I915_WRITE(PIPESTAT(pipe), 0);
                 I915_WRITE(PIPESTAT(pipe), I915_READ(PIPESTAT(pipe)));
@@ -4375,7 +4359,7 @@ static void i965_irq_preinstall(struct drm_device * dev)
         I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT));
 
         I915_WRITE(HWSTAM, 0xeffe);
-        for_each_pipe(pipe)
+        for_each_pipe(dev_priv, pipe)
                 I915_WRITE(PIPESTAT(pipe), 0);
         I915_WRITE(IMR, 0xffffffff);
         I915_WRITE(IER, 0x0);
@@ -4444,7 +4428,6 @@ static int i965_irq_postinstall(struct drm_device *dev)
 static void i915_hpd_irq_setup(struct drm_device *dev)
 {
         struct drm_i915_private *dev_priv = dev->dev_private;
-        struct drm_mode_config *mode_config = &dev->mode_config;
         struct intel_encoder *intel_encoder;
         u32 hotplug_en;
 
@@ -4455,7 +4438,7 @@ static void i915_hpd_irq_setup(struct drm_device *dev)
                 hotplug_en &= ~HOTPLUG_INT_EN_MASK;
                 /* Note HDMI and DP share hotplug bits */
                 /* enable bits are the same for all generations */
-                list_for_each_entry(intel_encoder, &mode_config->encoder_list, base.head)
+                for_each_intel_encoder(dev, intel_encoder)
                         if (dev_priv->hpd_stats[intel_encoder->hpd_pin].hpd_mark == HPD_ENABLED)
                                 hotplug_en |= hpd_mask_i915[intel_encoder->hpd_pin];
                 /* Programming the CRT detection parameters tends
@@ -4501,7 +4484,7 @@ static irqreturn_t i965_irq_handler(int irq, void *arg)
                                           "Command parser error, iir 0x%08x",
                                           iir);
 
-                for_each_pipe(pipe) {
+                for_each_pipe(dev_priv, pipe) {
                         int reg = PIPESTAT(pipe);
                         pipe_stats[pipe] = I915_READ(reg);
 
@@ -4532,7 +4515,7 @@ static irqreturn_t i965_irq_handler(int irq, void *arg)
                 if (iir & I915_BSD_USER_INTERRUPT)
                         notify_ring(dev, &dev_priv->ring[VCS]);
 
-                for_each_pipe(pipe) {
+                for_each_pipe(dev_priv, pipe) {
                         if (pipe_stats[pipe] & PIPE_START_VBLANK_INTERRUPT_STATUS &&
                             i915_handle_vblank(dev, pipe, pipe, iir))
                                 flip_mask &= ~DISPLAY_PLANE_FLIP_PENDING(pipe);
@@ -4589,12 +4572,12 @@ static void i965_irq_uninstall(struct drm_device * dev)
         I915_WRITE(PORT_HOTPLUG_STAT, I915_READ(PORT_HOTPLUG_STAT));
 
         I915_WRITE(HWSTAM, 0xffffffff);
-        for_each_pipe(pipe)
+        for_each_pipe(dev_priv, pipe)
                 I915_WRITE(PIPESTAT(pipe), 0);
         I915_WRITE(IMR, 0xffffffff);
         I915_WRITE(IER, 0x0);
 
-        for_each_pipe(pipe)
+        for_each_pipe(dev_priv, pipe)
                 I915_WRITE(PIPESTAT(pipe),
                            I915_READ(PIPESTAT(pipe)) & 0x8000ffff);
         I915_WRITE(IIR, I915_READ(IIR));
@@ -4652,8 +4635,8 @@ void intel_irq_init(struct drm_device *dev)
         INIT_WORK(&dev_priv->l3_parity.error_work, ivybridge_parity_work);
 
         /* Let's track the enabled rps events */
-        if (IS_VALLEYVIEW(dev))
-                /* WaGsvRC0ResidenncyMethod:VLV */
+        if (IS_VALLEYVIEW(dev) && !IS_CHERRYVIEW(dev))
+                /* WaGsvRC0ResidencyMethod:vlv */
                 dev_priv->pm_rps_events = GEN6_PM_RP_UP_EI_EXPIRED;
         else
                 dev_priv->pm_rps_events = GEN6_PM_RPS_EVENTS;
@@ -4680,6 +4663,14 @@ void intel_irq_init(struct drm_device *dev)
                 dev->max_vblank_count = 0xffffff; /* only 24 bits of frame count */
         }
 
+        /*
+         * Opt out of the vblank disable timer on everything except gen2.
+         * Gen2 doesn't have a hardware frame counter and so depends on
+         * vblank interrupts to produce sane vblank seuquence numbers.
+         */
+        if (!IS_GEN2(dev))
+                dev->vblank_disable_immediate = true;
+
         if (drm_core_check_feature(dev, DRIVER_MODESET)) {
                 dev->driver->get_vblank_timestamp = i915_get_vblank_timestamp;
                 dev->driver->get_scanout_position = i915_get_crtc_scanoutpos;
diff --git a/drivers/gpu/drm/i915/i915_params.c b/drivers/gpu/drm/i915/i915_params.c
index 9842fd2e742a..c91cb2033cc5 100644
--- a/drivers/gpu/drm/i915/i915_params.c
+++ b/drivers/gpu/drm/i915/i915_params.c
@@ -35,6 +35,7 @@ struct i915_params i915 __read_mostly = {
         .vbt_sdvo_panel_type = -1,
         .enable_rc6 = -1,
         .enable_fbc = -1,
+        .enable_execlists = 0,
         .enable_hangcheck = true,
         .enable_ppgtt = -1,
         .enable_psr = 0,
@@ -118,6 +119,11 @@ MODULE_PARM_DESC(enable_ppgtt,
         "Override PPGTT usage. "
         "(-1=auto [default], 0=disabled, 1=aliasing, 2=full)");
 
+module_param_named(enable_execlists, i915.enable_execlists, int, 0400);
+MODULE_PARM_DESC(enable_execlists,
+        "Override execlists usage. "
+        "(-1=auto, 0=disabled [default], 1=enabled)");
+
 module_param_named(enable_psr, i915.enable_psr, int, 0600);
 MODULE_PARM_DESC(enable_psr, "Enable PSR (default: false)");
 
diff --git a/drivers/gpu/drm/i915/i915_reg.h b/drivers/gpu/drm/i915/i915_reg.h
index f29b44c86a2f..c01e5f31430e 100644
--- a/drivers/gpu/drm/i915/i915_reg.h
+++ b/drivers/gpu/drm/i915/i915_reg.h
@@ -143,6 +143,14 @@
 #define GAB_CTL                         0x24000
 #define   GAB_CTL_CONT_AFTER_PAGEFAULT  (1<<8)
 
+#define GEN7_BIOS_RESERVED              0x1082C0
+#define GEN7_BIOS_RESERVED_1M           (0 << 5)
+#define GEN7_BIOS_RESERVED_256K         (1 << 5)
+#define GEN8_BIOS_RESERVED_SHIFT       7
+#define GEN7_BIOS_RESERVED_MASK        0x1
+#define GEN8_BIOS_RESERVED_MASK        0x3
+
+
 /* VGA stuff */
 
 #define VGA_ST01_MDA 0x3ba
@@ -272,6 +280,7 @@
 #define   MI_SEMAPHORE_POLL             (1<<15)
 #define   MI_SEMAPHORE_SAD_GTE_SDD      (1<<12)
 #define MI_STORE_DWORD_IMM      MI_INSTR(0x20, 1)
+#define MI_STORE_DWORD_IMM_GEN8 MI_INSTR(0x20, 2)
 #define   MI_MEM_VIRTUAL        (1 << 22) /* 965+ only */
 #define MI_STORE_DWORD_INDEX    MI_INSTR(0x21, 1)
 #define   MI_STORE_DWORD_INDEX_SHIFT 2
@@ -282,6 +291,7 @@
  *   address/value pairs. Don't overdue it, though, x <= 2^4 must hold!
  */
 #define MI_LOAD_REGISTER_IMM(x) MI_INSTR(0x22, 2*(x)-1)
+#define   MI_LRI_FORCE_POSTED           (1<<12)
 #define MI_STORE_REGISTER_MEM(x) MI_INSTR(0x24, 2*(x)-1)
 #define MI_STORE_REGISTER_MEM_GEN8(x) MI_INSTR(0x24, 3*(x)-1)
 #define   MI_SRM_LRM_GLOBAL_GTT         (1<<22)
@@ -501,10 +511,26 @@
 #define BUNIT_REG_BISOC                         0x11
 
 #define PUNIT_REG_DSPFREQ                       0x36
+#define   DSPFREQSTAT_SHIFT_CHV                 24
+#define   DSPFREQSTAT_MASK_CHV                  (0x1f << DSPFREQSTAT_SHIFT_CHV)
+#define   DSPFREQGUAR_SHIFT_CHV                 8
+#define   DSPFREQGUAR_MASK_CHV                  (0x1f << DSPFREQGUAR_SHIFT_CHV)
 #define   DSPFREQSTAT_SHIFT                     30
 #define   DSPFREQSTAT_MASK                      (0x3 << DSPFREQSTAT_SHIFT)
 #define   DSPFREQGUAR_SHIFT                     14
 #define   DSPFREQGUAR_MASK                      (0x3 << DSPFREQGUAR_SHIFT)
+#define   _DP_SSC(val, pipe)                    ((val) << (2 * (pipe)))
+#define   DP_SSC_MASK(pipe)                     _DP_SSC(0x3, (pipe))
+#define   DP_SSC_PWR_ON(pipe)                   _DP_SSC(0x0, (pipe))
+#define   DP_SSC_CLK_GATE(pipe)                 _DP_SSC(0x1, (pipe))
+#define   DP_SSC_RESET(pipe)                    _DP_SSC(0x2, (pipe))
+#define   DP_SSC_PWR_GATE(pipe)                 _DP_SSC(0x3, (pipe))
+#define   _DP_SSS(val, pipe)                    ((val) << (2 * (pipe) + 16))
+#define   DP_SSS_MASK(pipe)                     _DP_SSS(0x3, (pipe))
+#define   DP_SSS_PWR_ON(pipe)                   _DP_SSS(0x0, (pipe))
+#define   DP_SSS_CLK_GATE(pipe)                 _DP_SSS(0x1, (pipe))
+#define   DP_SSS_RESET(pipe)                    _DP_SSS(0x2, (pipe))
+#define   DP_SSS_PWR_GATE(pipe)                 _DP_SSS(0x3, (pipe))
 
 /* See the PUNIT HAS v0.8 for the below bits */
 enum punit_power_well {
@@ -518,6 +544,11 @@ enum punit_power_well {
         PUNIT_POWER_WELL_DPIO_TX_C_LANES_23     = 9,
         PUNIT_POWER_WELL_DPIO_RX0               = 10,
         PUNIT_POWER_WELL_DPIO_RX1               = 11,
+        PUNIT_POWER_WELL_DPIO_CMN_D             = 12,
+        /* FIXME: guesswork below */
+        PUNIT_POWER_WELL_DPIO_TX_D_LANES_01     = 13,
+        PUNIT_POWER_WELL_DPIO_TX_D_LANES_23     = 14,
+        PUNIT_POWER_WELL_DPIO_RX2               = 15,
 
         PUNIT_POWER_WELL_NUM,
 };
@@ -838,8 +869,8 @@ enum punit_power_well {
 
 #define _VLV_TX_DW2_CH0                 0x8288
 #define _VLV_TX_DW2_CH1                 0x8488
-#define   DPIO_SWING_MARGIN_SHIFT       16
-#define   DPIO_SWING_MARGIN_MASK        (0xff << DPIO_SWING_MARGIN_SHIFT)
+#define   DPIO_SWING_MARGIN000_SHIFT    16
+#define   DPIO_SWING_MARGIN000_MASK     (0xff << DPIO_SWING_MARGIN000_SHIFT)
 #define   DPIO_UNIQ_TRANS_SCALE_SHIFT   8
 #define VLV_TX_DW2(ch) _PORT(ch, _VLV_TX_DW2_CH0, _VLV_TX_DW2_CH1)
 
@@ -847,12 +878,16 @@ enum punit_power_well {
 #define _VLV_TX_DW3_CH1                 0x848c
 /* The following bit for CHV phy */
 #define   DPIO_TX_UNIQ_TRANS_SCALE_EN   (1<<27)
+#define   DPIO_SWING_MARGIN101_SHIFT    16
+#define   DPIO_SWING_MARGIN101_MASK     (0xff << DPIO_SWING_MARGIN101_SHIFT)
 #define VLV_TX_DW3(ch) _PORT(ch, _VLV_TX_DW3_CH0, _VLV_TX_DW3_CH1)
 
 #define _VLV_TX_DW4_CH0                 0x8290
 #define _VLV_TX_DW4_CH1                 0x8490
 #define   DPIO_SWING_DEEMPH9P5_SHIFT    24
 #define   DPIO_SWING_DEEMPH9P5_MASK     (0xff << DPIO_SWING_DEEMPH9P5_SHIFT)
+#define   DPIO_SWING_DEEMPH6P0_SHIFT    16
+#define   DPIO_SWING_DEEMPH6P0_MASK     (0xff << DPIO_SWING_DEEMPH6P0_SHIFT)
 #define VLV_TX_DW4(ch) _PORT(ch, _VLV_TX_DW4_CH0, _VLV_TX_DW4_CH1)
 
 #define _VLV_TX3_DW4_CH0                0x690
@@ -1003,6 +1038,13 @@ enum punit_power_well {
 #define   PGTBL_ADDRESS_LO_MASK 0xfffff000 /* bits [31:12] */
 #define   PGTBL_ADDRESS_HI_MASK 0x000000f0 /* bits [35:32] (gen4) */
 #define PGTBL_ER        0x02024
+#define PRB0_BASE (0x2030-0x30)
+#define PRB1_BASE (0x2040-0x30) /* 830,gen3 */
+#define PRB2_BASE (0x2050-0x30) /* gen3 */
+#define SRB0_BASE (0x2100-0x30) /* gen2 */
+#define SRB1_BASE (0x2110-0x30) /* gen2 */
+#define SRB2_BASE (0x2120-0x30) /* 830 */
+#define SRB3_BASE (0x2130-0x30) /* 830 */
 #define RENDER_RING_BASE        0x02000
 #define BSD_RING_BASE           0x04000
 #define GEN6_BSD_RING_BASE      0x12000
@@ -1064,6 +1106,7 @@ enum punit_power_well {
 #define RING_ACTHD_UDW(base)    ((base)+0x5c)
 #define RING_NOPID(base)        ((base)+0x94)
 #define RING_IMR(base)          ((base)+0xa8)
+#define RING_HWSTAM(base)       ((base)+0x98)
 #define RING_TIMESTAMP(base)    ((base)+0x358)
 #define   TAIL_ADDR             0x001FFFF8
 #define   HEAD_WRAP_COUNT       0xFFE00000
@@ -1248,6 +1291,10 @@ enum punit_power_well {
 #define   INSTPM_TLB_INVALIDATE (1<<9)
 #define   INSTPM_SYNC_FLUSH     (1<<5)
 #define ACTHD           0x020c8
+#define MEM_MODE        0x020cc
+#define   MEM_DISPLAY_B_TRICKLE_FEED_DISABLE (1<<3) /* 830 only */
+#define   MEM_DISPLAY_A_TRICKLE_FEED_DISABLE (1<<2) /* 830/845 only */
+#define   MEM_DISPLAY_TRICKLE_FEED_DISABLE (1<<2) /* 85x only */
 #define FW_BLC          0x020d8
 #define FW_BLC2         0x020dc
 #define FW_BLC_SELF     0x020e0 /* 915+ only */
@@ -1380,6 +1427,7 @@ enum punit_power_well {
 #define GT_BSD_CS_ERROR_INTERRUPT               (1 << 15)
 #define GT_BSD_USER_INTERRUPT                   (1 << 12)
 #define GT_RENDER_L3_PARITY_ERROR_INTERRUPT_S1  (1 << 11) /* hsw+; rsvd on snb, ivb, vlv */
+#define GT_CONTEXT_SWITCH_INTERRUPT             (1 <<  8)
 #define GT_RENDER_L3_PARITY_ERROR_INTERRUPT     (1 <<  5) /* !snb */
 #define GT_RENDER_PIPECTL_NOTIFY_INTERRUPT      (1 <<  4)
 #define GT_RENDER_CS_MASTER_ERROR_INTERRUPT     (1 <<  3)
@@ -1519,6 +1567,7 @@ enum punit_power_well {
 /* Framebuffer compression for Ironlake */
 #define ILK_DPFC_CB_BASE        0x43200
 #define ILK_DPFC_CONTROL        0x43208
+#define   FBC_CTL_FALSE_COLOR   (1<<10)
 /* The bit 28-8 is reserved */
 #define   DPFC_RESERVED         (0x1FFFFF00)
 #define ILK_DPFC_RECOMP_CTL     0x4320c
@@ -1675,12 +1724,9 @@ enum punit_power_well {
 #define DPIO_PHY_STATUS                 (VLV_DISPLAY_BASE + 0x6240)
 #define   DPLL_PORTD_READY_MASK         (0xf)
 #define DISPLAY_PHY_CONTROL (VLV_DISPLAY_BASE + 0x60100)
-#define   PHY_COM_LANE_RESET_DEASSERT(phy, val) \
-                                ((phy == DPIO_PHY0) ? (val | 1) : (val | 2))
-#define   PHY_COM_LANE_RESET_ASSERT(phy, val) \
-                                ((phy == DPIO_PHY0) ? (val & ~1) : (val & ~2))
+#define   PHY_COM_LANE_RESET_DEASSERT(phy) (1 << (phy))
 #define DISPLAY_PHY_STATUS (VLV_DISPLAY_BASE + 0x60104)
-#define   PHY_POWERGOOD(phy)    ((phy == DPIO_PHY0) ? (1<<31) : (1<<30))
+#define   PHY_POWERGOOD(phy)    (((phy) == DPIO_PHY0) ? (1<<31) : (1<<30))
 
 /*
  * The i830 generation, in LVDS mode, defines P1 as the bit number set within
@@ -2397,6 +2443,7 @@ enum punit_power_well {
 #define _PIPEASRC       0x6001c
 #define _BCLRPAT_A      0x60020
 #define _VSYNCSHIFT_A   0x60028
+#define _PIPE_MULT_A    0x6002c
 
 /* Pipe B timing regs */
 #define _HTOTAL_B       0x61000
@@ -2408,6 +2455,7 @@ enum punit_power_well {
 #define _PIPEBSRC       0x6101c
 #define _BCLRPAT_B      0x61020
 #define _VSYNCSHIFT_B   0x61028
+#define _PIPE_MULT_B    0x6102c
 
 #define TRANSCODER_A_OFFSET 0x60000
 #define TRANSCODER_B_OFFSET 0x61000
@@ -2428,6 +2476,7 @@ enum punit_power_well {
 #define BCLRPAT(trans) _TRANSCODER2(trans, _BCLRPAT_A)
 #define VSYNCSHIFT(trans) _TRANSCODER2(trans, _VSYNCSHIFT_A)
 #define PIPESRC(trans) _TRANSCODER2(trans, _PIPEASRC)
+#define PIPE_MULT(trans) _TRANSCODER2(trans, _PIPE_MULT_A)
 
 /* HSW+ eDP PSR registers */
 #define EDP_PSR_BASE(dev)                       (IS_HASWELL(dev) ? 0x64800 : 0x6f800)
@@ -3476,6 +3525,8 @@ enum punit_power_well {
 #define   DP_LINK_TRAIN_OFF             (3 << 28)
 #define   DP_LINK_TRAIN_MASK            (3 << 28)
 #define   DP_LINK_TRAIN_SHIFT           28
+#define   DP_LINK_TRAIN_PAT_3_CHV       (1 << 14)
+#define   DP_LINK_TRAIN_MASK_CHV        ((3 << 28)|(1<<14))
 
 /* CPT Link training mode */
 #define   DP_LINK_TRAIN_PAT_1_CPT       (0 << 8)
@@ -3732,7 +3783,6 @@ enum punit_power_well {
 #define   PIPE_VSYNC_INTERRUPT_STATUS           (1UL<<9)
 #define   PIPE_DISPLAY_LINE_COMPARE_STATUS      (1UL<<8)
 #define   PIPE_DPST_EVENT_STATUS                (1UL<<7)
-#define   PIPE_LEGACY_BLC_EVENT_STATUS          (1UL<<6)
 #define   PIPE_A_PSR_STATUS_VLV                 (1UL<<6)
 #define   PIPE_LEGACY_BLC_EVENT_STATUS          (1UL<<6)
 #define   PIPE_ODD_FIELD_INTERRUPT_STATUS       (1UL<<5)
@@ -3842,73 +3892,151 @@ enum punit_power_well {
 #define   DSPARB_BEND_SHIFT     9 /* on 855 */
 #define   DSPARB_AEND_SHIFT     0
 
+/* pnv/gen4/g4x/vlv/chv */
 #define DSPFW1                  (dev_priv->info.display_mmio_offset + 0x70034)
-#define   DSPFW_SR_SHIFT        23
-#define   DSPFW_SR_MASK         (0x1ff<<23)
-#define   DSPFW_CURSORB_SHIFT   16
-#define   DSPFW_CURSORB_MASK    (0x3f<<16)
-#define   DSPFW_PLANEB_SHIFT    8
-#define   DSPFW_PLANEB_MASK     (0x7f<<8)
-#define   DSPFW_PLANEA_MASK     (0x7f)
+#define   DSPFW_SR_SHIFT                23
+#define   DSPFW_SR_MASK                 (0x1ff<<23)
+#define   DSPFW_CURSORB_SHIFT           16
+#define   DSPFW_CURSORB_MASK            (0x3f<<16)
+#define   DSPFW_PLANEB_SHIFT            8
+#define   DSPFW_PLANEB_MASK             (0x7f<<8)
+#define   DSPFW_PLANEB_MASK_VLV         (0xff<<8) /* vlv/chv */
+#define   DSPFW_PLANEA_SHIFT            0
+#define   DSPFW_PLANEA_MASK             (0x7f<<0)
+#define   DSPFW_PLANEA_MASK_VLV         (0xff<<0) /* vlv/chv */
 #define DSPFW2                  (dev_priv->info.display_mmio_offset + 0x70038)
-#define   DSPFW_CURSORA_MASK    0x00003f00
-#define   DSPFW_CURSORA_SHIFT   8
-#define   DSPFW_PLANEC_MASK     (0x7f)
+#define   DSPFW_FBC_SR_EN               (1<<31)   /* g4x */
+#define   DSPFW_FBC_SR_SHIFT            28
+#define   DSPFW_FBC_SR_MASK             (0x7<<28) /* g4x */
+#define   DSPFW_FBC_HPLL_SR_SHIFT       24
+#define   DSPFW_FBC_HPLL_SR_MASK        (0xf<<24) /* g4x */
+#define   DSPFW_SPRITEB_SHIFT           (16)
+#define   DSPFW_SPRITEB_MASK            (0x7f<<16) /* g4x */
+#define   DSPFW_SPRITEB_MASK_VLV        (0xff<<16) /* vlv/chv */
+#define   DSPFW_CURSORA_SHIFT           8
+#define   DSPFW_CURSORA_MASK            (0x3f<<8)
+#define   DSPFW_PLANEC_SHIFT_OLD        0
+#define   DSPFW_PLANEC_MASK_OLD         (0x7f<<0) /* pre-gen4 sprite C */
+#define   DSPFW_SPRITEA_SHIFT           0
+#define   DSPFW_SPRITEA_MASK            (0x7f<<0) /* g4x */
+#define   DSPFW_SPRITEA_MASK_VLV        (0xff<<0) /* vlv/chv */
 #define DSPFW3                  (dev_priv->info.display_mmio_offset + 0x7003c)
-#define   DSPFW_HPLL_SR_EN      (1<<31)
-#define   DSPFW_CURSOR_SR_SHIFT 24
+#define   DSPFW_HPLL_SR_EN              (1<<31)
 #define   PINEVIEW_SELF_REFRESH_EN      (1<<30)
+#define   DSPFW_CURSOR_SR_SHIFT         24
 #define   DSPFW_CURSOR_SR_MASK          (0x3f<<24)
 #define   DSPFW_HPLL_CURSOR_SHIFT       16
 #define   DSPFW_HPLL_CURSOR_MASK        (0x3f<<16)
-#define   DSPFW_HPLL_SR_MASK            (0x1ff)
-#define DSPFW4                  (dev_priv->info.display_mmio_offset + 0x70070)
-#define DSPFW7                  (dev_priv->info.display_mmio_offset + 0x7007c)
+#define   DSPFW_HPLL_SR_SHIFT           0
+#define   DSPFW_HPLL_SR_MASK            (0x1ff<<0)
+
+/* vlv/chv */
+#define DSPFW4                  (VLV_DISPLAY_BASE + 0x70070)
+#define   DSPFW_SPRITEB_WM1_SHIFT       16
+#define   DSPFW_SPRITEB_WM1_MASK        (0xff<<16)
+#define   DSPFW_CURSORA_WM1_SHIFT       8
+#define   DSPFW_CURSORA_WM1_MASK        (0x3f<<8)
+#define   DSPFW_SPRITEA_WM1_SHIFT       0
+#define   DSPFW_SPRITEA_WM1_MASK        (0xff<<0)
+#define DSPFW5                  (VLV_DISPLAY_BASE + 0x70074)
+#define   DSPFW_PLANEB_WM1_SHIFT        24
+#define   DSPFW_PLANEB_WM1_MASK         (0xff<<24)
+#define   DSPFW_PLANEA_WM1_SHIFT        16
+#define   DSPFW_PLANEA_WM1_MASK         (0xff<<16)
+#define   DSPFW_CURSORB_WM1_SHIFT       8
+#define   DSPFW_CURSORB_WM1_MASK        (0x3f<<8)
+#define   DSPFW_CURSOR_SR_WM1_SHIFT     0
+#define   DSPFW_CURSOR_SR_WM1_MASK      (0x3f<<0)
+#define DSPFW6                  (VLV_DISPLAY_BASE + 0x70078)
+#define   DSPFW_SR_WM1_SHIFT            0
+#define   DSPFW_SR_WM1_MASK             (0x1ff<<0)
+#define DSPFW7                  (VLV_DISPLAY_BASE + 0x7007c)
+#define DSPFW7_CHV              (VLV_DISPLAY_BASE + 0x700b4) /* wtf #1? */
+#define   DSPFW_SPRITED_WM1_SHIFT       24
+#define   DSPFW_SPRITED_WM1_MASK        (0xff<<24)
+#define   DSPFW_SPRITED_SHIFT           16
+#define   DSPFW_SPRITED_MASK            (0xff<<16)
+#define   DSPFW_SPRITEC_WM1_SHIFT       8
+#define   DSPFW_SPRITEC_WM1_MASK        (0xff<<8)
+#define   DSPFW_SPRITEC_SHIFT           0
+#define   DSPFW_SPRITEC_MASK            (0xff<<0)
+#define DSPFW8_CHV              (VLV_DISPLAY_BASE + 0x700b8)
+#define   DSPFW_SPRITEF_WM1_SHIFT       24
+#define   DSPFW_SPRITEF_WM1_MASK        (0xff<<24)
+#define   DSPFW_SPRITEF_SHIFT           16
+#define   DSPFW_SPRITEF_MASK            (0xff<<16)
+#define   DSPFW_SPRITEE_WM1_SHIFT       8
+#define   DSPFW_SPRITEE_WM1_MASK        (0xff<<8)
+#define   DSPFW_SPRITEE_SHIFT           0
+#define   DSPFW_SPRITEE_MASK            (0xff<<0)
+#define DSPFW9_CHV              (VLV_DISPLAY_BASE + 0x7007c) /* wtf #2? */
+#define   DSPFW_PLANEC_WM1_SHIFT        24
+#define   DSPFW_PLANEC_WM1_MASK         (0xff<<24)
+#define   DSPFW_PLANEC_SHIFT            16
+#define   DSPFW_PLANEC_MASK             (0xff<<16)
+#define   DSPFW_CURSORC_WM1_SHIFT       8
+#define   DSPFW_CURSORC_WM1_MASK        (0x3f<<16)
+#define   DSPFW_CURSORC_SHIFT           0
+#define   DSPFW_CURSORC_MASK            (0x3f<<0)
+
+/* vlv/chv high order bits */
+#define DSPHOWM                 (VLV_DISPLAY_BASE + 0x70064)
+#define   DSPFW_SR_HI_SHIFT             24
+#define   DSPFW_SR_HI_MASK              (1<<24)
+#define   DSPFW_SPRITEF_HI_SHIFT        23
+#define   DSPFW_SPRITEF_HI_MASK         (1<<23)
+#define   DSPFW_SPRITEE_HI_SHIFT        22
+#define   DSPFW_SPRITEE_HI_MASK         (1<<22)
+#define   DSPFW_PLANEC_HI_SHIFT         21
+#define   DSPFW_PLANEC_HI_MASK          (1<<21)
+#define   DSPFW_SPRITED_HI_SHIFT        20
+#define   DSPFW_SPRITED_HI_MASK         (1<<20)
+#define   DSPFW_SPRITEC_HI_SHIFT        16
+#define   DSPFW_SPRITEC_HI_MASK         (1<<16)
+#define   DSPFW_PLANEB_HI_SHIFT         12
+#define   DSPFW_PLANEB_HI_MASK          (1<<12)
+#define   DSPFW_SPRITEB_HI_SHIFT        8
+#define   DSPFW_SPRITEB_HI_MASK         (1<<8)
+#define   DSPFW_SPRITEA_HI_SHIFT        4
+#define   DSPFW_SPRITEA_HI_MASK         (1<<4)
+#define   DSPFW_PLANEA_HI_SHIFT         0
+#define   DSPFW_PLANEA_HI_MASK          (1<<0)
+#define DSPHOWM1                (VLV_DISPLAY_BASE + 0x70068)
+#define   DSPFW_SR_WM1_HI_SHIFT         24
+#define   DSPFW_SR_WM1_HI_MASK          (1<<24)
+#define   DSPFW_SPRITEF_WM1_HI_SHIFT    23
+#define   DSPFW_SPRITEF_WM1_HI_MASK     (1<<23)
+#define   DSPFW_SPRITEE_WM1_HI_SHIFT    22
+#define   DSPFW_SPRITEE_WM1_HI_MASK     (1<<22)
+#define   DSPFW_PLANEC_WM1_HI_SHIFT     21
+#define   DSPFW_PLANEC_WM1_HI_MASK      (1<<21)
+#define   DSPFW_SPRITED_WM1_HI_SHIFT    20
+#define   DSPFW_SPRITED_WM1_HI_MASK     (1<<20)
+#define   DSPFW_SPRITEC_WM1_HI_SHIFT    16
+#define   DSPFW_SPRITEC_WM1_HI_MASK     (1<<16)
+#define   DSPFW_PLANEB_WM1_HI_SHIFT     12
+#define   DSPFW_PLANEB_WM1_HI_MASK      (1<<12)
+#define   DSPFW_SPRITEB_WM1_HI_SHIFT    8
+#define   DSPFW_SPRITEB_WM1_HI_MASK     (1<<8)
+#define   DSPFW_SPRITEA_WM1_HI_SHIFT    4
+#define   DSPFW_SPRITEA_WM1_HI_MASK     (1<<4)
+#define   DSPFW_PLANEA_WM1_HI_SHIFT     0
+#define   DSPFW_PLANEA_WM1_HI_MASK      (1<<0)
 
 /* drain latency register values*/
 #define DRAIN_LATENCY_PRECISION_32      32
 #define DRAIN_LATENCY_PRECISION_64      64
-#define VLV_DDL1                        (VLV_DISPLAY_BASE + 0x70050)
-#define DDL_CURSORA_PRECISION_64        (1<<31)
-#define DDL_CURSORA_PRECISION_32        (0<<31)
-#define DDL_CURSORA_SHIFT               24
-#define DDL_SPRITEB_PRECISION_64        (1<<23)
-#define DDL_SPRITEB_PRECISION_32        (0<<23)
-#define DDL_SPRITEB_SHIFT               16
-#define DDL_SPRITEA_PRECISION_64        (1<<15)
-#define DDL_SPRITEA_PRECISION_32        (0<<15)
-#define DDL_SPRITEA_SHIFT               8
-#define DDL_PLANEA_PRECISION_64         (1<<7)
-#define DDL_PLANEA_PRECISION_32         (0<<7)
-#define DDL_PLANEA_SHIFT                0
-
-#define VLV_DDL2                        (VLV_DISPLAY_BASE + 0x70054)
-#define DDL_CURSORB_PRECISION_64        (1<<31)
-#define DDL_CURSORB_PRECISION_32        (0<<31)
-#define DDL_CURSORB_SHIFT               24
-#define DDL_SPRITED_PRECISION_64        (1<<23)
-#define DDL_SPRITED_PRECISION_32        (0<<23)
-#define DDL_SPRITED_SHIFT               16
-#define DDL_SPRITEC_PRECISION_64        (1<<15)
-#define DDL_SPRITEC_PRECISION_32        (0<<15)
-#define DDL_SPRITEC_SHIFT               8
-#define DDL_PLANEB_PRECISION_64         (1<<7)
-#define DDL_PLANEB_PRECISION_32         (0<<7)
-#define DDL_PLANEB_SHIFT                0
-
-#define VLV_DDL3                        (VLV_DISPLAY_BASE + 0x70058)
-#define DDL_CURSORC_PRECISION_64        (1<<31)
-#define DDL_CURSORC_PRECISION_32        (0<<31)
-#define DDL_CURSORC_SHIFT               24
-#define DDL_SPRITEF_PRECISION_64        (1<<23)
-#define DDL_SPRITEF_PRECISION_32        (0<<23)
-#define DDL_SPRITEF_SHIFT               16
-#define DDL_SPRITEE_PRECISION_64        (1<<15)
-#define DDL_SPRITEE_PRECISION_32        (0<<15)
-#define DDL_SPRITEE_SHIFT               8
-#define DDL_PLANEC_PRECISION_64         (1<<7)
-#define DDL_PLANEC_PRECISION_32         (0<<7)
-#define DDL_PLANEC_SHIFT                0
+#define VLV_DDL(pipe)                   (VLV_DISPLAY_BASE + 0x70050 + 4 * (pipe))
+#define DDL_CURSOR_PRECISION_64         (1<<31)
+#define DDL_CURSOR_PRECISION_32         (0<<31)
+#define DDL_CURSOR_SHIFT                24
+#define DDL_SPRITE_PRECISION_64(sprite) (1<<(15+8*(sprite)))
+#define DDL_SPRITE_PRECISION_32(sprite) (0<<(15+8*(sprite)))
+#define DDL_SPRITE_SHIFT(sprite)        (8+8*(sprite))
+#define DDL_PLANE_PRECISION_64          (1<<7)
+#define DDL_PLANE_PRECISION_32          (0<<7)
+#define DDL_PLANE_SHIFT                 0
+#define DRAIN_LATENCY_MASK              0x7f
 
 /* FIFO watermark sizes etc */
 #define G4X_FIFO_LINE_SIZE      64
@@ -4026,7 +4154,8 @@ enum punit_power_well {
 /* Old style CUR*CNTR flags (desktop 8xx) */
 #define   CURSOR_ENABLE         0x80000000
 #define   CURSOR_GAMMA_ENABLE   0x40000000
-#define   CURSOR_STRIDE_MASK    0x30000000
+#define   CURSOR_STRIDE_SHIFT   28
+#define   CURSOR_STRIDE(x)      ((ffs(x)-9) << CURSOR_STRIDE_SHIFT) /* 256,512,1k,2k */
 #define   CURSOR_PIPE_CSC_ENABLE (1<<24)
 #define   CURSOR_FORMAT_SHIFT   24
 #define   CURSOR_FORMAT_MASK    (0x07 << CURSOR_FORMAT_SHIFT)
@@ -4111,6 +4240,7 @@ enum punit_power_well {
 #define   DISPPLANE_NO_LINE_DOUBLE              0
 #define   DISPPLANE_STEREO_POLARITY_FIRST       0
 #define   DISPPLANE_STEREO_POLARITY_SECOND      (1<<18)
+#define   DISPPLANE_ROTATE_180         (1<<15)
 #define   DISPPLANE_TRICKLE_FEED_DISABLE        (1<<14) /* Ironlake */
 #define   DISPPLANE_TILED                       (1<<10)
 #define _DSPAADDR                               0x70184
@@ -4195,6 +4325,7 @@ enum punit_power_well {
 #define   DVS_YUV_ORDER_UYVY    (1<<16)
 #define   DVS_YUV_ORDER_YVYU    (2<<16)
 #define   DVS_YUV_ORDER_VYUY    (3<<16)
+#define   DVS_ROTATE_180        (1<<15)
 #define   DVS_DEST_KEY          (1<<2)
 #define   DVS_TRICKLE_FEED_DISABLE (1<<14)
 #define   DVS_TILED             (1<<10)
@@ -4265,6 +4396,7 @@ enum punit_power_well {
 #define   SPRITE_YUV_ORDER_UYVY         (1<<16)
 #define   SPRITE_YUV_ORDER_YVYU         (2<<16)
 #define   SPRITE_YUV_ORDER_VYUY         (3<<16)
+#define   SPRITE_ROTATE_180             (1<<15)
 #define   SPRITE_TRICKLE_FEED_DISABLE   (1<<14)
 #define   SPRITE_INT_GAMMA_ENABLE       (1<<13)
 #define   SPRITE_TILED                  (1<<10)
@@ -4338,6 +4470,7 @@ enum punit_power_well {
 #define   SP_YUV_ORDER_UYVY             (1<<16)
 #define   SP_YUV_ORDER_YVYU             (2<<16)
 #define   SP_YUV_ORDER_VYUY             (3<<16)
+#define   SP_ROTATE_180                 (1<<15)
 #define   SP_TILED                      (1<<10)
 #define _SPALINOFF              (VLV_DISPLAY_BASE + 0x72184)
 #define _SPASTRIDE              (VLV_DISPLAY_BASE + 0x72188)
@@ -5246,8 +5379,7 @@ enum punit_power_well {
 #define PIPEA_PP_STATUS         (VLV_DISPLAY_BASE + 0x61200)
 #define PIPEA_PP_CONTROL        (VLV_DISPLAY_BASE + 0x61204)
 #define PIPEA_PP_ON_DELAYS      (VLV_DISPLAY_BASE + 0x61208)
-#define  PANEL_PORT_SELECT_DPB_VLV      (1 << 30)
-#define  PANEL_PORT_SELECT_DPC_VLV      (2 << 30)
+#define  PANEL_PORT_SELECT_VLV(port)    ((port) << 30)
 #define PIPEA_PP_OFF_DELAYS     (VLV_DISPLAY_BASE + 0x6120c)
 #define PIPEA_PP_DIVISOR        (VLV_DISPLAY_BASE + 0x61210)
 
@@ -5407,7 +5539,6 @@ enum punit_power_well {
 #define   VLV_GTLC_ALLOWWAKEERR                 (1 << 1)
 #define   VLV_GTLC_PW_MEDIA_STATUS_MASK         (1 << 5)
 #define   VLV_GTLC_PW_RENDER_STATUS_MASK        (1 << 7)
-#define VLV_GTLC_SURVIVABILITY_REG              0x130098
 #define  FORCEWAKE_MT                           0xa188 /* multi-threaded */
 #define   FORCEWAKE_KERNEL                      0x1
 #define   FORCEWAKE_USER                        0x2
@@ -5545,12 +5676,6 @@ enum punit_power_well {
                                                  GEN6_PM_RP_DOWN_THRESHOLD | \
                                                  GEN6_PM_RP_DOWN_TIMEOUT)
 
-#define CHV_CZ_CLOCK_FREQ_MODE_200                      200
-#define CHV_CZ_CLOCK_FREQ_MODE_267                      267
-#define CHV_CZ_CLOCK_FREQ_MODE_320                      320
-#define CHV_CZ_CLOCK_FREQ_MODE_333                      333
-#define CHV_CZ_CLOCK_FREQ_MODE_400                      400
-
 #define GEN7_GT_SCRATCH_BASE                    0x4F100
 #define GEN7_GT_SCRATCH_REG_NUM                 8
 
@@ -5866,15 +5991,7 @@ enum punit_power_well {
 #define DDI_BUF_CTL_B                           0x64100
 #define DDI_BUF_CTL(port) _PORT(port, DDI_BUF_CTL_A, DDI_BUF_CTL_B)
 #define  DDI_BUF_CTL_ENABLE                     (1<<31)
-#define  DDI_BUF_EMP_400MV_0DB_HSW              (0<<24)   /* Sel0 */
-#define  DDI_BUF_EMP_400MV_3_5DB_HSW            (1<<24)   /* Sel1 */
-#define  DDI_BUF_EMP_400MV_6DB_HSW              (2<<24)   /* Sel2 */
-#define  DDI_BUF_EMP_400MV_9_5DB_HSW            (3<<24)   /* Sel3 */
-#define  DDI_BUF_EMP_600MV_0DB_HSW              (4<<24)   /* Sel4 */
-#define  DDI_BUF_EMP_600MV_3_5DB_HSW            (5<<24)   /* Sel5 */
-#define  DDI_BUF_EMP_600MV_6DB_HSW              (6<<24)   /* Sel6 */
-#define  DDI_BUF_EMP_800MV_0DB_HSW              (7<<24)   /* Sel7 */
-#define  DDI_BUF_EMP_800MV_3_5DB_HSW            (8<<24)   /* Sel8 */
+#define  DDI_BUF_TRANS_SELECT(n)        ((n) << 24)
 #define  DDI_BUF_EMP_MASK                       (0xf<<24)
 #define  DDI_BUF_PORT_REVERSAL                  (1<<16)
 #define  DDI_BUF_IS_IDLE                        (1<<7)
diff --git a/drivers/gpu/drm/i915/i915_sysfs.c b/drivers/gpu/drm/i915/i915_sysfs.c
index ae7fd8fc27f0..503847f18fdd 100644
--- a/drivers/gpu/drm/i915/i915_sysfs.c
+++ b/drivers/gpu/drm/i915/i915_sysfs.c
@@ -540,7 +540,7 @@ static ssize_t error_state_read(struct file *filp, struct kobject *kobj,
 
         memset(&error_priv, 0, sizeof(error_priv));
 
-        ret = i915_error_state_buf_init(&error_str, count, off);
+        ret = i915_error_state_buf_init(&error_str, to_i915(dev), count, off);
         if (ret)
                 return ret;
 
diff --git a/drivers/gpu/drm/i915/intel_bios.c b/drivers/gpu/drm/i915/intel_bios.c
index afcc8dd40bdd..a4bd90f36a03 100644
--- a/drivers/gpu/drm/i915/intel_bios.c
+++ b/drivers/gpu/drm/i915/intel_bios.c
@@ -627,16 +627,16 @@ parse_edp(struct drm_i915_private *dev_priv, struct bdb_header *bdb)
 
         switch (edp_link_params->preemphasis) {
         case EDP_PREEMPHASIS_NONE:
-                dev_priv->vbt.edp_preemphasis = DP_TRAIN_PRE_EMPHASIS_0;
+                dev_priv->vbt.edp_preemphasis = DP_TRAIN_PRE_EMPH_LEVEL_0;
                 break;
         case EDP_PREEMPHASIS_3_5dB:
-                dev_priv->vbt.edp_preemphasis = DP_TRAIN_PRE_EMPHASIS_3_5;
+                dev_priv->vbt.edp_preemphasis = DP_TRAIN_PRE_EMPH_LEVEL_1;
                 break;
         case EDP_PREEMPHASIS_6dB:
-                dev_priv->vbt.edp_preemphasis = DP_TRAIN_PRE_EMPHASIS_6;
+                dev_priv->vbt.edp_preemphasis = DP_TRAIN_PRE_EMPH_LEVEL_2;
                 break;
         case EDP_PREEMPHASIS_9_5dB:
-                dev_priv->vbt.edp_preemphasis = DP_TRAIN_PRE_EMPHASIS_9_5;
+                dev_priv->vbt.edp_preemphasis = DP_TRAIN_PRE_EMPH_LEVEL_3;
                 break;
         default:
                 DRM_DEBUG_KMS("VBT has unknown eDP pre-emphasis value %u\n",
@@ -646,16 +646,16 @@ parse_edp(struct drm_i915_private *dev_priv, struct bdb_header *bdb)
 
         switch (edp_link_params->vswing) {
         case EDP_VSWING_0_4V:
-                dev_priv->vbt.edp_vswing = DP_TRAIN_VOLTAGE_SWING_400;
+                dev_priv->vbt.edp_vswing = DP_TRAIN_VOLTAGE_SWING_LEVEL_0;
                 break;
         case EDP_VSWING_0_6V:
-                dev_priv->vbt.edp_vswing = DP_TRAIN_VOLTAGE_SWING_600;
+                dev_priv->vbt.edp_vswing = DP_TRAIN_VOLTAGE_SWING_LEVEL_1;
                 break;
         case EDP_VSWING_0_8V:
-                dev_priv->vbt.edp_vswing = DP_TRAIN_VOLTAGE_SWING_800;
+                dev_priv->vbt.edp_vswing = DP_TRAIN_VOLTAGE_SWING_LEVEL_2;
                 break;
         case EDP_VSWING_1_2V:
-                dev_priv->vbt.edp_vswing = DP_TRAIN_VOLTAGE_SWING_1200;
+                dev_priv->vbt.edp_vswing = DP_TRAIN_VOLTAGE_SWING_LEVEL_3;
                 break;
         default:
                 DRM_DEBUG_KMS("VBT has unknown eDP voltage swing value %u\n",
@@ -976,12 +976,10 @@ static void parse_ddi_port(struct drm_i915_private *dev_priv, enum port port,
         if (bdb->version >= 158) {
                 /* The VBT HDMI level shift values match the table we have. */
                 hdmi_level_shift = child->raw[7] & 0xF;
-                if (hdmi_level_shift < 0xC) {
-                        DRM_DEBUG_KMS("VBT HDMI level shift for port %c: %d\n",
-                                      port_name(port),
-                                      hdmi_level_shift);
-                        info->hdmi_level_shift = hdmi_level_shift;
-                }
+                DRM_DEBUG_KMS("VBT HDMI level shift for port %c: %d\n",
+                              port_name(port),
+                              hdmi_level_shift);
+                info->hdmi_level_shift = hdmi_level_shift;
         }
 }
 
@@ -1114,8 +1112,7 @@ init_vbt_defaults(struct drm_i915_private *dev_priv)
                 struct ddi_vbt_port_info *info =
                         &dev_priv->vbt.ddi_port_info[port];
 
-                /* Recommended BSpec default: 800mV 0dB. */
-                info->hdmi_level_shift = 6;
+                info->hdmi_level_shift = HDMI_LEVEL_SHIFT_UNKNOWN;
 
                 info->supports_dvi = (port != PORT_A && port != PORT_E);
                 info->supports_hdmi = info->supports_dvi;
diff --git a/drivers/gpu/drm/i915/intel_bios.h b/drivers/gpu/drm/i915/intel_bios.h
index b98667796337..905999bee2ac 100644
--- a/drivers/gpu/drm/i915/intel_bios.h
+++ b/drivers/gpu/drm/i915/intel_bios.h
@@ -802,7 +802,8 @@ struct mipi_config {
 
         u16 rsvd4;
 
-        u8 rsvd5[5];
+        u8 rsvd5;
+        u32 target_burst_mode_freq;
         u32 dsi_ddr_clk;
         u32 bridge_ref_clk;
 
diff --git a/drivers/gpu/drm/i915/intel_ddi.c b/drivers/gpu/drm/i915/intel_ddi.c
index 5db0b5552e39..b63d4fa204a3 100644
--- a/drivers/gpu/drm/i915/intel_ddi.c
+++ b/drivers/gpu/drm/i915/intel_ddi.c
@@ -28,87 +28,103 @@
 #include "i915_drv.h"
 #include "intel_drv.h"
 
+struct ddi_buf_trans {
+        u32 trans1;     /* balance leg enable, de-emph level */
+        u32 trans2;     /* vref sel, vswing */
+};
+
 /* HDMI/DVI modes ignore everything but the last 2 items. So we share
  * them for both DP and FDI transports, allowing those ports to
  * automatically adapt to HDMI connections as well
  */
-static const u32 hsw_ddi_translations_dp[] = {
-        0x00FFFFFF, 0x0006000E,         /* DP parameters */
-        0x00D75FFF, 0x0005000A,
-        0x00C30FFF, 0x00040006,
-        0x80AAAFFF, 0x000B0000,
-        0x00FFFFFF, 0x0005000A,
-        0x00D75FFF, 0x000C0004,
-        0x80C30FFF, 0x000B0000,
-        0x00FFFFFF, 0x00040006,
-        0x80D75FFF, 0x000B0000,
+static const struct ddi_buf_trans hsw_ddi_translations_dp[] = {
+        { 0x00FFFFFF, 0x0006000E },
+        { 0x00D75FFF, 0x0005000A },
+        { 0x00C30FFF, 0x00040006 },
+        { 0x80AAAFFF, 0x000B0000 },
+        { 0x00FFFFFF, 0x0005000A },
+        { 0x00D75FFF, 0x000C0004 },
+        { 0x80C30FFF, 0x000B0000 },
+        { 0x00FFFFFF, 0x00040006 },
+        { 0x80D75FFF, 0x000B0000 },
 };
 
-static const u32 hsw_ddi_translations_fdi[] = {
-        0x00FFFFFF, 0x0007000E,         /* FDI parameters */
-        0x00D75FFF, 0x000F000A,
-        0x00C30FFF, 0x00060006,
-        0x00AAAFFF, 0x001E0000,
-        0x00FFFFFF, 0x000F000A,
-        0x00D75FFF, 0x00160004,
-        0x00C30FFF, 0x001E0000,
-        0x00FFFFFF, 0x00060006,
-        0x00D75FFF, 0x001E0000,
+static const struct ddi_buf_trans hsw_ddi_translations_fdi[] = {
+        { 0x00FFFFFF, 0x0007000E },
+        { 0x00D75FFF, 0x000F000A },
+        { 0x00C30FFF, 0x00060006 },
+        { 0x00AAAFFF, 0x001E0000 },
+        { 0x00FFFFFF, 0x000F000A },
+        { 0x00D75FFF, 0x00160004 },
+        { 0x00C30FFF, 0x001E0000 },
+        { 0x00FFFFFF, 0x00060006 },
+        { 0x00D75FFF, 0x001E0000 },
 };
 
-static const u32 hsw_ddi_translations_hdmi[] = {
-                                /* Idx  NT mV diff      T mV diff       db  */
-        0x00FFFFFF, 0x0006000E, /* 0:   400             400             0   */
-        0x00E79FFF, 0x000E000C, /* 1:   400             500             2   */
-        0x00D75FFF, 0x0005000A, /* 2:   400             600             3.5 */
-        0x00FFFFFF, 0x0005000A, /* 3:   600             600             0   */
-        0x00E79FFF, 0x001D0007, /* 4:   600             750             2   */
-        0x00D75FFF, 0x000C0004, /* 5:   600             900             3.5 */
-        0x00FFFFFF, 0x00040006, /* 6:   800             800             0   */
-        0x80E79FFF, 0x00030002, /* 7:   800             1000            2   */
-        0x00FFFFFF, 0x00140005, /* 8:   850             850             0   */
-        0x00FFFFFF, 0x000C0004, /* 9:   900             900             0   */
-        0x00FFFFFF, 0x001C0003, /* 10:  950             950             0   */
-        0x80FFFFFF, 0x00030002, /* 11:  1000            1000            0   */
+static const struct ddi_buf_trans hsw_ddi_translations_hdmi[] = {
+                                        /* Idx  NT mV d T mV d  db      */
+        { 0x00FFFFFF, 0x0006000E },     /* 0:   400     400     0       */
+        { 0x00E79FFF, 0x000E000C },     /* 1:   400     500     2       */
+        { 0x00D75FFF, 0x0005000A },     /* 2:   400     600     3.5     */
+        { 0x00FFFFFF, 0x0005000A },     /* 3:   600     600     0       */
+        { 0x00E79FFF, 0x001D0007 },     /* 4:   600     750     2       */
+        { 0x00D75FFF, 0x000C0004 },     /* 5:   600     900     3.5     */
+        { 0x00FFFFFF, 0x00040006 },     /* 6:   800     800     0       */
+        { 0x80E79FFF, 0x00030002 },     /* 7:   800     1000    2       */
+        { 0x00FFFFFF, 0x00140005 },     /* 8:   850     850     0       */
+        { 0x00FFFFFF, 0x000C0004 },     /* 9:   900     900     0       */
+        { 0x00FFFFFF, 0x001C0003 },     /* 10:  950     950     0       */
+        { 0x80FFFFFF, 0x00030002 },     /* 11:  1000    1000    0       */
 };
 
-static const u32 bdw_ddi_translations_edp[] = {
-        0x00FFFFFF, 0x00000012,         /* eDP parameters */
-        0x00EBAFFF, 0x00020011,
-        0x00C71FFF, 0x0006000F,
-        0x00AAAFFF, 0x000E000A,
-        0x00FFFFFF, 0x00020011,
-        0x00DB6FFF, 0x0005000F,
-        0x00BEEFFF, 0x000A000C,
-        0x00FFFFFF, 0x0005000F,
-        0x00DB6FFF, 0x000A000C,
-        0x00FFFFFF, 0x00140006          /* HDMI parameters 800mV 0dB*/
+static const struct ddi_buf_trans bdw_ddi_translations_edp[] = {
+        { 0x00FFFFFF, 0x00000012 },
+        { 0x00EBAFFF, 0x00020011 },
+        { 0x00C71FFF, 0x0006000F },
+        { 0x00AAAFFF, 0x000E000A },
+        { 0x00FFFFFF, 0x00020011 },
+        { 0x00DB6FFF, 0x0005000F },
+        { 0x00BEEFFF, 0x000A000C },
+        { 0x00FFFFFF, 0x0005000F },
+        { 0x00DB6FFF, 0x000A000C },
 };
 
-static const u32 bdw_ddi_translations_dp[] = {
-        0x00FFFFFF, 0x0007000E,         /* DP parameters */
-        0x00D75FFF, 0x000E000A,
-        0x00BEFFFF, 0x00140006,
-        0x80B2CFFF, 0x001B0002,
-        0x00FFFFFF, 0x000E000A,
-        0x00D75FFF, 0x00180004,
-        0x80CB2FFF, 0x001B0002,
-        0x00F7DFFF, 0x00180004,
-        0x80D75FFF, 0x001B0002,
-        0x00FFFFFF, 0x00140006          /* HDMI parameters 800mV 0dB*/
+static const struct ddi_buf_trans bdw_ddi_translations_dp[] = {
+        { 0x00FFFFFF, 0x0007000E },
+        { 0x00D75FFF, 0x000E000A },
+        { 0x00BEFFFF, 0x00140006 },
+        { 0x80B2CFFF, 0x001B0002 },
+        { 0x00FFFFFF, 0x000E000A },
+        { 0x00D75FFF, 0x00180004 },
+        { 0x80CB2FFF, 0x001B0002 },
+        { 0x00F7DFFF, 0x00180004 },
+        { 0x80D75FFF, 0x001B0002 },
 };
 
-static const u32 bdw_ddi_translations_fdi[] = {
-        0x00FFFFFF, 0x0001000E,         /* FDI parameters */
-        0x00D75FFF, 0x0004000A,
-        0x00C30FFF, 0x00070006,
-        0x00AAAFFF, 0x000C0000,
-        0x00FFFFFF, 0x0004000A,
-        0x00D75FFF, 0x00090004,
-        0x00C30FFF, 0x000C0000,
-        0x00FFFFFF, 0x00070006,
-        0x00D75FFF, 0x000C0000,
-        0x00FFFFFF, 0x00140006          /* HDMI parameters 800mV 0dB*/
+static const struct ddi_buf_trans bdw_ddi_translations_fdi[] = {
+        { 0x00FFFFFF, 0x0001000E },
+        { 0x00D75FFF, 0x0004000A },
+        { 0x00C30FFF, 0x00070006 },
+        { 0x00AAAFFF, 0x000C0000 },
+        { 0x00FFFFFF, 0x0004000A },
+        { 0x00D75FFF, 0x00090004 },
+        { 0x00C30FFF, 0x000C0000 },
+        { 0x00FFFFFF, 0x00070006 },
+        { 0x00D75FFF, 0x000C0000 },
+};
+
+static const struct ddi_buf_trans bdw_ddi_translations_hdmi[] = {
+                                        /* Idx  NT mV d T mV df db      */
+        { 0x00FFFFFF, 0x0007000E },     /* 0:   400     400     0       */
+        { 0x00D75FFF, 0x000E000A },     /* 1:   400     600     3.5     */
+        { 0x00BEFFFF, 0x00140006 },     /* 2:   400     800     6       */
+        { 0x00FFFFFF, 0x0009000D },     /* 3:   450     450     0       */
+        { 0x00FFFFFF, 0x000E000A },     /* 4:   600     600     0       */
+        { 0x00D7FFFF, 0x00140006 },     /* 5:   600     800     2.5     */
+        { 0x80CB2FFF, 0x001B0002 },     /* 6:   600     1000    4.5     */
+        { 0x00FFFFFF, 0x00140006 },     /* 7:   800     800     0       */
+        { 0x80E79FFF, 0x001B0002 },     /* 8:   800     1000    2       */
+        { 0x80FFFFFF, 0x001B0002 },     /* 9:   1000    1000    0       */
 };
 
 enum port intel_ddi_get_encoder_port(struct intel_encoder *intel_encoder)
@@ -145,26 +161,36 @@ static void intel_prepare_ddi_buffers(struct drm_device *dev, enum port port)
 {
         struct drm_i915_private *dev_priv = dev->dev_private;
         u32 reg;
-        int i;
+        int i, n_hdmi_entries, hdmi_800mV_0dB;
         int hdmi_level = dev_priv->vbt.ddi_port_info[port].hdmi_level_shift;
-        const u32 *ddi_translations_fdi;
-        const u32 *ddi_translations_dp;
-        const u32 *ddi_translations_edp;
-        const u32 *ddi_translations;
+        const struct ddi_buf_trans *ddi_translations_fdi;
+        const struct ddi_buf_trans *ddi_translations_dp;
+        const struct ddi_buf_trans *ddi_translations_edp;
+        const struct ddi_buf_trans *ddi_translations_hdmi;
+        const struct ddi_buf_trans *ddi_translations;
 
         if (IS_BROADWELL(dev)) {
                 ddi_translations_fdi = bdw_ddi_translations_fdi;
                 ddi_translations_dp = bdw_ddi_translations_dp;
                 ddi_translations_edp = bdw_ddi_translations_edp;
+                ddi_translations_hdmi = bdw_ddi_translations_hdmi;
+                n_hdmi_entries = ARRAY_SIZE(bdw_ddi_translations_hdmi);
+                hdmi_800mV_0dB = 7;
         } else if (IS_HASWELL(dev)) {
                 ddi_translations_fdi = hsw_ddi_translations_fdi;
                 ddi_translations_dp = hsw_ddi_translations_dp;
                 ddi_translations_edp = hsw_ddi_translations_dp;
+                ddi_translations_hdmi = hsw_ddi_translations_hdmi;
+                n_hdmi_entries = ARRAY_SIZE(hsw_ddi_translations_hdmi);
+                hdmi_800mV_0dB = 6;
         } else {
                 WARN(1, "ddi translation table missing\n");
                 ddi_translations_edp = bdw_ddi_translations_dp;
                 ddi_translations_fdi = bdw_ddi_translations_fdi;
                 ddi_translations_dp = bdw_ddi_translations_dp;
+                ddi_translations_hdmi = bdw_ddi_translations_hdmi;
+                n_hdmi_entries = ARRAY_SIZE(bdw_ddi_translations_hdmi);
+                hdmi_800mV_0dB = 7;
         }
 
         switch (port) {
@@ -190,14 +216,22 @@ static void intel_prepare_ddi_buffers(struct drm_device *dev, enum port port)
 
         for (i = 0, reg = DDI_BUF_TRANS(port);
              i < ARRAY_SIZE(hsw_ddi_translations_fdi); i++) {
-                I915_WRITE(reg, ddi_translations[i]);
+                I915_WRITE(reg, ddi_translations[i].trans1);
                 reg += 4;
-        }
-        /* Entry 9 is for HDMI: */
-        for (i = 0; i < 2; i++) {
-                I915_WRITE(reg, hsw_ddi_translations_hdmi[hdmi_level * 2 + i]);
+                I915_WRITE(reg, ddi_translations[i].trans2);
                 reg += 4;
         }
+
+        /* Choose a good default if VBT is badly populated */
+        if (hdmi_level == HDMI_LEVEL_SHIFT_UNKNOWN ||
+            hdmi_level >= n_hdmi_entries)
+                hdmi_level = hdmi_800mV_0dB;
+
+        /* Entry 9 is for HDMI: */
+        I915_WRITE(reg, ddi_translations_hdmi[hdmi_level].trans1);
+        reg += 4;
+        I915_WRITE(reg, ddi_translations_hdmi[hdmi_level].trans2);
+        reg += 4;
 }
 
 /* Program DDI buffers translations for DP. By default, program ports A-D in DP
@@ -214,18 +248,6 @@ void intel_prepare_ddi(struct drm_device *dev)
                 intel_prepare_ddi_buffers(dev, port);
 }
 
-static const long hsw_ddi_buf_ctl_values[] = {
-        DDI_BUF_EMP_400MV_0DB_HSW,
-        DDI_BUF_EMP_400MV_3_5DB_HSW,
-        DDI_BUF_EMP_400MV_6DB_HSW,
-        DDI_BUF_EMP_400MV_9_5DB_HSW,
-        DDI_BUF_EMP_600MV_0DB_HSW,
-        DDI_BUF_EMP_600MV_3_5DB_HSW,
-        DDI_BUF_EMP_600MV_6DB_HSW,
-        DDI_BUF_EMP_800MV_0DB_HSW,
-        DDI_BUF_EMP_800MV_3_5DB_HSW
-};
-
 static void intel_wait_ddi_buf_idle(struct drm_i915_private *dev_priv,
                                     enum port port)
 {
@@ -285,7 +307,7 @@ void hsw_fdi_link_train(struct drm_crtc *crtc)
 
         /* Start the training iterating through available voltages and emphasis,
          * testing each value twice. */
-        for (i = 0; i < ARRAY_SIZE(hsw_ddi_buf_ctl_values) * 2; i++) {
+        for (i = 0; i < ARRAY_SIZE(hsw_ddi_translations_fdi) * 2; i++) {
                 /* Configure DP_TP_CTL with auto-training */
                 I915_WRITE(DP_TP_CTL(PORT_E),
                                         DP_TP_CTL_FDI_AUTOTRAIN |
@@ -300,7 +322,7 @@ void hsw_fdi_link_train(struct drm_crtc *crtc)
                 I915_WRITE(DDI_BUF_CTL(PORT_E),
                            DDI_BUF_CTL_ENABLE |
                            ((intel_crtc->config.fdi_lanes - 1) << 1) |
-                           hsw_ddi_buf_ctl_values[i / 2]);
+                           DDI_BUF_TRANS_SELECT(i / 2));
                 POSTING_READ(DDI_BUF_CTL(PORT_E));
 
                 udelay(600);
@@ -375,7 +397,7 @@ void intel_ddi_init_dp_buf_reg(struct intel_encoder *encoder)
                 enc_to_dig_port(&encoder->base);
 
         intel_dp->DP = intel_dig_port->saved_port_bits |
-                DDI_BUF_CTL_ENABLE | DDI_BUF_EMP_400MV_0DB_HSW;
+                DDI_BUF_CTL_ENABLE | DDI_BUF_TRANS_SELECT(0);
         intel_dp->DP |= DDI_PORT_WIDTH(intel_dp->lane_count);
 
 }
@@ -402,7 +424,7 @@ intel_ddi_get_crtc_encoder(struct drm_crtc *crtc)
 }
 
 #define LC_FREQ 2700
-#define LC_FREQ_2K (LC_FREQ * 2000)
+#define LC_FREQ_2K U64_C(LC_FREQ * 2000)
 
 #define P_MIN 2
 #define P_MAX 64
@@ -414,7 +436,11 @@ intel_ddi_get_crtc_encoder(struct drm_crtc *crtc)
 #define VCO_MIN 2400
 #define VCO_MAX 4800
 
-#define ABS_DIFF(a, b) ((a > b) ? (a - b) : (b - a))
+#define abs_diff(a, b) ({                       \
+        typeof(a) __a = (a);                    \
+        typeof(b) __b = (b);                    \
+        (void) (&__a == &__b);                  \
+        __a > __b ? (__a - __b) : (__b - __a); })
 
 struct wrpll_rnp {
         unsigned p, n2, r2;
@@ -524,9 +550,9 @@ static void wrpll_update_rnp(uint64_t freq2k, unsigned budget,
          */
         a = freq2k * budget * p * r2;
         b = freq2k * budget * best->p * best->r2;
-        diff = ABS_DIFF((freq2k * p * r2), (LC_FREQ_2K * n2));
-        diff_best = ABS_DIFF((freq2k * best->p * best->r2),
-                             (LC_FREQ_2K * best->n2));
+        diff = abs_diff(freq2k * p * r2, LC_FREQ_2K * n2);
+        diff_best = abs_diff(freq2k * best->p * best->r2,
+                             LC_FREQ_2K * best->n2);
         c = 1000000 * diff;
         d = 1000000 * diff_best;
 
@@ -587,8 +613,8 @@ static int intel_ddi_calc_wrpll_link(struct drm_i915_private *dev_priv,
         return (refclk * n * 100) / (p * r);
 }
 
-void intel_ddi_clock_get(struct intel_encoder *encoder,
-                         struct intel_crtc_config *pipe_config)
+static void hsw_ddi_clock_get(struct intel_encoder *encoder,
+                              struct intel_crtc_config *pipe_config)
 {
         struct drm_i915_private *dev_priv = encoder->base.dev->dev_private;
         int link_clock = 0;
@@ -643,9 +669,15 @@ void intel_ddi_clock_get(struct intel_encoder *encoder,
                 pipe_config->adjusted_mode.crtc_clock = pipe_config->port_clock;
 }
 
+void intel_ddi_clock_get(struct intel_encoder *encoder,
+                         struct intel_crtc_config *pipe_config)
+{
+        hsw_ddi_clock_get(encoder, pipe_config);
+}
+
 static void
-intel_ddi_calculate_wrpll(int clock /* in Hz */,
-                          unsigned *r2_out, unsigned *n2_out, unsigned *p_out)
+hsw_ddi_calculate_wrpll(int clock /* in Hz */,
+                        unsigned *r2_out, unsigned *n2_out, unsigned *p_out)
 {
         uint64_t freq2k;
         unsigned p, n2, r2;
@@ -708,27 +740,17 @@ intel_ddi_calculate_wrpll(int clock /* in Hz */,
         *r2_out = best.r2;
 }
 
-/*
- * Tries to find a PLL for the CRTC. If it finds, it increases the refcount and
- * stores it in intel_crtc->ddi_pll_sel, so other mode sets won't be able to
- * steal the selected PLL. You need to call intel_ddi_pll_enable to actually
- * enable the PLL.
- */
-bool intel_ddi_pll_select(struct intel_crtc *intel_crtc)
+static bool
+hsw_ddi_pll_select(struct intel_crtc *intel_crtc,
+                   struct intel_encoder *intel_encoder,
+                   int clock)
 {
-        struct drm_crtc *crtc = &intel_crtc->base;
-        struct intel_encoder *intel_encoder = intel_ddi_get_crtc_encoder(crtc);
-        int type = intel_encoder->type;
-        int clock = intel_crtc->config.port_clock;
-
-        intel_put_shared_dpll(intel_crtc);
-
-        if (type == INTEL_OUTPUT_HDMI) {
+        if (intel_encoder->type == INTEL_OUTPUT_HDMI) {
                 struct intel_shared_dpll *pll;
                 uint32_t val;
                 unsigned p, n2, r2;
 
-                intel_ddi_calculate_wrpll(clock * 1000, &r2, &n2, &p);
+                hsw_ddi_calculate_wrpll(clock * 1000, &r2, &n2, &p);
 
                 val = WRPLL_PLL_ENABLE | WRPLL_PLL_LCPLL |
                       WRPLL_DIVIDER_REFERENCE(r2) | WRPLL_DIVIDER_FEEDBACK(n2) |
@@ -749,6 +771,25 @@ bool intel_ddi_pll_select(struct intel_crtc *intel_crtc)
         return true;
 }
 
+
+/*
+ * Tries to find a *shared* PLL for the CRTC and store it in
+ * intel_crtc->ddi_pll_sel.
+ *
+ * For private DPLLs, compute_config() should do the selection for us. This
+ * function should be folded into compute_config() eventually.
+ */
+bool intel_ddi_pll_select(struct intel_crtc *intel_crtc)
+{
+        struct drm_crtc *crtc = &intel_crtc->base;
+        struct intel_encoder *intel_encoder = intel_ddi_get_crtc_encoder(crtc);
+        int clock = intel_crtc->config.port_clock;
+
+        intel_put_shared_dpll(intel_crtc);
+
+        return hsw_ddi_pll_select(intel_crtc, intel_encoder, clock);
+}
+
 void intel_ddi_set_pipe_settings(struct drm_crtc *crtc)
 {
         struct drm_i915_private *dev_priv = crtc->dev->dev_private;
@@ -1183,31 +1224,52 @@ static void intel_disable_ddi(struct intel_encoder *intel_encoder)
         }
 }
 
-int intel_ddi_get_cdclk_freq(struct drm_i915_private *dev_priv)
+static int bdw_get_cdclk_freq(struct drm_i915_private *dev_priv)
+{
+        uint32_t lcpll = I915_READ(LCPLL_CTL);
+        uint32_t freq = lcpll & LCPLL_CLK_FREQ_MASK;
+
+        if (lcpll & LCPLL_CD_SOURCE_FCLK)
+                return 800000;
+        else if (I915_READ(FUSE_STRAP) & HSW_CDCLK_LIMIT)
+                return 450000;
+        else if (freq == LCPLL_CLK_FREQ_450)
+                return 450000;
+        else if (freq == LCPLL_CLK_FREQ_54O_BDW)
+                return 540000;
+        else if (freq == LCPLL_CLK_FREQ_337_5_BDW)
+                return 337500;
+        else
+                return 675000;
+}
+
+static int hsw_get_cdclk_freq(struct drm_i915_private *dev_priv)
 {
         struct drm_device *dev = dev_priv->dev;
         uint32_t lcpll = I915_READ(LCPLL_CTL);
         uint32_t freq = lcpll & LCPLL_CLK_FREQ_MASK;
 
-        if (lcpll & LCPLL_CD_SOURCE_FCLK) {
+        if (lcpll & LCPLL_CD_SOURCE_FCLK)
                 return 800000;
-        } else if (I915_READ(FUSE_STRAP) & HSW_CDCLK_LIMIT) {
+        else if (I915_READ(FUSE_STRAP) & HSW_CDCLK_LIMIT)
                 return 450000;
-        } else if (freq == LCPLL_CLK_FREQ_450) {
+        else if (freq == LCPLL_CLK_FREQ_450)
                 return 450000;
-        } else if (IS_HASWELL(dev)) {
-                if (IS_ULT(dev))
-                        return 337500;
-                else
-                        return 540000;
-        } else {
-                if (freq == LCPLL_CLK_FREQ_54O_BDW)
-                        return 540000;
-                else if (freq == LCPLL_CLK_FREQ_337_5_BDW)
-                        return 337500;
-                else
-                        return 675000;
-        }
+        else if (IS_ULT(dev))
+                return 337500;
+        else
+                return 540000;
+}
+
+int intel_ddi_get_cdclk_freq(struct drm_i915_private *dev_priv)
+{
+        struct drm_device *dev = dev_priv->dev;
+
+        if (IS_BROADWELL(dev))
+                return bdw_get_cdclk_freq(dev_priv);
+
+        /* Haswell */
+        return hsw_get_cdclk_freq(dev_priv);
 }
 
 static void hsw_ddi_pll_enable(struct drm_i915_private *dev_priv,
@@ -1248,10 +1310,8 @@ static const char * const hsw_ddi_pll_names[] = {
         "WRPLL 2",
 };
 
-void intel_ddi_pll_init(struct drm_device *dev)
+static void hsw_shared_dplls_init(struct drm_i915_private *dev_priv)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
-        uint32_t val = I915_READ(LCPLL_CTL);
         int i;
 
         dev_priv->num_shared_dpll = 2;
@@ -1264,6 +1324,14 @@ void intel_ddi_pll_init(struct drm_device *dev)
                 dev_priv->shared_dplls[i].get_hw_state =
                         hsw_ddi_pll_get_hw_state;
         }
+}
+
+void intel_ddi_pll_init(struct drm_device *dev)
+{
+        struct drm_i915_private *dev_priv = dev->dev_private;
+        uint32_t val = I915_READ(LCPLL_CTL);
+
+        hsw_shared_dplls_init(dev_priv);
 
         /* The LCPLL register should be turned on by the BIOS. For now let's
          * just check its state and print errors in case something is wrong.
@@ -1444,7 +1512,7 @@ void intel_ddi_get_config(struct intel_encoder *encoder,
                 dev_priv->vbt.edp_bpp = pipe_config->pipe_bpp;
         }
 
-        intel_ddi_clock_get(encoder, pipe_config);
+        hsw_ddi_clock_get(encoder, pipe_config);
 }
 
 static void intel_ddi_destroy(struct drm_encoder *encoder)
diff --git a/drivers/gpu/drm/i915/intel_display.c b/drivers/gpu/drm/i915/intel_display.c
index d8324c69fa86..507370513f3d 100644
--- a/drivers/gpu/drm/i915/intel_display.c
+++ b/drivers/gpu/drm/i915/intel_display.c
@@ -91,15 +91,16 @@ static int intel_framebuffer_init(struct drm_device *dev,
                                   struct intel_framebuffer *ifb,
                                   struct drm_mode_fb_cmd2 *mode_cmd,
                                   struct drm_i915_gem_object *obj);
-static void intel_dp_set_m_n(struct intel_crtc *crtc);
 static void i9xx_set_pipeconf(struct intel_crtc *intel_crtc);
 static void intel_set_pipe_timings(struct intel_crtc *intel_crtc);
 static void intel_cpu_transcoder_set_m_n(struct intel_crtc *crtc,
-                                         struct intel_link_m_n *m_n);
+                                         struct intel_link_m_n *m_n,
+                                         struct intel_link_m_n *m2_n2);
 static void ironlake_set_pipeconf(struct drm_crtc *crtc);
 static void haswell_set_pipeconf(struct drm_crtc *crtc);
 static void intel_set_pipe_csc(struct drm_crtc *crtc);
 static void vlv_prepare_pll(struct intel_crtc *crtc);
+static void chv_prepare_pll(struct intel_crtc *crtc);
 
 static struct intel_encoder *intel_find_encoder(struct intel_connector *connector, int pipe)
 {
@@ -899,7 +900,8 @@ static void g4x_wait_for_vblank(struct drm_device *dev, int pipe)
         frame = I915_READ(frame_reg);
 
         if (wait_for(I915_READ_NOTRACE(frame_reg) != frame, 50))
-                WARN(1, "vblank wait timed out\n");
+                WARN(1, "vblank wait on pipe %c timed out\n",
+                     pipe_name(pipe));
 }
 
 /**
@@ -940,7 +942,8 @@ void intel_wait_for_vblank(struct drm_device *dev, int pipe)
         if (wait_for(I915_READ(pipestat_reg) &
                      PIPE_VBLANK_INTERRUPT_STATUS,
                      50))
-                DRM_DEBUG_KMS("vblank wait timed out\n");
+                DRM_DEBUG_KMS("vblank wait on pipe %c timed out\n",
+                              pipe_name(pipe));
 }
 
 static bool pipe_dsl_stopped(struct drm_device *dev, enum pipe pipe)
@@ -964,8 +967,7 @@ static bool pipe_dsl_stopped(struct drm_device *dev, enum pipe pipe)
 
 /*
  * intel_wait_for_pipe_off - wait for pipe to turn off
- * @dev: drm device
- * @pipe: pipe to wait for
+ * @crtc: crtc whose pipe to wait for
  *
  * After disabling a pipe, we can't wait for vblank in the usual way,
  * spinning on the vblank interrupt status bit, since we won't actually
@@ -979,11 +981,12 @@ static bool pipe_dsl_stopped(struct drm_device *dev, enum pipe pipe)
  *   ends up stopping at the start of the next frame).
  *
  */
-void intel_wait_for_pipe_off(struct drm_device *dev, int pipe)
+static void intel_wait_for_pipe_off(struct intel_crtc *crtc)
 {
+        struct drm_device *dev = crtc->base.dev;
         struct drm_i915_private *dev_priv = dev->dev_private;
-        enum transcoder cpu_transcoder = intel_pipe_to_cpu_transcoder(dev_priv,
-                                                                      pipe);
+        enum transcoder cpu_transcoder = crtc->config.cpu_transcoder;
+        enum pipe pipe = crtc->pipe;
 
         if (INTEL_INFO(dev)->gen >= 4) {
                 int reg = PIPECONF(cpu_transcoder);
@@ -1192,27 +1195,40 @@ void assert_fdi_rx_pll(struct drm_i915_private *dev_priv,
 static void assert_panel_unlocked(struct drm_i915_private *dev_priv,
                                   enum pipe pipe)
 {
-        int pp_reg, lvds_reg;
+        struct drm_device *dev = dev_priv->dev;
+        int pp_reg;
         u32 val;
         enum pipe panel_pipe = PIPE_A;
         bool locked = true;
 
-        if (HAS_PCH_SPLIT(dev_priv->dev)) {
+        if (WARN_ON(HAS_DDI(dev)))
+                return;
+
+        if (HAS_PCH_SPLIT(dev)) {
+                u32 port_sel;
+
                 pp_reg = PCH_PP_CONTROL;
-                lvds_reg = PCH_LVDS;
+                port_sel = I915_READ(PCH_PP_ON_DELAYS) & PANEL_PORT_SELECT_MASK;
+
+                if (port_sel == PANEL_PORT_SELECT_LVDS &&
+                    I915_READ(PCH_LVDS) & LVDS_PIPEB_SELECT)
+                        panel_pipe = PIPE_B;
+                /* XXX: else fix for eDP */
+        } else if (IS_VALLEYVIEW(dev)) {
+                /* presumably write lock depends on pipe, not port select */
+                pp_reg = VLV_PIPE_PP_CONTROL(pipe);
+                panel_pipe = pipe;
         } else {
                 pp_reg = PP_CONTROL;
-                lvds_reg = LVDS;
+                if (I915_READ(LVDS) & LVDS_PIPEB_SELECT)
+                        panel_pipe = PIPE_B;
         }
 
         val = I915_READ(pp_reg);
         if (!(val & PANEL_POWER_ON) ||
-            ((val & PANEL_UNLOCK_REGS) == PANEL_UNLOCK_REGS))
+            ((val & PANEL_UNLOCK_MASK) == PANEL_UNLOCK_REGS))
                 locked = false;
 
-        if (I915_READ(lvds_reg) & LVDS_PIPEB_SELECT)
-                panel_pipe = PIPE_B;
-
         WARN(panel_pipe == pipe && locked,
              "panel assertion failure, pipe %c regs locked\n",
              pipe_name(pipe));
@@ -1245,8 +1261,9 @@ void assert_pipe(struct drm_i915_private *dev_priv,
         enum transcoder cpu_transcoder = intel_pipe_to_cpu_transcoder(dev_priv,
                                                                       pipe);
 
-        /* if we need the pipe A quirk it must be always on */
-        if (pipe == PIPE_A && dev_priv->quirks & QUIRK_PIPEA_FORCE)
+        /* if we need the pipe quirk it must be always on */
+        if ((pipe == PIPE_A && dev_priv->quirks & QUIRK_PIPEA_FORCE) ||
+            (pipe == PIPE_B && dev_priv->quirks & QUIRK_PIPEB_FORCE))
                 state = true;
 
         if (!intel_display_power_enabled(dev_priv,
@@ -1300,7 +1317,7 @@ static void assert_planes_disabled(struct drm_i915_private *dev_priv,
         }
 
         /* Need to check both planes against the pipe */
-        for_each_pipe(i) {
+        for_each_pipe(dev_priv, i) {
                 reg = DSPCNTR(i);
                 val = I915_READ(reg);
                 cur_pipe = (val & DISPPLANE_SEL_PIPE_MASK) >>
@@ -1341,6 +1358,12 @@ static void assert_sprites_disabled(struct drm_i915_private *dev_priv,
         }
 }
 
+static void assert_vblank_disabled(struct drm_crtc *crtc)
+{
+        if (WARN_ON(drm_crtc_vblank_get(crtc) == 0))
+                drm_crtc_vblank_put(crtc);
+}
+
 static void ibx_assert_pch_refclk_enabled(struct drm_i915_private *dev_priv)
 {
         u32 val;
@@ -1513,34 +1536,6 @@ static void intel_init_dpio(struct drm_device *dev)
         }
 }
 
-static void intel_reset_dpio(struct drm_device *dev)
-{
-        struct drm_i915_private *dev_priv = dev->dev_private;
-
-        if (IS_CHERRYVIEW(dev)) {
-                enum dpio_phy phy;
-                u32 val;
-
-                for (phy = DPIO_PHY0; phy < I915_NUM_PHYS_VLV; phy++) {
-                        /* Poll for phypwrgood signal */
-                        if (wait_for(I915_READ(DISPLAY_PHY_STATUS) &
-                                                PHY_POWERGOOD(phy), 1))
-                                DRM_ERROR("Display PHY %d is not power up\n", phy);
-
-                        /*
-                         * Deassert common lane reset for PHY.
-                         *
-                         * This should only be done on init and resume from S3
-                         * with both PLLs disabled, or we risk losing DPIO and
-                         * PLL synchronization.
-                         */
-                        val = I915_READ(DISPLAY_PHY_CONTROL);
-                        I915_WRITE(DISPLAY_PHY_CONTROL,
-                                PHY_COM_LANE_RESET_DEASSERT(phy, val));
-                }
-        }
-}
-
 static void vlv_enable_pll(struct intel_crtc *crtc)
 {
         struct drm_device *dev = crtc->base.dev;
@@ -1554,7 +1549,7 @@ static void vlv_enable_pll(struct intel_crtc *crtc)
         BUG_ON(!IS_VALLEYVIEW(dev_priv->dev));
 
         /* PLL is protected by panel, make sure we can write it */
-        if (IS_MOBILE(dev_priv->dev) && !IS_I830(dev_priv->dev))
+        if (IS_MOBILE(dev_priv->dev))
                 assert_panel_unlocked(dev_priv, crtc->pipe);
 
         I915_WRITE(reg, dpll);
@@ -1617,6 +1612,18 @@ static void chv_enable_pll(struct intel_crtc *crtc)
         mutex_unlock(&dev_priv->dpio_lock);
 }
 
+static int intel_num_dvo_pipes(struct drm_device *dev)
+{
+        struct intel_crtc *crtc;
+        int count = 0;
+
+        for_each_intel_crtc(dev, crtc)
+                count += crtc->active &&
+                        intel_pipe_has_type(&crtc->base, INTEL_OUTPUT_DVO);
+
+        return count;
+}
+
 static void i9xx_enable_pll(struct intel_crtc *crtc)
 {
         struct drm_device *dev = crtc->base.dev;
@@ -1633,7 +1640,18 @@ static void i9xx_enable_pll(struct intel_crtc *crtc)
         if (IS_MOBILE(dev) && !IS_I830(dev))
                 assert_panel_unlocked(dev_priv, crtc->pipe);
 
-        I915_WRITE(reg, dpll);
+        /* Enable DVO 2x clock on both PLLs if necessary */
+        if (IS_I830(dev) && intel_num_dvo_pipes(dev) > 0) {
+                /*
+                 * It appears to be important that we don't enable this
+                 * for the current pipe before otherwise configuring the
+                 * PLL. No idea how this should be handled if multiple
+                 * DVO outputs are enabled simultaneosly.
+                 */
+                dpll |= DPLL_DVO_2X_MODE;
+                I915_WRITE(DPLL(!crtc->pipe),
+                           I915_READ(DPLL(!crtc->pipe)) | DPLL_DVO_2X_MODE);
+        }
 
         /* Wait for the clocks to stabilize. */
         POSTING_READ(reg);
@@ -1672,10 +1690,25 @@ static void i9xx_enable_pll(struct intel_crtc *crtc)
  *
  * Note!  This is for pre-ILK only.
  */
-static void i9xx_disable_pll(struct drm_i915_private *dev_priv, enum pipe pipe)
+static void i9xx_disable_pll(struct intel_crtc *crtc)
 {
-        /* Don't disable pipe A or pipe A PLLs if needed */
-        if (pipe == PIPE_A && (dev_priv->quirks & QUIRK_PIPEA_FORCE))
+        struct drm_device *dev = crtc->base.dev;
+        struct drm_i915_private *dev_priv = dev->dev_private;
+        enum pipe pipe = crtc->pipe;
+
+        /* Disable DVO 2x clock on both PLLs if necessary */
+        if (IS_I830(dev) &&
+            intel_pipe_has_type(&crtc->base, INTEL_OUTPUT_DVO) &&
+            intel_num_dvo_pipes(dev) == 1) {
+                I915_WRITE(DPLL(PIPE_B),
+                           I915_READ(DPLL(PIPE_B)) & ~DPLL_DVO_2X_MODE);
+                I915_WRITE(DPLL(PIPE_A),
+                           I915_READ(DPLL(PIPE_A)) & ~DPLL_DVO_2X_MODE);
+        }
+
+        /* Don't disable pipe or pipe PLLs if needed */
+        if ((pipe == PIPE_A && dev_priv->quirks & QUIRK_PIPEA_FORCE) ||
+            (pipe == PIPE_B && dev_priv->quirks & QUIRK_PIPEB_FORCE))
                 return;
 
         /* Make sure the pipe isn't still relying on us */
@@ -1712,7 +1745,7 @@ static void chv_disable_pll(struct drm_i915_private *dev_priv, enum pipe pipe)
         assert_pipe_disabled(dev_priv, pipe);
 
         /* Set PLL en = 0 */
-        val = DPLL_SSC_REF_CLOCK_CHV;
+        val = DPLL_SSC_REF_CLOCK_CHV | DPLL_REFA_CLK_ENABLE_VLV;
         if (pipe != PIPE_A)
                 val |= DPLL_INTEGRATED_CRI_CLK_VLV;
         I915_WRITE(DPLL(pipe), val);
@@ -1806,7 +1839,7 @@ static void intel_enable_shared_dpll(struct intel_crtc *crtc)
         if (WARN_ON(pll->refcount == 0))
                 return;
 
-        DRM_DEBUG_KMS("enable %s (active %d, on? %d)for crtc %d\n",
+        DRM_DEBUG_KMS("enable %s (active %d, on? %d) for crtc %d\n",
                       pll->name, pll->active, pll->on,
                       crtc->base.base.id);
 
@@ -1824,7 +1857,7 @@ static void intel_enable_shared_dpll(struct intel_crtc *crtc)
         pll->on = true;
 }
 
-void intel_disable_shared_dpll(struct intel_crtc *crtc)
+static void intel_disable_shared_dpll(struct intel_crtc *crtc)
 {
         struct drm_device *dev = crtc->base.dev;
         struct drm_i915_private *dev_priv = dev->dev_private;
@@ -1868,7 +1901,7 @@ static void ironlake_enable_pch_transcoder(struct drm_i915_private *dev_priv,
         uint32_t reg, val, pipeconf_val;
 
         /* PCH only available on ILK+ */
-        BUG_ON(INTEL_INFO(dev)->gen < 5);
+        BUG_ON(!HAS_PCH_SPLIT(dev));
 
         /* Make sure PCH DPLL is enabled */
         assert_shared_dpll_enabled(dev_priv,
@@ -1921,7 +1954,7 @@ static void lpt_enable_pch_transcoder(struct drm_i915_private *dev_priv,
         u32 val, pipeconf_val;
 
         /* PCH only available on ILK+ */
-        BUG_ON(INTEL_INFO(dev_priv->dev)->gen < 5);
+        BUG_ON(!HAS_PCH_SPLIT(dev_priv->dev));
 
         /* FDI must be feeding us bits for PCH ports */
         assert_fdi_tx_enabled(dev_priv, (enum pipe) cpu_transcoder);
@@ -2043,8 +2076,8 @@ static void intel_enable_pipe(struct intel_crtc *crtc)
         reg = PIPECONF(cpu_transcoder);
         val = I915_READ(reg);
         if (val & PIPECONF_ENABLE) {
-                WARN_ON(!(pipe == PIPE_A &&
-                          dev_priv->quirks & QUIRK_PIPEA_FORCE));
+                WARN_ON(!((pipe == PIPE_A && dev_priv->quirks & QUIRK_PIPEA_FORCE) ||
+                          (pipe == PIPE_B && dev_priv->quirks & QUIRK_PIPEB_FORCE)));
                 return;
         }
 
@@ -2054,21 +2087,19 @@ static void intel_enable_pipe(struct intel_crtc *crtc)
 
 /**
  * intel_disable_pipe - disable a pipe, asserting requirements
- * @dev_priv: i915 private structure
- * @pipe: pipe to disable
+ * @crtc: crtc whose pipes is to be disabled
  *
- * Disable @pipe, making sure that various hardware specific requirements
- * are met, if applicable, e.g. plane disabled, panel fitter off, etc.
- *
- * @pipe should be %PIPE_A or %PIPE_B.
+ * Disable the pipe of @crtc, making sure that various hardware
+ * specific requirements are met, if applicable, e.g. plane
+ * disabled, panel fitter off, etc.
  *
  * Will wait until the pipe has shut down before returning.
  */
-static void intel_disable_pipe(struct drm_i915_private *dev_priv,
-                               enum pipe pipe)
+static void intel_disable_pipe(struct intel_crtc *crtc)
 {
-        enum transcoder cpu_transcoder = intel_pipe_to_cpu_transcoder(dev_priv,
-                                                                      pipe);
+        struct drm_i915_private *dev_priv = crtc->base.dev->dev_private;
+        enum transcoder cpu_transcoder = crtc->config.cpu_transcoder;
+        enum pipe pipe = crtc->pipe;
         int reg;
         u32 val;
 
@@ -2080,17 +2111,26 @@ static void intel_disable_pipe(struct drm_i915_private *dev_priv,
         assert_cursor_disabled(dev_priv, pipe);
         assert_sprites_disabled(dev_priv, pipe);
 
-        /* Don't disable pipe A or pipe A PLLs if needed */
-        if (pipe == PIPE_A && (dev_priv->quirks & QUIRK_PIPEA_FORCE))
-                return;
-
         reg = PIPECONF(cpu_transcoder);
         val = I915_READ(reg);
         if ((val & PIPECONF_ENABLE) == 0)
                 return;
 
-        I915_WRITE(reg, val & ~PIPECONF_ENABLE);
-        intel_wait_for_pipe_off(dev_priv->dev, pipe);
+        /*
+         * Double wide has implications for planes
+         * so best keep it disabled when not needed.
+         */
+        if (crtc->config.double_wide)
+                val &= ~PIPECONF_DOUBLE_WIDE;
+
+        /* Don't disable pipe or pipe PLLs if needed */
+        if (!(pipe == PIPE_A && dev_priv->quirks & QUIRK_PIPEA_FORCE) &&
+            !(pipe == PIPE_B && dev_priv->quirks & QUIRK_PIPEB_FORCE))
+                val &= ~PIPECONF_ENABLE;
+
+        I915_WRITE(reg, val);
+        if ((val & PIPECONF_ENABLE) == 0)
+                intel_wait_for_pipe_off(crtc);
 }
 
 /*
@@ -2109,35 +2149,28 @@ void intel_flush_primary_plane(struct drm_i915_private *dev_priv,
 
 /**
  * intel_enable_primary_hw_plane - enable the primary plane on a given pipe
- * @dev_priv: i915 private structure
- * @plane: plane to enable
- * @pipe: pipe being fed
+ * @plane:  plane to be enabled
+ * @crtc: crtc for the plane
  *
- * Enable @plane on @pipe, making sure that @pipe is running first.
+ * Enable @plane on @crtc, making sure that the pipe is running first.
  */
-static void intel_enable_primary_hw_plane(struct drm_i915_private *dev_priv,
-                                          enum plane plane, enum pipe pipe)
+static void intel_enable_primary_hw_plane(struct drm_plane *plane,
+                                          struct drm_crtc *crtc)
 {
-        struct drm_device *dev = dev_priv->dev;
-        struct intel_crtc *intel_crtc =
-                to_intel_crtc(dev_priv->pipe_to_crtc_mapping[pipe]);
-        int reg;
-        u32 val;
+        struct drm_device *dev = plane->dev;
+        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
 
         /* If the pipe isn't enabled, we can't pump pixels and may hang */
-        assert_pipe_enabled(dev_priv, pipe);
+        assert_pipe_enabled(dev_priv, intel_crtc->pipe);
 
         if (intel_crtc->primary_enabled)
                 return;
 
         intel_crtc->primary_enabled = true;
 
-        reg = DSPCNTR(plane);
-        val = I915_READ(reg);
-        WARN_ON(val & DISPLAY_PLANE_ENABLE);
-
-        I915_WRITE(reg, val | DISPLAY_PLANE_ENABLE);
-        intel_flush_primary_plane(dev_priv, plane);
+        dev_priv->display.update_primary_plane(crtc, plane->fb,
+                                               crtc->x, crtc->y);
 
         /*
          * BDW signals flip done immediately if the plane
@@ -2150,31 +2183,27 @@ static void intel_enable_primary_hw_plane(struct drm_i915_private *dev_priv,
 
 /**
  * intel_disable_primary_hw_plane - disable the primary hardware plane
- * @dev_priv: i915 private structure
- * @plane: plane to disable
- * @pipe: pipe consuming the data
+ * @plane: plane to be disabled
+ * @crtc: crtc for the plane
  *
- * Disable @plane; should be an independent operation.
+ * Disable @plane on @crtc, making sure that the pipe is running first.
  */
-static void intel_disable_primary_hw_plane(struct drm_i915_private *dev_priv,
-                                           enum plane plane, enum pipe pipe)
+static void intel_disable_primary_hw_plane(struct drm_plane *plane,
+                                           struct drm_crtc *crtc)
 {
-        struct intel_crtc *intel_crtc =
-                to_intel_crtc(dev_priv->pipe_to_crtc_mapping[pipe]);
-        int reg;
-        u32 val;
+        struct drm_device *dev = plane->dev;
+        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
+
+        assert_pipe_enabled(dev_priv, intel_crtc->pipe);
 
         if (!intel_crtc->primary_enabled)
                 return;
 
         intel_crtc->primary_enabled = false;
 
-        reg = DSPCNTR(plane);
-        val = I915_READ(reg);
-        WARN_ON((val & DISPLAY_PLANE_ENABLE) == 0);
-
-        I915_WRITE(reg, val & ~DISPLAY_PLANE_ENABLE);
-        intel_flush_primary_plane(dev_priv, plane);
+        dev_priv->display.update_primary_plane(crtc, plane->fb,
+                                               crtc->x, crtc->y);
 }
 
 static bool need_vtd_wa(struct drm_device *dev)
@@ -2422,16 +2451,46 @@ static void i9xx_update_primary_plane(struct drm_crtc *crtc,
         struct drm_device *dev = crtc->dev;
         struct drm_i915_private *dev_priv = dev->dev_private;
         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
-        struct drm_i915_gem_object *obj = intel_fb_obj(fb);
+        struct drm_i915_gem_object *obj;
         int plane = intel_crtc->plane;
         unsigned long linear_offset;
         u32 dspcntr;
-        u32 reg;
+        u32 reg = DSPCNTR(plane);
+        int pixel_size;
+
+        if (!intel_crtc->primary_enabled) {
+                I915_WRITE(reg, 0);
+                if (INTEL_INFO(dev)->gen >= 4)
+                        I915_WRITE(DSPSURF(plane), 0);
+                else
+                        I915_WRITE(DSPADDR(plane), 0);
+                POSTING_READ(reg);
+                return;
+        }
+
+        obj = intel_fb_obj(fb);
+        if (WARN_ON(obj == NULL))
+                return;
+
+        pixel_size = drm_format_plane_cpp(fb->pixel_format, 0);
+
+        dspcntr = DISPPLANE_GAMMA_ENABLE;
+
+        dspcntr |= DISPLAY_PLANE_ENABLE;
+
+        if (INTEL_INFO(dev)->gen < 4) {
+                if (intel_crtc->pipe == PIPE_B)
+                        dspcntr |= DISPPLANE_SEL_PIPE_B;
+
+                /* pipesrc and dspsize control the size that is scaled from,
+                 * which should always be the user's requested size.
+                 */
+                I915_WRITE(DSPSIZE(plane),
+                           ((intel_crtc->config.pipe_src_h - 1) << 16) |
+                           (intel_crtc->config.pipe_src_w - 1));
+                I915_WRITE(DSPPOS(plane), 0);
+        }
 
-        reg = DSPCNTR(plane);
-        dspcntr = I915_READ(reg);
-        /* Mask out pixel format bits in case we change it */
-        dspcntr &= ~DISPPLANE_PIXFORMAT_MASK;
         switch (fb->pixel_format) {
         case DRM_FORMAT_C8:
                 dspcntr |= DISPPLANE_8BPP;
@@ -2463,30 +2522,40 @@ static void i9xx_update_primary_plane(struct drm_crtc *crtc,
                 BUG();
         }
 
-        if (INTEL_INFO(dev)->gen >= 4) {
-                if (obj->tiling_mode != I915_TILING_NONE)
-                        dspcntr |= DISPPLANE_TILED;
-                else
-                        dspcntr &= ~DISPPLANE_TILED;
-        }
+        if (INTEL_INFO(dev)->gen >= 4 &&
+            obj->tiling_mode != I915_TILING_NONE)
+                dspcntr |= DISPPLANE_TILED;
 
         if (IS_G4X(dev))
                 dspcntr |= DISPPLANE_TRICKLE_FEED_DISABLE;
 
-        I915_WRITE(reg, dspcntr);
-
-        linear_offset = y * fb->pitches[0] + x * (fb->bits_per_pixel / 8);
+        linear_offset = y * fb->pitches[0] + x * pixel_size;
 
         if (INTEL_INFO(dev)->gen >= 4) {
                 intel_crtc->dspaddr_offset =
                         intel_gen4_compute_page_offset(&x, &y, obj->tiling_mode,
-                                                       fb->bits_per_pixel / 8,
+                                                       pixel_size,
                                                        fb->pitches[0]);
                 linear_offset -= intel_crtc->dspaddr_offset;
         } else {
                 intel_crtc->dspaddr_offset = linear_offset;
         }
 
+        if (to_intel_plane(crtc->primary)->rotation == BIT(DRM_ROTATE_180)) {
+                dspcntr |= DISPPLANE_ROTATE_180;
+
+                x += (intel_crtc->config.pipe_src_w - 1);
+                y += (intel_crtc->config.pipe_src_h - 1);
+
+                /* Finding the last pixel of the last line of the display
+                data and adding to linear_offset*/
+                linear_offset +=
+                        (intel_crtc->config.pipe_src_h - 1) * fb->pitches[0] +
+                        (intel_crtc->config.pipe_src_w - 1) * pixel_size;
+        }
+
+        I915_WRITE(reg, dspcntr);
+
         DRM_DEBUG_KMS("Writing base %08lX %08lX %d %d %d\n",
                       i915_gem_obj_ggtt_offset(obj), linear_offset, x, y,
                       fb->pitches[0]);
@@ -2508,16 +2577,33 @@ static void ironlake_update_primary_plane(struct drm_crtc *crtc,
         struct drm_device *dev = crtc->dev;
         struct drm_i915_private *dev_priv = dev->dev_private;
         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
-        struct drm_i915_gem_object *obj = intel_fb_obj(fb);
+        struct drm_i915_gem_object *obj;
         int plane = intel_crtc->plane;
         unsigned long linear_offset;
         u32 dspcntr;
-        u32 reg;
+        u32 reg = DSPCNTR(plane);
+        int pixel_size;
+
+        if (!intel_crtc->primary_enabled) {
+                I915_WRITE(reg, 0);
+                I915_WRITE(DSPSURF(plane), 0);
+                POSTING_READ(reg);
+                return;
+        }
+
+        obj = intel_fb_obj(fb);
+        if (WARN_ON(obj == NULL))
+                return;
+
+        pixel_size = drm_format_plane_cpp(fb->pixel_format, 0);
+
+        dspcntr = DISPPLANE_GAMMA_ENABLE;
+
+        dspcntr |= DISPLAY_PLANE_ENABLE;
+
+        if (IS_HASWELL(dev) || IS_BROADWELL(dev))
+                dspcntr |= DISPPLANE_PIPE_CSC_ENABLE;
 
-        reg = DSPCNTR(plane);
-        dspcntr = I915_READ(reg);
-        /* Mask out pixel format bits in case we change it */
-        dspcntr &= ~DISPPLANE_PIXFORMAT_MASK;
         switch (fb->pixel_format) {
         case DRM_FORMAT_C8:
                 dspcntr |= DISPPLANE_8BPP;
@@ -2547,22 +2633,32 @@ static void ironlake_update_primary_plane(struct drm_crtc *crtc,
 
         if (obj->tiling_mode != I915_TILING_NONE)
                 dspcntr |= DISPPLANE_TILED;
-        else
-                dspcntr &= ~DISPPLANE_TILED;
 
-        if (IS_HASWELL(dev) || IS_BROADWELL(dev))
-                dspcntr &= ~DISPPLANE_TRICKLE_FEED_DISABLE;
-        else
+        if (!IS_HASWELL(dev) && !IS_BROADWELL(dev))
                 dspcntr |= DISPPLANE_TRICKLE_FEED_DISABLE;
 
-        I915_WRITE(reg, dspcntr);
-
-        linear_offset = y * fb->pitches[0] + x * (fb->bits_per_pixel / 8);
+        linear_offset = y * fb->pitches[0] + x * pixel_size;
         intel_crtc->dspaddr_offset =
                 intel_gen4_compute_page_offset(&x, &y, obj->tiling_mode,
-                                               fb->bits_per_pixel / 8,
+                                               pixel_size,
                                                fb->pitches[0]);
         linear_offset -= intel_crtc->dspaddr_offset;
+        if (to_intel_plane(crtc->primary)->rotation == BIT(DRM_ROTATE_180)) {
+                dspcntr |= DISPPLANE_ROTATE_180;
+
+                if (!IS_HASWELL(dev) && !IS_BROADWELL(dev)) {
+                        x += (intel_crtc->config.pipe_src_w - 1);
+                        y += (intel_crtc->config.pipe_src_h - 1);
+
+                        /* Finding the last pixel of the last line of the display
+                        data and adding to linear_offset*/
+                        linear_offset +=
+                                (intel_crtc->config.pipe_src_h - 1) * fb->pitches[0] +
+                                (intel_crtc->config.pipe_src_w - 1) * pixel_size;
+                }
+        }
+
+        I915_WRITE(reg, dspcntr);
 
         DRM_DEBUG_KMS("Writing base %08lX %08lX %d %d %d\n",
                       i915_gem_obj_ggtt_offset(obj), linear_offset, x, y,
@@ -3346,23 +3442,54 @@ bool intel_has_pending_fb_unpin(struct drm_device *dev)
         return false;
 }
 
+static void page_flip_completed(struct intel_crtc *intel_crtc)
+{
+        struct drm_i915_private *dev_priv = to_i915(intel_crtc->base.dev);
+        struct intel_unpin_work *work = intel_crtc->unpin_work;
+
+        /* ensure that the unpin work is consistent wrt ->pending. */
+        smp_rmb();
+        intel_crtc->unpin_work = NULL;
+
+        if (work->event)
+                drm_send_vblank_event(intel_crtc->base.dev,
+                                      intel_crtc->pipe,
+                                      work->event);
+
+        drm_crtc_vblank_put(&intel_crtc->base);
+
+        wake_up_all(&dev_priv->pending_flip_queue);
+        queue_work(dev_priv->wq, &work->work);
+
+        trace_i915_flip_complete(intel_crtc->plane,
+                                 work->pending_flip_obj);
+}
+
 void intel_crtc_wait_for_pending_flips(struct drm_crtc *crtc)
 {
         struct drm_device *dev = crtc->dev;
         struct drm_i915_private *dev_priv = dev->dev_private;
 
-        if (crtc->primary->fb == NULL)
-                return;
-
         WARN_ON(waitqueue_active(&dev_priv->pending_flip_queue));
+        if (WARN_ON(wait_event_timeout(dev_priv->pending_flip_queue,
+                                       !intel_crtc_has_pending_flip(crtc),
+                                       60*HZ) == 0)) {
+                struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
+                unsigned long flags;
 
-        WARN_ON(wait_event_timeout(dev_priv->pending_flip_queue,
-                                   !intel_crtc_has_pending_flip(crtc),
-                                   60*HZ) == 0);
+                spin_lock_irqsave(&dev->event_lock, flags);
+                if (intel_crtc->unpin_work) {
+                        WARN_ONCE(1, "Removing stuck page flip\n");
+                        page_flip_completed(intel_crtc);
+                }
+                spin_unlock_irqrestore(&dev->event_lock, flags);
+        }
 
-        mutex_lock(&dev->struct_mutex);
-        intel_finish_fb(crtc->primary->fb);
-        mutex_unlock(&dev->struct_mutex);
+        if (crtc->primary->fb) {
+                mutex_lock(&dev->struct_mutex);
+                intel_finish_fb(crtc->primary->fb);
+                mutex_unlock(&dev->struct_mutex);
+        }
 }
 
 /* Program iCLKIP clock to the desired frequency */
@@ -3911,14 +4038,14 @@ static void intel_crtc_dpms_overlay(struct intel_crtc *intel_crtc, bool enable)
 static void intel_crtc_enable_planes(struct drm_crtc *crtc)
 {
         struct drm_device *dev = crtc->dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
         int pipe = intel_crtc->pipe;
-        int plane = intel_crtc->plane;
+
+        assert_vblank_disabled(crtc);
 
         drm_vblank_on(dev, pipe);
 
-        intel_enable_primary_hw_plane(dev_priv, plane, pipe);
+        intel_enable_primary_hw_plane(crtc->primary, crtc);
         intel_enable_planes(crtc);
         intel_crtc_update_cursor(crtc, true);
         intel_crtc_dpms_overlay(intel_crtc, true);
@@ -3955,7 +4082,7 @@ static void intel_crtc_disable_planes(struct drm_crtc *crtc)
         intel_crtc_dpms_overlay(intel_crtc, false);
         intel_crtc_update_cursor(crtc, false);
         intel_disable_planes(crtc);
-        intel_disable_primary_hw_plane(dev_priv, plane, pipe);
+        intel_disable_primary_hw_plane(crtc->primary, crtc);
 
         /*
          * FIXME: Once we grow proper nuclear flip support out of this we need
@@ -3965,6 +4092,8 @@ static void intel_crtc_disable_planes(struct drm_crtc *crtc)
         intel_frontbuffer_flip(dev, INTEL_FRONTBUFFER_ALL_MASK(pipe));
 
         drm_vblank_off(dev, pipe);
+
+        assert_vblank_disabled(crtc);
 }
 
 static void ironlake_crtc_enable(struct drm_crtc *crtc)
@@ -3974,7 +4103,6 @@ static void ironlake_crtc_enable(struct drm_crtc *crtc)
         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
         struct intel_encoder *encoder;
         int pipe = intel_crtc->pipe;
-        enum plane plane = intel_crtc->plane;
 
         WARN_ON(!crtc->enabled);
 
@@ -3991,18 +4119,11 @@ static void ironlake_crtc_enable(struct drm_crtc *crtc)
 
         if (intel_crtc->config.has_pch_encoder) {
                 intel_cpu_transcoder_set_m_n(intel_crtc,
-                                             &intel_crtc->config.fdi_m_n);
+                                     &intel_crtc->config.fdi_m_n, NULL);
         }
 
         ironlake_set_pipeconf(crtc);
 
-        /* Set up the display plane register */
-        I915_WRITE(DSPCNTR(plane), DISPPLANE_GAMMA_ENABLE);
-        POSTING_READ(DSPCNTR(plane));
-
-        dev_priv->display.update_primary_plane(crtc, crtc->primary->fb,
-                                               crtc->x, crtc->y);
-
         intel_crtc->active = true;
 
         intel_set_cpu_fifo_underrun_reporting(dev, pipe, true);
@@ -4087,7 +4208,6 @@ static void haswell_crtc_enable(struct drm_crtc *crtc)
         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
         struct intel_encoder *encoder;
         int pipe = intel_crtc->pipe;
-        enum plane plane = intel_crtc->plane;
 
         WARN_ON(!crtc->enabled);
 
@@ -4102,22 +4222,20 @@ static void haswell_crtc_enable(struct drm_crtc *crtc)
 
         intel_set_pipe_timings(intel_crtc);
 
+        if (intel_crtc->config.cpu_transcoder != TRANSCODER_EDP) {
+                I915_WRITE(PIPE_MULT(intel_crtc->config.cpu_transcoder),
+                           intel_crtc->config.pixel_multiplier - 1);
+        }
+
         if (intel_crtc->config.has_pch_encoder) {
                 intel_cpu_transcoder_set_m_n(intel_crtc,
-                                             &intel_crtc->config.fdi_m_n);
+                                     &intel_crtc->config.fdi_m_n, NULL);
         }
 
         haswell_set_pipeconf(crtc);
 
         intel_set_pipe_csc(crtc);
 
-        /* Set up the display plane register */
-        I915_WRITE(DSPCNTR(plane), DISPPLANE_GAMMA_ENABLE | DISPPLANE_PIPE_CSC_ENABLE);
-        POSTING_READ(DSPCNTR(plane));
-
-        dev_priv->display.update_primary_plane(crtc, crtc->primary->fb,
-                                               crtc->x, crtc->y);
-
         intel_crtc->active = true;
 
         intel_set_cpu_fifo_underrun_reporting(dev, pipe, true);
@@ -4198,7 +4316,8 @@ static void ironlake_crtc_disable(struct drm_crtc *crtc)
         if (intel_crtc->config.has_pch_encoder)
                 intel_set_pch_fifo_underrun_reporting(dev, pipe, false);
 
-        intel_disable_pipe(dev_priv, pipe);
+        intel_disable_pipe(intel_crtc);
+
         ironlake_pfit_disable(intel_crtc);
 
         for_each_encoder_on_crtc(dev, crtc, encoder)
@@ -4246,7 +4365,6 @@ static void haswell_crtc_disable(struct drm_crtc *crtc)
         struct drm_i915_private *dev_priv = dev->dev_private;
         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
         struct intel_encoder *encoder;
-        int pipe = intel_crtc->pipe;
         enum transcoder cpu_transcoder = intel_crtc->config.cpu_transcoder;
 
         if (!intel_crtc->active)
@@ -4261,7 +4379,7 @@ static void haswell_crtc_disable(struct drm_crtc *crtc)
 
         if (intel_crtc->config.has_pch_encoder)
                 intel_set_pch_fifo_underrun_reporting(dev, TRANSCODER_A, false);
-        intel_disable_pipe(dev_priv, pipe);
+        intel_disable_pipe(intel_crtc);
 
         if (intel_crtc->config.dp_encoder_is_mst)
                 intel_ddi_set_vc_payload_alloc(crtc, false);
@@ -4539,12 +4657,57 @@ static void valleyview_set_cdclk(struct drm_device *dev, int cdclk)
         vlv_update_cdclk(dev);
 }
 
+static void cherryview_set_cdclk(struct drm_device *dev, int cdclk)
+{
+        struct drm_i915_private *dev_priv = dev->dev_private;
+        u32 val, cmd;
+
+        WARN_ON(dev_priv->display.get_display_clock_speed(dev) != dev_priv->vlv_cdclk_freq);
+
+        switch (cdclk) {
+        case 400000:
+                cmd = 3;
+                break;
+        case 333333:
+        case 320000:
+                cmd = 2;
+                break;
+        case 266667:
+                cmd = 1;
+                break;
+        case 200000:
+                cmd = 0;
+                break;
+        default:
+                WARN_ON(1);
+                return;
+        }
+
+        mutex_lock(&dev_priv->rps.hw_lock);
+        val = vlv_punit_read(dev_priv, PUNIT_REG_DSPFREQ);
+        val &= ~DSPFREQGUAR_MASK_CHV;
+        val |= (cmd << DSPFREQGUAR_SHIFT_CHV);
+        vlv_punit_write(dev_priv, PUNIT_REG_DSPFREQ, val);
+        if (wait_for((vlv_punit_read(dev_priv, PUNIT_REG_DSPFREQ) &
+                      DSPFREQSTAT_MASK_CHV) == (cmd << DSPFREQSTAT_SHIFT_CHV),
+                     50)) {
+                DRM_ERROR("timed out waiting for CDclk change\n");
+        }
+        mutex_unlock(&dev_priv->rps.hw_lock);
+
+        vlv_update_cdclk(dev);
+}
+
 static int valleyview_calc_cdclk(struct drm_i915_private *dev_priv,
                                  int max_pixclk)
 {
         int vco = valleyview_get_vco(dev_priv);
         int freq_320 = (vco <<  1) % 320000 != 0 ? 333333 : 320000;
 
+        /* FIXME: Punit isn't quite ready yet */
+        if (IS_CHERRYVIEW(dev_priv->dev))
+                return 400000;
+
         /*
          * Really only a few cases to deal with, as only 4 CDclks are supported:
          *   200MHz
@@ -4607,21 +4770,23 @@ static void valleyview_modeset_global_resources(struct drm_device *dev)
         int max_pixclk = intel_mode_max_pixclk(dev_priv);
         int req_cdclk = valleyview_calc_cdclk(dev_priv, max_pixclk);
 
-        if (req_cdclk != dev_priv->vlv_cdclk_freq)
-                valleyview_set_cdclk(dev, req_cdclk);
+        if (req_cdclk != dev_priv->vlv_cdclk_freq) {
+                if (IS_CHERRYVIEW(dev))
+                        cherryview_set_cdclk(dev, req_cdclk);
+                else
+                        valleyview_set_cdclk(dev, req_cdclk);
+        }
+
         modeset_update_crtc_power_domains(dev);
 }
 
 static void valleyview_crtc_enable(struct drm_crtc *crtc)
 {
         struct drm_device *dev = crtc->dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
         struct intel_encoder *encoder;
         int pipe = intel_crtc->pipe;
-        int plane = intel_crtc->plane;
         bool is_dsi;
-        u32 dspcntr;
 
         WARN_ON(!crtc->enabled);
 
@@ -4630,33 +4795,20 @@ static void valleyview_crtc_enable(struct drm_crtc *crtc)
 
         is_dsi = intel_pipe_has_type(crtc, INTEL_OUTPUT_DSI);
 
-        if (!is_dsi && !IS_CHERRYVIEW(dev))
-                vlv_prepare_pll(intel_crtc);
-
-        /* Set up the display plane register */
-        dspcntr = DISPPLANE_GAMMA_ENABLE;
+        if (!is_dsi) {
+                if (IS_CHERRYVIEW(dev))
+                        chv_prepare_pll(intel_crtc);
+                else
+                        vlv_prepare_pll(intel_crtc);
+        }
 
         if (intel_crtc->config.has_dp_encoder)
                 intel_dp_set_m_n(intel_crtc);
 
         intel_set_pipe_timings(intel_crtc);
 
-        /* pipesrc and dspsize control the size that is scaled from,
-         * which should always be the user's requested size.
-         */
-        I915_WRITE(DSPSIZE(plane),
-                   ((intel_crtc->config.pipe_src_h - 1) << 16) |
-                   (intel_crtc->config.pipe_src_w - 1));
-        I915_WRITE(DSPPOS(plane), 0);
-
         i9xx_set_pipeconf(intel_crtc);
 
-        I915_WRITE(DSPCNTR(plane), dspcntr);
-        POSTING_READ(DSPCNTR(plane));
-
-        dev_priv->display.update_primary_plane(crtc, crtc->primary->fb,
-                                               crtc->x, crtc->y);
-
         intel_crtc->active = true;
 
         intel_set_cpu_fifo_underrun_reporting(dev, pipe, true);
@@ -4704,12 +4856,9 @@ static void i9xx_set_pll_dividers(struct intel_crtc *crtc)
 static void i9xx_crtc_enable(struct drm_crtc *crtc)
 {
         struct drm_device *dev = crtc->dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
         struct intel_encoder *encoder;
         int pipe = intel_crtc->pipe;
-        int plane = intel_crtc->plane;
-        u32 dspcntr;
 
         WARN_ON(!crtc->enabled);
 
@@ -4718,35 +4867,13 @@ static void i9xx_crtc_enable(struct drm_crtc *crtc)
 
         i9xx_set_pll_dividers(intel_crtc);
 
-        /* Set up the display plane register */
-        dspcntr = DISPPLANE_GAMMA_ENABLE;
-
-        if (pipe == 0)
-                dspcntr &= ~DISPPLANE_SEL_PIPE_MASK;
-        else
-                dspcntr |= DISPPLANE_SEL_PIPE_B;
-
         if (intel_crtc->config.has_dp_encoder)
                 intel_dp_set_m_n(intel_crtc);
 
         intel_set_pipe_timings(intel_crtc);
 
-        /* pipesrc and dspsize control the size that is scaled from,
-         * which should always be the user's requested size.
-         */
-        I915_WRITE(DSPSIZE(plane),
-                   ((intel_crtc->config.pipe_src_h - 1) << 16) |
-                   (intel_crtc->config.pipe_src_w - 1));
-        I915_WRITE(DSPPOS(plane), 0);
-
         i9xx_set_pipeconf(intel_crtc);
 
-        I915_WRITE(DSPCNTR(plane), dspcntr);
-        POSTING_READ(DSPCNTR(plane));
-
-        dev_priv->display.update_primary_plane(crtc, crtc->primary->fb,
-                                               crtc->x, crtc->y);
-
         intel_crtc->active = true;
 
         if (!IS_GEN2(dev))
@@ -4842,7 +4969,7 @@ static void i9xx_crtc_disable(struct drm_crtc *crtc)
          */
         intel_wait_for_vblank(dev, pipe);
 
-        intel_disable_pipe(dev_priv, pipe);
+        intel_disable_pipe(intel_crtc);
 
         i9xx_pfit_disable(intel_crtc);
 
@@ -4856,7 +4983,7 @@ static void i9xx_crtc_disable(struct drm_crtc *crtc)
                 else if (IS_VALLEYVIEW(dev))
                         vlv_disable_pll(dev_priv, pipe);
                 else
-                        i9xx_disable_pll(dev_priv, pipe);
+                        i9xx_disable_pll(intel_crtc);
         }
 
         if (!IS_GEN2(dev))
@@ -5275,6 +5402,10 @@ static int valleyview_get_display_clock_speed(struct drm_device *dev)
         u32 val;
         int divider;
 
+        /* FIXME: Punit isn't quite ready yet */
+        if (IS_CHERRYVIEW(dev))
+                return 400000;
+
         mutex_lock(&dev_priv->dpio_lock);
         val = vlv_cck_read(dev_priv, CCK_DISPLAY_CLOCK_CONTROL);
         mutex_unlock(&dev_priv->dpio_lock);
@@ -5519,7 +5650,8 @@ static void intel_pch_transcoder_set_m_n(struct intel_crtc *crtc,
 }
 
 static void intel_cpu_transcoder_set_m_n(struct intel_crtc *crtc,
-                                         struct intel_link_m_n *m_n)
+                                         struct intel_link_m_n *m_n,
+                                         struct intel_link_m_n *m2_n2)
 {
         struct drm_device *dev = crtc->base.dev;
         struct drm_i915_private *dev_priv = dev->dev_private;
@@ -5531,6 +5663,18 @@ static void intel_cpu_transcoder_set_m_n(struct intel_crtc *crtc,
                 I915_WRITE(PIPE_DATA_N1(transcoder), m_n->gmch_n);
                 I915_WRITE(PIPE_LINK_M1(transcoder), m_n->link_m);
                 I915_WRITE(PIPE_LINK_N1(transcoder), m_n->link_n);
+                /* M2_N2 registers to be set only for gen < 8 (M2_N2 available
+                 * for gen < 8) and if DRRS is supported (to make sure the
+                 * registers are not unnecessarily accessed).
+                 */
+                if (m2_n2 && INTEL_INFO(dev)->gen < 8 &&
+                        crtc->config.has_drrs) {
+                        I915_WRITE(PIPE_DATA_M2(transcoder),
+                                        TU_SIZE(m2_n2->tu) | m2_n2->gmch_m);
+                        I915_WRITE(PIPE_DATA_N2(transcoder), m2_n2->gmch_n);
+                        I915_WRITE(PIPE_LINK_M2(transcoder), m2_n2->link_m);
+                        I915_WRITE(PIPE_LINK_N2(transcoder), m2_n2->link_n);
+                }
         } else {
                 I915_WRITE(PIPE_DATA_M_G4X(pipe), TU_SIZE(m_n->tu) | m_n->gmch_m);
                 I915_WRITE(PIPE_DATA_N_G4X(pipe), m_n->gmch_n);
@@ -5539,12 +5683,13 @@ static void intel_cpu_transcoder_set_m_n(struct intel_crtc *crtc,
         }
 }
 
-static void intel_dp_set_m_n(struct intel_crtc *crtc)
+void intel_dp_set_m_n(struct intel_crtc *crtc)
 {
         if (crtc->config.has_pch_encoder)
                 intel_pch_transcoder_set_m_n(crtc, &crtc->config.dp_m_n);
         else
-                intel_cpu_transcoder_set_m_n(crtc, &crtc->config.dp_m_n);
+                intel_cpu_transcoder_set_m_n(crtc, &crtc->config.dp_m_n,
+                                                   &crtc->config.dp_m2_n2);
 }
 
 static void vlv_update_pll(struct intel_crtc *crtc)
@@ -5662,6 +5807,18 @@ static void vlv_prepare_pll(struct intel_crtc *crtc)
 
 static void chv_update_pll(struct intel_crtc *crtc)
 {
+        crtc->config.dpll_hw_state.dpll = DPLL_SSC_REF_CLOCK_CHV |
+                DPLL_REFA_CLK_ENABLE_VLV | DPLL_VGA_MODE_DIS |
+                DPLL_VCO_ENABLE;
+        if (crtc->pipe != PIPE_A)
+                crtc->config.dpll_hw_state.dpll |= DPLL_INTEGRATED_CRI_CLK_VLV;
+
+        crtc->config.dpll_hw_state.dpll_md =
+                (crtc->config.pixel_multiplier - 1) << DPLL_MD_UDI_MULTIPLIER_SHIFT;
+}
+
+static void chv_prepare_pll(struct intel_crtc *crtc)
+{
         struct drm_device *dev = crtc->base.dev;
         struct drm_i915_private *dev_priv = dev->dev_private;
         int pipe = crtc->pipe;
@@ -5671,15 +5828,6 @@ static void chv_update_pll(struct intel_crtc *crtc)
         u32 bestn, bestm1, bestm2, bestp1, bestp2, bestm2_frac;
         int refclk;
 
-        crtc->config.dpll_hw_state.dpll = DPLL_SSC_REF_CLOCK_CHV |
-                DPLL_REFA_CLK_ENABLE_VLV | DPLL_VGA_MODE_DIS |
-                DPLL_VCO_ENABLE;
-        if (pipe != PIPE_A)
-                crtc->config.dpll_hw_state.dpll |= DPLL_INTEGRATED_CRI_CLK_VLV;
-
-        crtc->config.dpll_hw_state.dpll_md =
-                (crtc->config.pixel_multiplier - 1) << DPLL_MD_UDI_MULTIPLIER_SHIFT;
-
         bestn = crtc->config.dpll.n;
         bestm2_frac = crtc->config.dpll.m2 & 0x3fffff;
         bestm1 = crtc->config.dpll.m1;
@@ -5839,7 +5987,7 @@ static void i8xx_update_pll(struct intel_crtc *crtc,
                         dpll |= PLL_P2_DIVIDE_BY_4;
         }
 
-        if (intel_pipe_has_type(&crtc->base, INTEL_OUTPUT_DVO))
+        if (!IS_I830(dev) && intel_pipe_has_type(&crtc->base, INTEL_OUTPUT_DVO))
                 dpll |= DPLL_DVO_2X_MODE;
 
         if (intel_pipe_has_type(&crtc->base, INTEL_OUTPUT_LVDS) &&
@@ -5990,9 +6138,9 @@ static void i9xx_set_pipeconf(struct intel_crtc *intel_crtc)
 
         pipeconf = 0;
 
-        if (dev_priv->quirks & QUIRK_PIPEA_FORCE &&
-            I915_READ(PIPECONF(intel_crtc->pipe)) & PIPECONF_ENABLE)
-                pipeconf |= PIPECONF_ENABLE;
+        if ((intel_crtc->pipe == PIPE_A && dev_priv->quirks & QUIRK_PIPEA_FORCE) ||
+            (intel_crtc->pipe == PIPE_B && dev_priv->quirks & QUIRK_PIPEB_FORCE))
+                pipeconf |= I915_READ(PIPECONF(intel_crtc->pipe)) & PIPECONF_ENABLE;
 
         if (intel_crtc->config.double_wide)
                 pipeconf |= PIPECONF_DOUBLE_WIDE;
@@ -6235,7 +6383,7 @@ static void i9xx_get_plane_config(struct intel_crtc *crtc,
         crtc->base.primary->fb->height = ((val >> 0) & 0xfff) + 1;
 
         val = I915_READ(DSPSTRIDE(pipe));
-        crtc->base.primary->fb->pitches[0] = val & 0xffffff80;
+        crtc->base.primary->fb->pitches[0] = val & 0xffffffc0;
 
         aligned_height = intel_align_height(dev, crtc->base.primary->fb->height,
                                             plane_config->tiled);
@@ -6345,6 +6493,14 @@ static bool i9xx_get_pipe_config(struct intel_crtc *crtc,
         }
         pipe_config->dpll_hw_state.dpll = I915_READ(DPLL(crtc->pipe));
         if (!IS_VALLEYVIEW(dev)) {
+                /*
+                 * DPLL_DVO_2X_MODE must be enabled for both DPLLs
+                 * on 830. Filter it out here so that we don't
+                 * report errors due to that.
+                 */
+                if (IS_I830(dev))
+                        pipe_config->dpll_hw_state.dpll &= ~DPLL_DVO_2X_MODE;
+
                 pipe_config->dpll_hw_state.fp0 = I915_READ(FP0(crtc->pipe));
                 pipe_config->dpll_hw_state.fp1 = I915_READ(FP1(crtc->pipe));
         } else {
@@ -6367,7 +6523,6 @@ static bool i9xx_get_pipe_config(struct intel_crtc *crtc,
 static void ironlake_init_pch_refclk(struct drm_device *dev)
 {
         struct drm_i915_private *dev_priv = dev->dev_private;
-        struct drm_mode_config *mode_config = &dev->mode_config;
         struct intel_encoder *encoder;
         u32 val, final;
         bool has_lvds = false;
@@ -6377,8 +6532,7 @@ static void ironlake_init_pch_refclk(struct drm_device *dev)
         bool can_ssc = false;
 
         /* We need to take the global config into account */
-        list_for_each_entry(encoder, &mode_config->encoder_list,
-                            base.head) {
+        for_each_intel_encoder(dev, encoder) {
                 switch (encoder->type) {
                 case INTEL_OUTPUT_LVDS:
                         has_panel = true;
@@ -6685,11 +6839,10 @@ static void lpt_disable_clkout_dp(struct drm_device *dev)
 
 static void lpt_init_pch_refclk(struct drm_device *dev)
 {
-        struct drm_mode_config *mode_config = &dev->mode_config;
         struct intel_encoder *encoder;
         bool has_vga = false;
 
-        list_for_each_entry(encoder, &mode_config->encoder_list, base.head) {
+        for_each_intel_encoder(dev, encoder) {
                 switch (encoder->type) {
                 case INTEL_OUTPUT_ANALOG:
                         has_vga = true;
@@ -7145,7 +7298,8 @@ static void intel_pch_transcoder_get_m_n(struct intel_crtc *crtc,
 
 static void intel_cpu_transcoder_get_m_n(struct intel_crtc *crtc,
                                          enum transcoder transcoder,
-                                         struct intel_link_m_n *m_n)
+                                         struct intel_link_m_n *m_n,
+                                         struct intel_link_m_n *m2_n2)
 {
         struct drm_device *dev = crtc->base.dev;
         struct drm_i915_private *dev_priv = dev->dev_private;
@@ -7159,6 +7313,20 @@ static void intel_cpu_transcoder_get_m_n(struct intel_crtc *crtc,
                 m_n->gmch_n = I915_READ(PIPE_DATA_N1(transcoder));
                 m_n->tu = ((I915_READ(PIPE_DATA_M1(transcoder))
                             & TU_SIZE_MASK) >> TU_SIZE_SHIFT) + 1;
+                /* Read M2_N2 registers only for gen < 8 (M2_N2 available for
+                 * gen < 8) and if DRRS is supported (to make sure the
+                 * registers are not unnecessarily read).
+                 */
+                if (m2_n2 && INTEL_INFO(dev)->gen < 8 &&
+                        crtc->config.has_drrs) {
+                        m2_n2->link_m = I915_READ(PIPE_LINK_M2(transcoder));
+                        m2_n2->link_n = I915_READ(PIPE_LINK_N2(transcoder));
+                        m2_n2->gmch_m = I915_READ(PIPE_DATA_M2(transcoder))
+                                        & ~TU_SIZE_MASK;
+                        m2_n2->gmch_n = I915_READ(PIPE_DATA_N2(transcoder));
+                        m2_n2->tu = ((I915_READ(PIPE_DATA_M2(transcoder))
+                                        & TU_SIZE_MASK) >> TU_SIZE_SHIFT) + 1;
+                }
         } else {
                 m_n->link_m = I915_READ(PIPE_LINK_M_G4X(pipe));
                 m_n->link_n = I915_READ(PIPE_LINK_N_G4X(pipe));
@@ -7177,14 +7345,15 @@ void intel_dp_get_m_n(struct intel_crtc *crtc,
                 intel_pch_transcoder_get_m_n(crtc, &pipe_config->dp_m_n);
         else
                 intel_cpu_transcoder_get_m_n(crtc, pipe_config->cpu_transcoder,
-                                             &pipe_config->dp_m_n);
+                                             &pipe_config->dp_m_n,
+                                             &pipe_config->dp_m2_n2);
 }
 
 static void ironlake_get_fdi_m_n_config(struct intel_crtc *crtc,
                                         struct intel_crtc_config *pipe_config)
 {
         intel_cpu_transcoder_get_m_n(crtc, pipe_config->cpu_transcoder,
-                                     &pipe_config->fdi_m_n);
+                                     &pipe_config->fdi_m_n, NULL);
 }
 
 static void ironlake_get_pfit_config(struct intel_crtc *crtc,
@@ -7255,7 +7424,7 @@ static void ironlake_get_plane_config(struct intel_crtc *crtc,
         crtc->base.primary->fb->height = ((val >> 0) & 0xfff) + 1;
 
         val = I915_READ(DSPSTRIDE(pipe));
-        crtc->base.primary->fb->pitches[0] = val & 0xffffff80;
+        crtc->base.primary->fb->pitches[0] = val & 0xffffffc0;
 
         aligned_height = intel_align_height(dev, crtc->base.primary->fb->height,
                                             plane_config->tiled);
@@ -7615,6 +7784,22 @@ static int haswell_crtc_mode_set(struct drm_crtc *crtc,
         return 0;
 }
 
+static void haswell_get_ddi_pll(struct drm_i915_private *dev_priv,
+                                enum port port,
+                                struct intel_crtc_config *pipe_config)
+{
+        pipe_config->ddi_pll_sel = I915_READ(PORT_CLK_SEL(port));
+
+        switch (pipe_config->ddi_pll_sel) {
+        case PORT_CLK_SEL_WRPLL1:
+                pipe_config->shared_dpll = DPLL_ID_WRPLL1;
+                break;
+        case PORT_CLK_SEL_WRPLL2:
+                pipe_config->shared_dpll = DPLL_ID_WRPLL2;
+                break;
+        }
+}
+
 static void haswell_get_ddi_port_state(struct intel_crtc *crtc,
                                        struct intel_crtc_config *pipe_config)
 {
@@ -7628,16 +7813,7 @@ static void haswell_get_ddi_port_state(struct intel_crtc *crtc,
 
         port = (tmp & TRANS_DDI_PORT_MASK) >> TRANS_DDI_PORT_SHIFT;
 
-        pipe_config->ddi_pll_sel = I915_READ(PORT_CLK_SEL(port));
-
-        switch (pipe_config->ddi_pll_sel) {
-        case PORT_CLK_SEL_WRPLL1:
-                pipe_config->shared_dpll = DPLL_ID_WRPLL1;
-                break;
-        case PORT_CLK_SEL_WRPLL2:
-                pipe_config->shared_dpll = DPLL_ID_WRPLL2;
-                break;
-        }
+        haswell_get_ddi_pll(dev_priv, port, pipe_config);
 
         if (pipe_config->shared_dpll >= 0) {
                 pll = &dev_priv->shared_dplls[pipe_config->shared_dpll];
@@ -7719,7 +7895,12 @@ static bool haswell_get_pipe_config(struct intel_crtc *crtc,
                 pipe_config->ips_enabled = hsw_crtc_supports_ips(crtc) &&
                         (I915_READ(IPS_CTL) & IPS_ENABLE);
 
-        pipe_config->pixel_multiplier = 1;
+        if (pipe_config->cpu_transcoder != TRANSCODER_EDP) {
+                pipe_config->pixel_multiplier =
+                        I915_READ(PIPE_MULT(pipe_config->cpu_transcoder)) + 1;
+        } else {
+                pipe_config->pixel_multiplier = 1;
+        }
 
         return true;
 }
@@ -8037,74 +8218,62 @@ static void i845_update_cursor(struct drm_crtc *crtc, u32 base)
         struct drm_device *dev = crtc->dev;
         struct drm_i915_private *dev_priv = dev->dev_private;
         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
-        uint32_t cntl;
+        uint32_t cntl = 0, size = 0;
 
-        if (base != intel_crtc->cursor_base) {
-                /* On these chipsets we can only modify the base whilst
-                 * the cursor is disabled.
-                 */
-                if (intel_crtc->cursor_cntl) {
-                        I915_WRITE(_CURACNTR, 0);
-                        POSTING_READ(_CURACNTR);
-                        intel_crtc->cursor_cntl = 0;
+        if (base) {
+                unsigned int width = intel_crtc->cursor_width;
+                unsigned int height = intel_crtc->cursor_height;
+                unsigned int stride = roundup_pow_of_two(width) * 4;
+
+                switch (stride) {
+                default:
+                        WARN_ONCE(1, "Invalid cursor width/stride, width=%u, stride=%u\n",
+                                  width, stride);
+                        stride = 256;
+                        /* fallthrough */
+                case 256:
+                case 512:
+                case 1024:
+                case 2048:
+                        break;
                 }
 
-                I915_WRITE(_CURABASE, base);
-                POSTING_READ(_CURABASE);
+                cntl |= CURSOR_ENABLE |
+                        CURSOR_GAMMA_ENABLE |
+                        CURSOR_FORMAT_ARGB |
+                        CURSOR_STRIDE(stride);
+
+                size = (height << 12) | width;
         }
 
-        /* XXX width must be 64, stride 256 => 0x00 << 28 */
-        cntl = 0;
-        if (base)
-                cntl = (CURSOR_ENABLE |
-                        CURSOR_GAMMA_ENABLE |
-                        CURSOR_FORMAT_ARGB);
-        if (intel_crtc->cursor_cntl != cntl) {
-                I915_WRITE(_CURACNTR, cntl);
+        if (intel_crtc->cursor_cntl != 0 &&
+            (intel_crtc->cursor_base != base ||
+             intel_crtc->cursor_size != size ||
+             intel_crtc->cursor_cntl != cntl)) {
+                /* On these chipsets we can only modify the base/size/stride
+                 * whilst the cursor is disabled.
+                 */
+                I915_WRITE(_CURACNTR, 0);
                 POSTING_READ(_CURACNTR);
-                intel_crtc->cursor_cntl = cntl;
+                intel_crtc->cursor_cntl = 0;
         }
-}
 
-static void i9xx_update_cursor(struct drm_crtc *crtc, u32 base)
-{
-        struct drm_device *dev = crtc->dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
-        struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
-        int pipe = intel_crtc->pipe;
-        uint32_t cntl;
+        if (intel_crtc->cursor_base != base)
+                I915_WRITE(_CURABASE, base);
 
-        cntl = 0;
-        if (base) {
-                cntl = MCURSOR_GAMMA_ENABLE;
-                switch (intel_crtc->cursor_width) {
-                        case 64:
-                                cntl |= CURSOR_MODE_64_ARGB_AX;
-                                break;
-                        case 128:
-                                cntl |= CURSOR_MODE_128_ARGB_AX;
-                                break;
-                        case 256:
-                                cntl |= CURSOR_MODE_256_ARGB_AX;
-                                break;
-                        default:
-                                WARN_ON(1);
-                                return;
-                }
-                cntl |= pipe << 28; /* Connect to correct pipe */
+        if (intel_crtc->cursor_size != size) {
+                I915_WRITE(CURSIZE, size);
+                intel_crtc->cursor_size = size;
         }
+
         if (intel_crtc->cursor_cntl != cntl) {
-                I915_WRITE(CURCNTR(pipe), cntl);
-                POSTING_READ(CURCNTR(pipe));
+                I915_WRITE(_CURACNTR, cntl);
+                POSTING_READ(_CURACNTR);
                 intel_crtc->cursor_cntl = cntl;
         }
-
-        /* and commit changes on next vblank */
-        I915_WRITE(CURBASE(pipe), base);
-        POSTING_READ(CURBASE(pipe));
 }
 
-static void ivb_update_cursor(struct drm_crtc *crtc, u32 base)
+static void i9xx_update_cursor(struct drm_crtc *crtc, u32 base)
 {
         struct drm_device *dev = crtc->dev;
         struct drm_i915_private *dev_priv = dev->dev_private;
@@ -8129,6 +8298,7 @@ static void ivb_update_cursor(struct drm_crtc *crtc, u32 base)
                                 WARN_ON(1);
                                 return;
                 }
+                cntl |= pipe << 28; /* Connect to correct pipe */
         }
         if (IS_HASWELL(dev) || IS_BROADWELL(dev))
                 cntl |= CURSOR_PIPE_CSC_ENABLE;
@@ -8188,15 +8358,50 @@ static void intel_crtc_update_cursor(struct drm_crtc *crtc,
 
         I915_WRITE(CURPOS(pipe), pos);
 
-        if (IS_IVYBRIDGE(dev) || IS_HASWELL(dev) || IS_BROADWELL(dev))
-                ivb_update_cursor(crtc, base);
-        else if (IS_845G(dev) || IS_I865G(dev))
+        if (IS_845G(dev) || IS_I865G(dev))
                 i845_update_cursor(crtc, base);
         else
                 i9xx_update_cursor(crtc, base);
         intel_crtc->cursor_base = base;
 }
 
+static bool cursor_size_ok(struct drm_device *dev,
+                           uint32_t width, uint32_t height)
+{
+        if (width == 0 || height == 0)
+                return false;
+
+        /*
+         * 845g/865g are special in that they are only limited by
+         * the width of their cursors, the height is arbitrary up to
+         * the precision of the register. Everything else requires
+         * square cursors, limited to a few power-of-two sizes.
+         */
+        if (IS_845G(dev) || IS_I865G(dev)) {
+                if ((width & 63) != 0)
+                        return false;
+
+                if (width > (IS_845G(dev) ? 64 : 512))
+                        return false;
+
+                if (height > 1023)
+                        return false;
+        } else {
+                switch (width | height) {
+                case 256:
+                case 128:
+                        if (IS_GEN2(dev))
+                                return false;
+                case 64:
+                        break;
+                default:
+                        return false;
+                }
+        }
+
+        return true;
+}
+
 /*
  * intel_crtc_cursor_set_obj - Set cursor to specified GEM object
  *
@@ -8212,7 +8417,7 @@ static int intel_crtc_cursor_set_obj(struct drm_crtc *crtc,
         struct drm_i915_private *dev_priv = dev->dev_private;
         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
         enum pipe pipe = intel_crtc->pipe;
-        unsigned old_width;
+        unsigned old_width, stride;
         uint32_t addr;
         int ret;
 
@@ -8220,20 +8425,18 @@ static int intel_crtc_cursor_set_obj(struct drm_crtc *crtc,
         if (!obj) {
                 DRM_DEBUG_KMS("cursor off\n");
                 addr = 0;
-                obj = NULL;
                 mutex_lock(&dev->struct_mutex);
                 goto finish;
         }
 
         /* Check for which cursor types we support */
-        if (!((width == 64 && height == 64) ||
-                        (width == 128 && height == 128 && !IS_GEN2(dev)) ||
-                        (width == 256 && height == 256 && !IS_GEN2(dev)))) {
+        if (!cursor_size_ok(dev, width, height)) {
                 DRM_DEBUG("Cursor dimension not supported\n");
                 return -EINVAL;
         }
 
-        if (obj->base.size < width * height * 4) {
+        stride = roundup_pow_of_two(width) * 4;
+        if (obj->base.size < stride * height) {
                 DRM_DEBUG_KMS("buffer is too small\n");
                 ret = -ENOMEM;
                 goto fail;
@@ -8295,9 +8498,6 @@ static int intel_crtc_cursor_set_obj(struct drm_crtc *crtc,
                 addr = obj->phys_handle->busaddr;
         }
 
-        if (IS_GEN2(dev))
-                I915_WRITE(CURSIZE, (height << 12) | width);
-
  finish:
         if (intel_crtc->cursor_bo) {
                 if (!INTEL_INFO(dev)->cursor_needs_physical)
@@ -8944,12 +9144,13 @@ static void intel_mark_fb_busy(struct drm_device *dev,
                                unsigned frontbuffer_bits,
                                struct intel_engine_cs *ring)
 {
+        struct drm_i915_private *dev_priv = dev->dev_private;
         enum pipe pipe;
 
         if (!i915.powersave)
                 return;
 
-        for_each_pipe(pipe) {
+        for_each_pipe(dev_priv, pipe) {
                 if (!(frontbuffer_bits & INTEL_FRONTBUFFER_ALL_MASK(pipe)))
                         continue;
 
@@ -9019,6 +9220,14 @@ void intel_frontbuffer_flush(struct drm_device *dev,
         intel_mark_fb_busy(dev, frontbuffer_bits, NULL);
 
         intel_edp_psr_flush(dev, frontbuffer_bits);
+
+        /*
+         * FIXME: Unconditional fbc flushing here is a rather gross hack and
+         * needs to be reworked into a proper frontbuffer tracking scheme like
+         * psr employs.
+         */
+        if (IS_BROADWELL(dev))
+                gen8_fbc_sw_flush(dev, FBC_REND_CACHE_CLEAN);
 }
 
 /**
@@ -9151,7 +9360,6 @@ static void intel_unpin_work_fn(struct work_struct *__work)
 static void do_intel_finish_page_flip(struct drm_device *dev,
                                       struct drm_crtc *crtc)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
         struct intel_unpin_work *work;
         unsigned long flags;
@@ -9171,23 +9379,9 @@ static void do_intel_finish_page_flip(struct drm_device *dev,
                 return;
         }
 
-        /* and that the unpin work is consistent wrt ->pending. */
-        smp_rmb();
-
-        intel_crtc->unpin_work = NULL;
-
-        if (work->event)
-                drm_send_vblank_event(dev, intel_crtc->pipe, work->event);
-
-        drm_crtc_vblank_put(crtc);
+        page_flip_completed(intel_crtc);
 
         spin_unlock_irqrestore(&dev->event_lock, flags);
-
-        wake_up_all(&dev_priv->pending_flip_queue);
-
-        queue_work(dev_priv->wq, &work->work);
-
-        trace_i915_flip_complete(intel_crtc->plane, work->pending_flip_obj);
 }
 
 void intel_finish_page_flip(struct drm_device *dev, int pipe)
@@ -9532,6 +9726,8 @@ static bool use_mmio_flip(struct intel_engine_cs *ring,
                 return false;
         else if (i915.use_mmio_flip > 0)
                 return true;
+        else if (i915.enable_execlists)
+                return true;
         else
                 return ring != obj->ring;
 }
@@ -9665,6 +9861,65 @@ static int intel_default_queue_flip(struct drm_device *dev,
         return -ENODEV;
 }
 
+static bool __intel_pageflip_stall_check(struct drm_device *dev,
+                                         struct drm_crtc *crtc)
+{
+        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
+        struct intel_unpin_work *work = intel_crtc->unpin_work;
+        u32 addr;
+
+        if (atomic_read(&work->pending) >= INTEL_FLIP_COMPLETE)
+                return true;
+
+        if (!work->enable_stall_check)
+                return false;
+
+        if (work->flip_ready_vblank == 0) {
+                if (work->flip_queued_ring &&
+                    !i915_seqno_passed(work->flip_queued_ring->get_seqno(work->flip_queued_ring, true),
+                                       work->flip_queued_seqno))
+                        return false;
+
+                work->flip_ready_vblank = drm_vblank_count(dev, intel_crtc->pipe);
+        }
+
+        if (drm_vblank_count(dev, intel_crtc->pipe) - work->flip_ready_vblank < 3)
+                return false;
+
+        /* Potential stall - if we see that the flip has happened,
+         * assume a missed interrupt. */
+        if (INTEL_INFO(dev)->gen >= 4)
+                addr = I915_HI_DISPBASE(I915_READ(DSPSURF(intel_crtc->plane)));
+        else
+                addr = I915_READ(DSPADDR(intel_crtc->plane));
+
+        /* There is a potential issue here with a false positive after a flip
+         * to the same address. We could address this by checking for a
+         * non-incrementing frame counter.
+         */
+        return addr == work->gtt_offset;
+}
+
+void intel_check_page_flip(struct drm_device *dev, int pipe)
+{
+        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_crtc *crtc = dev_priv->pipe_to_crtc_mapping[pipe];
+        struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
+        unsigned long flags;
+
+        if (crtc == NULL)
+                return;
+
+        spin_lock_irqsave(&dev->event_lock, flags);
+        if (intel_crtc->unpin_work && __intel_pageflip_stall_check(dev, crtc)) {
+                WARN_ONCE(1, "Kicking stuck page flip: queued at %d, now %d\n",
+                         intel_crtc->unpin_work->flip_queued_vblank, drm_vblank_count(dev, pipe));
+                page_flip_completed(intel_crtc);
+        }
+        spin_unlock_irqrestore(&dev->event_lock, flags);
+}
+
 static int intel_crtc_page_flip(struct drm_crtc *crtc,
                                 struct drm_framebuffer *fb,
                                 struct drm_pending_vblank_event *event,
@@ -9721,12 +9976,20 @@ static int intel_crtc_page_flip(struct drm_crtc *crtc,
         /* We borrow the event spin lock for protecting unpin_work */
         spin_lock_irqsave(&dev->event_lock, flags);
         if (intel_crtc->unpin_work) {
-                spin_unlock_irqrestore(&dev->event_lock, flags);
-                kfree(work);
-                drm_crtc_vblank_put(crtc);
+                /* Before declaring the flip queue wedged, check if
+                 * the hardware completed the operation behind our backs.
+                 */
+                if (__intel_pageflip_stall_check(dev, crtc)) {
+                        DRM_DEBUG_DRIVER("flip queue: previous flip completed, continuing\n");
+                        page_flip_completed(intel_crtc);
+                } else {
+                        DRM_DEBUG_DRIVER("flip queue: crtc already busy\n");
+                        spin_unlock_irqrestore(&dev->event_lock, flags);
 
-                DRM_DEBUG_DRIVER("flip queue: crtc already busy\n");
-                return -EBUSY;
+                        drm_crtc_vblank_put(crtc);
+                        kfree(work);
+                        return -EBUSY;
+                }
         }
         intel_crtc->unpin_work = work;
         spin_unlock_irqrestore(&dev->event_lock, flags);
@@ -9746,8 +10009,6 @@ static int intel_crtc_page_flip(struct drm_crtc *crtc,
 
         work->pending_flip_obj = obj;
 
-        work->enable_stall_check = true;
-
         atomic_inc(&intel_crtc->unpin_work_count);
         intel_crtc->reset_counter = atomic_read(&dev_priv->gpu_error.reset_counter);
 
@@ -9776,14 +10037,26 @@ static int intel_crtc_page_flip(struct drm_crtc *crtc,
         work->gtt_offset =
                 i915_gem_obj_ggtt_offset(obj) + intel_crtc->dspaddr_offset;
 
-        if (use_mmio_flip(ring, obj))
+        if (use_mmio_flip(ring, obj)) {
                 ret = intel_queue_mmio_flip(dev, crtc, fb, obj, ring,
                                             page_flip_flags);
-        else
+                if (ret)
+                        goto cleanup_unpin;
+
+                work->flip_queued_seqno = obj->last_write_seqno;
+                work->flip_queued_ring = obj->ring;
+        } else {
                 ret = dev_priv->display.queue_flip(dev, crtc, fb, obj, ring,
-                                page_flip_flags);
-        if (ret)
-                goto cleanup_unpin;
+                                                   page_flip_flags);
+                if (ret)
+                        goto cleanup_unpin;
+
+                work->flip_queued_seqno = intel_ring_get_seqno(ring);
+                work->flip_queued_ring = ring;
+        }
+
+        work->flip_queued_vblank = drm_vblank_count(dev, intel_crtc->pipe);
+        work->enable_stall_check = true;
 
         i915_gem_track_fb(work->old_fb_obj, obj,
                           INTEL_FRONTBUFFER_PRIMARY(pipe));
@@ -9818,8 +10091,11 @@ free_work:
 out_hang:
                 intel_crtc_wait_for_pending_flips(crtc);
                 ret = intel_pipe_set_base(crtc, crtc->x, crtc->y, fb);
-                if (ret == 0 && event)
+                if (ret == 0 && event) {
+                        spin_lock_irqsave(&dev->event_lock, flags);
                         drm_send_vblank_event(dev, pipe, event);
+                        spin_unlock_irqrestore(&dev->event_lock, flags);
+                }
         }
         return ret;
 }
@@ -9847,8 +10123,7 @@ static void intel_modeset_update_staged_output_state(struct drm_device *dev)
                         to_intel_encoder(connector->base.encoder);
         }
 
-        list_for_each_entry(encoder, &dev->mode_config.encoder_list,
-                            base.head) {
+        for_each_intel_encoder(dev, encoder) {
                 encoder->new_crtc =
                         to_intel_crtc(encoder->base.crtc);
         }
@@ -9879,8 +10154,7 @@ static void intel_modeset_commit_output_state(struct drm_device *dev)
                 connector->base.encoder = &connector->new_encoder->base;
         }
 
-        list_for_each_entry(encoder, &dev->mode_config.encoder_list,
-                            base.head) {
+        for_each_intel_encoder(dev, encoder) {
                 encoder->base.crtc = &encoder->new_crtc->base;
         }
 
@@ -10007,6 +10281,15 @@ static void intel_dump_pipe_config(struct intel_crtc *crtc,
                       pipe_config->dp_m_n.gmch_m, pipe_config->dp_m_n.gmch_n,
                       pipe_config->dp_m_n.link_m, pipe_config->dp_m_n.link_n,
                       pipe_config->dp_m_n.tu);
+
+        DRM_DEBUG_KMS("dp: %i, gmch_m2: %u, gmch_n2: %u, link_m2: %u, link_n2: %u, tu2: %u\n",
+                      pipe_config->has_dp_encoder,
+                      pipe_config->dp_m2_n2.gmch_m,
+                      pipe_config->dp_m2_n2.gmch_n,
+                      pipe_config->dp_m2_n2.link_m,
+                      pipe_config->dp_m2_n2.link_n,
+                      pipe_config->dp_m2_n2.tu);
+
         DRM_DEBUG_KMS("requested mode:\n");
         drm_mode_debug_printmodeline(&pipe_config->requested_mode);
         DRM_DEBUG_KMS("adjusted mode:\n");
@@ -10041,8 +10324,7 @@ static bool check_single_encoder_cloning(struct intel_crtc *crtc,
         struct drm_device *dev = crtc->base.dev;
         struct intel_encoder *source_encoder;
 
-        list_for_each_entry(source_encoder,
-                            &dev->mode_config.encoder_list, base.head) {
+        for_each_intel_encoder(dev, source_encoder) {
                 if (source_encoder->new_crtc != crtc)
                         continue;
 
@@ -10058,8 +10340,7 @@ static bool check_encoder_cloning(struct intel_crtc *crtc)
         struct drm_device *dev = crtc->base.dev;
         struct intel_encoder *encoder;
 
-        list_for_each_entry(encoder,
-                            &dev->mode_config.encoder_list, base.head) {
+        for_each_intel_encoder(dev, encoder) {
                 if (encoder->new_crtc != crtc)
                         continue;
 
@@ -10143,8 +10424,7 @@ encoder_retry:
          * adjust it according to limitations or connector properties, and also
          * a chance to reject the mode entirely.
          */
-        list_for_each_entry(encoder, &dev->mode_config.encoder_list,
-                            base.head) {
+        for_each_intel_encoder(dev, encoder) {
 
                 if (&encoder->new_crtc->base != crtc)
                         continue;
@@ -10222,8 +10502,7 @@ intel_modeset_affected_pipes(struct drm_crtc *crtc, unsigned *modeset_pipes,
                                 1 << connector->new_encoder->new_crtc->pipe;
         }
 
-        list_for_each_entry(encoder, &dev->mode_config.encoder_list,
-                            base.head) {
+        for_each_intel_encoder(dev, encoder) {
                 if (encoder->base.crtc == &encoder->new_crtc->base)
                         continue;
 
@@ -10297,8 +10576,7 @@ intel_modeset_update_state(struct drm_device *dev, unsigned prepare_pipes)
         struct intel_crtc *intel_crtc;
         struct drm_connector *connector;
 
-        list_for_each_entry(intel_encoder, &dev->mode_config.encoder_list,
-                            base.head) {
+        for_each_intel_encoder(dev, intel_encoder) {
                 if (!intel_encoder->base.crtc)
                         continue;
 
@@ -10387,6 +10665,22 @@ intel_pipe_config_compare(struct drm_device *dev,
                 return false; \
         }
 
+/* This is required for BDW+ where there is only one set of registers for
+ * switching between high and low RR.
+ * This macro can be used whenever a comparison has to be made between one
+ * hw state and multiple sw state variables.
+ */
+#define PIPE_CONF_CHECK_I_ALT(name, alt_name) \
+        if ((current_config->name != pipe_config->name) && \
+                (current_config->alt_name != pipe_config->name)) { \
+                        DRM_ERROR("mismatch in " #name " " \
+                                  "(expected %i or %i, found %i)\n", \
+                                  current_config->name, \
+                                  current_config->alt_name, \
+                                  pipe_config->name); \
+                        return false; \
+        }
+
 #define PIPE_CONF_CHECK_FLAGS(name, mask)       \
         if ((current_config->name ^ pipe_config->name) & (mask)) { \
                 DRM_ERROR("mismatch in " #name "(" #mask ") "      \
@@ -10419,11 +10713,28 @@ intel_pipe_config_compare(struct drm_device *dev,
         PIPE_CONF_CHECK_I(fdi_m_n.tu);
 
         PIPE_CONF_CHECK_I(has_dp_encoder);
-        PIPE_CONF_CHECK_I(dp_m_n.gmch_m);
-        PIPE_CONF_CHECK_I(dp_m_n.gmch_n);
-        PIPE_CONF_CHECK_I(dp_m_n.link_m);
-        PIPE_CONF_CHECK_I(dp_m_n.link_n);
-        PIPE_CONF_CHECK_I(dp_m_n.tu);
+
+        if (INTEL_INFO(dev)->gen < 8) {
+                PIPE_CONF_CHECK_I(dp_m_n.gmch_m);
+                PIPE_CONF_CHECK_I(dp_m_n.gmch_n);
+                PIPE_CONF_CHECK_I(dp_m_n.link_m);
+                PIPE_CONF_CHECK_I(dp_m_n.link_n);
+                PIPE_CONF_CHECK_I(dp_m_n.tu);
+
+                if (current_config->has_drrs) {
+                        PIPE_CONF_CHECK_I(dp_m2_n2.gmch_m);
+                        PIPE_CONF_CHECK_I(dp_m2_n2.gmch_n);
+                        PIPE_CONF_CHECK_I(dp_m2_n2.link_m);
+                        PIPE_CONF_CHECK_I(dp_m2_n2.link_n);
+                        PIPE_CONF_CHECK_I(dp_m2_n2.tu);
+                }
+        } else {
+                PIPE_CONF_CHECK_I_ALT(dp_m_n.gmch_m, dp_m2_n2.gmch_m);
+                PIPE_CONF_CHECK_I_ALT(dp_m_n.gmch_n, dp_m2_n2.gmch_n);
+                PIPE_CONF_CHECK_I_ALT(dp_m_n.link_m, dp_m2_n2.link_m);
+                PIPE_CONF_CHECK_I_ALT(dp_m_n.link_n, dp_m2_n2.link_n);
+                PIPE_CONF_CHECK_I_ALT(dp_m_n.tu, dp_m2_n2.tu);
+        }
 
         PIPE_CONF_CHECK_I(adjusted_mode.crtc_hdisplay);
         PIPE_CONF_CHECK_I(adjusted_mode.crtc_htotal);
@@ -10509,6 +10820,7 @@ intel_pipe_config_compare(struct drm_device *dev,
 
 #undef PIPE_CONF_CHECK_X
 #undef PIPE_CONF_CHECK_I
+#undef PIPE_CONF_CHECK_I_ALT
 #undef PIPE_CONF_CHECK_FLAGS
 #undef PIPE_CONF_CHECK_CLOCK_FUZZY
 #undef PIPE_CONF_QUIRK
@@ -10538,8 +10850,7 @@ check_encoder_state(struct drm_device *dev)
         struct intel_encoder *encoder;
         struct intel_connector *connector;
 
-        list_for_each_entry(encoder, &dev->mode_config.encoder_list,
-                            base.head) {
+        for_each_intel_encoder(dev, encoder) {
                 bool enabled = false;
                 bool active = false;
                 enum pipe pipe, tracked_pipe;
@@ -10618,8 +10929,7 @@ check_crtc_state(struct drm_device *dev)
                 WARN(crtc->active && !crtc->base.enabled,
                      "active crtc, but not enabled in sw tracking\n");
 
-                list_for_each_entry(encoder, &dev->mode_config.encoder_list,
-                                    base.head) {
+                for_each_intel_encoder(dev, encoder) {
                         if (encoder->base.crtc != &crtc->base)
                                 continue;
                         enabled = true;
@@ -10637,12 +10947,12 @@ check_crtc_state(struct drm_device *dev)
                 active = dev_priv->display.get_pipe_config(crtc,
                                                            &pipe_config);
 
-                /* hw state is inconsistent with the pipe A quirk */
-                if (crtc->pipe == PIPE_A && dev_priv->quirks & QUIRK_PIPEA_FORCE)
+                /* hw state is inconsistent with the pipe quirk */
+                if ((crtc->pipe == PIPE_A && dev_priv->quirks & QUIRK_PIPEA_FORCE) ||
+                    (crtc->pipe == PIPE_B && dev_priv->quirks & QUIRK_PIPEB_FORCE))
                         active = crtc->active;
 
-                list_for_each_entry(encoder, &dev->mode_config.encoder_list,
-                                    base.head) {
+                for_each_intel_encoder(dev, encoder) {
                         enum pipe pipe;
                         if (encoder->base.crtc != &crtc->base)
                                 continue;
@@ -11010,7 +11320,7 @@ static void intel_set_config_restore_state(struct drm_device *dev,
         }
 
         count = 0;
-        list_for_each_entry(encoder, &dev->mode_config.encoder_list, base.head) {
+        for_each_intel_encoder(dev, encoder) {
                 encoder->new_crtc =
                         to_intel_crtc(config->save_encoder_crtcs[count++]);
         }
@@ -11169,8 +11479,7 @@ intel_modeset_stage_output_state(struct drm_device *dev,
         }
 
         /* Check for any encoders that needs to be disabled. */
-        list_for_each_entry(encoder, &dev->mode_config.encoder_list,
-                            base.head) {
+        for_each_intel_encoder(dev, encoder) {
                 int num_connectors = 0;
                 list_for_each_entry(connector,
                                     &dev->mode_config.connector_list,
@@ -11203,9 +11512,7 @@ intel_modeset_stage_output_state(struct drm_device *dev,
         for_each_intel_crtc(dev, crtc) {
                 crtc->new_enabled = false;
 
-                list_for_each_entry(encoder,
-                                    &dev->mode_config.encoder_list,
-                                    base.head) {
+                for_each_intel_encoder(dev, encoder) {
                         if (encoder->new_crtc == crtc) {
                                 crtc->new_enabled = true;
                                 break;
@@ -11242,7 +11549,7 @@ static void disable_crtc_nofb(struct intel_crtc *crtc)
                         connector->new_encoder = NULL;
         }
 
-        list_for_each_entry(encoder, &dev->mode_config.encoder_list, base.head) {
+        for_each_intel_encoder(dev, encoder) {
                 if (encoder->new_crtc == crtc)
                         encoder->new_crtc = NULL;
         }
@@ -11305,7 +11612,6 @@ static int intel_crtc_set_config(struct drm_mode_set *set)
                 ret = intel_set_mode(set->crtc, set->mode,
                                      set->x, set->y, set->fb);
         } else if (config->fb_changed) {
-                struct drm_i915_private *dev_priv = dev->dev_private;
                 struct intel_crtc *intel_crtc = to_intel_crtc(set->crtc);
 
                 intel_crtc_wait_for_pending_flips(set->crtc);
@@ -11319,8 +11625,7 @@ static int intel_crtc_set_config(struct drm_mode_set *set)
                  */
                 if (!intel_crtc->primary_enabled && ret == 0) {
                         WARN_ON(!intel_crtc->active);
-                        intel_enable_primary_hw_plane(dev_priv, intel_crtc->plane,
-                                                      intel_crtc->pipe);
+                        intel_enable_primary_hw_plane(set->crtc->primary, set->crtc);
                 }
 
                 /*
@@ -11473,8 +11778,6 @@ static int
 intel_primary_plane_disable(struct drm_plane *plane)
 {
         struct drm_device *dev = plane->dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
-        struct intel_plane *intel_plane = to_intel_plane(plane);
         struct intel_crtc *intel_crtc;
 
         if (!plane->fb)
@@ -11497,8 +11800,8 @@ intel_primary_plane_disable(struct drm_plane *plane)
                 goto disable_unpin;
 
         intel_crtc_wait_for_pending_flips(plane->crtc);
-        intel_disable_primary_hw_plane(dev_priv, intel_plane->plane,
-                                       intel_plane->pipe);
+        intel_disable_primary_hw_plane(plane, plane->crtc);
+
 disable_unpin:
         mutex_lock(&dev->struct_mutex);
         i915_gem_track_fb(intel_fb_obj(plane->fb), NULL,
@@ -11520,7 +11823,6 @@ intel_primary_plane_setplane(struct drm_plane *plane, struct drm_crtc *crtc,
         struct drm_device *dev = crtc->dev;
         struct drm_i915_private *dev_priv = dev->dev_private;
         struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
-        struct intel_plane *intel_plane = to_intel_plane(plane);
         struct drm_i915_gem_object *obj = intel_fb_obj(fb);
         struct drm_i915_gem_object *old_obj = intel_fb_obj(plane->fb);
         struct drm_rect dest = {
@@ -11542,6 +11844,21 @@ intel_primary_plane_setplane(struct drm_plane *plane, struct drm_crtc *crtc,
                 .x2 = intel_crtc->active ? intel_crtc->config.pipe_src_w : 0,
                 .y2 = intel_crtc->active ? intel_crtc->config.pipe_src_h : 0,
         };
+        const struct {
+                int crtc_x, crtc_y;
+                unsigned int crtc_w, crtc_h;
+                uint32_t src_x, src_y, src_w, src_h;
+        } orig = {
+                .crtc_x = crtc_x,
+                .crtc_y = crtc_y,
+                .crtc_w = crtc_w,
+                .crtc_h = crtc_h,
+                .src_x = src_x,
+                .src_y = src_y,
+                .src_w = src_w,
+                .src_h = src_h,
+        };
+        struct intel_plane *intel_plane = to_intel_plane(plane);
         bool visible;
         int ret;
 
@@ -11607,9 +11924,7 @@ intel_primary_plane_setplane(struct drm_plane *plane, struct drm_crtc *crtc,
                                   INTEL_FRONTBUFFER_PRIMARY(intel_crtc->pipe));
 
                 if (intel_crtc->primary_enabled)
-                        intel_disable_primary_hw_plane(dev_priv,
-                                                       intel_plane->plane,
-                                                       intel_plane->pipe);
+                        intel_disable_primary_hw_plane(plane, crtc);
 
 
                 if (plane->fb != fb)
@@ -11618,16 +11933,42 @@ intel_primary_plane_setplane(struct drm_plane *plane, struct drm_crtc *crtc,
 
                 mutex_unlock(&dev->struct_mutex);
 
-                return 0;
-        }
+        } else {
+                if (intel_crtc && intel_crtc->active &&
+                    intel_crtc->primary_enabled) {
+                        /*
+                         * FBC does not work on some platforms for rotated
+                         * planes, so disable it when rotation is not 0 and
+                         * update it when rotation is set back to 0.
+                         *
+                         * FIXME: This is redundant with the fbc update done in
+                         * the primary plane enable function except that that
+                         * one is done too late. We eventually need to unify
+                         * this.
+                         */
+                        if (INTEL_INFO(dev)->gen <= 4 && !IS_G4X(dev) &&
+                            dev_priv->fbc.plane == intel_crtc->plane &&
+                            intel_plane->rotation != BIT(DRM_ROTATE_0)) {
+                                intel_disable_fbc(dev);
+                        }
+                }
+                ret = intel_pipe_set_base(crtc, src.x1, src.y1, fb);
+                if (ret)
+                        return ret;
 
-        ret = intel_pipe_set_base(crtc, src.x1, src.y1, fb);
-        if (ret)
-                return ret;
+                if (!intel_crtc->primary_enabled)
+                        intel_enable_primary_hw_plane(plane, crtc);
+        }
 
-        if (!intel_crtc->primary_enabled)
-                intel_enable_primary_hw_plane(dev_priv, intel_crtc->plane,
-                                              intel_crtc->pipe);
+        intel_plane->crtc_x = orig.crtc_x;
+        intel_plane->crtc_y = orig.crtc_y;
+        intel_plane->crtc_w = orig.crtc_w;
+        intel_plane->crtc_h = orig.crtc_h;
+        intel_plane->src_x = orig.src_x;
+        intel_plane->src_y = orig.src_y;
+        intel_plane->src_w = orig.src_w;
+        intel_plane->src_h = orig.src_h;
+        intel_plane->obj = obj;
 
         return 0;
 }
@@ -11644,6 +11985,7 @@ static const struct drm_plane_funcs intel_primary_plane_funcs = {
         .update_plane = intel_primary_plane_setplane,
         .disable_plane = intel_primary_plane_disable,
         .destroy = intel_plane_destroy,
+        .set_property = intel_plane_set_property
 };
 
 static struct drm_plane *intel_primary_plane_create(struct drm_device *dev,
@@ -11661,6 +12003,7 @@ static struct drm_plane *intel_primary_plane_create(struct drm_device *dev,
         primary->max_downscale = 1;
         primary->pipe = pipe;
         primary->plane = pipe;
+        primary->rotation = BIT(DRM_ROTATE_0);
         if (HAS_FBC(dev) && INTEL_INFO(dev)->gen < 4)
                 primary->plane = !pipe;
 
@@ -11676,6 +12019,19 @@ static struct drm_plane *intel_primary_plane_create(struct drm_device *dev,
                                  &intel_primary_plane_funcs,
                                  intel_primary_formats, num_formats,
                                  DRM_PLANE_TYPE_PRIMARY);
+
+        if (INTEL_INFO(dev)->gen >= 4) {
+                if (!dev->mode_config.rotation_property)
+                        dev->mode_config.rotation_property =
+                                drm_mode_create_rotation_property(dev,
+                                                        BIT(DRM_ROTATE_0) |
+                                                        BIT(DRM_ROTATE_180));
+                if (dev->mode_config.rotation_property)
+                        drm_object_attach_property(&primary->base.base,
+                                dev->mode_config.rotation_property,
+                                primary->rotation);
+        }
+
         return &primary->base;
 }
 
@@ -11736,6 +12092,10 @@ intel_cursor_plane_update(struct drm_plane *plane, struct drm_crtc *crtc,
                 return intel_crtc_cursor_set_obj(crtc, obj, crtc_w, crtc_h);
         } else {
                 intel_crtc_update_cursor(crtc, visible);
+
+                intel_frontbuffer_flip(crtc->dev,
+                                       INTEL_FRONTBUFFER_CURSOR(intel_crtc->pipe));
+
                 return 0;
         }
 }
@@ -11812,8 +12172,7 @@ static void intel_crtc_init(struct drm_device *dev, int pipe)
 
         intel_crtc->cursor_base = ~0;
         intel_crtc->cursor_cntl = ~0;
-
-        init_waitqueue_head(&intel_crtc->vbl_wait);
+        intel_crtc->cursor_size = ~0;
 
         BUG_ON(pipe >= ARRAY_SIZE(dev_priv->plane_to_crtc_mapping) ||
                dev_priv->plane_to_crtc_mapping[intel_crtc->plane] != NULL);
@@ -11876,8 +12235,7 @@ static int intel_encoder_clones(struct intel_encoder *encoder)
         int index_mask = 0;
         int entry = 0;
 
-        list_for_each_entry(source_encoder,
-                            &dev->mode_config.encoder_list, base.head) {
+        for_each_intel_encoder(dev, source_encoder) {
                 if (encoders_cloneable(encoder, source_encoder))
                         index_mask |= (1 << entry);
 
@@ -12066,7 +12424,7 @@ static void intel_setup_outputs(struct drm_device *dev)
 
         intel_edp_psr_init(dev);
 
-        list_for_each_entry(encoder, &dev->mode_config.encoder_list, base.head) {
+        for_each_intel_encoder(dev, encoder) {
                 encoder->base.possible_crtcs = encoder->crtc_mask;
                 encoder->base.possible_clones =
                         intel_encoder_clones(encoder);
@@ -12332,29 +12690,27 @@ static void intel_init_display(struct drm_device *dev)
                 dev_priv->display.get_display_clock_speed =
                         i830_get_display_clock_speed;
 
-        if (HAS_PCH_SPLIT(dev)) {
-                if (IS_GEN5(dev)) {
-                        dev_priv->display.fdi_link_train = ironlake_fdi_link_train;
-                        dev_priv->display.write_eld = ironlake_write_eld;
-                } else if (IS_GEN6(dev)) {
-                        dev_priv->display.fdi_link_train = gen6_fdi_link_train;
-                        dev_priv->display.write_eld = ironlake_write_eld;
-                        dev_priv->display.modeset_global_resources =
-                                snb_modeset_global_resources;
-                } else if (IS_IVYBRIDGE(dev)) {
-                        /* FIXME: detect B0+ stepping and use auto training */
-                        dev_priv->display.fdi_link_train = ivb_manual_fdi_link_train;
-                        dev_priv->display.write_eld = ironlake_write_eld;
-                        dev_priv->display.modeset_global_resources =
-                                ivb_modeset_global_resources;
-                } else if (IS_HASWELL(dev) || IS_GEN8(dev)) {
-                        dev_priv->display.fdi_link_train = hsw_fdi_link_train;
-                        dev_priv->display.write_eld = haswell_write_eld;
-                        dev_priv->display.modeset_global_resources =
-                                haswell_modeset_global_resources;
-                }
-        } else if (IS_G4X(dev)) {
+        if (IS_G4X(dev)) {
                 dev_priv->display.write_eld = g4x_write_eld;
+        } else if (IS_GEN5(dev)) {
+                dev_priv->display.fdi_link_train = ironlake_fdi_link_train;
+                dev_priv->display.write_eld = ironlake_write_eld;
+        } else if (IS_GEN6(dev)) {
+                dev_priv->display.fdi_link_train = gen6_fdi_link_train;
+                dev_priv->display.write_eld = ironlake_write_eld;
+                dev_priv->display.modeset_global_resources =
+                        snb_modeset_global_resources;
+        } else if (IS_IVYBRIDGE(dev)) {
+                /* FIXME: detect B0+ stepping and use auto training */
+                dev_priv->display.fdi_link_train = ivb_manual_fdi_link_train;
+                dev_priv->display.write_eld = ironlake_write_eld;
+                dev_priv->display.modeset_global_resources =
+                        ivb_modeset_global_resources;
+        } else if (IS_HASWELL(dev) || IS_BROADWELL(dev)) {
+                dev_priv->display.fdi_link_train = hsw_fdi_link_train;
+                dev_priv->display.write_eld = haswell_write_eld;
+                dev_priv->display.modeset_global_resources =
+                        haswell_modeset_global_resources;
         } else if (IS_VALLEYVIEW(dev)) {
                 dev_priv->display.modeset_global_resources =
                         valleyview_modeset_global_resources;
@@ -12388,6 +12744,8 @@ static void intel_init_display(struct drm_device *dev)
         }
 
         intel_panel_init_backlight_funcs(dev);
+
+        mutex_init(&dev_priv->pps_mutex);
 }
 
 /*
@@ -12403,6 +12761,14 @@ static void quirk_pipea_force(struct drm_device *dev)
         DRM_INFO("applying pipe a force quirk\n");
 }
 
+static void quirk_pipeb_force(struct drm_device *dev)
+{
+        struct drm_i915_private *dev_priv = dev->dev_private;
+
+        dev_priv->quirks |= QUIRK_PIPEB_FORCE;
+        DRM_INFO("applying pipe b force quirk\n");
+}
+
 /*
  * Some machines (Lenovo U160) do not work with SSC on LVDS for some reason
  */
@@ -12477,6 +12843,12 @@ static struct intel_quirk intel_quirks[] = {
         /* ThinkPad T60 needs pipe A force quirk (bug #16494) */
         { 0x2782, 0x17aa, 0x201a, quirk_pipea_force },
 
+        /* 830 needs to leave pipe A & dpll A up */
+        { 0x3577, PCI_ANY_ID, PCI_ANY_ID, quirk_pipea_force },
+
+        /* 830 needs to leave pipe B & dpll B up */
+        { 0x3577, PCI_ANY_ID, PCI_ANY_ID, quirk_pipeb_force },
+
         /* Lenovo U160 cannot use SSC on LVDS */
         { 0x0046, 0x17aa, 0x3920, quirk_ssc_force_disable },
 
@@ -12550,7 +12922,11 @@ static void i915_disable_vga(struct drm_device *dev)
         vga_put(dev->pdev, VGA_RSRC_LEGACY_IO);
         udelay(300);
 
-        I915_WRITE(vga_reg, VGA_DISP_DISABLE);
+        /*
+         * Fujitsu-Siemens Lifebook S6010 (830) has problems resuming
+         * from S3 without preserving (some of?) the other bits.
+         */
+        I915_WRITE(vga_reg, dev_priv->bios_vgacntr | VGA_DISP_DISABLE);
         POSTING_READ(vga_reg);
 }
 
@@ -12563,8 +12939,6 @@ void intel_modeset_init_hw(struct drm_device *dev)
 
         intel_init_clock_gating(dev);
 
-        intel_reset_dpio(dev);
-
         intel_enable_gt_powersave(dev);
 }
 
@@ -12610,7 +12984,10 @@ void intel_modeset_init(struct drm_device *dev)
                 dev->mode_config.max_height = 8192;
         }
 
-        if (IS_GEN2(dev)) {
+        if (IS_845G(dev) || IS_I865G(dev)) {
+                dev->mode_config.cursor_width = IS_845G(dev) ? 64 : 512;
+                dev->mode_config.cursor_height = 1023;
+        } else if (IS_GEN2(dev)) {
                 dev->mode_config.cursor_width = GEN2_CURSOR_WIDTH;
                 dev->mode_config.cursor_height = GEN2_CURSOR_HEIGHT;
         } else {
@@ -12624,7 +13001,7 @@ void intel_modeset_init(struct drm_device *dev)
                       INTEL_INFO(dev)->num_pipes,
                       INTEL_INFO(dev)->num_pipes > 1 ? "s" : "");
 
-        for_each_pipe(pipe) {
+        for_each_pipe(dev_priv, pipe) {
                 intel_crtc_init(dev, pipe);
                 for_each_sprite(pipe, sprite) {
                         ret = intel_plane_init(dev, pipe, sprite);
@@ -12635,10 +13012,11 @@ void intel_modeset_init(struct drm_device *dev)
         }
 
         intel_init_dpio(dev);
-        intel_reset_dpio(dev);
 
         intel_shared_dpll_init(dev);
 
+        /* save the BIOS value before clobbering it */
+        dev_priv->bios_vgacntr = I915_READ(i915_vgacntrl_reg(dev));
         /* Just disable it once at startup */
         i915_disable_vga(dev);
         intel_setup_outputs(dev);
@@ -12730,9 +13108,10 @@ static void intel_sanitize_crtc(struct intel_crtc *crtc)
         I915_WRITE(reg, I915_READ(reg) & ~PIPECONF_FRAME_START_DELAY_MASK);
 
         /* restore vblank interrupts to correct state */
-        if (crtc->active)
+        if (crtc->active) {
+                update_scanline_offset(crtc);
                 drm_vblank_on(dev, crtc->pipe);
-        else
+        } else
                 drm_vblank_off(dev, crtc->pipe);
 
         /* We need to sanitize the plane -> pipe mapping first because this will
@@ -12815,7 +13194,7 @@ static void intel_sanitize_crtc(struct intel_crtc *crtc)
                 }
         }
 
-        if (crtc->active || IS_VALLEYVIEW(dev) || INTEL_INFO(dev)->gen < 5) {
+        if (crtc->active || HAS_GMCH_DISPLAY(dev)) {
                 /*
                  * We start out with underrun reporting disabled to avoid races.
                  * For correct bookkeeping mark this on active crtcs.
@@ -12831,8 +13210,6 @@ static void intel_sanitize_crtc(struct intel_crtc *crtc)
                  */
                 crtc->cpu_fifo_underrun_disabled = true;
                 crtc->pch_fifo_underrun_disabled = true;
-
-                update_scanline_offset(crtc);
         }
 }
 
@@ -12964,8 +13341,7 @@ static void intel_modeset_readout_hw_state(struct drm_device *dev)
                         intel_display_power_get(dev_priv, POWER_DOMAIN_PLLS);
         }
 
-        list_for_each_entry(encoder, &dev->mode_config.encoder_list,
-                            base.head) {
+        for_each_intel_encoder(dev, encoder) {
                 pipe = 0;
 
                 if (encoder->get_hw_state(encoder, &pipe)) {
@@ -13029,12 +13405,11 @@ void intel_modeset_setup_hw_state(struct drm_device *dev,
         }
 
         /* HW state is read out, now we need to sanitize this mess. */
-        list_for_each_entry(encoder, &dev->mode_config.encoder_list,
-                            base.head) {
+        for_each_intel_encoder(dev, encoder) {
                 intel_sanitize_encoder(encoder);
         }
 
-        for_each_pipe(pipe) {
+        for_each_pipe(dev_priv, pipe) {
                 crtc = to_intel_crtc(dev_priv->pipe_to_crtc_mapping[pipe]);
                 intel_sanitize_crtc(crtc);
                 intel_dump_pipe_config(crtc, &crtc->config, "[setup_hw_state]");
@@ -13062,7 +13437,7 @@ void intel_modeset_setup_hw_state(struct drm_device *dev,
                  * We need to use raw interfaces for restoring state to avoid
                  * checking (bogus) intermediate states.
                  */
-                for_each_pipe(pipe) {
+                for_each_pipe(dev_priv, pipe) {
                         struct drm_crtc *crtc =
                                 dev_priv->pipe_to_crtc_mapping[pipe];
 
@@ -13283,7 +13658,7 @@ intel_display_capture_error_state(struct drm_device *dev)
         if (IS_HASWELL(dev) || IS_BROADWELL(dev))
                 error->power_well_driver = I915_READ(HSW_PWR_WELL_DRIVER);
 
-        for_each_pipe(i) {
+        for_each_pipe(dev_priv, i) {
                 error->pipe[i].power_domain_on =
                         intel_display_power_enabled_unlocked(dev_priv,
                                                            POWER_DOMAIN_PIPE(i));
@@ -13347,6 +13722,7 @@ intel_display_print_error_state(struct drm_i915_error_state_buf *m,
                                 struct drm_device *dev,
                                 struct intel_display_error_state *error)
 {
+        struct drm_i915_private *dev_priv = dev->dev_private;
         int i;
 
         if (!error)
@@ -13356,7 +13732,7 @@ intel_display_print_error_state(struct drm_i915_error_state_buf *m,
         if (IS_HASWELL(dev) || IS_BROADWELL(dev))
                 err_printf(m, "PWR_WELL_CTL2: %08x\n",
                            error->power_well_driver);
-        for_each_pipe(i) {
+        for_each_pipe(dev_priv, i) {
                 err_printf(m, "Pipe [%d]:\n", i);
                 err_printf(m, "  Power: %s\n",
                            error->pipe[i].power_domain_on ? "on" : "off");
@@ -13397,3 +13773,25 @@ intel_display_print_error_state(struct drm_i915_error_state_buf *m,
                 err_printf(m, "  VSYNC: %08x\n", error->transcoder[i].vsync);
         }
 }
+
+void intel_modeset_preclose(struct drm_device *dev, struct drm_file *file)
+{
+        struct intel_crtc *crtc;
+
+        for_each_intel_crtc(dev, crtc) {
+                struct intel_unpin_work *work;
+                unsigned long irqflags;
+
+                spin_lock_irqsave(&dev->event_lock, irqflags);
+
+                work = crtc->unpin_work;
+
+                if (work && work->event &&
+                    work->event->base.file_priv == file) {
+                        kfree(work->event);
+                        work->event = NULL;
+                }
+
+                spin_unlock_irqrestore(&dev->event_lock, irqflags);
+        }
+}
diff --git a/drivers/gpu/drm/i915/intel_dp.c b/drivers/gpu/drm/i915/intel_dp.c
index fdff1d420c14..f6a3fdd5589e 100644
--- a/drivers/gpu/drm/i915/intel_dp.c
+++ b/drivers/gpu/drm/i915/intel_dp.c
@@ -111,7 +111,7 @@ static struct intel_dp *intel_attached_dp(struct drm_connector *connector)
 }
 
 static void intel_dp_link_down(struct intel_dp *intel_dp);
-static bool _edp_panel_vdd_on(struct intel_dp *intel_dp);
+static bool edp_panel_vdd_on(struct intel_dp *intel_dp);
 static void edp_panel_vdd_off(struct intel_dp *intel_dp, bool sync);
 
 int
@@ -290,32 +290,201 @@ intel_dp_init_panel_power_sequencer_registers(struct drm_device *dev,
                                               struct intel_dp *intel_dp,
                                               struct edp_power_seq *out);
 
+static void pps_lock(struct intel_dp *intel_dp)
+{
+        struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp);
+        struct intel_encoder *encoder = &intel_dig_port->base;
+        struct drm_device *dev = encoder->base.dev;
+        struct drm_i915_private *dev_priv = dev->dev_private;
+        enum intel_display_power_domain power_domain;
+
+        /*
+         * See vlv_power_sequencer_reset() why we need
+         * a power domain reference here.
+         */
+        power_domain = intel_display_port_power_domain(encoder);
+        intel_display_power_get(dev_priv, power_domain);
+
+        mutex_lock(&dev_priv->pps_mutex);
+}
+
+static void pps_unlock(struct intel_dp *intel_dp)
+{
+        struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp);
+        struct intel_encoder *encoder = &intel_dig_port->base;
+        struct drm_device *dev = encoder->base.dev;
+        struct drm_i915_private *dev_priv = dev->dev_private;
+        enum intel_display_power_domain power_domain;
+
+        mutex_unlock(&dev_priv->pps_mutex);
+
+        power_domain = intel_display_port_power_domain(encoder);
+        intel_display_power_put(dev_priv, power_domain);
+}
+
 static enum pipe
 vlv_power_sequencer_pipe(struct intel_dp *intel_dp)
 {
         struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp);
-        struct drm_crtc *crtc = intel_dig_port->base.base.crtc;
         struct drm_device *dev = intel_dig_port->base.base.dev;
         struct drm_i915_private *dev_priv = dev->dev_private;
-        enum port port = intel_dig_port->port;
-        enum pipe pipe;
+        struct intel_encoder *encoder;
+        unsigned int pipes = (1 << PIPE_A) | (1 << PIPE_B);
+        struct edp_power_seq power_seq;
+
+        lockdep_assert_held(&dev_priv->pps_mutex);
+
+        if (intel_dp->pps_pipe != INVALID_PIPE)
+                return intel_dp->pps_pipe;
+
+        /*
+         * We don't have power sequencer currently.
+         * Pick one that's not used by other ports.
+         */
+        list_for_each_entry(encoder, &dev->mode_config.encoder_list,
+                            base.head) {
+                struct intel_dp *tmp;
+
+                if (encoder->type != INTEL_OUTPUT_EDP)
+                        continue;
+
+                tmp = enc_to_intel_dp(&encoder->base);
+
+                if (tmp->pps_pipe != INVALID_PIPE)
+                        pipes &= ~(1 << tmp->pps_pipe);
+        }
+
+        /*
+         * Didn't find one. This should not happen since there
+         * are two power sequencers and up to two eDP ports.
+         */
+        if (WARN_ON(pipes == 0))
+                return PIPE_A;
+
+        intel_dp->pps_pipe = ffs(pipes) - 1;
+
+        DRM_DEBUG_KMS("picked pipe %c power sequencer for port %c\n",
+                      pipe_name(intel_dp->pps_pipe),
+                      port_name(intel_dig_port->port));
 
-        /* modeset should have pipe */
-        if (crtc)
-                return to_intel_crtc(crtc)->pipe;
+        /* init power sequencer on this pipe and port */
+        intel_dp_init_panel_power_sequencer(dev, intel_dp, &power_seq);
+        intel_dp_init_panel_power_sequencer_registers(dev, intel_dp,
+                                                      &power_seq);
+
+        return intel_dp->pps_pipe;
+}
+
+typedef bool (*vlv_pipe_check)(struct drm_i915_private *dev_priv,
+                               enum pipe pipe);
+
+static bool vlv_pipe_has_pp_on(struct drm_i915_private *dev_priv,
+                               enum pipe pipe)
+{
+        return I915_READ(VLV_PIPE_PP_STATUS(pipe)) & PP_ON;
+}
+
+static bool vlv_pipe_has_vdd_on(struct drm_i915_private *dev_priv,
+                                enum pipe pipe)
+{
+        return I915_READ(VLV_PIPE_PP_CONTROL(pipe)) & EDP_FORCE_VDD;
+}
+
+static bool vlv_pipe_any(struct drm_i915_private *dev_priv,
+                         enum pipe pipe)
+{
+        return true;
+}
+
+static enum pipe
+vlv_initial_pps_pipe(struct drm_i915_private *dev_priv,
+                     enum port port,
+                     vlv_pipe_check pipe_check)
+{
+        enum pipe pipe;
 
-        /* init time, try to find a pipe with this port selected */
         for (pipe = PIPE_A; pipe <= PIPE_B; pipe++) {
                 u32 port_sel = I915_READ(VLV_PIPE_PP_ON_DELAYS(pipe)) &
                         PANEL_PORT_SELECT_MASK;
-                if (port_sel == PANEL_PORT_SELECT_DPB_VLV && port == PORT_B)
-                        return pipe;
-                if (port_sel == PANEL_PORT_SELECT_DPC_VLV && port == PORT_C)
-                        return pipe;
+
+                if (port_sel != PANEL_PORT_SELECT_VLV(port))
+                        continue;
+
+                if (!pipe_check(dev_priv, pipe))
+                        continue;
+
+                return pipe;
+        }
+
+        return INVALID_PIPE;
+}
+
+static void
+vlv_initial_power_sequencer_setup(struct intel_dp *intel_dp)
+{
+        struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp);
+        struct drm_device *dev = intel_dig_port->base.base.dev;
+        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct edp_power_seq power_seq;
+        enum port port = intel_dig_port->port;
+
+        lockdep_assert_held(&dev_priv->pps_mutex);
+
+        /* try to find a pipe with this port selected */
+        /* first pick one where the panel is on */
+        intel_dp->pps_pipe = vlv_initial_pps_pipe(dev_priv, port,
+                                                  vlv_pipe_has_pp_on);
+        /* didn't find one? pick one where vdd is on */
+        if (intel_dp->pps_pipe == INVALID_PIPE)
+                intel_dp->pps_pipe = vlv_initial_pps_pipe(dev_priv, port,
+                                                          vlv_pipe_has_vdd_on);
+        /* didn't find one? pick one with just the correct port */
+        if (intel_dp->pps_pipe == INVALID_PIPE)
+                intel_dp->pps_pipe = vlv_initial_pps_pipe(dev_priv, port,
+                                                          vlv_pipe_any);
+
+        /* didn't find one? just let vlv_power_sequencer_pipe() pick one when needed */
+        if (intel_dp->pps_pipe == INVALID_PIPE) {
+                DRM_DEBUG_KMS("no initial power sequencer for port %c\n",
+                              port_name(port));
+                return;
         }
 
-        /* shrug */
-        return PIPE_A;
+        DRM_DEBUG_KMS("initial power sequencer for port %c: pipe %c\n",
+                      port_name(port), pipe_name(intel_dp->pps_pipe));
+
+        intel_dp_init_panel_power_sequencer(dev, intel_dp, &power_seq);
+        intel_dp_init_panel_power_sequencer_registers(dev, intel_dp,
+                                                      &power_seq);
+}
+
+void vlv_power_sequencer_reset(struct drm_i915_private *dev_priv)
+{
+        struct drm_device *dev = dev_priv->dev;
+        struct intel_encoder *encoder;
+
+        if (WARN_ON(!IS_VALLEYVIEW(dev)))
+                return;
+
+        /*
+         * We can't grab pps_mutex here due to deadlock with power_domain
+         * mutex when power_domain functions are called while holding pps_mutex.
+         * That also means that in order to use pps_pipe the code needs to
+         * hold both a power domain reference and pps_mutex, and the power domain
+         * reference get/put must be done while _not_ holding pps_mutex.
+         * pps_{lock,unlock}() do these steps in the correct order, so one
+         * should use them always.
+         */
+
+        list_for_each_entry(encoder, &dev->mode_config.encoder_list, base.head) {
+                struct intel_dp *intel_dp;
+
+                if (encoder->type != INTEL_OUTPUT_EDP)
+                        continue;
+
+                intel_dp = enc_to_intel_dp(&encoder->base);
+                intel_dp->pps_pipe = INVALID_PIPE;
+        }
 }
 
 static u32 _pp_ctrl_reg(struct intel_dp *intel_dp)
@@ -349,12 +518,15 @@ static int edp_notify_handler(struct notifier_block *this, unsigned long code,
         struct drm_i915_private *dev_priv = dev->dev_private;
         u32 pp_div;
         u32 pp_ctrl_reg, pp_div_reg;
-        enum pipe pipe = vlv_power_sequencer_pipe(intel_dp);
 
         if (!is_edp(intel_dp) || code != SYS_RESTART)
                 return 0;
 
+        pps_lock(intel_dp);
+
         if (IS_VALLEYVIEW(dev)) {
+                enum pipe pipe = vlv_power_sequencer_pipe(intel_dp);
+
                 pp_ctrl_reg = VLV_PIPE_PP_CONTROL(pipe);
                 pp_div_reg  = VLV_PIPE_PP_DIVISOR(pipe);
                 pp_div = I915_READ(pp_div_reg);
@@ -366,6 +538,8 @@ static int edp_notify_handler(struct notifier_block *this, unsigned long code,
                 msleep(intel_dp->panel_power_cycle_delay);
         }
 
+        pps_unlock(intel_dp);
+
         return 0;
 }
 
@@ -374,6 +548,8 @@ static bool edp_have_panel_power(struct intel_dp *intel_dp)
         struct drm_device *dev = intel_dp_to_dev(intel_dp);
         struct drm_i915_private *dev_priv = dev->dev_private;
 
+        lockdep_assert_held(&dev_priv->pps_mutex);
+
         return (I915_READ(_pp_stat_reg(intel_dp)) & PP_ON) != 0;
 }
 
@@ -381,13 +557,10 @@ static bool edp_have_panel_vdd(struct intel_dp *intel_dp)
 {
         struct drm_device *dev = intel_dp_to_dev(intel_dp);
         struct drm_i915_private *dev_priv = dev->dev_private;
-        struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp);
-        struct intel_encoder *intel_encoder = &intel_dig_port->base;
-        enum intel_display_power_domain power_domain;
 
-        power_domain = intel_display_port_power_domain(intel_encoder);
-        return intel_display_power_enabled(dev_priv, power_domain) &&
-               (I915_READ(_pp_ctrl_reg(intel_dp)) & EDP_FORCE_VDD) != 0;
+        lockdep_assert_held(&dev_priv->pps_mutex);
+
+        return I915_READ(_pp_ctrl_reg(intel_dp)) & EDP_FORCE_VDD;
 }
 
 static void
@@ -535,7 +708,15 @@ intel_dp_aux_ch(struct intel_dp *intel_dp,
         bool has_aux_irq = HAS_AUX_IRQ(dev);
         bool vdd;
 
-        vdd = _edp_panel_vdd_on(intel_dp);
+        pps_lock(intel_dp);
+
+        /*
+         * We will be called with VDD already enabled for dpcd/edid/oui reads.
+         * In such cases we want to leave VDD enabled and it's up to upper layers
+         * to turn it off. But for eg. i2c-dev access we need to turn it on/off
+         * ourselves.
+         */
+        vdd = edp_panel_vdd_on(intel_dp);
 
         /* dp aux is extremely sensitive to irq latency, hence request the
          * lowest possible wakeup latency and so prevent the cpu from going into
@@ -644,6 +825,8 @@ out:
         if (vdd)
                 edp_panel_vdd_off(intel_dp, false);
 
+        pps_unlock(intel_dp);
+
         return ret;
 }
 
@@ -828,20 +1011,6 @@ intel_dp_set_clock(struct intel_encoder *encoder,
         }
 }
 
-static void
-intel_dp_set_m2_n2(struct intel_crtc *crtc, struct intel_link_m_n *m_n)
-{
-        struct drm_device *dev = crtc->base.dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
-        enum transcoder transcoder = crtc->config.cpu_transcoder;
-
-        I915_WRITE(PIPE_DATA_M2(transcoder),
-                TU_SIZE(m_n->tu) | m_n->gmch_m);
-        I915_WRITE(PIPE_DATA_N2(transcoder), m_n->gmch_n);
-        I915_WRITE(PIPE_LINK_M2(transcoder), m_n->link_m);
-        I915_WRITE(PIPE_LINK_N2(transcoder), m_n->link_n);
-}
-
 bool
 intel_dp_compute_config(struct intel_encoder *encoder,
                         struct intel_crtc_config *pipe_config)
@@ -867,6 +1036,7 @@ intel_dp_compute_config(struct intel_encoder *encoder,
                 pipe_config->has_pch_encoder = true;
 
         pipe_config->has_dp_encoder = true;
+        pipe_config->has_drrs = false;
         pipe_config->has_audio = intel_dp->has_audio;
 
         if (is_edp(intel_dp) && intel_connector->panel.fixed_mode) {
@@ -898,23 +1068,15 @@ intel_dp_compute_config(struct intel_encoder *encoder,
                         bpp = dev_priv->vbt.edp_bpp;
                 }
 
-                if (IS_BROADWELL(dev)) {
-                        /* Yes, it's an ugly hack. */
-                        min_lane_count = max_lane_count;
-                        DRM_DEBUG_KMS("forcing lane count to max (%u) on BDW\n",
-                                      min_lane_count);
-                } else if (dev_priv->vbt.edp_lanes) {
-                        min_lane_count = min(dev_priv->vbt.edp_lanes,
-                                             max_lane_count);
-                        DRM_DEBUG_KMS("using min %u lanes per VBT\n",
-                                      min_lane_count);
-                }
-
-                if (dev_priv->vbt.edp_rate) {
-                        min_clock = min(dev_priv->vbt.edp_rate >> 3, max_clock);
-                        DRM_DEBUG_KMS("using min %02x link bw per VBT\n",
-                                      bws[min_clock]);
-                }
+                /*
+                 * Use the maximum clock and number of lanes the eDP panel
+                 * advertizes being capable of. The panels are generally
+                 * designed to support only a single clock and lane
+                 * configuration, and typically these values correspond to the
+                 * native resolution of the panel.
+                 */
+                min_lane_count = max_lane_count;
+                min_clock = max_clock;
         }
 
         for (; bpp >= 6*3; bpp -= 2*3) {
@@ -970,13 +1132,14 @@ found:
 
         if (intel_connector->panel.downclock_mode != NULL &&
                 intel_dp->drrs_state.type == SEAMLESS_DRRS_SUPPORT) {
+                        pipe_config->has_drrs = true;
                         intel_link_compute_m_n(bpp, lane_count,
                                 intel_connector->panel.downclock_mode->clock,
                                 pipe_config->port_clock,
                                 &pipe_config->dp_m2_n2);
         }
 
-        if (HAS_DDI(dev))
+        if (IS_HASWELL(dev) || IS_BROADWELL(dev))
                 hsw_dp_set_ddi_pll_sel(pipe_config, intel_dp->link_bw);
         else
                 intel_dp_set_clock(encoder, pipe_config, intel_dp->link_bw);
@@ -1110,6 +1273,8 @@ static void wait_panel_status(struct intel_dp *intel_dp,
         struct drm_i915_private *dev_priv = dev->dev_private;
         u32 pp_stat_reg, pp_ctrl_reg;
 
+        lockdep_assert_held(&dev_priv->pps_mutex);
+
         pp_stat_reg = _pp_stat_reg(intel_dp);
         pp_ctrl_reg = _pp_ctrl_reg(intel_dp);
 
@@ -1173,13 +1338,20 @@ static  u32 ironlake_get_pp_control(struct intel_dp *intel_dp)
         struct drm_i915_private *dev_priv = dev->dev_private;
         u32 control;
 
+        lockdep_assert_held(&dev_priv->pps_mutex);
+
         control = I915_READ(_pp_ctrl_reg(intel_dp));
         control &= ~PANEL_UNLOCK_MASK;
         control |= PANEL_UNLOCK_REGS;
         return control;
 }
 
-static bool _edp_panel_vdd_on(struct intel_dp *intel_dp)
+/*
+ * Must be paired with edp_panel_vdd_off().
+ * Must hold pps_mutex around the whole on/off sequence.
+ * Can be nested with intel_edp_panel_vdd_{on,off}() calls.
+ */
+static bool edp_panel_vdd_on(struct intel_dp *intel_dp)
 {
         struct drm_device *dev = intel_dp_to_dev(intel_dp);
         struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp);
@@ -1190,6 +1362,8 @@ static bool _edp_panel_vdd_on(struct intel_dp *intel_dp)
         u32 pp_stat_reg, pp_ctrl_reg;
         bool need_to_disable = !intel_dp->want_panel_vdd;
 
+        lockdep_assert_held(&dev_priv->pps_mutex);
+
         if (!is_edp(intel_dp))
                 return false;
 
@@ -1227,62 +1401,76 @@ static bool _edp_panel_vdd_on(struct intel_dp *intel_dp)
         return need_to_disable;
 }
 
+/*
+ * Must be paired with intel_edp_panel_vdd_off() or
+ * intel_edp_panel_off().
+ * Nested calls to these functions are not allowed since
+ * we drop the lock. Caller must use some higher level
+ * locking to prevent nested calls from other threads.
+ */
 void intel_edp_panel_vdd_on(struct intel_dp *intel_dp)
 {
-        if (is_edp(intel_dp)) {
-                bool vdd = _edp_panel_vdd_on(intel_dp);
+        bool vdd;
 
-                WARN(!vdd, "eDP VDD already requested on\n");
-        }
+        if (!is_edp(intel_dp))
+                return;
+
+        pps_lock(intel_dp);
+        vdd = edp_panel_vdd_on(intel_dp);
+        pps_unlock(intel_dp);
+
+        WARN(!vdd, "eDP VDD already requested on\n");
 }
 
 static void edp_panel_vdd_off_sync(struct intel_dp *intel_dp)
 {
         struct drm_device *dev = intel_dp_to_dev(intel_dp);
         struct drm_i915_private *dev_priv = dev->dev_private;
+        struct intel_digital_port *intel_dig_port =
+                dp_to_dig_port(intel_dp);
+        struct intel_encoder *intel_encoder = &intel_dig_port->base;
+        enum intel_display_power_domain power_domain;
         u32 pp;
         u32 pp_stat_reg, pp_ctrl_reg;
 
-        WARN_ON(!drm_modeset_is_locked(&dev->mode_config.connection_mutex));
+        lockdep_assert_held(&dev_priv->pps_mutex);
 
-        if (!intel_dp->want_panel_vdd && edp_have_panel_vdd(intel_dp)) {
-                struct intel_digital_port *intel_dig_port =
-                                                dp_to_dig_port(intel_dp);
-                struct intel_encoder *intel_encoder = &intel_dig_port->base;
-                enum intel_display_power_domain power_domain;
+        WARN_ON(intel_dp->want_panel_vdd);
+
+        if (!edp_have_panel_vdd(intel_dp))
+                return;
 
-                DRM_DEBUG_KMS("Turning eDP VDD off\n");
+        DRM_DEBUG_KMS("Turning eDP VDD off\n");
 
-                pp = ironlake_get_pp_control(intel_dp);
-                pp &= ~EDP_FORCE_VDD;
+        pp = ironlake_get_pp_control(intel_dp);
+        pp &= ~EDP_FORCE_VDD;
 
-                pp_ctrl_reg = _pp_ctrl_reg(intel_dp);
-                pp_stat_reg = _pp_stat_reg(intel_dp);
+        pp_ctrl_reg = _pp_ctrl_reg(intel_dp);
+        pp_stat_reg = _pp_stat_reg(intel_dp);
 
-                I915_WRITE(pp_ctrl_reg, pp);
-                POSTING_READ(pp_ctrl_reg);
+        I915_WRITE(pp_ctrl_reg, pp);
+        POSTING_READ(pp_ctrl_reg);
 
-                /* Make sure sequencer is idle before allowing subsequent activity */
-                DRM_DEBUG_KMS("PP_STATUS: 0x%08x PP_CONTROL: 0x%08x\n",
-                I915_READ(pp_stat_reg), I915_READ(pp_ctrl_reg));
+        /* Make sure sequencer is idle before allowing subsequent activity */
+        DRM_DEBUG_KMS("PP_STATUS: 0x%08x PP_CONTROL: 0x%08x\n",
+        I915_READ(pp_stat_reg), I915_READ(pp_ctrl_reg));
 
-                if ((pp & POWER_TARGET_ON) == 0)
-                        intel_dp->last_power_cycle = jiffies;
+        if ((pp & POWER_TARGET_ON) == 0)
+                intel_dp->last_power_cycle = jiffies;
 
-                power_domain = intel_display_port_power_domain(intel_encoder);
-                intel_display_power_put(dev_priv, power_domain);
-        }
+        power_domain = intel_display_port_power_domain(intel_encoder);
+        intel_display_power_put(dev_priv, power_domain);
 }
 
 static void edp_panel_vdd_work(struct work_struct *__work)
 {
         struct intel_dp *intel_dp = container_of(to_delayed_work(__work),
                                                  struct intel_dp, panel_vdd_work);
-        struct drm_device *dev = intel_dp_to_dev(intel_dp);
 
-        drm_modeset_lock(&dev->mode_config.connection_mutex, NULL);
-        edp_panel_vdd_off_sync(intel_dp);
-        drm_modeset_unlock(&dev->mode_config.connection_mutex);
+        pps_lock(intel_dp);
+        if (!intel_dp->want_panel_vdd)
+                edp_panel_vdd_off_sync(intel_dp);
+        pps_unlock(intel_dp);
 }
 
 static void edp_panel_vdd_schedule_off(struct intel_dp *intel_dp)
@@ -1298,8 +1486,18 @@ static void edp_panel_vdd_schedule_off(struct intel_dp *intel_dp)
         schedule_delayed_work(&intel_dp->panel_vdd_work, delay);
 }
 
+/*
+ * Must be paired with edp_panel_vdd_on().
+ * Must hold pps_mutex around the whole on/off sequence.
+ * Can be nested with intel_edp_panel_vdd_{on,off}() calls.
+ */
 static void edp_panel_vdd_off(struct intel_dp *intel_dp, bool sync)
 {
+        struct drm_i915_private *dev_priv =
+                intel_dp_to_dev(intel_dp)->dev_private;
+
+        lockdep_assert_held(&dev_priv->pps_mutex);
+
         if (!is_edp(intel_dp))
                 return;
 
@@ -1313,6 +1511,22 @@ static void edp_panel_vdd_off(struct intel_dp *intel_dp, bool sync)
                 edp_panel_vdd_schedule_off(intel_dp);
 }
 
+/*
+ * Must be paired with intel_edp_panel_vdd_on().
+ * Nested calls to these functions are not allowed since
+ * we drop the lock. Caller must use some higher level
+ * locking to prevent nested calls from other threads.
+ */
+static void intel_edp_panel_vdd_off(struct intel_dp *intel_dp, bool sync)
+{
+        if (!is_edp(intel_dp))
+                return;
+
+        pps_lock(intel_dp);
+        edp_panel_vdd_off(intel_dp, sync);
+        pps_unlock(intel_dp);
+}
+
 void intel_edp_panel_on(struct intel_dp *intel_dp)
 {
         struct drm_device *dev = intel_dp_to_dev(intel_dp);
@@ -1325,9 +1539,11 @@ void intel_edp_panel_on(struct intel_dp *intel_dp)
 
         DRM_DEBUG_KMS("Turn eDP power on\n");
 
+        pps_lock(intel_dp);
+
         if (edp_have_panel_power(intel_dp)) {
                 DRM_DEBUG_KMS("eDP power already on\n");
-                return;
+                goto out;
         }
 
         wait_panel_power_cycle(intel_dp);
@@ -1356,6 +1572,9 @@ void intel_edp_panel_on(struct intel_dp *intel_dp)
                 I915_WRITE(pp_ctrl_reg, pp);
                 POSTING_READ(pp_ctrl_reg);
         }
+
+ out:
+        pps_unlock(intel_dp);
 }
 
 void intel_edp_panel_off(struct intel_dp *intel_dp)
@@ -1373,6 +1592,8 @@ void intel_edp_panel_off(struct intel_dp *intel_dp)
 
         DRM_DEBUG_KMS("Turn eDP power off\n");
 
+        pps_lock(intel_dp);
+
         WARN(!intel_dp->want_panel_vdd, "Need VDD to turn off panel\n");
 
         pp = ironlake_get_pp_control(intel_dp);
@@ -1394,9 +1615,12 @@ void intel_edp_panel_off(struct intel_dp *intel_dp)
         /* We got a reference when we enabled the VDD. */
         power_domain = intel_display_port_power_domain(intel_encoder);
         intel_display_power_put(dev_priv, power_domain);
+
+        pps_unlock(intel_dp);
 }
 
-void intel_edp_backlight_on(struct intel_dp *intel_dp)
+/* Enable backlight in the panel power control. */
+static void _intel_edp_backlight_on(struct intel_dp *intel_dp)
 {
         struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp);
         struct drm_device *dev = intel_dig_port->base.base.dev;
@@ -1404,13 +1628,6 @@ void intel_edp_backlight_on(struct intel_dp *intel_dp)
         u32 pp;
         u32 pp_ctrl_reg;
 
-        if (!is_edp(intel_dp))
-                return;
-
-        DRM_DEBUG_KMS("\n");
-
-        intel_panel_enable_backlight(intel_dp->attached_connector);
-
         /*
          * If we enable the backlight right away following a panel power
          * on, we may see slight flicker as the panel syncs with the eDP
@@ -1418,6 +1635,9 @@ void intel_edp_backlight_on(struct intel_dp *intel_dp)
          * allowing it to appear.
          */
         wait_backlight_on(intel_dp);
+
+        pps_lock(intel_dp);
+
         pp = ironlake_get_pp_control(intel_dp);
         pp |= EDP_BLC_ENABLE;
 
@@ -1425,9 +1645,24 @@ void intel_edp_backlight_on(struct intel_dp *intel_dp)
 
         I915_WRITE(pp_ctrl_reg, pp);
         POSTING_READ(pp_ctrl_reg);
+
+        pps_unlock(intel_dp);
 }
 
-void intel_edp_backlight_off(struct intel_dp *intel_dp)
+/* Enable backlight PWM and backlight PP control. */
+void intel_edp_backlight_on(struct intel_dp *intel_dp)
+{
+        if (!is_edp(intel_dp))
+                return;
+
+        DRM_DEBUG_KMS("\n");
+
+        intel_panel_enable_backlight(intel_dp->attached_connector);
+        _intel_edp_backlight_on(intel_dp);
+}
+
+/* Disable backlight in the panel power control. */
+static void _intel_edp_backlight_off(struct intel_dp *intel_dp)
 {
         struct drm_device *dev = intel_dp_to_dev(intel_dp);
         struct drm_i915_private *dev_priv = dev->dev_private;
@@ -1437,7 +1672,8 @@ void intel_edp_backlight_off(struct intel_dp *intel_dp)
         if (!is_edp(intel_dp))
                 return;
 
-        DRM_DEBUG_KMS("\n");
+        pps_lock(intel_dp);
+
         pp = ironlake_get_pp_control(intel_dp);
         pp &= ~EDP_BLC_ENABLE;
 
@@ -1445,13 +1681,51 @@ void intel_edp_backlight_off(struct intel_dp *intel_dp)
 
         I915_WRITE(pp_ctrl_reg, pp);
         POSTING_READ(pp_ctrl_reg);
-        intel_dp->last_backlight_off = jiffies;
 
+        pps_unlock(intel_dp);
+
+        intel_dp->last_backlight_off = jiffies;
         edp_wait_backlight_off(intel_dp);
+}
+
+/* Disable backlight PP control and backlight PWM. */
+void intel_edp_backlight_off(struct intel_dp *intel_dp)
+{
+        if (!is_edp(intel_dp))
+                return;
+
+        DRM_DEBUG_KMS("\n");
 
+        _intel_edp_backlight_off(intel_dp);
         intel_panel_disable_backlight(intel_dp->attached_connector);
 }
 
+/*
+ * Hook for controlling the panel power control backlight through the bl_power
+ * sysfs attribute. Take care to handle multiple calls.
+ */
+static void intel_edp_backlight_power(struct intel_connector *connector,
+                                      bool enable)
+{
+        struct intel_dp *intel_dp = intel_attached_dp(&connector->base);
+        bool is_enabled;
+
+        pps_lock(intel_dp);
+        is_enabled = ironlake_get_pp_control(intel_dp) & EDP_BLC_ENABLE;
+        pps_unlock(intel_dp);
+
+        if (is_enabled == enable)
+                return;
+
+        DRM_DEBUG_KMS("panel power control backlight %s\n",
+                      enable ? "enable" : "disable");
+
+        if (enable)
+                _intel_edp_backlight_on(intel_dp);
+        else
+                _intel_edp_backlight_off(intel_dp);
+}
+
 static void ironlake_edp_pll_on(struct intel_dp *intel_dp)
 {
         struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp);
@@ -1515,8 +1789,6 @@ void intel_dp_sink_dpms(struct intel_dp *intel_dp, int mode)
         if (mode != DRM_MODE_DPMS_ON) {
                 ret = drm_dp_dpcd_writeb(&intel_dp->aux, DP_SET_POWER,
                                          DP_SET_POWER_D3);
-                if (ret != 1)
-                        DRM_DEBUG_DRIVER("failed to write sink power state\n");
         } else {
                 /*
                  * When turning on, we need to retry for 1ms to give the sink
@@ -1530,6 +1802,10 @@ void intel_dp_sink_dpms(struct intel_dp *intel_dp, int mode)
                         msleep(1);
                 }
         }
+
+        if (ret != 1)
+                DRM_DEBUG_KMS("failed to %s sink power state\n",
+                              mode == DRM_MODE_DPMS_ON ? "enable" : "disable");
 }
 
 static bool intel_dp_get_hw_state(struct intel_encoder *encoder,
@@ -1576,7 +1852,7 @@ static bool intel_dp_get_hw_state(struct intel_encoder *encoder,
                         return true;
                 }
 
-                for_each_pipe(i) {
+                for_each_pipe(dev_priv, i) {
                         trans_dp = I915_READ(TRANS_DP_CTL(i));
                         if ((trans_dp & TRANS_DP_PORT_SEL_MASK) == trans_sel) {
                                 *pipe = i;
@@ -2036,7 +2312,6 @@ void intel_edp_psr_init(struct drm_device *dev)
 static void intel_disable_dp(struct intel_encoder *encoder)
 {
         struct intel_dp *intel_dp = enc_to_intel_dp(&encoder->base);
-        enum port port = dp_to_dig_port(intel_dp)->port;
         struct drm_device *dev = encoder->base.dev;
 
         /* Make sure the panel is off before trying to change the mode. But also
@@ -2046,21 +2321,19 @@ static void intel_disable_dp(struct intel_encoder *encoder)
         intel_dp_sink_dpms(intel_dp, DRM_MODE_DPMS_OFF);
         intel_edp_panel_off(intel_dp);
 
-        /* cpu edp my only be disable _after_ the cpu pipe/plane is disabled. */
-        if (!(port == PORT_A || IS_VALLEYVIEW(dev)))
+        /* disable the port before the pipe on g4x */
+        if (INTEL_INFO(dev)->gen < 5)
                 intel_dp_link_down(intel_dp);
 }
 
-static void g4x_post_disable_dp(struct intel_encoder *encoder)
+static void ilk_post_disable_dp(struct intel_encoder *encoder)
 {
         struct intel_dp *intel_dp = enc_to_intel_dp(&encoder->base);
         enum port port = dp_to_dig_port(intel_dp)->port;
 
-        if (port != PORT_A)
-                return;
-
         intel_dp_link_down(intel_dp);
-        ironlake_edp_pll_off(intel_dp);
+        if (port == PORT_A)
+                ironlake_edp_pll_off(intel_dp);
 }
 
 static void vlv_post_disable_dp(struct intel_encoder *encoder)
@@ -2106,6 +2379,104 @@ static void chv_post_disable_dp(struct intel_encoder *encoder)
         mutex_unlock(&dev_priv->dpio_lock);
 }
 
+static void
+_intel_dp_set_link_train(struct intel_dp *intel_dp,
+                         uint32_t *DP,
+                         uint8_t dp_train_pat)
+{
+        struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp);
+        struct drm_device *dev = intel_dig_port->base.base.dev;
+        struct drm_i915_private *dev_priv = dev->dev_private;
+        enum port port = intel_dig_port->port;
+
+        if (HAS_DDI(dev)) {
+                uint32_t temp = I915_READ(DP_TP_CTL(port));
+
+                if (dp_train_pat & DP_LINK_SCRAMBLING_DISABLE)
+                        temp |= DP_TP_CTL_SCRAMBLE_DISABLE;
+                else
+                        temp &= ~DP_TP_CTL_SCRAMBLE_DISABLE;
+
+                temp &= ~DP_TP_CTL_LINK_TRAIN_MASK;
+                switch (dp_train_pat & DP_TRAINING_PATTERN_MASK) {
+                case DP_TRAINING_PATTERN_DISABLE:
+                        temp |= DP_TP_CTL_LINK_TRAIN_NORMAL;
+
+                        break;
+                case DP_TRAINING_PATTERN_1:
+                        temp |= DP_TP_CTL_LINK_TRAIN_PAT1;
+                        break;
+                case DP_TRAINING_PATTERN_2:
+                        temp |= DP_TP_CTL_LINK_TRAIN_PAT2;
+                        break;
+                case DP_TRAINING_PATTERN_3:
+                        temp |= DP_TP_CTL_LINK_TRAIN_PAT3;
+                        break;
+                }
+                I915_WRITE(DP_TP_CTL(port), temp);
+
+        } else if (HAS_PCH_CPT(dev) && (IS_GEN7(dev) || port != PORT_A)) {
+                *DP &= ~DP_LINK_TRAIN_MASK_CPT;
+
+                switch (dp_train_pat & DP_TRAINING_PATTERN_MASK) {
+                case DP_TRAINING_PATTERN_DISABLE:
+                        *DP |= DP_LINK_TRAIN_OFF_CPT;
+                        break;
+                case DP_TRAINING_PATTERN_1:
+                        *DP |= DP_LINK_TRAIN_PAT_1_CPT;
+                        break;
+                case DP_TRAINING_PATTERN_2:
+                        *DP |= DP_LINK_TRAIN_PAT_2_CPT;
+                        break;
+                case DP_TRAINING_PATTERN_3:
+                        DRM_ERROR("DP training pattern 3 not supported\n");
+                        *DP |= DP_LINK_TRAIN_PAT_2_CPT;
+                        break;
+                }
+
+        } else {
+                if (IS_CHERRYVIEW(dev))
+                        *DP &= ~DP_LINK_TRAIN_MASK_CHV;
+                else
+                        *DP &= ~DP_LINK_TRAIN_MASK;
+
+                switch (dp_train_pat & DP_TRAINING_PATTERN_MASK) {
+                case DP_TRAINING_PATTERN_DISABLE:
+                        *DP |= DP_LINK_TRAIN_OFF;
+                        break;
+                case DP_TRAINING_PATTERN_1:
+                        *DP |= DP_LINK_TRAIN_PAT_1;
+                        break;
+                case DP_TRAINING_PATTERN_2:
+                        *DP |= DP_LINK_TRAIN_PAT_2;
+                        break;
+                case DP_TRAINING_PATTERN_3:
+                        if (IS_CHERRYVIEW(dev)) {
+                                *DP |= DP_LINK_TRAIN_PAT_3_CHV;
+                        } else {
+                                DRM_ERROR("DP training pattern 3 not supported\n");
+                                *DP |= DP_LINK_TRAIN_PAT_2;
+                        }
+                        break;
+                }
+        }
+}
+
+static void intel_dp_enable_port(struct intel_dp *intel_dp)
+{
+        struct drm_device *dev = intel_dp_to_dev(intel_dp);
+        struct drm_i915_private *dev_priv = dev->dev_private;
+
+        intel_dp->DP |= DP_PORT_EN;
+
+        /* enable with pattern 1 (as per spec) */
+        _intel_dp_set_link_train(intel_dp, &intel_dp->DP,
+                                 DP_TRAINING_PATTERN_1);
+
+        I915_WRITE(intel_dp->output_reg, intel_dp->DP);
+        POSTING_READ(intel_dp->output_reg);
+}
+
 static void intel_enable_dp(struct intel_encoder *encoder)
 {
         struct intel_dp *intel_dp = enc_to_intel_dp(&encoder->base);
@@ -2116,11 +2487,12 @@ static void intel_enable_dp(struct intel_encoder *encoder)
         if (WARN_ON(dp_reg & DP_PORT_EN))
                 return;
 
+        intel_dp_enable_port(intel_dp);
         intel_edp_panel_vdd_on(intel_dp);
+        intel_edp_panel_on(intel_dp);
+        intel_edp_panel_vdd_off(intel_dp, true);
         intel_dp_sink_dpms(intel_dp, DRM_MODE_DPMS_ON);
         intel_dp_start_link_train(intel_dp);
-        intel_edp_panel_on(intel_dp);
-        edp_panel_vdd_off(intel_dp, true);
         intel_dp_complete_link_train(intel_dp);
         intel_dp_stop_link_train(intel_dp);
 }
@@ -2154,6 +2526,78 @@ static void g4x_pre_enable_dp(struct intel_encoder *encoder)
         }
 }
 
+static void vlv_steal_power_sequencer(struct drm_device *dev,
+                                      enum pipe pipe)
+{
+        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct intel_encoder *encoder;
+
+        lockdep_assert_held(&dev_priv->pps_mutex);
+
+        list_for_each_entry(encoder, &dev->mode_config.encoder_list,
+                            base.head) {
+                struct intel_dp *intel_dp;
+                enum port port;
+
+                if (encoder->type != INTEL_OUTPUT_EDP)
+                        continue;
+
+                intel_dp = enc_to_intel_dp(&encoder->base);
+                port = dp_to_dig_port(intel_dp)->port;
+
+                if (intel_dp->pps_pipe != pipe)
+                        continue;
+
+                DRM_DEBUG_KMS("stealing pipe %c power sequencer from port %c\n",
+                              pipe_name(pipe), port_name(port));
+
+                /* make sure vdd is off before we steal it */
+                edp_panel_vdd_off_sync(intel_dp);
+
+                intel_dp->pps_pipe = INVALID_PIPE;
+        }
+}
+
+static void vlv_init_panel_power_sequencer(struct intel_dp *intel_dp)
+{
+        struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp);
+        struct intel_encoder *encoder = &intel_dig_port->base;
+        struct drm_device *dev = encoder->base.dev;
+        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct intel_crtc *crtc = to_intel_crtc(encoder->base.crtc);
+        struct edp_power_seq power_seq;
+
+        lockdep_assert_held(&dev_priv->pps_mutex);
+
+        if (intel_dp->pps_pipe == crtc->pipe)
+                return;
+
+        /*
+         * If another power sequencer was being used on this
+         * port previously make sure to turn off vdd there while
+         * we still have control of it.
+         */
+        if (intel_dp->pps_pipe != INVALID_PIPE)
+                edp_panel_vdd_off_sync(intel_dp);
+
+        /*
+         * We may be stealing the power
+         * sequencer from another port.
+         */
+        vlv_steal_power_sequencer(dev, crtc->pipe);
+
+        /* now it's all ours */
+        intel_dp->pps_pipe = crtc->pipe;
+
+        DRM_DEBUG_KMS("initializing pipe %c power sequencer for port %c\n",
+                      pipe_name(intel_dp->pps_pipe), port_name(intel_dig_port->port));
+
+        /* init power sequencer on this pipe and port */
+        intel_dp_init_panel_power_sequencer(dev, intel_dp, &power_seq);
+        intel_dp_init_panel_power_sequencer_registers(dev, intel_dp,
+                                                      &power_seq);
+}
+
 static void vlv_pre_enable_dp(struct intel_encoder *encoder)
 {
         struct intel_dp *intel_dp = enc_to_intel_dp(&encoder->base);
@@ -2163,7 +2607,6 @@ static void vlv_pre_enable_dp(struct intel_encoder *encoder)
         struct intel_crtc *intel_crtc = to_intel_crtc(encoder->base.crtc);
         enum dpio_channel port = vlv_dport_to_channel(dport);
         int pipe = intel_crtc->pipe;
-        struct edp_power_seq power_seq;
         u32 val;
 
         mutex_lock(&dev_priv->dpio_lock);
@@ -2182,10 +2625,9 @@ static void vlv_pre_enable_dp(struct intel_encoder *encoder)
         mutex_unlock(&dev_priv->dpio_lock);
 
         if (is_edp(intel_dp)) {
-                /* init power sequencer on this pipe and port */
-                intel_dp_init_panel_power_sequencer(dev, intel_dp, &power_seq);
-                intel_dp_init_panel_power_sequencer_registers(dev, intel_dp,
-                                                              &power_seq);
+                pps_lock(intel_dp);
+                vlv_init_panel_power_sequencer(intel_dp);
+                pps_unlock(intel_dp);
         }
 
         intel_enable_dp(encoder);
@@ -2229,7 +2671,6 @@ static void chv_pre_enable_dp(struct intel_encoder *encoder)
         struct intel_digital_port *dport = dp_to_dig_port(intel_dp);
         struct drm_device *dev = encoder->base.dev;
         struct drm_i915_private *dev_priv = dev->dev_private;
-        struct edp_power_seq power_seq;
         struct intel_crtc *intel_crtc =
                 to_intel_crtc(encoder->base.crtc);
         enum dpio_channel ch = vlv_dport_to_channel(dport);
@@ -2275,10 +2716,9 @@ static void chv_pre_enable_dp(struct intel_encoder *encoder)
         mutex_unlock(&dev_priv->dpio_lock);
 
         if (is_edp(intel_dp)) {
-                /* init power sequencer on this pipe and port */
-                intel_dp_init_panel_power_sequencer(dev, intel_dp, &power_seq);
-                intel_dp_init_panel_power_sequencer_registers(dev, intel_dp,
-                                                              &power_seq);
+                pps_lock(intel_dp);
+                vlv_init_panel_power_sequencer(intel_dp);
+                pps_unlock(intel_dp);
         }
 
         intel_enable_dp(encoder);
@@ -2297,6 +2737,8 @@ static void chv_dp_pre_pll_enable(struct intel_encoder *encoder)
         enum pipe pipe = intel_crtc->pipe;
         u32 val;
 
+        intel_dp_prepare(encoder);
+
         mutex_lock(&dev_priv->dpio_lock);
 
         /* program left/right clock distribution */
@@ -2395,13 +2837,13 @@ intel_dp_voltage_max(struct intel_dp *intel_dp)
         enum port port = dp_to_dig_port(intel_dp)->port;
 
         if (IS_VALLEYVIEW(dev))
-                return DP_TRAIN_VOLTAGE_SWING_1200;
+                return DP_TRAIN_VOLTAGE_SWING_LEVEL_3;
         else if (IS_GEN7(dev) && port == PORT_A)
-                return DP_TRAIN_VOLTAGE_SWING_800;
+                return DP_TRAIN_VOLTAGE_SWING_LEVEL_2;
         else if (HAS_PCH_CPT(dev) && port != PORT_A)
-                return DP_TRAIN_VOLTAGE_SWING_1200;
+                return DP_TRAIN_VOLTAGE_SWING_LEVEL_3;
         else
-                return DP_TRAIN_VOLTAGE_SWING_800;
+                return DP_TRAIN_VOLTAGE_SWING_LEVEL_2;
 }
 
 static uint8_t
@@ -2412,49 +2854,49 @@ intel_dp_pre_emphasis_max(struct intel_dp *intel_dp, uint8_t voltage_swing)
 
         if (IS_HASWELL(dev) || IS_BROADWELL(dev)) {
                 switch (voltage_swing & DP_TRAIN_VOLTAGE_SWING_MASK) {
-                case DP_TRAIN_VOLTAGE_SWING_400:
-                        return DP_TRAIN_PRE_EMPHASIS_9_5;
-                case DP_TRAIN_VOLTAGE_SWING_600:
-                        return DP_TRAIN_PRE_EMPHASIS_6;
-                case DP_TRAIN_VOLTAGE_SWING_800:
-                        return DP_TRAIN_PRE_EMPHASIS_3_5;
-                case DP_TRAIN_VOLTAGE_SWING_1200:
+                case DP_TRAIN_VOLTAGE_SWING_LEVEL_0:
+                        return DP_TRAIN_PRE_EMPH_LEVEL_3;
+                case DP_TRAIN_VOLTAGE_SWING_LEVEL_1:
+                        return DP_TRAIN_PRE_EMPH_LEVEL_2;
+                case DP_TRAIN_VOLTAGE_SWING_LEVEL_2:
+                        return DP_TRAIN_PRE_EMPH_LEVEL_1;
+                case DP_TRAIN_VOLTAGE_SWING_LEVEL_3:
                 default:
-                        return DP_TRAIN_PRE_EMPHASIS_0;
+                        return DP_TRAIN_PRE_EMPH_LEVEL_0;
                 }
         } else if (IS_VALLEYVIEW(dev)) {
                 switch (voltage_swing & DP_TRAIN_VOLTAGE_SWING_MASK) {
-                case DP_TRAIN_VOLTAGE_SWING_400:
-                        return DP_TRAIN_PRE_EMPHASIS_9_5;
-                case DP_TRAIN_VOLTAGE_SWING_600:
-                        return DP_TRAIN_PRE_EMPHASIS_6;
-                case DP_TRAIN_VOLTAGE_SWING_800:
-                        return DP_TRAIN_PRE_EMPHASIS_3_5;
-                case DP_TRAIN_VOLTAGE_SWING_1200:
+                case DP_TRAIN_VOLTAGE_SWING_LEVEL_0:
+                        return DP_TRAIN_PRE_EMPH_LEVEL_3;
+                case DP_TRAIN_VOLTAGE_SWING_LEVEL_1:
+                        return DP_TRAIN_PRE_EMPH_LEVEL_2;
+                case DP_TRAIN_VOLTAGE_SWING_LEVEL_2:
+                        return DP_TRAIN_PRE_EMPH_LEVEL_1;
+                case DP_TRAIN_VOLTAGE_SWING_LEVEL_3:
                 default:
-                        return DP_TRAIN_PRE_EMPHASIS_0;
+                        return DP_TRAIN_PRE_EMPH_LEVEL_0;
                 }
         } else if (IS_GEN7(dev) && port == PORT_A) {
                 switch (voltage_swing & DP_TRAIN_VOLTAGE_SWING_MASK) {
-                case DP_TRAIN_VOLTAGE_SWING_400:
-                        return DP_TRAIN_PRE_EMPHASIS_6;
-                case DP_TRAIN_VOLTAGE_SWING_600:
-                case DP_TRAIN_VOLTAGE_SWING_800:
-                        return DP_TRAIN_PRE_EMPHASIS_3_5;
+                case DP_TRAIN_VOLTAGE_SWING_LEVEL_0:
+                        return DP_TRAIN_PRE_EMPH_LEVEL_2;
+                case DP_TRAIN_VOLTAGE_SWING_LEVEL_1:
+                case DP_TRAIN_VOLTAGE_SWING_LEVEL_2:
+                        return DP_TRAIN_PRE_EMPH_LEVEL_1;
                 default:
-                        return DP_TRAIN_PRE_EMPHASIS_0;
+                        return DP_TRAIN_PRE_EMPH_LEVEL_0;
                 }
         } else {
                 switch (voltage_swing & DP_TRAIN_VOLTAGE_SWING_MASK) {
-                case DP_TRAIN_VOLTAGE_SWING_400:
-                        return DP_TRAIN_PRE_EMPHASIS_6;
-                case DP_TRAIN_VOLTAGE_SWING_600:
-                        return DP_TRAIN_PRE_EMPHASIS_6;
-                case DP_TRAIN_VOLTAGE_SWING_800:
-                        return DP_TRAIN_PRE_EMPHASIS_3_5;
-                case DP_TRAIN_VOLTAGE_SWING_1200:
+                case DP_TRAIN_VOLTAGE_SWING_LEVEL_0:
+                        return DP_TRAIN_PRE_EMPH_LEVEL_2;
+                case DP_TRAIN_VOLTAGE_SWING_LEVEL_1:
+                        return DP_TRAIN_PRE_EMPH_LEVEL_2;
+                case DP_TRAIN_VOLTAGE_SWING_LEVEL_2:
+                        return DP_TRAIN_PRE_EMPH_LEVEL_1;
+                case DP_TRAIN_VOLTAGE_SWING_LEVEL_3:
                 default:
-                        return DP_TRAIN_PRE_EMPHASIS_0;
+                        return DP_TRAIN_PRE_EMPH_LEVEL_0;
                 }
         }
 }
@@ -2473,22 +2915,22 @@ static uint32_t intel_vlv_signal_levels(struct intel_dp *intel_dp)
         int pipe = intel_crtc->pipe;
 
         switch (train_set & DP_TRAIN_PRE_EMPHASIS_MASK) {
-        case DP_TRAIN_PRE_EMPHASIS_0:
+        case DP_TRAIN_PRE_EMPH_LEVEL_0:
                 preemph_reg_value = 0x0004000;
                 switch (train_set & DP_TRAIN_VOLTAGE_SWING_MASK) {
-                case DP_TRAIN_VOLTAGE_SWING_400:
+                case DP_TRAIN_VOLTAGE_SWING_LEVEL_0:
                         demph_reg_value = 0x2B405555;
                         uniqtranscale_reg_value = 0x552AB83A;
                         break;
-                case DP_TRAIN_VOLTAGE_SWING_600:
+                case DP_TRAIN_VOLTAGE_SWING_LEVEL_1:
                         demph_reg_value = 0x2B404040;
                         uniqtranscale_reg_value = 0x5548B83A;
                         break;
-                case DP_TRAIN_VOLTAGE_SWING_800:
+                case DP_TRAIN_VOLTAGE_SWING_LEVEL_2:
                         demph_reg_value = 0x2B245555;
                         uniqtranscale_reg_value = 0x5560B83A;
                         break;
-                case DP_TRAIN_VOLTAGE_SWING_1200:
+                case DP_TRAIN_VOLTAGE_SWING_LEVEL_3:
                         demph_reg_value = 0x2B405555;
                         uniqtranscale_reg_value = 0x5598DA3A;
                         break;
@@ -2496,18 +2938,18 @@ static uint32_t intel_vlv_signal_levels(struct intel_dp *intel_dp)
                         return 0;
                 }
                 break;
-        case DP_TRAIN_PRE_EMPHASIS_3_5:
+        case DP_TRAIN_PRE_EMPH_LEVEL_1:
                 preemph_reg_value = 0x0002000;
                 switch (train_set & DP_TRAIN_VOLTAGE_SWING_MASK) {
-                case DP_TRAIN_VOLTAGE_SWING_400:
+                case DP_TRAIN_VOLTAGE_SWING_LEVEL_0:
                         demph_reg_value = 0x2B404040;
                         uniqtranscale_reg_value = 0x5552B83A;
                         break;
-                case DP_TRAIN_VOLTAGE_SWING_600:
+                case DP_TRAIN_VOLTAGE_SWING_LEVEL_1:
                         demph_reg_value = 0x2B404848;
                         uniqtranscale_reg_value = 0x5580B83A;
                         break;
-                case DP_TRAIN_VOLTAGE_SWING_800:
+                case DP_TRAIN_VOLTAGE_SWING_LEVEL_2:
                         demph_reg_value = 0x2B404040;
                         uniqtranscale_reg_value = 0x55ADDA3A;
                         break;
@@ -2515,14 +2957,14 @@ static uint32_t intel_vlv_signal_levels(struct intel_dp *intel_dp)
                         return 0;
                 }
                 break;
-        case DP_TRAIN_PRE_EMPHASIS_6:
+        case DP_TRAIN_PRE_EMPH_LEVEL_2:
                 preemph_reg_value = 0x0000000;
                 switch (train_set & DP_TRAIN_VOLTAGE_SWING_MASK) {
-                case DP_TRAIN_VOLTAGE_SWING_400:
+                case DP_TRAIN_VOLTAGE_SWING_LEVEL_0:
                         demph_reg_value = 0x2B305555;
                         uniqtranscale_reg_value = 0x5570B83A;
                         break;
-                case DP_TRAIN_VOLTAGE_SWING_600:
+                case DP_TRAIN_VOLTAGE_SWING_LEVEL_1:
                         demph_reg_value = 0x2B2B4040;
                         uniqtranscale_reg_value = 0x55ADDA3A;
                         break;
@@ -2530,10 +2972,10 @@ static uint32_t intel_vlv_signal_levels(struct intel_dp *intel_dp)
                         return 0;
                 }
                 break;
-        case DP_TRAIN_PRE_EMPHASIS_9_5:
+        case DP_TRAIN_PRE_EMPH_LEVEL_3:
                 preemph_reg_value = 0x0006000;
                 switch (train_set & DP_TRAIN_VOLTAGE_SWING_MASK) {
-                case DP_TRAIN_VOLTAGE_SWING_400:
+                case DP_TRAIN_VOLTAGE_SWING_LEVEL_0:
                         demph_reg_value = 0x1B405555;
                         uniqtranscale_reg_value = 0x55ADDA3A;
                         break;
@@ -2572,21 +3014,21 @@ static uint32_t intel_chv_signal_levels(struct intel_dp *intel_dp)
         int i;
 
         switch (train_set & DP_TRAIN_PRE_EMPHASIS_MASK) {
-        case DP_TRAIN_PRE_EMPHASIS_0:
+        case DP_TRAIN_PRE_EMPH_LEVEL_0:
                 switch (train_set & DP_TRAIN_VOLTAGE_SWING_MASK) {
-                case DP_TRAIN_VOLTAGE_SWING_400:
+                case DP_TRAIN_VOLTAGE_SWING_LEVEL_0:
                         deemph_reg_value = 128;
                         margin_reg_value = 52;
                         break;
-                case DP_TRAIN_VOLTAGE_SWING_600:
+                case DP_TRAIN_VOLTAGE_SWING_LEVEL_1:
                         deemph_reg_value = 128;
                         margin_reg_value = 77;
                         break;
-                case DP_TRAIN_VOLTAGE_SWING_800:
+                case DP_TRAIN_VOLTAGE_SWING_LEVEL_2:
                         deemph_reg_value = 128;
                         margin_reg_value = 102;
                         break;
-                case DP_TRAIN_VOLTAGE_SWING_1200:
+                case DP_TRAIN_VOLTAGE_SWING_LEVEL_3:
                         deemph_reg_value = 128;
                         margin_reg_value = 154;
                         /* FIXME extra to set for 1200 */
@@ -2595,17 +3037,17 @@ static uint32_t intel_chv_signal_levels(struct intel_dp *intel_dp)
                         return 0;
                 }
                 break;
-        case DP_TRAIN_PRE_EMPHASIS_3_5:
+        case DP_TRAIN_PRE_EMPH_LEVEL_1:
                 switch (train_set & DP_TRAIN_VOLTAGE_SWING_MASK) {
-                case DP_TRAIN_VOLTAGE_SWING_400:
+                case DP_TRAIN_VOLTAGE_SWING_LEVEL_0:
                         deemph_reg_value = 85;
                         margin_reg_value = 78;
                         break;
-                case DP_TRAIN_VOLTAGE_SWING_600:
+                case DP_TRAIN_VOLTAGE_SWING_LEVEL_1:
                         deemph_reg_value = 85;
                         margin_reg_value = 116;
                         break;
-                case DP_TRAIN_VOLTAGE_SWING_800:
+                case DP_TRAIN_VOLTAGE_SWING_LEVEL_2:
                         deemph_reg_value = 85;
                         margin_reg_value = 154;
                         break;
@@ -2613,13 +3055,13 @@ static uint32_t intel_chv_signal_levels(struct intel_dp *intel_dp)
                         return 0;
                 }
                 break;
-        case DP_TRAIN_PRE_EMPHASIS_6:
+        case DP_TRAIN_PRE_EMPH_LEVEL_2:
                 switch (train_set & DP_TRAIN_VOLTAGE_SWING_MASK) {
-                case DP_TRAIN_VOLTAGE_SWING_400:
+                case DP_TRAIN_VOLTAGE_SWING_LEVEL_0:
                         deemph_reg_value = 64;
                         margin_reg_value = 104;
                         break;
-                case DP_TRAIN_VOLTAGE_SWING_600:
+                case DP_TRAIN_VOLTAGE_SWING_LEVEL_1:
                         deemph_reg_value = 64;
                         margin_reg_value = 154;
                         break;
@@ -2627,9 +3069,9 @@ static uint32_t intel_chv_signal_levels(struct intel_dp *intel_dp)
                         return 0;
                 }
                 break;
-        case DP_TRAIN_PRE_EMPHASIS_9_5:
+        case DP_TRAIN_PRE_EMPH_LEVEL_3:
                 switch (train_set & DP_TRAIN_VOLTAGE_SWING_MASK) {
-                case DP_TRAIN_VOLTAGE_SWING_400:
+                case DP_TRAIN_VOLTAGE_SWING_LEVEL_0:
                         deemph_reg_value = 43;
                         margin_reg_value = 154;
                         break;
@@ -2663,8 +3105,8 @@ static uint32_t intel_chv_signal_levels(struct intel_dp *intel_dp)
         /* Program swing margin */
         for (i = 0; i < 4; i++) {
                 val = vlv_dpio_read(dev_priv, pipe, CHV_TX_DW2(ch, i));
-                val &= ~DPIO_SWING_MARGIN_MASK;
-                val |= margin_reg_value << DPIO_SWING_MARGIN_SHIFT;
+                val &= ~DPIO_SWING_MARGIN000_MASK;
+                val |= margin_reg_value << DPIO_SWING_MARGIN000_SHIFT;
                 vlv_dpio_write(dev_priv, pipe, CHV_TX_DW2(ch, i), val);
         }
 
@@ -2676,9 +3118,9 @@ static uint32_t intel_chv_signal_levels(struct intel_dp *intel_dp)
         }
 
         if (((train_set & DP_TRAIN_PRE_EMPHASIS_MASK)
-                        == DP_TRAIN_PRE_EMPHASIS_0) &&
+                        == DP_TRAIN_PRE_EMPH_LEVEL_0) &&
                 ((train_set & DP_TRAIN_VOLTAGE_SWING_MASK)
-                        == DP_TRAIN_VOLTAGE_SWING_1200)) {
+                        == DP_TRAIN_VOLTAGE_SWING_LEVEL_3)) {
 
                 /*
                  * The document said it needs to set bit 27 for ch0 and bit 26
@@ -2757,32 +3199,32 @@ intel_gen4_signal_levels(uint8_t train_set)
         uint32_t        signal_levels = 0;
 
         switch (train_set & DP_TRAIN_VOLTAGE_SWING_MASK) {
-        case DP_TRAIN_VOLTAGE_SWING_400:
+        case DP_TRAIN_VOLTAGE_SWING_LEVEL_0:
         default:
                 signal_levels |= DP_VOLTAGE_0_4;
                 break;
-        case DP_TRAIN_VOLTAGE_SWING_600:
+        case DP_TRAIN_VOLTAGE_SWING_LEVEL_1:
                 signal_levels |= DP_VOLTAGE_0_6;
                 break;
-        case DP_TRAIN_VOLTAGE_SWING_800:
+        case DP_TRAIN_VOLTAGE_SWING_LEVEL_2:
                 signal_levels |= DP_VOLTAGE_0_8;
                 break;
-        case DP_TRAIN_VOLTAGE_SWING_1200:
+        case DP_TRAIN_VOLTAGE_SWING_LEVEL_3:
                 signal_levels |= DP_VOLTAGE_1_2;
                 break;
         }
         switch (train_set & DP_TRAIN_PRE_EMPHASIS_MASK) {
-        case DP_TRAIN_PRE_EMPHASIS_0:
+        case DP_TRAIN_PRE_EMPH_LEVEL_0:
         default:
                 signal_levels |= DP_PRE_EMPHASIS_0;
                 break;
-        case DP_TRAIN_PRE_EMPHASIS_3_5:
+        case DP_TRAIN_PRE_EMPH_LEVEL_1:
                 signal_levels |= DP_PRE_EMPHASIS_3_5;
                 break;
-        case DP_TRAIN_PRE_EMPHASIS_6:
+        case DP_TRAIN_PRE_EMPH_LEVEL_2:
                 signal_levels |= DP_PRE_EMPHASIS_6;
                 break;
-        case DP_TRAIN_PRE_EMPHASIS_9_5:
+        case DP_TRAIN_PRE_EMPH_LEVEL_3:
                 signal_levels |= DP_PRE_EMPHASIS_9_5;
                 break;
         }
@@ -2796,19 +3238,19 @@ intel_gen6_edp_signal_levels(uint8_t train_set)
         int signal_levels = train_set & (DP_TRAIN_VOLTAGE_SWING_MASK |
                                          DP_TRAIN_PRE_EMPHASIS_MASK);
         switch (signal_levels) {
-        case DP_TRAIN_VOLTAGE_SWING_400 | DP_TRAIN_PRE_EMPHASIS_0:
-        case DP_TRAIN_VOLTAGE_SWING_600 | DP_TRAIN_PRE_EMPHASIS_0:
+        case DP_TRAIN_VOLTAGE_SWING_LEVEL_0 | DP_TRAIN_PRE_EMPH_LEVEL_0:
+        case DP_TRAIN_VOLTAGE_SWING_LEVEL_1 | DP_TRAIN_PRE_EMPH_LEVEL_0:
                 return EDP_LINK_TRAIN_400_600MV_0DB_SNB_B;
-        case DP_TRAIN_VOLTAGE_SWING_400 | DP_TRAIN_PRE_EMPHASIS_3_5:
+        case DP_TRAIN_VOLTAGE_SWING_LEVEL_0 | DP_TRAIN_PRE_EMPH_LEVEL_1:
                 return EDP_LINK_TRAIN_400MV_3_5DB_SNB_B;
-        case DP_TRAIN_VOLTAGE_SWING_400 | DP_TRAIN_PRE_EMPHASIS_6:
-        case DP_TRAIN_VOLTAGE_SWING_600 | DP_TRAIN_PRE_EMPHASIS_6:
+        case DP_TRAIN_VOLTAGE_SWING_LEVEL_0 | DP_TRAIN_PRE_EMPH_LEVEL_2:
+        case DP_TRAIN_VOLTAGE_SWING_LEVEL_1 | DP_TRAIN_PRE_EMPH_LEVEL_2:
                 return EDP_LINK_TRAIN_400_600MV_6DB_SNB_B;
-        case DP_TRAIN_VOLTAGE_SWING_600 | DP_TRAIN_PRE_EMPHASIS_3_5:
-        case DP_TRAIN_VOLTAGE_SWING_800 | DP_TRAIN_PRE_EMPHASIS_3_5:
+        case DP_TRAIN_VOLTAGE_SWING_LEVEL_1 | DP_TRAIN_PRE_EMPH_LEVEL_1:
+        case DP_TRAIN_VOLTAGE_SWING_LEVEL_2 | DP_TRAIN_PRE_EMPH_LEVEL_1:
                 return EDP_LINK_TRAIN_600_800MV_3_5DB_SNB_B;
-        case DP_TRAIN_VOLTAGE_SWING_800 | DP_TRAIN_PRE_EMPHASIS_0:
-        case DP_TRAIN_VOLTAGE_SWING_1200 | DP_TRAIN_PRE_EMPHASIS_0:
+        case DP_TRAIN_VOLTAGE_SWING_LEVEL_2 | DP_TRAIN_PRE_EMPH_LEVEL_0:
+        case DP_TRAIN_VOLTAGE_SWING_LEVEL_3 | DP_TRAIN_PRE_EMPH_LEVEL_0:
                 return EDP_LINK_TRAIN_800_1200MV_0DB_SNB_B;
         default:
                 DRM_DEBUG_KMS("Unsupported voltage swing/pre-emphasis level:"
@@ -2824,21 +3266,21 @@ intel_gen7_edp_signal_levels(uint8_t train_set)
         int signal_levels = train_set & (DP_TRAIN_VOLTAGE_SWING_MASK |
                                          DP_TRAIN_PRE_EMPHASIS_MASK);
         switch (signal_levels) {
-        case DP_TRAIN_VOLTAGE_SWING_400 | DP_TRAIN_PRE_EMPHASIS_0:
+        case DP_TRAIN_VOLTAGE_SWING_LEVEL_0 | DP_TRAIN_PRE_EMPH_LEVEL_0:
                 return EDP_LINK_TRAIN_400MV_0DB_IVB;
-        case DP_TRAIN_VOLTAGE_SWING_400 | DP_TRAIN_PRE_EMPHASIS_3_5:
+        case DP_TRAIN_VOLTAGE_SWING_LEVEL_0 | DP_TRAIN_PRE_EMPH_LEVEL_1:
                 return EDP_LINK_TRAIN_400MV_3_5DB_IVB;
-        case DP_TRAIN_VOLTAGE_SWING_400 | DP_TRAIN_PRE_EMPHASIS_6:
+        case DP_TRAIN_VOLTAGE_SWING_LEVEL_0 | DP_TRAIN_PRE_EMPH_LEVEL_2:
                 return EDP_LINK_TRAIN_400MV_6DB_IVB;
 
-        case DP_TRAIN_VOLTAGE_SWING_600 | DP_TRAIN_PRE_EMPHASIS_0:
+        case DP_TRAIN_VOLTAGE_SWING_LEVEL_1 | DP_TRAIN_PRE_EMPH_LEVEL_0:
                 return EDP_LINK_TRAIN_600MV_0DB_IVB;
-        case DP_TRAIN_VOLTAGE_SWING_600 | DP_TRAIN_PRE_EMPHASIS_3_5:
+        case DP_TRAIN_VOLTAGE_SWING_LEVEL_1 | DP_TRAIN_PRE_EMPH_LEVEL_1:
                 return EDP_LINK_TRAIN_600MV_3_5DB_IVB;
 
-        case DP_TRAIN_VOLTAGE_SWING_800 | DP_TRAIN_PRE_EMPHASIS_0:
+        case DP_TRAIN_VOLTAGE_SWING_LEVEL_2 | DP_TRAIN_PRE_EMPH_LEVEL_0:
                 return EDP_LINK_TRAIN_800MV_0DB_IVB;
-        case DP_TRAIN_VOLTAGE_SWING_800 | DP_TRAIN_PRE_EMPHASIS_3_5:
+        case DP_TRAIN_VOLTAGE_SWING_LEVEL_2 | DP_TRAIN_PRE_EMPH_LEVEL_1:
                 return EDP_LINK_TRAIN_800MV_3_5DB_IVB;
 
         default:
@@ -2855,30 +3297,30 @@ intel_hsw_signal_levels(uint8_t train_set)
         int signal_levels = train_set & (DP_TRAIN_VOLTAGE_SWING_MASK |
                                          DP_TRAIN_PRE_EMPHASIS_MASK);
         switch (signal_levels) {
-        case DP_TRAIN_VOLTAGE_SWING_400 | DP_TRAIN_PRE_EMPHASIS_0:
-                return DDI_BUF_EMP_400MV_0DB_HSW;
-        case DP_TRAIN_VOLTAGE_SWING_400 | DP_TRAIN_PRE_EMPHASIS_3_5:
-                return DDI_BUF_EMP_400MV_3_5DB_HSW;
-        case DP_TRAIN_VOLTAGE_SWING_400 | DP_TRAIN_PRE_EMPHASIS_6:
-                return DDI_BUF_EMP_400MV_6DB_HSW;
-        case DP_TRAIN_VOLTAGE_SWING_400 | DP_TRAIN_PRE_EMPHASIS_9_5:
-                return DDI_BUF_EMP_400MV_9_5DB_HSW;
-
-        case DP_TRAIN_VOLTAGE_SWING_600 | DP_TRAIN_PRE_EMPHASIS_0:
-                return DDI_BUF_EMP_600MV_0DB_HSW;
-        case DP_TRAIN_VOLTAGE_SWING_600 | DP_TRAIN_PRE_EMPHASIS_3_5:
-                return DDI_BUF_EMP_600MV_3_5DB_HSW;
-        case DP_TRAIN_VOLTAGE_SWING_600 | DP_TRAIN_PRE_EMPHASIS_6:
-                return DDI_BUF_EMP_600MV_6DB_HSW;
-
-        case DP_TRAIN_VOLTAGE_SWING_800 | DP_TRAIN_PRE_EMPHASIS_0:
-                return DDI_BUF_EMP_800MV_0DB_HSW;
-        case DP_TRAIN_VOLTAGE_SWING_800 | DP_TRAIN_PRE_EMPHASIS_3_5:
-                return DDI_BUF_EMP_800MV_3_5DB_HSW;
+        case DP_TRAIN_VOLTAGE_SWING_LEVEL_0 | DP_TRAIN_PRE_EMPH_LEVEL_0:
+                return DDI_BUF_TRANS_SELECT(0);
+        case DP_TRAIN_VOLTAGE_SWING_LEVEL_0 | DP_TRAIN_PRE_EMPH_LEVEL_1:
+                return DDI_BUF_TRANS_SELECT(1);
+        case DP_TRAIN_VOLTAGE_SWING_LEVEL_0 | DP_TRAIN_PRE_EMPH_LEVEL_2:
+                return DDI_BUF_TRANS_SELECT(2);
+        case DP_TRAIN_VOLTAGE_SWING_LEVEL_0 | DP_TRAIN_PRE_EMPH_LEVEL_3:
+                return DDI_BUF_TRANS_SELECT(3);
+
+        case DP_TRAIN_VOLTAGE_SWING_LEVEL_1 | DP_TRAIN_PRE_EMPH_LEVEL_0:
+                return DDI_BUF_TRANS_SELECT(4);
+        case DP_TRAIN_VOLTAGE_SWING_LEVEL_1 | DP_TRAIN_PRE_EMPH_LEVEL_1:
+                return DDI_BUF_TRANS_SELECT(5);
+        case DP_TRAIN_VOLTAGE_SWING_LEVEL_1 | DP_TRAIN_PRE_EMPH_LEVEL_2:
+                return DDI_BUF_TRANS_SELECT(6);
+
+        case DP_TRAIN_VOLTAGE_SWING_LEVEL_2 | DP_TRAIN_PRE_EMPH_LEVEL_0:
+                return DDI_BUF_TRANS_SELECT(7);
+        case DP_TRAIN_VOLTAGE_SWING_LEVEL_2 | DP_TRAIN_PRE_EMPH_LEVEL_1:
+                return DDI_BUF_TRANS_SELECT(8);
         default:
                 DRM_DEBUG_KMS("Unsupported voltage swing/pre-emphasis level:"
                               "0x%x\n", signal_levels);
-                return DDI_BUF_EMP_400MV_0DB_HSW;
+                return DDI_BUF_TRANS_SELECT(0);
         }
 }
 
@@ -2925,74 +3367,10 @@ intel_dp_set_link_train(struct intel_dp *intel_dp,
         struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp);
         struct drm_device *dev = intel_dig_port->base.base.dev;
         struct drm_i915_private *dev_priv = dev->dev_private;
-        enum port port = intel_dig_port->port;
         uint8_t buf[sizeof(intel_dp->train_set) + 1];
         int ret, len;
 
-        if (HAS_DDI(dev)) {
-                uint32_t temp = I915_READ(DP_TP_CTL(port));
-
-                if (dp_train_pat & DP_LINK_SCRAMBLING_DISABLE)
-                        temp |= DP_TP_CTL_SCRAMBLE_DISABLE;
-                else
-                        temp &= ~DP_TP_CTL_SCRAMBLE_DISABLE;
-
-                temp &= ~DP_TP_CTL_LINK_TRAIN_MASK;
-                switch (dp_train_pat & DP_TRAINING_PATTERN_MASK) {
-                case DP_TRAINING_PATTERN_DISABLE:
-                        temp |= DP_TP_CTL_LINK_TRAIN_NORMAL;
-
-                        break;
-                case DP_TRAINING_PATTERN_1:
-                        temp |= DP_TP_CTL_LINK_TRAIN_PAT1;
-                        break;
-                case DP_TRAINING_PATTERN_2:
-                        temp |= DP_TP_CTL_LINK_TRAIN_PAT2;
-                        break;
-                case DP_TRAINING_PATTERN_3:
-                        temp |= DP_TP_CTL_LINK_TRAIN_PAT3;
-                        break;
-                }
-                I915_WRITE(DP_TP_CTL(port), temp);
-
-        } else if (HAS_PCH_CPT(dev) && (IS_GEN7(dev) || port != PORT_A)) {
-                *DP &= ~DP_LINK_TRAIN_MASK_CPT;
-
-                switch (dp_train_pat & DP_TRAINING_PATTERN_MASK) {
-                case DP_TRAINING_PATTERN_DISABLE:
-                        *DP |= DP_LINK_TRAIN_OFF_CPT;
-                        break;
-                case DP_TRAINING_PATTERN_1:
-                        *DP |= DP_LINK_TRAIN_PAT_1_CPT;
-                        break;
-                case DP_TRAINING_PATTERN_2:
-                        *DP |= DP_LINK_TRAIN_PAT_2_CPT;
-                        break;
-                case DP_TRAINING_PATTERN_3:
-                        DRM_ERROR("DP training pattern 3 not supported\n");
-                        *DP |= DP_LINK_TRAIN_PAT_2_CPT;
-                        break;
-                }
-
-        } else {
-                *DP &= ~DP_LINK_TRAIN_MASK;
-
-                switch (dp_train_pat & DP_TRAINING_PATTERN_MASK) {
-                case DP_TRAINING_PATTERN_DISABLE:
-                        *DP |= DP_LINK_TRAIN_OFF;
-                        break;
-                case DP_TRAINING_PATTERN_1:
-                        *DP |= DP_LINK_TRAIN_PAT_1;
-                        break;
-                case DP_TRAINING_PATTERN_2:
-                        *DP |= DP_LINK_TRAIN_PAT_2;
-                        break;
-                case DP_TRAINING_PATTERN_3:
-                        DRM_ERROR("DP training pattern 3 not supported\n");
-                        *DP |= DP_LINK_TRAIN_PAT_2;
-                        break;
-                }
-        }
+        _intel_dp_set_link_train(intel_dp, DP, dp_train_pat);
 
         I915_WRITE(intel_dp->output_reg, *DP);
         POSTING_READ(intel_dp->output_reg);
@@ -3276,7 +3654,10 @@ intel_dp_link_down(struct intel_dp *intel_dp)
                 DP &= ~DP_LINK_TRAIN_MASK_CPT;
                 I915_WRITE(intel_dp->output_reg, DP | DP_LINK_TRAIN_PAT_IDLE_CPT);
         } else {
-                DP &= ~DP_LINK_TRAIN_MASK;
+                if (IS_CHERRYVIEW(dev))
+                        DP &= ~DP_LINK_TRAIN_MASK_CHV;
+                else
+                        DP &= ~DP_LINK_TRAIN_MASK;
                 I915_WRITE(intel_dp->output_reg, DP | DP_LINK_TRAIN_PAT_IDLE);
         }
         POSTING_READ(intel_dp->output_reg);
@@ -3322,15 +3703,11 @@ intel_dp_get_dpcd(struct intel_dp *intel_dp)
         struct drm_device *dev = dig_port->base.base.dev;
         struct drm_i915_private *dev_priv = dev->dev_private;
 
-        char dpcd_hex_dump[sizeof(intel_dp->dpcd) * 3];
-
         if (intel_dp_dpcd_read_wake(&intel_dp->aux, 0x000, intel_dp->dpcd,
                                     sizeof(intel_dp->dpcd)) < 0)
                 return false; /* aux transfer failed */
 
-        hex_dump_to_buffer(intel_dp->dpcd, sizeof(intel_dp->dpcd),
-                           32, 1, dpcd_hex_dump, sizeof(dpcd_hex_dump), false);
-        DRM_DEBUG_KMS("DPCD: %s\n", dpcd_hex_dump);
+        DRM_DEBUG_KMS("DPCD: %*ph\n", (int) sizeof(intel_dp->dpcd), intel_dp->dpcd);
 
         if (intel_dp->dpcd[DP_DPCD_REV] == 0)
                 return false; /* DPCD not present */
@@ -3351,7 +3728,7 @@ intel_dp_get_dpcd(struct intel_dp *intel_dp)
         if (intel_dp->dpcd[DP_DPCD_REV] >= 0x12 &&
             intel_dp->dpcd[DP_MAX_LANE_COUNT] & DP_TPS3_SUPPORTED) {
                 intel_dp->use_tps3 = true;
-                DRM_DEBUG_KMS("Displayport TPS3 supported");
+                DRM_DEBUG_KMS("Displayport TPS3 supported\n");
         } else
                 intel_dp->use_tps3 = false;
 
@@ -3388,7 +3765,7 @@ intel_dp_probe_oui(struct intel_dp *intel_dp)
                 DRM_DEBUG_KMS("Branch OUI: %02hx%02hx%02hx\n",
                               buf[0], buf[1], buf[2]);
 
-        edp_panel_vdd_off(intel_dp, false);
+        intel_edp_panel_vdd_off(intel_dp, false);
 }
 
 static bool
@@ -3402,7 +3779,7 @@ intel_dp_probe_mst(struct intel_dp *intel_dp)
         if (intel_dp->dpcd[DP_DPCD_REV] < 0x12)
                 return false;
 
-        _edp_panel_vdd_on(intel_dp);
+        intel_edp_panel_vdd_on(intel_dp);
         if (intel_dp_dpcd_read_wake(&intel_dp->aux, DP_MSTM_CAP, buf, 1)) {
                 if (buf[0] & DP_MST_CAP) {
                         DRM_DEBUG_KMS("Sink is MST capable\n");
@@ -3412,7 +3789,7 @@ intel_dp_probe_mst(struct intel_dp *intel_dp)
                         intel_dp->is_mst = false;
                 }
         }
-        edp_panel_vdd_off(intel_dp, false);
+        intel_edp_panel_vdd_off(intel_dp, false);
 
         drm_dp_mst_topology_mgr_set_mst(&intel_dp->mst_mgr, intel_dp->is_mst);
         return intel_dp->is_mst;
@@ -3427,21 +3804,21 @@ int intel_dp_sink_crc(struct intel_dp *intel_dp, u8 *crc)
         u8 buf[1];
 
         if (drm_dp_dpcd_readb(&intel_dp->aux, DP_TEST_SINK_MISC, buf) < 0)
-                return -EAGAIN;
+                return -EIO;
 
         if (!(buf[0] & DP_TEST_CRC_SUPPORTED))
                 return -ENOTTY;
 
         if (drm_dp_dpcd_writeb(&intel_dp->aux, DP_TEST_SINK,
                                DP_TEST_SINK_START) < 0)
-                return -EAGAIN;
+                return -EIO;
 
         /* Wait 2 vblanks to be sure we will have the correct CRC value */
         intel_wait_for_vblank(dev, intel_crtc->pipe);
         intel_wait_for_vblank(dev, intel_crtc->pipe);
 
         if (drm_dp_dpcd_read(&intel_dp->aux, DP_TEST_CRC_R_CR, crc, 6) < 0)
-                return -EAGAIN;
+                return -EIO;
 
         drm_dp_dpcd_writeb(&intel_dp->aux, DP_TEST_SINK, 0);
         return 0;
@@ -3644,20 +4021,24 @@ intel_dp_detect_dpcd(struct intel_dp *intel_dp)
 }
 
 static enum drm_connector_status
+edp_detect(struct intel_dp *intel_dp)
+{
+        struct drm_device *dev = intel_dp_to_dev(intel_dp);
+        enum drm_connector_status status;
+
+        status = intel_panel_detect(dev);
+        if (status == connector_status_unknown)
+                status = connector_status_connected;
+
+        return status;
+}
+
+static enum drm_connector_status
 ironlake_dp_detect(struct intel_dp *intel_dp)
 {
         struct drm_device *dev = intel_dp_to_dev(intel_dp);
         struct drm_i915_private *dev_priv = dev->dev_private;
         struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp);
-        enum drm_connector_status status;
-
-        /* Can't disconnect eDP, but you can close the lid... */
-        if (is_edp(intel_dp)) {
-                status = intel_panel_detect(dev);
-                if (status == connector_status_unknown)
-                        status = connector_status_connected;
-                return status;
-        }
 
         if (!ibx_digital_port_connected(dev_priv, intel_dig_port))
                 return connector_status_disconnected;
@@ -3733,9 +4114,9 @@ g4x_dp_detect(struct intel_dp *intel_dp)
 }
 
 static struct edid *
-intel_dp_get_edid(struct drm_connector *connector, struct i2c_adapter *adapter)
+intel_dp_get_edid(struct intel_dp *intel_dp)
 {
-        struct intel_connector *intel_connector = to_intel_connector(connector);
+        struct intel_connector *intel_connector = intel_dp->attached_connector;
 
         /* use cached edid if we have one */
         if (intel_connector->edid) {
@@ -3744,27 +4125,55 @@ intel_dp_get_edid(struct drm_connector *connector, struct i2c_adapter *adapter)
                         return NULL;
 
                 return drm_edid_duplicate(intel_connector->edid);
-        }
+        } else
+                return drm_get_edid(&intel_connector->base,
+                                    &intel_dp->aux.ddc);
+}
+
+static void
+intel_dp_set_edid(struct intel_dp *intel_dp)
+{
+        struct intel_connector *intel_connector = intel_dp->attached_connector;
+        struct edid *edid;
+
+        edid = intel_dp_get_edid(intel_dp);
+        intel_connector->detect_edid = edid;
 
-        return drm_get_edid(connector, adapter);
+        if (intel_dp->force_audio != HDMI_AUDIO_AUTO)
+                intel_dp->has_audio = intel_dp->force_audio == HDMI_AUDIO_ON;
+        else
+                intel_dp->has_audio = drm_detect_monitor_audio(edid);
 }
 
-static int
-intel_dp_get_edid_modes(struct drm_connector *connector, struct i2c_adapter *adapter)
+static void
+intel_dp_unset_edid(struct intel_dp *intel_dp)
 {
-        struct intel_connector *intel_connector = to_intel_connector(connector);
+        struct intel_connector *intel_connector = intel_dp->attached_connector;
 
-        /* use cached edid if we have one */
-        if (intel_connector->edid) {
-                /* invalid edid */
-                if (IS_ERR(intel_connector->edid))
-                        return 0;
+        kfree(intel_connector->detect_edid);
+        intel_connector->detect_edid = NULL;
 
-                return intel_connector_update_modes(connector,
-                                                    intel_connector->edid);
-        }
+        intel_dp->has_audio = false;
+}
+
+static enum intel_display_power_domain
+intel_dp_power_get(struct intel_dp *dp)
+{
+        struct intel_encoder *encoder = &dp_to_dig_port(dp)->base;
+        enum intel_display_power_domain power_domain;
+
+        power_domain = intel_display_port_power_domain(encoder);
+        intel_display_power_get(to_i915(encoder->base.dev), power_domain);
+
+        return power_domain;
+}
 
-        return intel_ddc_get_modes(connector, adapter);
+static void
+intel_dp_power_put(struct intel_dp *dp,
+                   enum intel_display_power_domain power_domain)
+{
+        struct intel_encoder *encoder = &dp_to_dig_port(dp)->base;
+        intel_display_power_put(to_i915(encoder->base.dev), power_domain);
 }
 
 static enum drm_connector_status
@@ -3774,33 +4183,30 @@ intel_dp_detect(struct drm_connector *connector, bool force)
         struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp);
         struct intel_encoder *intel_encoder = &intel_dig_port->base;
         struct drm_device *dev = connector->dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
         enum drm_connector_status status;
         enum intel_display_power_domain power_domain;
-        struct edid *edid = NULL;
         bool ret;
 
-        power_domain = intel_display_port_power_domain(intel_encoder);
-        intel_display_power_get(dev_priv, power_domain);
-
         DRM_DEBUG_KMS("[CONNECTOR:%d:%s]\n",
                       connector->base.id, connector->name);
+        intel_dp_unset_edid(intel_dp);
 
         if (intel_dp->is_mst) {
                 /* MST devices are disconnected from a monitor POV */
                 if (intel_encoder->type != INTEL_OUTPUT_EDP)
                         intel_encoder->type = INTEL_OUTPUT_DISPLAYPORT;
-                status = connector_status_disconnected;
-                goto out;
+                return connector_status_disconnected;
         }
 
-        intel_dp->has_audio = false;
+        power_domain = intel_dp_power_get(intel_dp);
 
-        if (HAS_PCH_SPLIT(dev))
+        /* Can't disconnect eDP, but you can close the lid... */
+        if (is_edp(intel_dp))
+                status = edp_detect(intel_dp);
+        else if (HAS_PCH_SPLIT(dev))
                 status = ironlake_dp_detect(intel_dp);
         else
                 status = g4x_dp_detect(intel_dp);
-
         if (status != connector_status_connected)
                 goto out;
 
@@ -3816,82 +4222,78 @@ intel_dp_detect(struct drm_connector *connector, bool force)
                 goto out;
         }
 
-        if (intel_dp->force_audio != HDMI_AUDIO_AUTO) {
-                intel_dp->has_audio = (intel_dp->force_audio == HDMI_AUDIO_ON);
-        } else {
-                edid = intel_dp_get_edid(connector, &intel_dp->aux.ddc);
-                if (edid) {
-                        intel_dp->has_audio = drm_detect_monitor_audio(edid);
-                        kfree(edid);
-                }
-        }
+        intel_dp_set_edid(intel_dp);
 
         if (intel_encoder->type != INTEL_OUTPUT_EDP)
                 intel_encoder->type = INTEL_OUTPUT_DISPLAYPORT;
         status = connector_status_connected;
 
 out:
-        intel_display_power_put(dev_priv, power_domain);
+        intel_dp_power_put(intel_dp, power_domain);
         return status;
 }
 
-static int intel_dp_get_modes(struct drm_connector *connector)
+static void
+intel_dp_force(struct drm_connector *connector)
 {
         struct intel_dp *intel_dp = intel_attached_dp(connector);
-        struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp);
-        struct intel_encoder *intel_encoder = &intel_dig_port->base;
-        struct intel_connector *intel_connector = to_intel_connector(connector);
-        struct drm_device *dev = connector->dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct intel_encoder *intel_encoder = &dp_to_dig_port(intel_dp)->base;
         enum intel_display_power_domain power_domain;
-        int ret;
 
-        /* We should parse the EDID data and find out if it has an audio sink
-         */
+        DRM_DEBUG_KMS("[CONNECTOR:%d:%s]\n",
+                      connector->base.id, connector->name);
+        intel_dp_unset_edid(intel_dp);
 
-        power_domain = intel_display_port_power_domain(intel_encoder);
-        intel_display_power_get(dev_priv, power_domain);
+        if (connector->status != connector_status_connected)
+                return;
 
-        ret = intel_dp_get_edid_modes(connector, &intel_dp->aux.ddc);
-        intel_display_power_put(dev_priv, power_domain);
-        if (ret)
-                return ret;
+        power_domain = intel_dp_power_get(intel_dp);
+
+        intel_dp_set_edid(intel_dp);
+
+        intel_dp_power_put(intel_dp, power_domain);
+
+        if (intel_encoder->type != INTEL_OUTPUT_EDP)
+                intel_encoder->type = INTEL_OUTPUT_DISPLAYPORT;
+}
+
+static int intel_dp_get_modes(struct drm_connector *connector)
+{
+        struct intel_connector *intel_connector = to_intel_connector(connector);
+        struct edid *edid;
+
+        edid = intel_connector->detect_edid;
+        if (edid) {
+                int ret = intel_connector_update_modes(connector, edid);
+                if (ret)
+                        return ret;
+        }
 
         /* if eDP has no EDID, fall back to fixed mode */
-        if (is_edp(intel_dp) && intel_connector->panel.fixed_mode) {
+        if (is_edp(intel_attached_dp(connector)) &&
+            intel_connector->panel.fixed_mode) {
                 struct drm_display_mode *mode;
-                mode = drm_mode_duplicate(dev,
+
+                mode = drm_mode_duplicate(connector->dev,
                                           intel_connector->panel.fixed_mode);
                 if (mode) {
                         drm_mode_probed_add(connector, mode);
                         return 1;
                 }
         }
+
         return 0;
 }
 
 static bool
 intel_dp_detect_audio(struct drm_connector *connector)
 {
-        struct intel_dp *intel_dp = intel_attached_dp(connector);
-        struct intel_digital_port *intel_dig_port = dp_to_dig_port(intel_dp);
-        struct intel_encoder *intel_encoder = &intel_dig_port->base;
-        struct drm_device *dev = connector->dev;
-        struct drm_i915_private *dev_priv = dev->dev_private;
-        enum intel_display_power_domain power_domain;
-        struct edid *edid;
         bool has_audio = false;
+        struct edid *edid;
 
-        power_domain = intel_display_port_power_domain(intel_encoder);
-        intel_display_power_get(dev_priv, power_domain);
-
-        edid = intel_dp_get_edid(connector, &intel_dp->aux.ddc);
-        if (edid) {
+        edid = to_intel_connector(connector)->detect_edid;
+        if (edid)
                 has_audio = drm_detect_monitor_audio(edid);
-                kfree(edid);
-        }
-
-        intel_display_power_put(dev_priv, power_domain);
 
         return has_audio;
 }
@@ -3989,6 +4391,8 @@ intel_dp_connector_destroy(struct drm_connector *connector)
 {
         struct intel_connector *intel_connector = to_intel_connector(connector);
 
+        kfree(intel_connector->detect_edid);
+
         if (!IS_ERR_OR_NULL(intel_connector->edid))
                 kfree(intel_connector->edid);
 
@@ -4005,16 +4409,20 @@ void intel_dp_encoder_destroy(struct drm_encoder *encoder)
 {
         struct intel_digital_port *intel_dig_port = enc_to_dig_port(encoder);
         struct intel_dp *intel_dp = &intel_dig_port->dp;
-        struct drm_device *dev = intel_dp_to_dev(intel_dp);
 
         drm_dp_aux_unregister(&intel_dp->aux);
         intel_dp_mst_encoder_cleanup(intel_dig_port);
         drm_encoder_cleanup(encoder);
         if (is_edp(intel_dp)) {
                 cancel_delayed_work_sync(&intel_dp->panel_vdd_work);
-                drm_modeset_lock(&dev->mode_config.connection_mutex, NULL);
+                /*
+                 * vdd might still be enabled do to the delayed vdd off.
+                 * Make sure vdd is actually turned off here.
+                 */
+                pps_lock(intel_dp);
                 edp_panel_vdd_off_sync(intel_dp);
-                drm_modeset_unlock(&dev->mode_config.connection_mutex);
+                pps_unlock(intel_dp);
+
                 if (intel_dp->edp_notifier.notifier_call) {
                         unregister_reboot_notifier(&intel_dp->edp_notifier);
                         intel_dp->edp_notifier.notifier_call = NULL;
@@ -4030,7 +4438,13 @@ static void intel_dp_encoder_suspend(struct intel_encoder *intel_encoder)
         if (!is_edp(intel_dp))
                 return;
 
+        /*
+         * vdd might still be enabled do to the delayed vdd off.
+         * Make sure vdd is actually turned off here.
+         */
+        pps_lock(intel_dp);
         edp_panel_vdd_off_sync(intel_dp);
+        pps_unlock(intel_dp);
 }
 
 static void intel_dp_encoder_reset(struct drm_encoder *encoder)
@@ -4041,6 +4455,7 @@ static void intel_dp_encoder_reset(struct drm_encoder *encoder)
 static const struct drm_connector_funcs intel_dp_connector_funcs = {
         .dpms = intel_connector_dpms,
         .detect = intel_dp_detect,
+        .force = intel_dp_force,
         .fill_modes = drm_helper_probe_single_connector_modes,
         .set_property = intel_dp_set_property,
         .destroy = intel_dp_connector_destroy,
@@ -4076,7 +4491,8 @@ intel_dp_hpd_pulse(struct intel_digital_port *intel_dig_port, bool long_hpd)
         if (intel_dig_port->base.type != INTEL_OUTPUT_EDP)
                 intel_dig_port->base.type = INTEL_OUTPUT_DISPLAYPORT;
 
-        DRM_DEBUG_KMS("got hpd irq on port %d - %s\n", intel_dig_port->port,
+        DRM_DEBUG_KMS("got hpd irq on port %c - %s\n",
+                      port_name(intel_dig_port->port),
                       long_hpd ? "long" : "short");
 
         power_domain = intel_display_port_power_domain(intel_encoder);
@@ -4216,6 +4632,8 @@ intel_dp_init_panel_power_sequencer(struct drm_device *dev,
         u32 pp_on, pp_off, pp_div, pp;
         int pp_ctrl_reg, pp_on_reg, pp_off_reg, pp_div_reg;
 
+        lockdep_assert_held(&dev_priv->pps_mutex);
+
         if (HAS_PCH_SPLIT(dev)) {
                 pp_ctrl_reg = PCH_PP_CONTROL;
                 pp_on_reg = PCH_PP_ON_DELAYS;
@@ -4315,6 +4733,9 @@ intel_dp_init_panel_power_sequencer_registers(struct drm_device *dev,
         u32 pp_on, pp_off, pp_div, port_sel = 0;
         int div = HAS_PCH_SPLIT(dev) ? intel_pch_rawclk(dev) : intel_hrawclk(dev);
         int pp_on_reg, pp_off_reg, pp_div_reg;
+        enum port port = dp_to_dig_port(intel_dp)->port;
+
+        lockdep_assert_held(&dev_priv->pps_mutex);
 
         if (HAS_PCH_SPLIT(dev)) {
                 pp_on_reg = PCH_PP_ON_DELAYS;
@@ -4349,12 +4770,9 @@ intel_dp_init_panel_power_sequencer_registers(struct drm_device *dev,
         /* Haswell doesn't have any port selection bits for the panel
          * power sequencer any more. */
         if (IS_VALLEYVIEW(dev)) {
-                if (dp_to_dig_port(intel_dp)->port == PORT_B)
-                        port_sel = PANEL_PORT_SELECT_DPB_VLV;
-                else
-                        port_sel = PANEL_PORT_SELECT_DPC_VLV;
+                port_sel = PANEL_PORT_SELECT_VLV(port);
         } else if (HAS_PCH_IBX(dev) || HAS_PCH_CPT(dev)) {
-                if (dp_to_dig_port(intel_dp)->port == PORT_A)
+                if (port == PORT_A)
                         port_sel = PANEL_PORT_SELECT_DPA;
                 else
                         port_sel = PANEL_PORT_SELECT_DPD;
@@ -4438,7 +4856,7 @@ void intel_dp_set_drrs_state(struct drm_device *dev, int refresh_rate)
                 val = I915_READ(reg);
                 if (index > DRRS_HIGH_RR) {
                         val |= PIPECONF_EDP_RR_MODE_SWITCH;
-                        intel_dp_set_m2_n2(intel_crtc, &config->dp_m2_n2);
+                        intel_dp_set_m_n(intel_crtc);
                 } else {
                         val &= ~PIPECONF_EDP_RR_MODE_SWITCH;
                 }
@@ -4478,7 +4896,7 @@ intel_dp_drrs_init(struct intel_digital_port *intel_dig_port,
         }
 
         if (dev_priv->vbt.drrs_type != SEAMLESS_DRRS_SUPPORT) {
-                DRM_INFO("VBT doesn't support DRRS\n");
+                DRM_DEBUG_KMS("VBT doesn't support DRRS\n");
                 return NULL;
         }
 
@@ -4486,7 +4904,7 @@ intel_dp_drrs_init(struct intel_digital_port *intel_dig_port,
                                         (dev, fixed_mode, connector);
 
         if (!downclock_mode) {
-                DRM_INFO("DRRS not supported\n");
+                DRM_DEBUG_KMS("DRRS not supported\n");
                 return NULL;
         }
 
@@ -4497,7 +4915,7 @@ intel_dp_drrs_init(struct intel_digital_port *intel_dig_port,
         intel_dp->drrs_state.type = dev_priv->vbt.drrs_type;
 
         intel_dp->drrs_state.refresh_rate_type = DRRS_HIGH_RR;
-        DRM_INFO("seamless DRRS supported for eDP panel.\n");
+        DRM_DEBUG_KMS("seamless DRRS supported for eDP panel.\n");
         return downclock_mode;
 }
 
@@ -4512,8 +4930,11 @@ void intel_edp_panel_vdd_sanitize(struct intel_encoder *intel_encoder)
                 return;
 
         intel_dp = enc_to_intel_dp(&intel_encoder->base);
+
+        pps_lock(intel_dp);
+
         if (!edp_have_panel_vdd(intel_dp))
-                return;
+                goto out;
         /*
          * The VDD bit needs a power domain reference, so if the bit is
          * already enabled when we boot or resume, grab this reference and
@@ -4525,6 +4946,8 @@ void intel_edp_panel_vdd_sanitize(struct intel_encoder *intel_encoder)
         intel_display_power_get(dev_priv, power_domain);
 
         edp_panel_vdd_schedule_off(intel_dp);
+ out:
+        pps_unlock(intel_dp);
 }
 
 static bool intel_edp_init_connector(struct intel_dp *intel_dp,
@@ -4552,7 +4975,7 @@ static bool intel_edp_init_connector(struct intel_dp *intel_dp,
         /* Cache DPCD and EDID for edp. */
         intel_edp_panel_vdd_on(intel_dp);
         has_dpcd = intel_dp_get_dpcd(intel_dp);
-        edp_panel_vdd_off(intel_dp, false);
+        intel_edp_panel_vdd_off(intel_dp, false);
 
         if (has_dpcd) {
                 if (intel_dp->dpcd[DP_DPCD_REV] >= 0x11)
@@ -4566,7 +4989,9 @@ static bool intel_edp_init_connector(struct intel_dp *intel_dp,
         }
 
         /* We now know it's not a ghost, init power sequence regs. */
+        pps_lock(intel_dp);
         intel_dp_init_panel_power_sequencer_registers(dev, intel_dp, power_seq);
+        pps_unlock(intel_dp);
 
         mutex_lock(&dev->mode_config.mutex);
         edid = drm_get_edid(connector, &intel_dp->aux.ddc);
@@ -4610,6 +5035,7 @@ static bool intel_edp_init_connector(struct intel_dp *intel_dp,
         }
 
         intel_panel_init(&intel_connector->panel, fixed_mode, downclock_mode);
+        intel_connector->panel.backlight_power = intel_edp_backlight_power;
         intel_panel_setup_backlight(connector);
 
         return true;
@@ -4628,6 +5054,8 @@ intel_dp_init_connector(struct intel_digital_port *intel_dig_port,
         struct edp_power_seq power_seq = { 0 };
         int type;
 
+        intel_dp->pps_pipe = INVALID_PIPE;
+
         /* intel_dp vfuncs */
         if (IS_VALLEYVIEW(dev))
                 intel_dp->get_aux_clock_divider = vlv_get_aux_clock_divider;
@@ -4698,8 +5126,15 @@ intel_dp_init_connector(struct intel_digital_port *intel_dig_port,
         }
 
         if (is_edp(intel_dp)) {
-                intel_dp_init_panel_power_timestamps(intel_dp);
-                intel_dp_init_panel_power_sequencer(dev, intel_dp, &power_seq);
+                pps_lock(intel_dp);
+                if (IS_VALLEYVIEW(dev)) {
+                        vlv_initial_power_sequencer_setup(intel_dp);
+                } else {
+                        intel_dp_init_panel_power_timestamps(intel_dp);
+                        intel_dp_init_panel_power_sequencer(dev, intel_dp,
+                                                            &power_seq);
+                }
+                pps_unlock(intel_dp);
         }
 
         intel_dp_aux_init(intel_dp, intel_connector);
@@ -4707,7 +5142,8 @@ intel_dp_init_connector(struct intel_digital_port *intel_dig_port,
         /* init MST on ports that can support it */
         if (IS_HASWELL(dev) || IS_BROADWELL(dev)) {
                 if (port == PORT_B || port == PORT_C || port == PORT_D) {
-                        intel_dp_mst_encoder_init(intel_dig_port, intel_connector->base.base.id);
+                        intel_dp_mst_encoder_init(intel_dig_port,
+                                                  intel_connector->base.base.id);
                 }
         }
 
@@ -4715,9 +5151,13 @@ intel_dp_init_connector(struct intel_digital_port *intel_dig_port,
                 drm_dp_aux_unregister(&intel_dp->aux);
                 if (is_edp(intel_dp)) {
                         cancel_delayed_work_sync(&intel_dp->panel_vdd_work);
-                        drm_modeset_lock(&dev->mode_config.connection_mutex, NULL);
+                        /*
+                         * vdd might still be enabled do to the delayed vdd off.
+                         * Make sure vdd is actually turned off here.
+                         */
+                        pps_lock(intel_dp);
                         edp_panel_vdd_off_sync(intel_dp);
-                        drm_modeset_unlock(&dev->mode_config.connection_mutex);
+                        pps_unlock(intel_dp);
                 }
                 drm_connector_unregister(connector);
                 drm_connector_cleanup(connector);
@@ -4781,7 +5221,8 @@ intel_dp_init(struct drm_device *dev, int output_reg, enum port port)
         } else {
                 intel_encoder->pre_enable = g4x_pre_enable_dp;
                 intel_encoder->enable = g4x_enable_dp;
-                intel_encoder->post_disable = g4x_post_disable_dp;
+                if (INTEL_INFO(dev)->gen >= 5)
+                        intel_encoder->post_disable = ilk_post_disable_dp;
         }
 
         intel_dig_port->port = port;
diff --git a/drivers/gpu/drm/i915/intel_drv.h b/drivers/gpu/drm/i915/intel_drv.h
index b8c8bbd8e5f9..07ce04683c30 100644
--- a/drivers/gpu/drm/i915/intel_drv.h
+++ b/drivers/gpu/drm/i915/intel_drv.h
@@ -25,6 +25,7 @@
 #ifndef __INTEL_DRV_H__
 #define __INTEL_DRV_H__
 
+#include <linux/async.h>
 #include <linux/i2c.h>
 #include <linux/hdmi.h>
 #include <drm/i915_drm.h>
@@ -179,6 +180,8 @@ struct intel_panel {
                 bool active_low_pwm;
                 struct backlight_device *device;
         } backlight;
+
+        void (*backlight_power)(struct intel_connector *, bool enable);
 };
 
 struct intel_connector {
@@ -211,6 +214,7 @@ struct intel_connector {
 
         /* Cached EDID for eDP and LVDS. May hold ERR_PTR for invalid EDID. */
         struct edid *edid;
+        struct edid *detect_edid;
 
         /* since POLL and HPD connectors may use the same HPD line keep the native
            state of connector->polled in case hotplug storm detection changes it */
@@ -330,6 +334,7 @@ struct intel_crtc_config {
 
         /* m2_n2 for eDP downclock */
         struct intel_link_m_n dp_m2_n2;
+        bool has_drrs;
 
         /*
          * Frequence the dpll for the port should run at. Differs from the
@@ -410,6 +415,7 @@ struct intel_crtc {
         uint32_t cursor_addr;
         int16_t cursor_width, cursor_height;
         uint32_t cursor_cntl;
+        uint32_t cursor_size;
         uint32_t cursor_base;
 
         struct intel_plane_config plane_config;
@@ -430,8 +436,6 @@ struct intel_crtc {
                 struct intel_pipe_wm active;
         } wm;
 
-        wait_queue_head_t vbl_wait;
-
         int scanline_offset;
         struct intel_mmio_flip mmio_flip;
 };
@@ -455,6 +459,7 @@ struct intel_plane {
         unsigned int crtc_w, crtc_h;
         uint32_t src_x, src_y;
         uint32_t src_w, src_h;
+        unsigned int rotation;
 
         /* Since we need to change the watermarks before/after
          * enabling/disabling the planes, we need to store the parameters here
@@ -565,6 +570,12 @@ struct intel_dp {
 
         struct notifier_block edp_notifier;
 
+        /*
+         * Pipe whose power sequencer is currently locked into
+         * this port. Only relevant on VLV/CHV.
+         */
+        enum pipe pps_pipe;
+
         bool use_tps3;
         bool can_mst; /* this port supports mst */
         bool is_mst;
@@ -663,6 +674,10 @@ struct intel_unpin_work {
 #define INTEL_FLIP_COMPLETE     2
         u32 flip_count;
         u32 gtt_offset;
+        struct intel_engine_cs *flip_queued_ring;
+        u32 flip_queued_seqno;
+        int flip_queued_vblank;
+        int flip_ready_vblank;
         bool enable_stall_check;
 };
 
@@ -827,7 +842,6 @@ int intel_get_pipe_from_crtc_id(struct drm_device *dev, void *data,
 enum transcoder intel_pipe_to_cpu_transcoder(struct drm_i915_private *dev_priv,
                                              enum pipe pipe);
 void intel_wait_for_vblank(struct drm_device *dev, int pipe);
-void intel_wait_for_pipe_off(struct drm_device *dev, int pipe);
 int ironlake_get_lanes_required(int target_clock, int link_bw, int bpp);
 void vlv_wait_port_ready(struct drm_i915_private *dev_priv,
                          struct intel_digital_port *dport);
@@ -848,6 +862,7 @@ __intel_framebuffer_create(struct drm_device *dev,
 void intel_prepare_page_flip(struct drm_device *dev, int plane);
 void intel_finish_page_flip(struct drm_device *dev, int pipe);
 void intel_finish_page_flip_plane(struct drm_device *dev, int plane);
+void intel_check_page_flip(struct drm_device *dev, int pipe);
 
 /* shared dpll functions */
 struct intel_shared_dpll *intel_crtc_to_shared_dpll(struct intel_crtc *crtc);
@@ -882,6 +897,7 @@ void hsw_enable_pc8(struct drm_i915_private *dev_priv);
 void hsw_disable_pc8(struct drm_i915_private *dev_priv);
 void intel_dp_get_m_n(struct intel_crtc *crtc,
                       struct intel_crtc_config *pipe_config);
+void intel_dp_set_m_n(struct intel_crtc *crtc);
 int intel_dotclock_calculate(int link_freq, const struct intel_link_m_n *m_n);
 void
 ironlake_check_encoder_dotclock(const struct intel_crtc_config *pipe_config,
@@ -896,7 +912,7 @@ void intel_mode_from_pipe_config(struct drm_display_mode *mode,
                                  struct intel_crtc_config *pipe_config);
 int intel_format_to_fourcc(int format);
 void intel_crtc_wait_for_pending_flips(struct drm_crtc *crtc);
-
+void intel_modeset_preclose(struct drm_device *dev, struct drm_file *file);
 
 /* intel_dp.c */
 void intel_dp_init(struct drm_device *dev, int output_reg, enum port port);
@@ -935,6 +951,7 @@ void intel_dp_mst_suspend(struct drm_device *dev);
 void intel_dp_mst_resume(struct drm_device *dev);
 int intel_dp_max_link_bw(struct intel_dp *intel_dp);
 void intel_dp_hot_plug(struct intel_encoder *intel_encoder);
+void vlv_power_sequencer_reset(struct drm_i915_private *dev_priv);
 /* intel_dp_mst.c */
 int intel_dp_mst_encoder_init(struct intel_digital_port *intel_dig_port, int conn_id);
 void intel_dp_mst_encoder_cleanup(struct intel_digital_port *intel_dig_port);
@@ -949,9 +966,9 @@ void intel_dvo_init(struct drm_device *dev);
 /* legacy fbdev emulation in intel_fbdev.c */
 #ifdef CONFIG_DRM_I915_FBDEV
 extern int intel_fbdev_init(struct drm_device *dev);
-extern void intel_fbdev_initial_config(struct drm_device *dev);
+extern void intel_fbdev_initial_config(void *data, async_cookie_t cookie);
 extern void intel_fbdev_fini(struct drm_device *dev);
-extern void intel_fbdev_set_suspend(struct drm_device *dev, int state);
+extern void intel_fbdev_set_suspend(struct drm_device *dev, int state, bool synchronous);
 extern void intel_fbdev_output_poll_changed(struct drm_device *dev);
 extern void intel_fbdev_restore_mode(struct drm_device *dev);
 #else
@@ -960,7 +977,7 @@ static inline int intel_fbdev_init(struct drm_device *dev)
         return 0;
 }
 
-static inline void intel_fbdev_initial_config(struct drm_device *dev)
+static inline void intel_fbdev_initial_config(void *data, async_cookie_t cookie)
 {
 }
 
@@ -968,7 +985,7 @@ static inline void intel_fbdev_fini(struct drm_device *dev)
 {
 }
 
-static inline void intel_fbdev_set_suspend(struct drm_device *dev, int state)
+static inline void intel_fbdev_set_suspend(struct drm_device *dev, int state, bool synchronous)
 {
 }
 
@@ -1091,7 +1108,10 @@ bool intel_sdvo_init(struct drm_device *dev, uint32_t sdvo_reg, bool is_sdvob);
 int intel_plane_init(struct drm_device *dev, enum pipe pipe, int plane);
 void intel_flush_primary_plane(struct drm_i915_private *dev_priv,
                                enum plane plane);
-void intel_plane_restore(struct drm_plane *plane);
+int intel_plane_set_property(struct drm_plane *plane,
+                             struct drm_property *prop,
+                             uint64_t val);
+int intel_plane_restore(struct drm_plane *plane);
 void intel_plane_disable(struct drm_plane *plane);
 int intel_sprite_set_colorkey(struct drm_device *dev, void *data,
                               struct drm_file *file_priv);
diff --git a/drivers/gpu/drm/i915/intel_dsi.c b/drivers/gpu/drm/i915/intel_dsi.c
index 670c29a7b5dd..5bd9e09ad3c5 100644
--- a/drivers/gpu/drm/i915/intel_dsi.c
+++ b/drivers/gpu/drm/i915/intel_dsi.c
@@ -184,7 +184,7 @@ static void intel_dsi_pre_enable(struct intel_encoder *encoder)
 
         /* update the hw state for DPLL */
         intel_crtc->config.dpll_hw_state.dpll = DPLL_INTEGRATED_CLOCK_VLV |
-                                                DPLL_REFA_CLK_ENABLE_VLV;
+                DPLL_REFA_CLK_ENABLE_VLV;
 
         tmp = I915_READ(DSPCLK_GATE_D);
         tmp |= DPOUNIT_CLOCK_GATE_DISABLE;
@@ -259,8 +259,8 @@ static void intel_dsi_disable(struct intel_encoder *encoder)
         temp = I915_READ(MIPI_CTRL(pipe));
         temp &= ~ESCAPE_CLOCK_DIVIDER_MASK;
         I915_WRITE(MIPI_CTRL(pipe), temp |
-                        intel_dsi->escape_clk_div <<
-                        ESCAPE_CLOCK_DIVIDER_SHIFT);
+                   intel_dsi->escape_clk_div <<
+                   ESCAPE_CLOCK_DIVIDER_SHIFT);
 
         I915_WRITE(MIPI_EOT_DISABLE(pipe), CLOCKSTOP);
 
@@ -297,7 +297,7 @@ static void intel_dsi_clear_device_ready(struct intel_encoder *encoder)
         usleep_range(2000, 2500);
 
         if (wait_for(((I915_READ(MIPI_PORT_CTRL(pipe)) & AFE_LATCHOUT)
-                                        == 0x00000), 30))
+                      == 0x00000), 30))
                 DRM_ERROR("DSI LP not going Low\n");
 
         val = I915_READ(MIPI_PORT_CTRL(pipe));
@@ -423,9 +423,11 @@ static u16 txclkesc(u32 divider, unsigned int us)
 }
 
 /* return pixels in terms of txbyteclkhs */
-static u16 txbyteclkhs(u16 pixels, int bpp, int lane_count)
+static u16 txbyteclkhs(u16 pixels, int bpp, int lane_count,
+                       u16 burst_mode_ratio)
 {
-        return DIV_ROUND_UP(DIV_ROUND_UP(pixels * bpp, 8), lane_count);
+        return DIV_ROUND_UP(DIV_ROUND_UP(pixels * bpp * burst_mode_ratio,
+                                         8 * 100), lane_count);
 }
 
 static void set_dsi_timings(struct drm_encoder *encoder,
@@ -451,10 +453,12 @@ static void set_dsi_timings(struct drm_encoder *encoder,
         vbp = mode->vtotal - mode->vsync_end;
 
         /* horizontal values are in terms of high speed byte clock */
-        hactive = txbyteclkhs(hactive, bpp, lane_count);
-        hfp = txbyteclkhs(hfp, bpp, lane_count);
-        hsync = txbyteclkhs(hsync, bpp, lane_count);
-        hbp = txbyteclkhs(hbp, bpp, lane_count);
+        hactive = txbyteclkhs(hactive, bpp, lane_count,
+                              intel_dsi->burst_mode_ratio);
+        hfp = txbyteclkhs(hfp, bpp, lane_count, intel_dsi->burst_mode_ratio);
+        hsync = txbyteclkhs(hsync, bpp, lane_count,
+                            intel_dsi->burst_mode_ratio);
+        hbp = txbyteclkhs(hbp, bpp, lane_count, intel_dsi->burst_mode_ratio);
 
         I915_WRITE(MIPI_HACTIVE_AREA_COUNT(pipe), hactive);
         I915_WRITE(MIPI_HFP_COUNT(pipe), hfp);
@@ -541,12 +545,14 @@ static void intel_dsi_prepare(struct intel_encoder *intel_encoder)
             intel_dsi->video_mode_format == VIDEO_MODE_BURST) {
                 I915_WRITE(MIPI_HS_TX_TIMEOUT(pipe),
                            txbyteclkhs(adjusted_mode->htotal, bpp,
-                                       intel_dsi->lane_count) + 1);
+                                       intel_dsi->lane_count,
+                                       intel_dsi->burst_mode_ratio) + 1);
         } else {
                 I915_WRITE(MIPI_HS_TX_TIMEOUT(pipe),
                            txbyteclkhs(adjusted_mode->vtotal *
                                        adjusted_mode->htotal,
-                                       bpp, intel_dsi->lane_count) + 1);
+                                       bpp, intel_dsi->lane_count,
+                                       intel_dsi->burst_mode_ratio) + 1);
         }
         I915_WRITE(MIPI_LP_RX_TIMEOUT(pipe), intel_dsi->lp_rx_timeout);
         I915_WRITE(MIPI_TURN_AROUND_TIMEOUT(pipe), intel_dsi->turn_arnd_val);
@@ -576,7 +582,7 @@ static void intel_dsi_prepare(struct intel_encoder *intel_encoder)
          * XXX: write MIPI_STOP_STATE_STALL?
          */
         I915_WRITE(MIPI_HIGH_LOW_SWITCH_COUNT(pipe),
-                                                intel_dsi->hs_to_lp_count);
+                   intel_dsi->hs_to_lp_count);
 
         /* XXX: low power clock equivalence in terms of byte clock. the number
          * of byte clocks occupied in one low power clock. based on txbyteclkhs
@@ -601,10 +607,10 @@ static void intel_dsi_prepare(struct intel_encoder *intel_encoder)
                  * 64 like 1366 x 768. Enable RANDOM resolution support for such
                  * panels by default */
                 I915_WRITE(MIPI_VIDEO_MODE_FORMAT(pipe),
-                                intel_dsi->video_frmt_cfg_bits |
-                                intel_dsi->video_mode_format |
-                                IP_TG_CONFIG |
-                                RANDOM_DPI_DISPLAY_RESOLUTION);
+                           intel_dsi->video_frmt_cfg_bits |
+                           intel_dsi->video_mode_format |
+                           IP_TG_CONFIG |
+                           RANDOM_DPI_DISPLAY_RESOLUTION);
 }
 
 static void intel_dsi_pre_pll_enable(struct intel_encoder *encoder)
diff --git a/drivers/gpu/drm/i915/intel_dsi.h b/drivers/gpu/drm/i915/intel_dsi.h
index fd51867fd0d3..657eb5c1b9d8 100644
--- a/drivers/gpu/drm/i915/intel_dsi.h
+++ b/drivers/gpu/drm/i915/intel_dsi.h
@@ -116,6 +116,8 @@ struct intel_dsi {
         u16 clk_hs_to_lp_count;
 
         u16 init_count;
+        u32 pclk;
+        u16 burst_mode_ratio;
 
         /* all delays in ms */
         u16 backlight_off_delay;
diff --git a/drivers/gpu/drm/i915/intel_dsi_cmd.c b/drivers/gpu/drm/i915/intel_dsi_cmd.c
index 7f1430ac8543..f4767fd2ebeb 100644
--- a/drivers/gpu/drm/i915/intel_dsi_cmd.c
+++ b/drivers/gpu/drm/i915/intel_dsi_cmd.c
@@ -430,7 +430,7 @@ void wait_for_dsi_fifo_empty(struct intel_dsi *intel_dsi)
         u32 mask;
 
         mask = LP_CTRL_FIFO_EMPTY | HS_CTRL_FIFO_EMPTY |
-                                        LP_DATA_FIFO_EMPTY | HS_DATA_FIFO_EMPTY;
+                LP_DATA_FIFO_EMPTY | HS_DATA_FIFO_EMPTY;
 
         if (wait_for((I915_READ(MIPI_GEN_FIFO_STAT(pipe)) & mask) == mask, 100))
                 DRM_ERROR("DPI FIFOs are not empty\n");
diff --git a/drivers/gpu/drm/i915/intel_dsi_panel_vbt.c b/drivers/gpu/drm/i915/intel_dsi_panel_vbt.c
index 47c7584a4aa0..f6bdd44069ce 100644
--- a/drivers/gpu/drm/i915/intel_dsi_panel_vbt.c
+++ b/drivers/gpu/drm/i915/intel_dsi_panel_vbt.c
@@ -271,6 +271,8 @@ static bool generic_init(struct intel_dsi_device *dsi)
         u32 ths_prepare_ns, tclk_trail_ns;
         u32 tclk_prepare_clkzero, ths_prepare_hszero;
         u32 lp_to_hs_switch, hs_to_lp_switch;
+        u32 pclk, computed_ddr;
+        u16 burst_mode_ratio;
 
         DRM_DEBUG_KMS("\n");
 
@@ -284,8 +286,6 @@ static bool generic_init(struct intel_dsi_device *dsi)
         else if (intel_dsi->pixel_format == VID_MODE_FORMAT_RGB565)
                 bits_per_pixel = 16;
 
-        bitrate = (mode->clock * bits_per_pixel) / intel_dsi->lane_count;
-
         intel_dsi->operation_mode = mipi_config->is_cmd_mode;
         intel_dsi->video_mode_format = mipi_config->video_transfer_mode;
         intel_dsi->escape_clk_div = mipi_config->byte_clk_sel;
@@ -297,6 +297,40 @@ static bool generic_init(struct intel_dsi_device *dsi)
         intel_dsi->video_frmt_cfg_bits =
                 mipi_config->bta_enabled ? DISABLE_VIDEO_BTA : 0;
 
+        pclk = mode->clock;
+
+        /* Burst Mode Ratio
+         * Target ddr frequency from VBT / non burst ddr freq
+         * multiply by 100 to preserve remainder
+         */
+        if (intel_dsi->video_mode_format == VIDEO_MODE_BURST) {
+                if (mipi_config->target_burst_mode_freq) {
+                        computed_ddr =
+                                (pclk * bits_per_pixel) / intel_dsi->lane_count;
+
+                        if (mipi_config->target_burst_mode_freq <
+                                                                computed_ddr) {
+                                DRM_ERROR("Burst mode freq is less than computed\n");
+                                return false;
+                        }
+
+                        burst_mode_ratio = DIV_ROUND_UP(
+                                mipi_config->target_burst_mode_freq * 100,
+                                computed_ddr);
+
+                        pclk = DIV_ROUND_UP(pclk * burst_mode_ratio, 100);
+                } else {
+                        DRM_ERROR("Burst mode target is not set\n");
+                        return false;
+                }
+        } else
+                burst_mode_ratio = 100;
+
+        intel_dsi->burst_mode_ratio = burst_mode_ratio;
+        intel_dsi->pclk = pclk;
+
+        bitrate = (pclk * bits_per_pixel) / intel_dsi->lane_count;
+
         switch (intel_dsi->escape_clk_div) {
         case 0:
                 tlpx_ns = 50;
diff --git a/drivers/gpu/drm/i915/intel_dsi_pll.c b/drivers/gpu/drm/i915/intel_dsi_pll.c
index d8bb1ea2f0da..fa7a6ca34cd6 100644
--- a/drivers/gpu/drm/i915/intel_dsi_pll.c
+++ b/drivers/gpu/drm/i915/intel_dsi_pll.c
@@ -134,8 +134,7 @@ static u32 dsi_rr_formula(const struct drm_display_mode *mode,
 #else
 
 /* Get DSI clock from pixel clock */
-static u32 dsi_clk_from_pclk(const struct drm_display_mode *mode,
-                          int pixel_format, int lane_count)
+static u32 dsi_clk_from_pclk(u32 pclk, int pixel_format, int lane_count)
 {
         u32 dsi_clk_khz;
         u32 bpp;
@@ -156,7 +155,7 @@ static u32 dsi_clk_from_pclk(const struct drm_display_mode *mode,
 
         /* DSI data rate = pixel clock * bits per pixel / lane count
            pixel clock is converted from KHz to Hz */
-        dsi_clk_khz = DIV_ROUND_CLOSEST(mode->clock * bpp, lane_count);
+        dsi_clk_khz = DIV_ROUND_CLOSEST(pclk * bpp, lane_count);
 
         return dsi_clk_khz;
 }
@@ -191,7 +190,7 @@ static int dsi_calc_mnp(u32 dsi_clk, struct dsi_mnp *dsi_mnp)
         for (m = 62; m <= 92; m++) {
                 for (p = 2; p <= 6; p++) {
                         /* Find the optimal m and p divisors
-                        with minimal error +/- the required clock */
+                           with minimal error +/- the required clock */
                         calc_dsi_clk = (m * ref_clk) / p;
                         if (calc_dsi_clk == target_dsi_clk) {
                                 calc_m = m;
@@ -228,15 +227,13 @@ static int dsi_calc_mnp(u32 dsi_clk, struct dsi_mnp *dsi_mnp)
 static void vlv_configure_dsi_pll(struct intel_encoder *encoder)
 {
         struct drm_i915_private *dev_priv = encoder->base.dev->dev_private;
-        struct intel_crtc *intel_crtc = to_intel_crtc(encoder->base.crtc);
-        const struct drm_display_mode *mode = &intel_crtc->config.adjusted_mode;
         struct intel_dsi *intel_dsi = enc_to_intel_dsi(&encoder->base);
         int ret;
         struct dsi_mnp dsi_mnp;
         u32 dsi_clk;
 
-        dsi_clk = dsi_clk_from_pclk(mode, intel_dsi->pixel_format,
-                                                intel_dsi->lane_count);
+        dsi_clk = dsi_clk_from_pclk(intel_dsi->pclk, intel_dsi->pixel_format,
+                                    intel_dsi->lane_count);
 
         ret = dsi_calc_mnp(dsi_clk, &dsi_mnp);
         if (ret) {
@@ -318,8 +315,8 @@ static void assert_bpp_mismatch(int pixel_format, int pipe_bpp)
         }
 
         WARN(bpp != pipe_bpp,
-                "bpp match assertion failure (expected %d, current %d)\n",
-                bpp, pipe_bpp);
+             "bpp match assertion failure (expected %d, current %d)\n",
+             bpp, pipe_bpp);
 }
 
 u32 vlv_get_dsi_pclk(struct intel_encoder *encoder, int pipe_bpp)
diff --git a/drivers/gpu/drm/i915/intel_dvo.c b/drivers/gpu/drm/i915/intel_dvo.c
index 56b47d2ffaf7..e40e3df33517 100644
--- a/drivers/gpu/drm/i915/intel_dvo.c
+++ b/drivers/gpu/drm/i915/intel_dvo.c
@@ -85,7 +85,7 @@ static const struct intel_dvo_device intel_dvo_devices[] = {
         {
                 .type = INTEL_DVO_CHIP_TMDS,
                 .name = "ns2501",
-                .dvo_reg = DVOC,
+                .dvo_reg = DVOB,
                 .slave_addr = NS2501_ADDR,
                 .dev_ops = &ns2501_ops,
        }
@@ -185,12 +185,13 @@ static void intel_enable_dvo(struct intel_encoder *encoder)
         u32 dvo_reg = intel_dvo->dev.dvo_reg;
         u32 temp = I915_READ(dvo_reg);
 
-        I915_WRITE(dvo_reg, temp | DVO_ENABLE);
-        I915_READ(dvo_reg);
         intel_dvo->dev.dev_ops->mode_set(&intel_dvo->dev,
                                          &crtc->config.requested_mode,
                                          &crtc->config.adjusted_mode);
 
+        I915_WRITE(dvo_reg, temp | DVO_ENABLE);
+        I915_READ(dvo_reg);
+
         intel_dvo->dev.dev_ops->dpms(&intel_dvo->dev, true);
 }
 
@@ -226,10 +227,6 @@ static void intel_dvo_dpms(struct drm_connector *connector, int mode)
 
                 intel_crtc_update_dpms(crtc);
 
-                intel_dvo->dev.dev_ops->mode_set(&intel_dvo->dev,
-                                                 &config->requested_mode,
-                                                 &config->adjusted_mode);
-
                 intel_dvo->dev.dev_ops->dpms(&intel_dvo->dev, true);
         } else {
                 intel_dvo->dev.dev_ops->dpms(&intel_dvo->dev, false);
diff --git a/drivers/gpu/drm/i915/intel_fbdev.c b/drivers/gpu/drm/i915/intel_fbdev.c
index f475414671d8..9b584f3fbb99 100644
--- a/drivers/gpu/drm/i915/intel_fbdev.c
+++ b/drivers/gpu/drm/i915/intel_fbdev.c
@@ -24,8 +24,10 @@
  *     David Airlie
  */
 
+#include <linux/async.h>
 #include <linux/module.h>
 #include <linux/kernel.h>
+#include <linux/console.h>
 #include <linux/errno.h>
 #include <linux/string.h>
 #include <linux/mm.h>
@@ -331,24 +333,6 @@ static bool intel_fb_initial_config(struct drm_fb_helper *fb_helper,
         int num_connectors_enabled = 0;
         int num_connectors_detected = 0;
 
-        /*
-         * If the user specified any force options, just bail here
-         * and use that config.
-         */
-        for (i = 0; i < fb_helper->connector_count; i++) {
-                struct drm_fb_helper_connector *fb_conn;
-                struct drm_connector *connector;
-
-                fb_conn = fb_helper->connector_info[i];
-                connector = fb_conn->connector;
-
-                if (!enabled[i])
-                        continue;
-
-                if (connector->force != DRM_FORCE_UNSPECIFIED)
-                        return false;
-        }
-
         save_enabled = kcalloc(dev->mode_config.num_connector, sizeof(bool),
                                GFP_KERNEL);
         if (!save_enabled)
@@ -374,8 +358,18 @@ static bool intel_fb_initial_config(struct drm_fb_helper *fb_helper,
                         continue;
                 }
 
+                if (connector->force == DRM_FORCE_OFF) {
+                        DRM_DEBUG_KMS("connector %s is disabled by user, skipping\n",
+                                      connector->name);
+                        enabled[i] = false;
+                        continue;
+                }
+
                 encoder = connector->encoder;
                 if (!encoder || WARN_ON(!encoder->crtc)) {
+                        if (connector->force > DRM_FORCE_OFF)
+                                goto bail;
+
                         DRM_DEBUG_KMS("connector %s has no encoder or crtc, skipping\n",
                                       connector->name);
                         enabled[i] = false;
@@ -394,8 +388,7 @@ static bool intel_fb_initial_config(struct drm_fb_helper *fb_helper,
                 for (j = 0; j < fb_helper->connector_count; j++) {
                         if (crtcs[j] == new_crtc) {
                                 DRM_DEBUG_KMS("fallback: cloned configuration\n");
-                                fallback = true;
-                                goto out;
+                                goto bail;
                         }
                 }
 
@@ -466,8 +459,8 @@ static bool intel_fb_initial_config(struct drm_fb_helper *fb_helper,
                 fallback = true;
         }
 
-out:
         if (fallback) {
+bail:
                 DRM_DEBUG_KMS("Not using firmware configuration\n");
                 memcpy(enabled, save_enabled, dev->mode_config.num_connector);
                 kfree(save_enabled);
@@ -636,6 +629,15 @@ out:
         return false;
 }
 
+static void intel_fbdev_suspend_worker(struct work_struct *work)
+{
+        intel_fbdev_set_suspend(container_of(work,
+                                             struct drm_i915_private,
+                                             fbdev_suspend_work)->dev,
+                                FBINFO_STATE_RUNNING,
+                                true);
+}
+
 int intel_fbdev_init(struct drm_device *dev)
 {
         struct intel_fbdev *ifbdev;
@@ -662,14 +664,16 @@ int intel_fbdev_init(struct drm_device *dev)
         }
 
         dev_priv->fbdev = ifbdev;
+        INIT_WORK(&dev_priv->fbdev_suspend_work, intel_fbdev_suspend_worker);
+
         drm_fb_helper_single_add_all_connectors(&ifbdev->helper);
 
         return 0;
 }
 
-void intel_fbdev_initial_config(struct drm_device *dev)
+void intel_fbdev_initial_config(void *data, async_cookie_t cookie)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_private *dev_priv = data;
         struct intel_fbdev *ifbdev = dev_priv->fbdev;
 
         /* Due to peculiar init order wrt to hpd handling this is separate. */
@@ -682,12 +686,15 @@ void intel_fbdev_fini(struct drm_device *dev)
         if (!dev_priv->fbdev)
                 return;
 
+        flush_work(&dev_priv->fbdev_suspend_work);
+
+        async_synchronize_full();
         intel_fbdev_destroy(dev, dev_priv->fbdev);
         kfree(dev_priv->fbdev);
         dev_priv->fbdev = NULL;
 }
 
-void intel_fbdev_set_suspend(struct drm_device *dev, int state)
+void intel_fbdev_set_suspend(struct drm_device *dev, int state, bool synchronous)
 {
         struct drm_i915_private *dev_priv = dev->dev_private;
         struct intel_fbdev *ifbdev = dev_priv->fbdev;
@@ -698,6 +705,33 @@ void intel_fbdev_set_suspend(struct drm_device *dev, int state)
 
         info = ifbdev->helper.fbdev;
 
+        if (synchronous) {
+                /* Flush any pending work to turn the console on, and then
+                 * wait to turn it off. It must be synchronous as we are
+                 * about to suspend or unload the driver.
+                 *
+                 * Note that from within the work-handler, we cannot flush
+                 * ourselves, so only flush outstanding work upon suspend!
+                 */
+                if (state != FBINFO_STATE_RUNNING)
+                        flush_work(&dev_priv->fbdev_suspend_work);
+                console_lock();
+        } else {
+                /*
+                 * The console lock can be pretty contented on resume due
+                 * to all the printk activity.  Try to keep it out of the hot
+                 * path of resume if possible.
+                 */
+                WARN_ON(state != FBINFO_STATE_RUNNING);
+                if (!console_trylock()) {
+                        /* Don't block our own workqueue as this can
+                         * be run in parallel with other i915.ko tasks.
+                         */
+                        schedule_work(&dev_priv->fbdev_suspend_work);
+                        return;
+                }
+        }
+
         /* On resume from hibernation: If the object is shmemfs backed, it has
          * been restored from swap. If the object is stolen however, it will be
          * full of whatever garbage was left in there.
@@ -706,6 +740,7 @@ void intel_fbdev_set_suspend(struct drm_device *dev, int state)
                 memset_io(info->screen_base, 0, info->screen_size);
 
         fb_set_suspend(info, state);
+        console_unlock();
 }
 
 void intel_fbdev_output_poll_changed(struct drm_device *dev)
diff --git a/drivers/gpu/drm/i915/intel_hdmi.c b/drivers/gpu/drm/i915/intel_hdmi.c
index 5a9de21637b7..29ec1535992d 100644
--- a/drivers/gpu/drm/i915/intel_hdmi.c
+++ b/drivers/gpu/drm/i915/intel_hdmi.c
@@ -869,10 +869,15 @@ static enum drm_mode_status
 intel_hdmi_mode_valid(struct drm_connector *connector,
                       struct drm_display_mode *mode)
 {
-        if (mode->clock > hdmi_portclock_limit(intel_attached_hdmi(connector),
-                                               true))
+        int clock = mode->clock;
+
+        if (mode->flags & DRM_MODE_FLAG_DBLCLK)
+                clock *= 2;
+
+        if (clock > hdmi_portclock_limit(intel_attached_hdmi(connector),
+                                         true))
                 return MODE_CLOCK_HIGH;
-        if (mode->clock < 20000)
+        if (clock < 20000)
                 return MODE_CLOCK_LOW;
 
         if (mode->flags & DRM_MODE_FLAG_DBLSCAN)
@@ -890,7 +895,7 @@ static bool hdmi_12bpc_possible(struct intel_crtc *crtc)
         if (HAS_GMCH_DISPLAY(dev))
                 return false;
 
-        list_for_each_entry(encoder, &dev->mode_config.encoder_list, base.head) {
+        for_each_intel_encoder(dev, encoder) {
                 if (encoder->new_crtc != crtc)
                         continue;
 
@@ -926,6 +931,10 @@ bool intel_hdmi_compute_config(struct intel_encoder *encoder,
                         intel_hdmi->color_range = 0;
         }
 
+        if (adjusted_mode->flags & DRM_MODE_FLAG_DBLCLK) {
+                pipe_config->pixel_multiplier = 2;
+        }
+
         if (intel_hdmi->color_range)
                 pipe_config->limited_color_range = true;
 
@@ -967,104 +976,117 @@ bool intel_hdmi_compute_config(struct intel_encoder *encoder,
         return true;
 }
 
-static enum drm_connector_status
-intel_hdmi_detect(struct drm_connector *connector, bool force)
+static void
+intel_hdmi_unset_edid(struct drm_connector *connector)
 {
-        struct drm_device *dev = connector->dev;
         struct intel_hdmi *intel_hdmi = intel_attached_hdmi(connector);
-        struct intel_digital_port *intel_dig_port =
-                hdmi_to_dig_port(intel_hdmi);
-        struct intel_encoder *intel_encoder = &intel_dig_port->base;
-        struct drm_i915_private *dev_priv = dev->dev_private;
-        struct edid *edid;
-        enum intel_display_power_domain power_domain;
-        enum drm_connector_status status = connector_status_disconnected;
 
-        DRM_DEBUG_KMS("[CONNECTOR:%d:%s]\n",
-                      connector->base.id, connector->name);
+        intel_hdmi->has_hdmi_sink = false;
+        intel_hdmi->has_audio = false;
+        intel_hdmi->rgb_quant_range_selectable = false;
+
+        kfree(to_intel_connector(connector)->detect_edid);
+        to_intel_connector(connector)->detect_edid = NULL;
+}
+
+static bool
+intel_hdmi_set_edid(struct drm_connector *connector)
+{
+        struct drm_i915_private *dev_priv = to_i915(connector->dev);
+        struct intel_hdmi *intel_hdmi = intel_attached_hdmi(connector);
+        struct intel_encoder *intel_encoder =
+                &hdmi_to_dig_port(intel_hdmi)->base;
+        enum intel_display_power_domain power_domain;
+        struct edid *edid;
+        bool connected = false;
 
         power_domain = intel_display_port_power_domain(intel_encoder);
         intel_display_power_get(dev_priv, power_domain);
 
-        intel_hdmi->has_hdmi_sink = false;
-        intel_hdmi->has_audio = false;
-        intel_hdmi->rgb_quant_range_selectable = false;
         edid = drm_get_edid(connector,
                             intel_gmbus_get_adapter(dev_priv,
                                                     intel_hdmi->ddc_bus));
 
-        if (edid) {
-                if (edid->input & DRM_EDID_INPUT_DIGITAL) {
-                        status = connector_status_connected;
-                        if (intel_hdmi->force_audio != HDMI_AUDIO_OFF_DVI)
-                                intel_hdmi->has_hdmi_sink =
-                                                drm_detect_hdmi_monitor(edid);
-                        intel_hdmi->has_audio = drm_detect_monitor_audio(edid);
-                        intel_hdmi->rgb_quant_range_selectable =
-                                drm_rgb_quant_range_selectable(edid);
-                }
-                kfree(edid);
-        }
+        intel_display_power_put(dev_priv, power_domain);
+
+        to_intel_connector(connector)->detect_edid = edid;
+        if (edid && edid->input & DRM_EDID_INPUT_DIGITAL) {
+                intel_hdmi->rgb_quant_range_selectable =
+                        drm_rgb_quant_range_selectable(edid);
 
-        if (status == connector_status_connected) {
+                intel_hdmi->has_audio = drm_detect_monitor_audio(edid);
                 if (intel_hdmi->force_audio != HDMI_AUDIO_AUTO)
                         intel_hdmi->has_audio =
-                                (intel_hdmi->force_audio == HDMI_AUDIO_ON);
-                intel_encoder->type = INTEL_OUTPUT_HDMI;
+                                intel_hdmi->force_audio == HDMI_AUDIO_ON;
+
+                if (intel_hdmi->force_audio != HDMI_AUDIO_OFF_DVI)
+                        intel_hdmi->has_hdmi_sink =
+                                drm_detect_hdmi_monitor(edid);
+
+                connected = true;
         }
 
-        intel_display_power_put(dev_priv, power_domain);
+        return connected;
+}
+
+static enum drm_connector_status
+intel_hdmi_detect(struct drm_connector *connector, bool force)
+{
+        enum drm_connector_status status;
+
+        DRM_DEBUG_KMS("[CONNECTOR:%d:%s]\n",
+                      connector->base.id, connector->name);
+
+        intel_hdmi_unset_edid(connector);
+
+        if (intel_hdmi_set_edid(connector)) {
+                struct intel_hdmi *intel_hdmi = intel_attached_hdmi(connector);
+
+                hdmi_to_dig_port(intel_hdmi)->base.type = INTEL_OUTPUT_HDMI;
+                status = connector_status_connected;
+        } else
+                status = connector_status_disconnected;
 
         return status;
 }
 
-static int intel_hdmi_get_modes(struct drm_connector *connector)
+static void
+intel_hdmi_force(struct drm_connector *connector)
 {
-        struct intel_encoder *intel_encoder = intel_attached_encoder(connector);
-        struct intel_hdmi *intel_hdmi = enc_to_intel_hdmi(&intel_encoder->base);
-        struct drm_i915_private *dev_priv = connector->dev->dev_private;
-        enum intel_display_power_domain power_domain;
-        int ret;
+        struct intel_hdmi *intel_hdmi = intel_attached_hdmi(connector);
 
-        /* We should parse the EDID data and find out if it's an HDMI sink so
-         * we can send audio to it.
-         */
+        DRM_DEBUG_KMS("[CONNECTOR:%d:%s]\n",
+                      connector->base.id, connector->name);
 
-        power_domain = intel_display_port_power_domain(intel_encoder);
-        intel_display_power_get(dev_priv, power_domain);
+        intel_hdmi_unset_edid(connector);
 
-        ret = intel_ddc_get_modes(connector,
-                                   intel_gmbus_get_adapter(dev_priv,
-                                                           intel_hdmi->ddc_bus));
+        if (connector->status != connector_status_connected)
+                return;
 
-        intel_display_power_put(dev_priv, power_domain);
+        intel_hdmi_set_edid(connector);
+        hdmi_to_dig_port(intel_hdmi)->base.type = INTEL_OUTPUT_HDMI;
+}
 
-        return ret;
+static int intel_hdmi_get_modes(struct drm_connector *connector)
+{
+        struct edid *edid;
+
+        edid = to_intel_connector(connector)->detect_edid;
+        if (edid == NULL)
+                return 0;
+
+        return intel_connector_update_modes(connector, edid);
 }
 
 static bool
 intel_hdmi_detect_audio(struct drm_connector *connector)
 {
-        struct intel_encoder *intel_encoder = intel_attached_encoder(connector);
-        struct intel_hdmi *intel_hdmi = enc_to_intel_hdmi(&intel_encoder->base);
-        struct drm_i915_private *dev_priv = connector->dev->dev_private;
-        enum intel_display_power_domain power_domain;
-        struct edid *edid;
         bool has_audio = false;
+        struct edid *edid;
 
-        power_domain = intel_display_port_power_domain(intel_encoder);
-        intel_display_power_get(dev_priv, power_domain);
-
-        edid = drm_get_edid(connector,
-                            intel_gmbus_get_adapter(dev_priv,
-                                                    intel_hdmi->ddc_bus));
-        if (edid) {
-                if (edid->input & DRM_EDID_INPUT_DIGITAL)
-                        has_audio = drm_detect_monitor_audio(edid);
-                kfree(edid);
-        }
-
-        intel_display_power_put(dev_priv, power_domain);
+        edid = to_intel_connector(connector)->detect_edid;
+        if (edid && edid->input & DRM_EDID_INPUT_DIGITAL)
+                has_audio = drm_detect_monitor_audio(edid);
 
         return has_audio;
 }
@@ -1265,6 +1287,8 @@ static void chv_hdmi_pre_pll_enable(struct intel_encoder *encoder)
         enum pipe pipe = intel_crtc->pipe;
         u32 val;
 
+        intel_hdmi_prepare(encoder);
+
         mutex_lock(&dev_priv->dpio_lock);
 
         /* program left/right clock distribution */
@@ -1434,8 +1458,8 @@ static void chv_hdmi_pre_enable(struct intel_encoder *encoder)
 
         for (i = 0; i < 4; i++) {
                 val = vlv_dpio_read(dev_priv, pipe, CHV_TX_DW2(ch, i));
-                val &= ~DPIO_SWING_MARGIN_MASK;
-                val |= 102 << DPIO_SWING_MARGIN_SHIFT;
+                val &= ~DPIO_SWING_MARGIN000_MASK;
+                val |= 102 << DPIO_SWING_MARGIN000_SHIFT;
                 vlv_dpio_write(dev_priv, pipe, CHV_TX_DW2(ch, i), val);
         }
 
@@ -1482,6 +1506,7 @@ static void chv_hdmi_pre_enable(struct intel_encoder *encoder)
 
 static void intel_hdmi_destroy(struct drm_connector *connector)
 {
+        kfree(to_intel_connector(connector)->detect_edid);
         drm_connector_cleanup(connector);
         kfree(connector);
 }
@@ -1489,6 +1514,7 @@ static void intel_hdmi_destroy(struct drm_connector *connector)
 static const struct drm_connector_funcs intel_hdmi_connector_funcs = {
         .dpms = intel_connector_dpms,
         .detect = intel_hdmi_detect,
+        .force = intel_hdmi_force,
         .fill_modes = drm_helper_probe_single_connector_modes,
         .set_property = intel_hdmi_set_property,
         .destroy = intel_hdmi_destroy,
diff --git a/drivers/gpu/drm/i915/intel_lrc.c b/drivers/gpu/drm/i915/intel_lrc.c
new file mode 100644
index 000000000000..bafd38b5703e
--- /dev/null
+++ b/drivers/gpu/drm/i915/intel_lrc.c
@@ -0,0 +1,1766 @@
+/*
+ * Copyright © 2014 Intel Corporation
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+ * IN THE SOFTWARE.
+ *
+ * Authors:
+ *    Ben Widawsky <ben@bwidawsk.net>
+ *    Michel Thierry <michel.thierry@intel.com>
+ *    Thomas Daniel <thomas.daniel@intel.com>
+ *    Oscar Mateo <oscar.mateo@intel.com>
+ *
+ */
+
+/**
+ * DOC: Logical Rings, Logical Ring Contexts and Execlists
+ *
+ * Motivation:
+ * GEN8 brings an expansion of the HW contexts: "Logical Ring Contexts".
+ * These expanded contexts enable a number of new abilities, especially
+ * "Execlists" (also implemented in this file).
+ *
+ * One of the main differences with the legacy HW contexts is that logical
+ * ring contexts incorporate many more things to the context's state, like
+ * PDPs or ringbuffer control registers:
+ *
+ * The reason why PDPs are included in the context is straightforward: as
+ * PPGTTs (per-process GTTs) are actually per-context, having the PDPs
+ * contained there mean you don't need to do a ppgtt->switch_mm yourself,
+ * instead, the GPU will do it for you on the context switch.
+ *
+ * But, what about the ringbuffer control registers (head, tail, etc..)?
+ * shouldn't we just need a set of those per engine command streamer? This is
+ * where the name "Logical Rings" starts to make sense: by virtualizing the
+ * rings, the engine cs shifts to a new "ring buffer" with every context
+ * switch. When you want to submit a workload to the GPU you: A) choose your
+ * context, B) find its appropriate virtualized ring, C) write commands to it
+ * and then, finally, D) tell the GPU to switch to that context.
+ *
+ * Instead of the legacy MI_SET_CONTEXT, the way you tell the GPU to switch
+ * to a contexts is via a context execution list, ergo "Execlists".
+ *
+ * LRC implementation:
+ * Regarding the creation of contexts, we have:
+ *
+ * - One global default context.
+ * - One local default context for each opened fd.
+ * - One local extra context for each context create ioctl call.
+ *
+ * Now that ringbuffers belong per-context (and not per-engine, like before)
+ * and that contexts are uniquely tied to a given engine (and not reusable,
+ * like before) we need:
+ *
+ * - One ringbuffer per-engine inside each context.
+ * - One backing object per-engine inside each context.
+ *
+ * The global default context starts its life with these new objects fully
+ * allocated and populated. The local default context for each opened fd is
+ * more complex, because we don't know at creation time which engine is going
+ * to use them. To handle this, we have implemented a deferred creation of LR
+ * contexts:
+ *
+ * The local context starts its life as a hollow or blank holder, that only
+ * gets populated for a given engine once we receive an execbuffer. If later
+ * on we receive another execbuffer ioctl for the same context but a different
+ * engine, we allocate/populate a new ringbuffer and context backing object and
+ * so on.
+ *
+ * Finally, regarding local contexts created using the ioctl call: as they are
+ * only allowed with the render ring, we can allocate & populate them right
+ * away (no need to defer anything, at least for now).
+ *
+ * Execlists implementation:
+ * Execlists are the new method by which, on gen8+ hardware, workloads are
+ * submitted for execution (as opposed to the legacy, ringbuffer-based, method).
+ * This method works as follows:
+ *
+ * When a request is committed, its commands (the BB start and any leading or
+ * trailing commands, like the seqno breadcrumbs) are placed in the ringbuffer
+ * for the appropriate context. The tail pointer in the hardware context is not
+ * updated at this time, but instead, kept by the driver in the ringbuffer
+ * structure. A structure representing this request is added to a request queue
+ * for the appropriate engine: this structure contains a copy of the context's
+ * tail after the request was written to the ring buffer and a pointer to the
+ * context itself.
+ *
+ * If the engine's request queue was empty before the request was added, the
+ * queue is processed immediately. Otherwise the queue will be processed during
+ * a context switch interrupt. In any case, elements on the queue will get sent
+ * (in pairs) to the GPU's ExecLists Submit Port (ELSP, for short) with a
+ * globally unique 20-bits submission ID.
+ *
+ * When execution of a request completes, the GPU updates the context status
+ * buffer with a context complete event and generates a context switch interrupt.
+ * During the interrupt handling, the driver examines the events in the buffer:
+ * for each context complete event, if the announced ID matches that on the head
+ * of the request queue, then that request is retired and removed from the queue.
+ *
+ * After processing, if any requests were retired and the queue is not empty
+ * then a new execution list can be submitted. The two requests at the front of
+ * the queue are next to be submitted but since a context may not occur twice in
+ * an execution list, if subsequent requests have the same ID as the first then
+ * the two requests must be combined. This is done simply by discarding requests
+ * at the head of the queue until either only one requests is left (in which case
+ * we use a NULL second context) or the first two requests have unique IDs.
+ *
+ * By always executing the first two requests in the queue the driver ensures
+ * that the GPU is kept as busy as possible. In the case where a single context
+ * completes but a second context is still executing, the request for this second
+ * context will be at the head of the queue when we remove the first one. This
+ * request will then be resubmitted along with a new request for a different context,
+ * which will cause the hardware to continue executing the second request and queue
+ * the new request (the GPU detects the condition of a context getting preempted
+ * with the same context and optimizes the context switch flow by not doing
+ * preemption, but just sampling the new tail pointer).
+ *
+ */
+
+#include <drm/drmP.h>
+#include <drm/i915_drm.h>
+#include "i915_drv.h"
+
+#define GEN8_LR_CONTEXT_RENDER_SIZE (20 * PAGE_SIZE)
+#define GEN8_LR_CONTEXT_OTHER_SIZE (2 * PAGE_SIZE)
+
+#define GEN8_LR_CONTEXT_ALIGN 4096
+
+#define RING_EXECLIST_QFULL             (1 << 0x2)
+#define RING_EXECLIST1_VALID            (1 << 0x3)
+#define RING_EXECLIST0_VALID            (1 << 0x4)
+#define RING_EXECLIST_ACTIVE_STATUS     (3 << 0xE)
+#define RING_EXECLIST1_ACTIVE           (1 << 0x11)
+#define RING_EXECLIST0_ACTIVE           (1 << 0x12)
+
+#define GEN8_CTX_STATUS_IDLE_ACTIVE     (1 << 0)
+#define GEN8_CTX_STATUS_PREEMPTED       (1 << 1)
+#define GEN8_CTX_STATUS_ELEMENT_SWITCH  (1 << 2)
+#define GEN8_CTX_STATUS_ACTIVE_IDLE     (1 << 3)
+#define GEN8_CTX_STATUS_COMPLETE        (1 << 4)
+#define GEN8_CTX_STATUS_LITE_RESTORE    (1 << 15)
+
+#define CTX_LRI_HEADER_0                0x01
+#define CTX_CONTEXT_CONTROL             0x02
+#define CTX_RING_HEAD                   0x04
+#define CTX_RING_TAIL                   0x06
+#define CTX_RING_BUFFER_START           0x08
+#define CTX_RING_BUFFER_CONTROL         0x0a
+#define CTX_BB_HEAD_U                   0x0c
+#define CTX_BB_HEAD_L                   0x0e
+#define CTX_BB_STATE                    0x10
+#define CTX_SECOND_BB_HEAD_U            0x12
+#define CTX_SECOND_BB_HEAD_L            0x14
+#define CTX_SECOND_BB_STATE             0x16
+#define CTX_BB_PER_CTX_PTR              0x18
+#define CTX_RCS_INDIRECT_CTX            0x1a
+#define CTX_RCS_INDIRECT_CTX_OFFSET     0x1c
+#define CTX_LRI_HEADER_1                0x21
+#define CTX_CTX_TIMESTAMP               0x22
+#define CTX_PDP3_UDW                    0x24
+#define CTX_PDP3_LDW                    0x26
+#define CTX_PDP2_UDW                    0x28
+#define CTX_PDP2_LDW                    0x2a
+#define CTX_PDP1_UDW                    0x2c
+#define CTX_PDP1_LDW                    0x2e
+#define CTX_PDP0_UDW                    0x30
+#define CTX_PDP0_LDW                    0x32
+#define CTX_LRI_HEADER_2                0x41
+#define CTX_R_PWR_CLK_STATE             0x42
+#define CTX_GPGPU_CSR_BASE_ADDRESS      0x44
+
+#define GEN8_CTX_VALID (1<<0)
+#define GEN8_CTX_FORCE_PD_RESTORE (1<<1)
+#define GEN8_CTX_FORCE_RESTORE (1<<2)
+#define GEN8_CTX_L3LLC_COHERENT (1<<5)
+#define GEN8_CTX_PRIVILEGE (1<<8)
+enum {
+        ADVANCED_CONTEXT = 0,
+        LEGACY_CONTEXT,
+        ADVANCED_AD_CONTEXT,
+        LEGACY_64B_CONTEXT
+};
+#define GEN8_CTX_MODE_SHIFT 3
+enum {
+        FAULT_AND_HANG = 0,
+        FAULT_AND_HALT, /* Debug only */
+        FAULT_AND_STREAM,
+        FAULT_AND_CONTINUE /* Unsupported */
+};
+#define GEN8_CTX_ID_SHIFT 32
+
+/**
+ * intel_sanitize_enable_execlists() - sanitize i915.enable_execlists
+ * @dev: DRM device.
+ * @enable_execlists: value of i915.enable_execlists module parameter.
+ *
+ * Only certain platforms support Execlists (the prerequisites being
+ * support for Logical Ring Contexts and Aliasing PPGTT or better),
+ * and only when enabled via module parameter.
+ *
+ * Return: 1 if Execlists is supported and has to be enabled.
+ */
+int intel_sanitize_enable_execlists(struct drm_device *dev, int enable_execlists)
+{
+        WARN_ON(i915.enable_ppgtt == -1);
+
+        if (enable_execlists == 0)
+                return 0;
+
+        if (HAS_LOGICAL_RING_CONTEXTS(dev) && USES_PPGTT(dev) &&
+            i915.use_mmio_flip >= 0)
+                return 1;
+
+        return 0;
+}
+
+/**
+ * intel_execlists_ctx_id() - get the Execlists Context ID
+ * @ctx_obj: Logical Ring Context backing object.
+ *
+ * Do not confuse with ctx->id! Unfortunately we have a name overload
+ * here: the old context ID we pass to userspace as a handler so that
+ * they can refer to a context, and the new context ID we pass to the
+ * ELSP so that the GPU can inform us of the context status via
+ * interrupts.
+ *
+ * Return: 20-bits globally unique context ID.
+ */
+u32 intel_execlists_ctx_id(struct drm_i915_gem_object *ctx_obj)
+{
+        u32 lrca = i915_gem_obj_ggtt_offset(ctx_obj);
+
+        /* LRCA is required to be 4K aligned so the more significant 20 bits
+         * are globally unique */
+        return lrca >> 12;
+}
+
+static uint64_t execlists_ctx_descriptor(struct drm_i915_gem_object *ctx_obj)
+{
+        uint64_t desc;
+        uint64_t lrca = i915_gem_obj_ggtt_offset(ctx_obj);
+
+        WARN_ON(lrca & 0xFFFFFFFF00000FFFULL);
+
+        desc = GEN8_CTX_VALID;
+        desc |= LEGACY_CONTEXT << GEN8_CTX_MODE_SHIFT;
+        desc |= GEN8_CTX_L3LLC_COHERENT;
+        desc |= GEN8_CTX_PRIVILEGE;
+        desc |= lrca;
+        desc |= (u64)intel_execlists_ctx_id(ctx_obj) << GEN8_CTX_ID_SHIFT;
+
+        /* TODO: WaDisableLiteRestore when we start using semaphore
+         * signalling between Command Streamers */
+        /* desc |= GEN8_CTX_FORCE_RESTORE; */
+
+        return desc;
+}
+
+static void execlists_elsp_write(struct intel_engine_cs *ring,
+                                 struct drm_i915_gem_object *ctx_obj0,
+                                 struct drm_i915_gem_object *ctx_obj1)
+{
+        struct drm_i915_private *dev_priv = ring->dev->dev_private;
+        uint64_t temp = 0;
+        uint32_t desc[4];
+        unsigned long flags;
+
+        /* XXX: You must always write both descriptors in the order below. */
+        if (ctx_obj1)
+                temp = execlists_ctx_descriptor(ctx_obj1);
+        else
+                temp = 0;
+        desc[1] = (u32)(temp >> 32);
+        desc[0] = (u32)temp;
+
+        temp = execlists_ctx_descriptor(ctx_obj0);
+        desc[3] = (u32)(temp >> 32);
+        desc[2] = (u32)temp;
+
+        /* Set Force Wakeup bit to prevent GT from entering C6 while ELSP writes
+         * are in progress.
+         *
+         * The other problem is that we can't just call gen6_gt_force_wake_get()
+         * because that function calls intel_runtime_pm_get(), which might sleep.
+         * Instead, we do the runtime_pm_get/put when creating/destroying requests.
+         */
+        spin_lock_irqsave(&dev_priv->uncore.lock, flags);
+        if (IS_CHERRYVIEW(dev_priv->dev)) {
+                if (dev_priv->uncore.fw_rendercount++ == 0)
+                        dev_priv->uncore.funcs.force_wake_get(dev_priv,
+                                                              FORCEWAKE_RENDER);
+                if (dev_priv->uncore.fw_mediacount++ == 0)
+                        dev_priv->uncore.funcs.force_wake_get(dev_priv,
+                                                              FORCEWAKE_MEDIA);
+        } else {
+                if (dev_priv->uncore.forcewake_count++ == 0)
+                        dev_priv->uncore.funcs.force_wake_get(dev_priv,
+                                                              FORCEWAKE_ALL);
+        }
+        spin_unlock_irqrestore(&dev_priv->uncore.lock, flags);
+
+        I915_WRITE(RING_ELSP(ring), desc[1]);
+        I915_WRITE(RING_ELSP(ring), desc[0]);
+        I915_WRITE(RING_ELSP(ring), desc[3]);
+        /* The context is automatically loaded after the following */
+        I915_WRITE(RING_ELSP(ring), desc[2]);
+
+        /* ELSP is a wo register, so use another nearby reg for posting instead */
+        POSTING_READ(RING_EXECLIST_STATUS(ring));
+
+        /* Release Force Wakeup (see the big comment above). */
+        spin_lock_irqsave(&dev_priv->uncore.lock, flags);
+        if (IS_CHERRYVIEW(dev_priv->dev)) {
+                if (--dev_priv->uncore.fw_rendercount == 0)
+                        dev_priv->uncore.funcs.force_wake_put(dev_priv,
+                                                              FORCEWAKE_RENDER);
+                if (--dev_priv->uncore.fw_mediacount == 0)
+                        dev_priv->uncore.funcs.force_wake_put(dev_priv,
+                                                              FORCEWAKE_MEDIA);
+        } else {
+                if (--dev_priv->uncore.forcewake_count == 0)
+                        dev_priv->uncore.funcs.force_wake_put(dev_priv,
+                                                              FORCEWAKE_ALL);
+        }
+
+        spin_unlock_irqrestore(&dev_priv->uncore.lock, flags);
+}
+
+static int execlists_ctx_write_tail(struct drm_i915_gem_object *ctx_obj, u32 tail)
+{
+        struct page *page;
+        uint32_t *reg_state;
+
+        page = i915_gem_object_get_page(ctx_obj, 1);
+        reg_state = kmap_atomic(page);
+
+        reg_state[CTX_RING_TAIL+1] = tail;
+
+        kunmap_atomic(reg_state);
+
+        return 0;
+}
+
+static int execlists_submit_context(struct intel_engine_cs *ring,
+                                    struct intel_context *to0, u32 tail0,
+                                    struct intel_context *to1, u32 tail1)
+{
+        struct drm_i915_gem_object *ctx_obj0;
+        struct drm_i915_gem_object *ctx_obj1 = NULL;
+
+        ctx_obj0 = to0->engine[ring->id].state;
+        BUG_ON(!ctx_obj0);
+        WARN_ON(!i915_gem_obj_is_pinned(ctx_obj0));
+
+        execlists_ctx_write_tail(ctx_obj0, tail0);
+
+        if (to1) {
+                ctx_obj1 = to1->engine[ring->id].state;
+                BUG_ON(!ctx_obj1);
+                WARN_ON(!i915_gem_obj_is_pinned(ctx_obj1));
+
+                execlists_ctx_write_tail(ctx_obj1, tail1);
+        }
+
+        execlists_elsp_write(ring, ctx_obj0, ctx_obj1);
+
+        return 0;
+}
+
+static void execlists_context_unqueue(struct intel_engine_cs *ring)
+{
+        struct intel_ctx_submit_request *req0 = NULL, *req1 = NULL;
+        struct intel_ctx_submit_request *cursor = NULL, *tmp = NULL;
+        struct drm_i915_private *dev_priv = ring->dev->dev_private;
+
+        assert_spin_locked(&ring->execlist_lock);
+
+        if (list_empty(&ring->execlist_queue))
+                return;
+
+        /* Try to read in pairs */
+        list_for_each_entry_safe(cursor, tmp, &ring->execlist_queue,
+                                 execlist_link) {
+                if (!req0) {
+                        req0 = cursor;
+                } else if (req0->ctx == cursor->ctx) {
+                        /* Same ctx: ignore first request, as second request
+                         * will update tail past first request's workload */
+                        cursor->elsp_submitted = req0->elsp_submitted;
+                        list_del(&req0->execlist_link);
+                        queue_work(dev_priv->wq, &req0->work);
+                        req0 = cursor;
+                } else {
+                        req1 = cursor;
+                        break;
+                }
+        }
+
+        WARN_ON(req1 && req1->elsp_submitted);
+
+        WARN_ON(execlists_submit_context(ring, req0->ctx, req0->tail,
+                                         req1 ? req1->ctx : NULL,
+                                         req1 ? req1->tail : 0));
+
+        req0->elsp_submitted++;
+        if (req1)
+                req1->elsp_submitted++;
+}
+
+static bool execlists_check_remove_request(struct intel_engine_cs *ring,
+                                           u32 request_id)
+{
+        struct drm_i915_private *dev_priv = ring->dev->dev_private;
+        struct intel_ctx_submit_request *head_req;
+
+        assert_spin_locked(&ring->execlist_lock);
+
+        head_req = list_first_entry_or_null(&ring->execlist_queue,
+                                            struct intel_ctx_submit_request,
+                                            execlist_link);
+
+        if (head_req != NULL) {
+                struct drm_i915_gem_object *ctx_obj =
+                                head_req->ctx->engine[ring->id].state;
+                if (intel_execlists_ctx_id(ctx_obj) == request_id) {
+                        WARN(head_req->elsp_submitted == 0,
+                             "Never submitted head request\n");
+
+                        if (--head_req->elsp_submitted <= 0) {
+                                list_del(&head_req->execlist_link);
+                                queue_work(dev_priv->wq, &head_req->work);
+                                return true;
+                        }
+                }
+        }
+
+        return false;
+}
+
+/**
+ * intel_execlists_handle_ctx_events() - handle Context Switch interrupts
+ * @ring: Engine Command Streamer to handle.
+ *
+ * Check the unread Context Status Buffers and manage the submission of new
+ * contexts to the ELSP accordingly.
+ */
+void intel_execlists_handle_ctx_events(struct intel_engine_cs *ring)
+{
+        struct drm_i915_private *dev_priv = ring->dev->dev_private;
+        u32 status_pointer;
+        u8 read_pointer;
+        u8 write_pointer;
+        u32 status;
+        u32 status_id;
+        u32 submit_contexts = 0;
+
+        status_pointer = I915_READ(RING_CONTEXT_STATUS_PTR(ring));
+
+        read_pointer = ring->next_context_status_buffer;
+        write_pointer = status_pointer & 0x07;
+        if (read_pointer > write_pointer)
+                write_pointer += 6;
+
+        spin_lock(&ring->execlist_lock);
+
+        while (read_pointer < write_pointer) {
+                read_pointer++;
+                status = I915_READ(RING_CONTEXT_STATUS_BUF(ring) +
+                                (read_pointer % 6) * 8);
+                status_id = I915_READ(RING_CONTEXT_STATUS_BUF(ring) +
+                                (read_pointer % 6) * 8 + 4);
+
+                if (status & GEN8_CTX_STATUS_PREEMPTED) {
+                        if (status & GEN8_CTX_STATUS_LITE_RESTORE) {
+                                if (execlists_check_remove_request(ring, status_id))
+                                        WARN(1, "Lite Restored request removed from queue\n");
+                        } else
+                                WARN(1, "Preemption without Lite Restore\n");
+                }
+
+                 if ((status & GEN8_CTX_STATUS_ACTIVE_IDLE) ||
+                     (status & GEN8_CTX_STATUS_ELEMENT_SWITCH)) {
+                        if (execlists_check_remove_request(ring, status_id))
+                                submit_contexts++;
+                }
+        }
+
+        if (submit_contexts != 0)
+                execlists_context_unqueue(ring);
+
+        spin_unlock(&ring->execlist_lock);
+
+        WARN(submit_contexts > 2, "More than two context complete events?\n");
+        ring->next_context_status_buffer = write_pointer % 6;
+
+        I915_WRITE(RING_CONTEXT_STATUS_PTR(ring),
+                   ((u32)ring->next_context_status_buffer & 0x07) << 8);
+}
+
+static void execlists_free_request_task(struct work_struct *work)
+{
+        struct intel_ctx_submit_request *req =
+                container_of(work, struct intel_ctx_submit_request, work);
+        struct drm_device *dev = req->ring->dev;
+        struct drm_i915_private *dev_priv = dev->dev_private;
+
+        intel_runtime_pm_put(dev_priv);
+
+        mutex_lock(&dev->struct_mutex);
+        i915_gem_context_unreference(req->ctx);
+        mutex_unlock(&dev->struct_mutex);
+
+        kfree(req);
+}
+
+static int execlists_context_queue(struct intel_engine_cs *ring,
+                                   struct intel_context *to,
+                                   u32 tail)
+{
+        struct intel_ctx_submit_request *req = NULL, *cursor;
+        struct drm_i915_private *dev_priv = ring->dev->dev_private;
+        unsigned long flags;
+        int num_elements = 0;
+
+        req = kzalloc(sizeof(*req), GFP_KERNEL);
+        if (req == NULL)
+                return -ENOMEM;
+        req->ctx = to;
+        i915_gem_context_reference(req->ctx);
+        req->ring = ring;
+        req->tail = tail;
+        INIT_WORK(&req->work, execlists_free_request_task);
+
+        intel_runtime_pm_get(dev_priv);
+
+        spin_lock_irqsave(&ring->execlist_lock, flags);
+
+        list_for_each_entry(cursor, &ring->execlist_queue, execlist_link)
+                if (++num_elements > 2)
+                        break;
+
+        if (num_elements > 2) {
+                struct intel_ctx_submit_request *tail_req;
+
+                tail_req = list_last_entry(&ring->execlist_queue,
+                                           struct intel_ctx_submit_request,
+                                           execlist_link);
+
+                if (to == tail_req->ctx) {
+                        WARN(tail_req->elsp_submitted != 0,
+                             "More than 2 already-submitted reqs queued\n");
+                        list_del(&tail_req->execlist_link);
+                        queue_work(dev_priv->wq, &tail_req->work);
+                }
+        }
+
+        list_add_tail(&req->execlist_link, &ring->execlist_queue);
+        if (num_elements == 0)
+                execlists_context_unqueue(ring);
+
+        spin_unlock_irqrestore(&ring->execlist_lock, flags);
+
+        return 0;
+}
+
+static int logical_ring_invalidate_all_caches(struct intel_ringbuffer *ringbuf)
+{
+        struct intel_engine_cs *ring = ringbuf->ring;
+        uint32_t flush_domains;
+        int ret;
+
+        flush_domains = 0;
+        if (ring->gpu_caches_dirty)
+                flush_domains = I915_GEM_GPU_DOMAINS;
+
+        ret = ring->emit_flush(ringbuf, I915_GEM_GPU_DOMAINS, flush_domains);
+        if (ret)
+                return ret;
+
+        ring->gpu_caches_dirty = false;
+        return 0;
+}
+
+static int execlists_move_to_gpu(struct intel_ringbuffer *ringbuf,
+                                 struct list_head *vmas)
+{
+        struct intel_engine_cs *ring = ringbuf->ring;
+        struct i915_vma *vma;
+        uint32_t flush_domains = 0;
+        bool flush_chipset = false;
+        int ret;
+
+        list_for_each_entry(vma, vmas, exec_list) {
+                struct drm_i915_gem_object *obj = vma->obj;
+
+                ret = i915_gem_object_sync(obj, ring);
+                if (ret)
+                        return ret;
+
+                if (obj->base.write_domain & I915_GEM_DOMAIN_CPU)
+                        flush_chipset |= i915_gem_clflush_object(obj, false);
+
+                flush_domains |= obj->base.write_domain;
+        }
+
+        if (flush_domains & I915_GEM_DOMAIN_GTT)
+                wmb();
+
+        /* Unconditionally invalidate gpu caches and ensure that we do flush
+         * any residual writes from the previous batch.
+         */
+        return logical_ring_invalidate_all_caches(ringbuf);
+}
+
+/**
+ * execlists_submission() - submit a batchbuffer for execution, Execlists style
+ * @dev: DRM device.
+ * @file: DRM file.
+ * @ring: Engine Command Streamer to submit to.
+ * @ctx: Context to employ for this submission.
+ * @args: execbuffer call arguments.
+ * @vmas: list of vmas.
+ * @batch_obj: the batchbuffer to submit.
+ * @exec_start: batchbuffer start virtual address pointer.
+ * @flags: translated execbuffer call flags.
+ *
+ * This is the evil twin version of i915_gem_ringbuffer_submission. It abstracts
+ * away the submission details of the execbuffer ioctl call.
+ *
+ * Return: non-zero if the submission fails.
+ */
+int intel_execlists_submission(struct drm_device *dev, struct drm_file *file,
+                               struct intel_engine_cs *ring,
+                               struct intel_context *ctx,
+                               struct drm_i915_gem_execbuffer2 *args,
+                               struct list_head *vmas,
+                               struct drm_i915_gem_object *batch_obj,
+                               u64 exec_start, u32 flags)
+{
+        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct intel_ringbuffer *ringbuf = ctx->engine[ring->id].ringbuf;
+        int instp_mode;
+        u32 instp_mask;
+        int ret;
+
+        instp_mode = args->flags & I915_EXEC_CONSTANTS_MASK;
+        instp_mask = I915_EXEC_CONSTANTS_MASK;
+        switch (instp_mode) {
+        case I915_EXEC_CONSTANTS_REL_GENERAL:
+        case I915_EXEC_CONSTANTS_ABSOLUTE:
+        case I915_EXEC_CONSTANTS_REL_SURFACE:
+                if (instp_mode != 0 && ring != &dev_priv->ring[RCS]) {
+                        DRM_DEBUG("non-0 rel constants mode on non-RCS\n");
+                        return -EINVAL;
+                }
+
+                if (instp_mode != dev_priv->relative_constants_mode) {
+                        if (instp_mode == I915_EXEC_CONSTANTS_REL_SURFACE) {
+                                DRM_DEBUG("rel surface constants mode invalid on gen5+\n");
+                                return -EINVAL;
+                        }
+
+                        /* The HW changed the meaning on this bit on gen6 */
+                        instp_mask &= ~I915_EXEC_CONSTANTS_REL_SURFACE;
+                }
+                break;
+        default:
+                DRM_DEBUG("execbuf with unknown constants: %d\n", instp_mode);
+                return -EINVAL;
+        }
+
+        if (args->num_cliprects != 0) {
+                DRM_DEBUG("clip rectangles are only valid on pre-gen5\n");
+                return -EINVAL;
+        } else {
+                if (args->DR4 == 0xffffffff) {
+                        DRM_DEBUG("UXA submitting garbage DR4, fixing up\n");
+                        args->DR4 = 0;
+                }
+
+                if (args->DR1 || args->DR4 || args->cliprects_ptr) {
+                        DRM_DEBUG("0 cliprects but dirt in cliprects fields\n");
+                        return -EINVAL;
+                }
+        }
+
+        if (args->flags & I915_EXEC_GEN7_SOL_RESET) {
+                DRM_DEBUG("sol reset is gen7 only\n");
+                return -EINVAL;
+        }
+
+        ret = execlists_move_to_gpu(ringbuf, vmas);
+        if (ret)
+                return ret;
+
+        if (ring == &dev_priv->ring[RCS] &&
+            instp_mode != dev_priv->relative_constants_mode) {
+                ret = intel_logical_ring_begin(ringbuf, 4);
+                if (ret)
+                        return ret;
+
+                intel_logical_ring_emit(ringbuf, MI_NOOP);
+                intel_logical_ring_emit(ringbuf, MI_LOAD_REGISTER_IMM(1));
+                intel_logical_ring_emit(ringbuf, INSTPM);
+                intel_logical_ring_emit(ringbuf, instp_mask << 16 | instp_mode);
+                intel_logical_ring_advance(ringbuf);
+
+                dev_priv->relative_constants_mode = instp_mode;
+        }
+
+        ret = ring->emit_bb_start(ringbuf, exec_start, flags);
+        if (ret)
+                return ret;
+
+        i915_gem_execbuffer_move_to_active(vmas, ring);
+        i915_gem_execbuffer_retire_commands(dev, file, ring, batch_obj);
+
+        return 0;
+}
+
+void intel_logical_ring_stop(struct intel_engine_cs *ring)
+{
+        struct drm_i915_private *dev_priv = ring->dev->dev_private;
+        int ret;
+
+        if (!intel_ring_initialized(ring))
+                return;
+
+        ret = intel_ring_idle(ring);
+        if (ret && !i915_reset_in_progress(&to_i915(ring->dev)->gpu_error))
+                DRM_ERROR("failed to quiesce %s whilst cleaning up: %d\n",
+                          ring->name, ret);
+
+        /* TODO: Is this correct with Execlists enabled? */
+        I915_WRITE_MODE(ring, _MASKED_BIT_ENABLE(STOP_RING));
+        if (wait_for_atomic((I915_READ_MODE(ring) & MODE_IDLE) != 0, 1000)) {
+                DRM_ERROR("%s :timed out trying to stop ring\n", ring->name);
+                return;
+        }
+        I915_WRITE_MODE(ring, _MASKED_BIT_DISABLE(STOP_RING));
+}
+
+int logical_ring_flush_all_caches(struct intel_ringbuffer *ringbuf)
+{
+        struct intel_engine_cs *ring = ringbuf->ring;
+        int ret;
+
+        if (!ring->gpu_caches_dirty)
+                return 0;
+
+        ret = ring->emit_flush(ringbuf, 0, I915_GEM_GPU_DOMAINS);
+        if (ret)
+                return ret;
+
+        ring->gpu_caches_dirty = false;
+        return 0;
+}
+
+/**
+ * intel_logical_ring_advance_and_submit() - advance the tail and submit the workload
+ * @ringbuf: Logical Ringbuffer to advance.
+ *
+ * The tail is updated in our logical ringbuffer struct, not in the actual context. What
+ * really happens during submission is that the context and current tail will be placed
+ * on a queue waiting for the ELSP to be ready to accept a new context submission. At that
+ * point, the tail *inside* the context is updated and the ELSP written to.
+ */
+void intel_logical_ring_advance_and_submit(struct intel_ringbuffer *ringbuf)
+{
+        struct intel_engine_cs *ring = ringbuf->ring;
+        struct intel_context *ctx = ringbuf->FIXME_lrc_ctx;
+
+        intel_logical_ring_advance(ringbuf);
+
+        if (intel_ring_stopped(ring))
+                return;
+
+        execlists_context_queue(ring, ctx, ringbuf->tail);
+}
+
+static int logical_ring_alloc_seqno(struct intel_engine_cs *ring,
+                                    struct intel_context *ctx)
+{
+        if (ring->outstanding_lazy_seqno)
+                return 0;
+
+        if (ring->preallocated_lazy_request == NULL) {
+                struct drm_i915_gem_request *request;
+
+                request = kmalloc(sizeof(*request), GFP_KERNEL);
+                if (request == NULL)
+                        return -ENOMEM;
+
+                /* Hold a reference to the context this request belongs to
+                 * (we will need it when the time comes to emit/retire the
+                 * request).
+                 */
+                request->ctx = ctx;
+                i915_gem_context_reference(request->ctx);
+
+                ring->preallocated_lazy_request = request;
+        }
+
+        return i915_gem_get_seqno(ring->dev, &ring->outstanding_lazy_seqno);
+}
+
+static int logical_ring_wait_request(struct intel_ringbuffer *ringbuf,
+                                     int bytes)
+{
+        struct intel_engine_cs *ring = ringbuf->ring;
+        struct drm_i915_gem_request *request;
+        u32 seqno = 0;
+        int ret;
+
+        if (ringbuf->last_retired_head != -1) {
+                ringbuf->head = ringbuf->last_retired_head;
+                ringbuf->last_retired_head = -1;
+
+                ringbuf->space = intel_ring_space(ringbuf);
+                if (ringbuf->space >= bytes)
+                        return 0;
+        }
+
+        list_for_each_entry(request, &ring->request_list, list) {
+                if (__intel_ring_space(request->tail, ringbuf->tail,
+                                       ringbuf->size) >= bytes) {
+                        seqno = request->seqno;
+                        break;
+                }
+        }
+
+        if (seqno == 0)
+                return -ENOSPC;
+
+        ret = i915_wait_seqno(ring, seqno);
+        if (ret)
+                return ret;
+
+        i915_gem_retire_requests_ring(ring);
+        ringbuf->head = ringbuf->last_retired_head;
+        ringbuf->last_retired_head = -1;
+
+        ringbuf->space = intel_ring_space(ringbuf);
+        return 0;
+}
+
+static int logical_ring_wait_for_space(struct intel_ringbuffer *ringbuf,
+                                       int bytes)
+{
+        struct intel_engine_cs *ring = ringbuf->ring;
+        struct drm_device *dev = ring->dev;
+        struct drm_i915_private *dev_priv = dev->dev_private;
+        unsigned long end;
+        int ret;
+
+        ret = logical_ring_wait_request(ringbuf, bytes);
+        if (ret != -ENOSPC)
+                return ret;
+
+        /* Force the context submission in case we have been skipping it */
+        intel_logical_ring_advance_and_submit(ringbuf);
+
+        /* With GEM the hangcheck timer should kick us out of the loop,
+         * leaving it early runs the risk of corrupting GEM state (due
+         * to running on almost untested codepaths). But on resume
+         * timers don't work yet, so prevent a complete hang in that
+         * case by choosing an insanely large timeout. */
+        end = jiffies + 60 * HZ;
+
+        do {
+                ringbuf->head = I915_READ_HEAD(ring);
+                ringbuf->space = intel_ring_space(ringbuf);
+                if (ringbuf->space >= bytes) {
+                        ret = 0;
+                        break;
+                }
+
+                msleep(1);
+
+                if (dev_priv->mm.interruptible && signal_pending(current)) {
+                        ret = -ERESTARTSYS;
+                        break;
+                }
+
+                ret = i915_gem_check_wedge(&dev_priv->gpu_error,
+                                           dev_priv->mm.interruptible);
+                if (ret)
+                        break;
+
+                if (time_after(jiffies, end)) {
+                        ret = -EBUSY;
+                        break;
+                }
+        } while (1);
+
+        return ret;
+}
+
+static int logical_ring_wrap_buffer(struct intel_ringbuffer *ringbuf)
+{
+        uint32_t __iomem *virt;
+        int rem = ringbuf->size - ringbuf->tail;
+
+        if (ringbuf->space < rem) {
+                int ret = logical_ring_wait_for_space(ringbuf, rem);
+
+                if (ret)
+                        return ret;
+        }
+
+        virt = ringbuf->virtual_start + ringbuf->tail;
+        rem /= 4;
+        while (rem--)
+                iowrite32(MI_NOOP, virt++);
+
+        ringbuf->tail = 0;
+        ringbuf->space = intel_ring_space(ringbuf);
+
+        return 0;
+}
+
+static int logical_ring_prepare(struct intel_ringbuffer *ringbuf, int bytes)
+{
+        int ret;
+
+        if (unlikely(ringbuf->tail + bytes > ringbuf->effective_size)) {
+                ret = logical_ring_wrap_buffer(ringbuf);
+                if (unlikely(ret))
+                        return ret;
+        }
+
+        if (unlikely(ringbuf->space < bytes)) {
+                ret = logical_ring_wait_for_space(ringbuf, bytes);
+                if (unlikely(ret))
+                        return ret;
+        }
+
+        return 0;
+}
+
+/**
+ * intel_logical_ring_begin() - prepare the logical ringbuffer to accept some commands
+ *
+ * @ringbuf: Logical ringbuffer.
+ * @num_dwords: number of DWORDs that we plan to write to the ringbuffer.
+ *
+ * The ringbuffer might not be ready to accept the commands right away (maybe it needs to
+ * be wrapped, or wait a bit for the tail to be updated). This function takes care of that
+ * and also preallocates a request (every workload submission is still mediated through
+ * requests, same as it did with legacy ringbuffer submission).
+ *
+ * Return: non-zero if the ringbuffer is not ready to be written to.
+ */
+int intel_logical_ring_begin(struct intel_ringbuffer *ringbuf, int num_dwords)
+{
+        struct intel_engine_cs *ring = ringbuf->ring;
+        struct drm_device *dev = ring->dev;
+        struct drm_i915_private *dev_priv = dev->dev_private;
+        int ret;
+
+        ret = i915_gem_check_wedge(&dev_priv->gpu_error,
+                                   dev_priv->mm.interruptible);
+        if (ret)
+                return ret;
+
+        ret = logical_ring_prepare(ringbuf, num_dwords * sizeof(uint32_t));
+        if (ret)
+                return ret;
+
+        /* Preallocate the olr before touching the ring */
+        ret = logical_ring_alloc_seqno(ring, ringbuf->FIXME_lrc_ctx);
+        if (ret)
+                return ret;
+
+        ringbuf->space -= num_dwords * sizeof(uint32_t);
+        return 0;
+}
+
+static int gen8_init_common_ring(struct intel_engine_cs *ring)
+{
+        struct drm_device *dev = ring->dev;
+        struct drm_i915_private *dev_priv = dev->dev_private;
+
+        I915_WRITE_IMR(ring, ~(ring->irq_enable_mask | ring->irq_keep_mask));
+        I915_WRITE(RING_HWSTAM(ring->mmio_base), 0xffffffff);
+
+        I915_WRITE(RING_MODE_GEN7(ring),
+                   _MASKED_BIT_DISABLE(GFX_REPLAY_MODE) |
+                   _MASKED_BIT_ENABLE(GFX_RUN_LIST_ENABLE));
+        POSTING_READ(RING_MODE_GEN7(ring));
+        DRM_DEBUG_DRIVER("Execlists enabled for %s\n", ring->name);
+
+        memset(&ring->hangcheck, 0, sizeof(ring->hangcheck));
+
+        return 0;
+}
+
+static int gen8_init_render_ring(struct intel_engine_cs *ring)
+{
+        struct drm_device *dev = ring->dev;
+        struct drm_i915_private *dev_priv = dev->dev_private;
+        int ret;
+
+        ret = gen8_init_common_ring(ring);
+        if (ret)
+                return ret;
+
+        /* We need to disable the AsyncFlip performance optimisations in order
+         * to use MI_WAIT_FOR_EVENT within the CS. It should already be
+         * programmed to '1' on all products.
+         *
+         * WaDisableAsyncFlipPerfMode:snb,ivb,hsw,vlv,bdw,chv
+         */
+        I915_WRITE(MI_MODE, _MASKED_BIT_ENABLE(ASYNC_FLIP_PERF_DISABLE));
+
+        ret = intel_init_pipe_control(ring);
+        if (ret)
+                return ret;
+
+        I915_WRITE(INSTPM, _MASKED_BIT_ENABLE(INSTPM_FORCE_ORDERING));
+
+        return ret;
+}
+
+static int gen8_emit_bb_start(struct intel_ringbuffer *ringbuf,
+                              u64 offset, unsigned flags)
+{
+        bool ppgtt = !(flags & I915_DISPATCH_SECURE);
+        int ret;
+
+        ret = intel_logical_ring_begin(ringbuf, 4);
+        if (ret)
+                return ret;
+
+        /* FIXME(BDW): Address space and security selectors. */
+        intel_logical_ring_emit(ringbuf, MI_BATCH_BUFFER_START_GEN8 | (ppgtt<<8));
+        intel_logical_ring_emit(ringbuf, lower_32_bits(offset));
+        intel_logical_ring_emit(ringbuf, upper_32_bits(offset));
+        intel_logical_ring_emit(ringbuf, MI_NOOP);
+        intel_logical_ring_advance(ringbuf);
+
+        return 0;
+}
+
+static bool gen8_logical_ring_get_irq(struct intel_engine_cs *ring)
+{
+        struct drm_device *dev = ring->dev;
+        struct drm_i915_private *dev_priv = dev->dev_private;
+        unsigned long flags;
+
+        if (!dev->irq_enabled)
+                return false;
+
+        spin_lock_irqsave(&dev_priv->irq_lock, flags);
+        if (ring->irq_refcount++ == 0) {
+                I915_WRITE_IMR(ring, ~(ring->irq_enable_mask | ring->irq_keep_mask));
+                POSTING_READ(RING_IMR(ring->mmio_base));
+        }
+        spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
+
+        return true;
+}
+
+static void gen8_logical_ring_put_irq(struct intel_engine_cs *ring)
+{
+        struct drm_device *dev = ring->dev;
+        struct drm_i915_private *dev_priv = dev->dev_private;
+        unsigned long flags;
+
+        spin_lock_irqsave(&dev_priv->irq_lock, flags);
+        if (--ring->irq_refcount == 0) {
+                I915_WRITE_IMR(ring, ~ring->irq_keep_mask);
+                POSTING_READ(RING_IMR(ring->mmio_base));
+        }
+        spin_unlock_irqrestore(&dev_priv->irq_lock, flags);
+}
+
+static int gen8_emit_flush(struct intel_ringbuffer *ringbuf,
+                           u32 invalidate_domains,
+                           u32 unused)
+{
+        struct intel_engine_cs *ring = ringbuf->ring;
+        struct drm_device *dev = ring->dev;
+        struct drm_i915_private *dev_priv = dev->dev_private;
+        uint32_t cmd;
+        int ret;
+
+        ret = intel_logical_ring_begin(ringbuf, 4);
+        if (ret)
+                return ret;
+
+        cmd = MI_FLUSH_DW + 1;
+
+        if (ring == &dev_priv->ring[VCS]) {
+                if (invalidate_domains & I915_GEM_GPU_DOMAINS)
+                        cmd |= MI_INVALIDATE_TLB | MI_INVALIDATE_BSD |
+                                MI_FLUSH_DW_STORE_INDEX |
+                                MI_FLUSH_DW_OP_STOREDW;
+        } else {
+                if (invalidate_domains & I915_GEM_DOMAIN_RENDER)
+                        cmd |= MI_INVALIDATE_TLB | MI_FLUSH_DW_STORE_INDEX |
+                                MI_FLUSH_DW_OP_STOREDW;
+        }
+
+        intel_logical_ring_emit(ringbuf, cmd);
+        intel_logical_ring_emit(ringbuf,
+                                I915_GEM_HWS_SCRATCH_ADDR |
+                                MI_FLUSH_DW_USE_GTT);
+        intel_logical_ring_emit(ringbuf, 0); /* upper addr */
+        intel_logical_ring_emit(ringbuf, 0); /* value */
+        intel_logical_ring_advance(ringbuf);
+
+        return 0;
+}
+
+static int gen8_emit_flush_render(struct intel_ringbuffer *ringbuf,
+                                  u32 invalidate_domains,
+                                  u32 flush_domains)
+{
+        struct intel_engine_cs *ring = ringbuf->ring;
+        u32 scratch_addr = ring->scratch.gtt_offset + 2 * CACHELINE_BYTES;
+        u32 flags = 0;
+        int ret;
+
+        flags |= PIPE_CONTROL_CS_STALL;
+
+        if (flush_domains) {
+                flags |= PIPE_CONTROL_RENDER_TARGET_CACHE_FLUSH;
+                flags |= PIPE_CONTROL_DEPTH_CACHE_FLUSH;
+        }
+
+        if (invalidate_domains) {
+                flags |= PIPE_CONTROL_TLB_INVALIDATE;
+                flags |= PIPE_CONTROL_INSTRUCTION_CACHE_INVALIDATE;
+                flags |= PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE;
+                flags |= PIPE_CONTROL_VF_CACHE_INVALIDATE;
+                flags |= PIPE_CONTROL_CONST_CACHE_INVALIDATE;
+                flags |= PIPE_CONTROL_STATE_CACHE_INVALIDATE;
+                flags |= PIPE_CONTROL_QW_WRITE;
+                flags |= PIPE_CONTROL_GLOBAL_GTT_IVB;
+        }
+
+        ret = intel_logical_ring_begin(ringbuf, 6);
+        if (ret)
+                return ret;
+
+        intel_logical_ring_emit(ringbuf, GFX_OP_PIPE_CONTROL(6));
+        intel_logical_ring_emit(ringbuf, flags);
+        intel_logical_ring_emit(ringbuf, scratch_addr);
+        intel_logical_ring_emit(ringbuf, 0);
+        intel_logical_ring_emit(ringbuf, 0);
+        intel_logical_ring_emit(ringbuf, 0);
+        intel_logical_ring_advance(ringbuf);
+
+        return 0;
+}
+
+static u32 gen8_get_seqno(struct intel_engine_cs *ring, bool lazy_coherency)
+{
+        return intel_read_status_page(ring, I915_GEM_HWS_INDEX);
+}
+
+static void gen8_set_seqno(struct intel_engine_cs *ring, u32 seqno)
+{
+        intel_write_status_page(ring, I915_GEM_HWS_INDEX, seqno);
+}
+
+static int gen8_emit_request(struct intel_ringbuffer *ringbuf)
+{
+        struct intel_engine_cs *ring = ringbuf->ring;
+        u32 cmd;
+        int ret;
+
+        ret = intel_logical_ring_begin(ringbuf, 6);
+        if (ret)
+                return ret;
+
+        cmd = MI_STORE_DWORD_IMM_GEN8;
+        cmd |= MI_GLOBAL_GTT;
+
+        intel_logical_ring_emit(ringbuf, cmd);
+        intel_logical_ring_emit(ringbuf,
+                                (ring->status_page.gfx_addr +
+                                (I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT)));
+        intel_logical_ring_emit(ringbuf, 0);
+        intel_logical_ring_emit(ringbuf, ring->outstanding_lazy_seqno);
+        intel_logical_ring_emit(ringbuf, MI_USER_INTERRUPT);
+        intel_logical_ring_emit(ringbuf, MI_NOOP);
+        intel_logical_ring_advance_and_submit(ringbuf);
+
+        return 0;
+}
+
+/**
+ * intel_logical_ring_cleanup() - deallocate the Engine Command Streamer
+ *
+ * @ring: Engine Command Streamer.
+ *
+ */
+void intel_logical_ring_cleanup(struct intel_engine_cs *ring)
+{
+        struct drm_i915_private *dev_priv = ring->dev->dev_private;
+
+        if (!intel_ring_initialized(ring))
+                return;
+
+        intel_logical_ring_stop(ring);
+        WARN_ON((I915_READ_MODE(ring) & MODE_IDLE) == 0);
+        ring->preallocated_lazy_request = NULL;
+        ring->outstanding_lazy_seqno = 0;
+
+        if (ring->cleanup)
+                ring->cleanup(ring);
+
+        i915_cmd_parser_fini_ring(ring);
+
+        if (ring->status_page.obj) {
+                kunmap(sg_page(ring->status_page.obj->pages->sgl));
+                ring->status_page.obj = NULL;
+        }
+}
+
+static int logical_ring_init(struct drm_device *dev, struct intel_engine_cs *ring)
+{
+        int ret;
+
+        /* Intentionally left blank. */
+        ring->buffer = NULL;
+
+        ring->dev = dev;
+        INIT_LIST_HEAD(&ring->active_list);
+        INIT_LIST_HEAD(&ring->request_list);
+        init_waitqueue_head(&ring->irq_queue);
+
+        INIT_LIST_HEAD(&ring->execlist_queue);
+        spin_lock_init(&ring->execlist_lock);
+        ring->next_context_status_buffer = 0;
+
+        ret = i915_cmd_parser_init_ring(ring);
+        if (ret)
+                return ret;
+
+        if (ring->init) {
+                ret = ring->init(ring);
+                if (ret)
+                        return ret;
+        }
+
+        ret = intel_lr_context_deferred_create(ring->default_context, ring);
+
+        return ret;
+}
+
+static int logical_render_ring_init(struct drm_device *dev)
+{
+        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct intel_engine_cs *ring = &dev_priv->ring[RCS];
+
+        ring->name = "render ring";
+        ring->id = RCS;
+        ring->mmio_base = RENDER_RING_BASE;
+        ring->irq_enable_mask =
+                GT_RENDER_USER_INTERRUPT << GEN8_RCS_IRQ_SHIFT;
+        ring->irq_keep_mask =
+                GT_CONTEXT_SWITCH_INTERRUPT << GEN8_RCS_IRQ_SHIFT;
+        if (HAS_L3_DPF(dev))
+                ring->irq_keep_mask |= GT_RENDER_L3_PARITY_ERROR_INTERRUPT;
+
+        ring->init = gen8_init_render_ring;
+        ring->cleanup = intel_fini_pipe_control;
+        ring->get_seqno = gen8_get_seqno;
+        ring->set_seqno = gen8_set_seqno;
+        ring->emit_request = gen8_emit_request;
+        ring->emit_flush = gen8_emit_flush_render;
+        ring->irq_get = gen8_logical_ring_get_irq;
+        ring->irq_put = gen8_logical_ring_put_irq;
+        ring->emit_bb_start = gen8_emit_bb_start;
+
+        return logical_ring_init(dev, ring);
+}
+
+static int logical_bsd_ring_init(struct drm_device *dev)
+{
+        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct intel_engine_cs *ring = &dev_priv->ring[VCS];
+
+        ring->name = "bsd ring";
+        ring->id = VCS;
+        ring->mmio_base = GEN6_BSD_RING_BASE;
+        ring->irq_enable_mask =
+                GT_RENDER_USER_INTERRUPT << GEN8_VCS1_IRQ_SHIFT;
+        ring->irq_keep_mask =
+                GT_CONTEXT_SWITCH_INTERRUPT << GEN8_VCS1_IRQ_SHIFT;
+
+        ring->init = gen8_init_common_ring;
+        ring->get_seqno = gen8_get_seqno;
+        ring->set_seqno = gen8_set_seqno;
+        ring->emit_request = gen8_emit_request;
+        ring->emit_flush = gen8_emit_flush;
+        ring->irq_get = gen8_logical_ring_get_irq;
+        ring->irq_put = gen8_logical_ring_put_irq;
+        ring->emit_bb_start = gen8_emit_bb_start;
+
+        return logical_ring_init(dev, ring);
+}
+
+static int logical_bsd2_ring_init(struct drm_device *dev)
+{
+        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct intel_engine_cs *ring = &dev_priv->ring[VCS2];
+
+        ring->name = "bds2 ring";
+        ring->id = VCS2;
+        ring->mmio_base = GEN8_BSD2_RING_BASE;
+        ring->irq_enable_mask =
+                GT_RENDER_USER_INTERRUPT << GEN8_VCS2_IRQ_SHIFT;
+        ring->irq_keep_mask =
+                GT_CONTEXT_SWITCH_INTERRUPT << GEN8_VCS2_IRQ_SHIFT;
+
+        ring->init = gen8_init_common_ring;
+        ring->get_seqno = gen8_get_seqno;
+        ring->set_seqno = gen8_set_seqno;
+        ring->emit_request = gen8_emit_request;
+        ring->emit_flush = gen8_emit_flush;
+        ring->irq_get = gen8_logical_ring_get_irq;
+        ring->irq_put = gen8_logical_ring_put_irq;
+        ring->emit_bb_start = gen8_emit_bb_start;
+
+        return logical_ring_init(dev, ring);
+}
+
+static int logical_blt_ring_init(struct drm_device *dev)
+{
+        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct intel_engine_cs *ring = &dev_priv->ring[BCS];
+
+        ring->name = "blitter ring";
+        ring->id = BCS;
+        ring->mmio_base = BLT_RING_BASE;
+        ring->irq_enable_mask =
+                GT_RENDER_USER_INTERRUPT << GEN8_BCS_IRQ_SHIFT;
+        ring->irq_keep_mask =
+                GT_CONTEXT_SWITCH_INTERRUPT << GEN8_BCS_IRQ_SHIFT;
+
+        ring->init = gen8_init_common_ring;
+        ring->get_seqno = gen8_get_seqno;
+        ring->set_seqno = gen8_set_seqno;
+        ring->emit_request = gen8_emit_request;
+        ring->emit_flush = gen8_emit_flush;
+        ring->irq_get = gen8_logical_ring_get_irq;
+        ring->irq_put = gen8_logical_ring_put_irq;
+        ring->emit_bb_start = gen8_emit_bb_start;
+
+        return logical_ring_init(dev, ring);
+}
+
+static int logical_vebox_ring_init(struct drm_device *dev)
+{
+        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct intel_engine_cs *ring = &dev_priv->ring[VECS];
+
+        ring->name = "video enhancement ring";
+        ring->id = VECS;
+        ring->mmio_base = VEBOX_RING_BASE;
+        ring->irq_enable_mask =
+                GT_RENDER_USER_INTERRUPT << GEN8_VECS_IRQ_SHIFT;
+        ring->irq_keep_mask =
+                GT_CONTEXT_SWITCH_INTERRUPT << GEN8_VECS_IRQ_SHIFT;
+
+        ring->init = gen8_init_common_ring;
+        ring->get_seqno = gen8_get_seqno;
+        ring->set_seqno = gen8_set_seqno;
+        ring->emit_request = gen8_emit_request;
+        ring->emit_flush = gen8_emit_flush;
+        ring->irq_get = gen8_logical_ring_get_irq;
+        ring->irq_put = gen8_logical_ring_put_irq;
+        ring->emit_bb_start = gen8_emit_bb_start;
+
+        return logical_ring_init(dev, ring);
+}
+
+/**
+ * intel_logical_rings_init() - allocate, populate and init the Engine Command Streamers
+ * @dev: DRM device.
+ *
+ * This function inits the engines for an Execlists submission style (the equivalent in the
+ * legacy ringbuffer submission world would be i915_gem_init_rings). It does it only for
+ * those engines that are present in the hardware.
+ *
+ * Return: non-zero if the initialization failed.
+ */
+int intel_logical_rings_init(struct drm_device *dev)
+{
+        struct drm_i915_private *dev_priv = dev->dev_private;
+        int ret;
+
+        ret = logical_render_ring_init(dev);
+        if (ret)
+                return ret;
+
+        if (HAS_BSD(dev)) {
+                ret = logical_bsd_ring_init(dev);
+                if (ret)
+                        goto cleanup_render_ring;
+        }
+
+        if (HAS_BLT(dev)) {
+                ret = logical_blt_ring_init(dev);
+                if (ret)
+                        goto cleanup_bsd_ring;
+        }
+
+        if (HAS_VEBOX(dev)) {
+                ret = logical_vebox_ring_init(dev);
+                if (ret)
+                        goto cleanup_blt_ring;
+        }
+
+        if (HAS_BSD2(dev)) {
+                ret = logical_bsd2_ring_init(dev);
+                if (ret)
+                        goto cleanup_vebox_ring;
+        }
+
+        ret = i915_gem_set_seqno(dev, ((u32)~0 - 0x1000));
+        if (ret)
+                goto cleanup_bsd2_ring;
+
+        return 0;
+
+cleanup_bsd2_ring:
+        intel_logical_ring_cleanup(&dev_priv->ring[VCS2]);
+cleanup_vebox_ring:
+        intel_logical_ring_cleanup(&dev_priv->ring[VECS]);
+cleanup_blt_ring:
+        intel_logical_ring_cleanup(&dev_priv->ring[BCS]);
+cleanup_bsd_ring:
+        intel_logical_ring_cleanup(&dev_priv->ring[VCS]);
+cleanup_render_ring:
+        intel_logical_ring_cleanup(&dev_priv->ring[RCS]);
+
+        return ret;
+}
+
+int intel_lr_context_render_state_init(struct intel_engine_cs *ring,
+                                       struct intel_context *ctx)
+{
+        struct intel_ringbuffer *ringbuf = ctx->engine[ring->id].ringbuf;
+        struct render_state so;
+        struct drm_i915_file_private *file_priv = ctx->file_priv;
+        struct drm_file *file = file_priv ? file_priv->file : NULL;
+        int ret;
+
+        ret = i915_gem_render_state_prepare(ring, &so);
+        if (ret)
+                return ret;
+
+        if (so.rodata == NULL)
+                return 0;
+
+        ret = ring->emit_bb_start(ringbuf,
+                        so.ggtt_offset,
+                        I915_DISPATCH_SECURE);
+        if (ret)
+                goto out;
+
+        i915_vma_move_to_active(i915_gem_obj_to_ggtt(so.obj), ring);
+
+        ret = __i915_add_request(ring, file, so.obj, NULL);
+        /* intel_logical_ring_add_request moves object to inactive if it
+         * fails */
+out:
+        i915_gem_render_state_fini(&so);
+        return ret;
+}
+
+static int
+populate_lr_context(struct intel_context *ctx, struct drm_i915_gem_object *ctx_obj,
+                    struct intel_engine_cs *ring, struct intel_ringbuffer *ringbuf)
+{
+        struct drm_device *dev = ring->dev;
+        struct drm_i915_private *dev_priv = dev->dev_private;
+        struct drm_i915_gem_object *ring_obj = ringbuf->obj;
+        struct i915_hw_ppgtt *ppgtt = ctx->ppgtt;
+        struct page *page;
+        uint32_t *reg_state;
+        int ret;
+
+        if (!ppgtt)
+                ppgtt = dev_priv->mm.aliasing_ppgtt;
+
+        ret = i915_gem_object_set_to_cpu_domain(ctx_obj, true);
+        if (ret) {
+                DRM_DEBUG_DRIVER("Could not set to CPU domain\n");
+                return ret;
+        }
+
+        ret = i915_gem_object_get_pages(ctx_obj);
+        if (ret) {
+                DRM_DEBUG_DRIVER("Could not get object pages\n");
+                return ret;
+        }
+
+        i915_gem_object_pin_pages(ctx_obj);
+
+        /* The second page of the context object contains some fields which must
+         * be set up prior to the first execution. */
+        page = i915_gem_object_get_page(ctx_obj, 1);
+        reg_state = kmap_atomic(page);
+
+        /* A context is actually a big batch buffer with several MI_LOAD_REGISTER_IMM
+         * commands followed by (reg, value) pairs. The values we are setting here are
+         * only for the first context restore: on a subsequent save, the GPU will
+         * recreate this batchbuffer with new values (including all the missing
+         * MI_LOAD_REGISTER_IMM commands that we are not initializing here). */
+        if (ring->id == RCS)
+                reg_state[CTX_LRI_HEADER_0] = MI_LOAD_REGISTER_IMM(14);
+        else
+                reg_state[CTX_LRI_HEADER_0] = MI_LOAD_REGISTER_IMM(11);
+        reg_state[CTX_LRI_HEADER_0] |= MI_LRI_FORCE_POSTED;
+        reg_state[CTX_CONTEXT_CONTROL] = RING_CONTEXT_CONTROL(ring);
+        reg_state[CTX_CONTEXT_CONTROL+1] =
+                        _MASKED_BIT_ENABLE((1<<3) | MI_RESTORE_INHIBIT);
+        reg_state[CTX_RING_HEAD] = RING_HEAD(ring->mmio_base);
+        reg_state[CTX_RING_HEAD+1] = 0;
+        reg_state[CTX_RING_TAIL] = RING_TAIL(ring->mmio_base);
+        reg_state[CTX_RING_TAIL+1] = 0;
+        reg_state[CTX_RING_BUFFER_START] = RING_START(ring->mmio_base);
+        reg_state[CTX_RING_BUFFER_START+1] = i915_gem_obj_ggtt_offset(ring_obj);
+        reg_state[CTX_RING_BUFFER_CONTROL] = RING_CTL(ring->mmio_base);
+        reg_state[CTX_RING_BUFFER_CONTROL+1] =
+                        ((ringbuf->size - PAGE_SIZE) & RING_NR_PAGES) | RING_VALID;
+        reg_state[CTX_BB_HEAD_U] = ring->mmio_base + 0x168;
+        reg_state[CTX_BB_HEAD_U+1] = 0;
+        reg_state[CTX_BB_HEAD_L] = ring->mmio_base + 0x140;
+        reg_state[CTX_BB_HEAD_L+1] = 0;
+        reg_state[CTX_BB_STATE] = ring->mmio_base + 0x110;
+        reg_state[CTX_BB_STATE+1] = (1<<5);
+        reg_state[CTX_SECOND_BB_HEAD_U] = ring->mmio_base + 0x11c;
+        reg_state[CTX_SECOND_BB_HEAD_U+1] = 0;
+        reg_state[CTX_SECOND_BB_HEAD_L] = ring->mmio_base + 0x114;
+        reg_state[CTX_SECOND_BB_HEAD_L+1] = 0;
+        reg_state[CTX_SECOND_BB_STATE] = ring->mmio_base + 0x118;
+        reg_state[CTX_SECOND_BB_STATE+1] = 0;
+        if (ring->id == RCS) {
+                /* TODO: according to BSpec, the register state context
+                 * for CHV does not have these. OTOH, these registers do
+                 * exist in CHV. I'm waiting for a clarification */
+                reg_state[CTX_BB_PER_CTX_PTR] = ring->mmio_base + 0x1c0;
+                reg_state[CTX_BB_PER_CTX_PTR+1] = 0;
+                reg_state[CTX_RCS_INDIRECT_CTX] = ring->mmio_base + 0x1c4;
+                reg_state[CTX_RCS_INDIRECT_CTX+1] = 0;
+                reg_state[CTX_RCS_INDIRECT_CTX_OFFSET] = ring->mmio_base + 0x1c8;
+                reg_state[CTX_RCS_INDIRECT_CTX_OFFSET+1] = 0;
+        }
+        reg_state[CTX_LRI_HEADER_1] = MI_LOAD_REGISTER_IMM(9);
+        reg_state[CTX_LRI_HEADER_1] |= MI_LRI_FORCE_POSTED;
+        reg_state[CTX_CTX_TIMESTAMP] = ring->mmio_base + 0x3a8;
+        reg_state[CTX_CTX_TIMESTAMP+1] = 0;
+        reg_state[CTX_PDP3_UDW] = GEN8_RING_PDP_UDW(ring, 3);
+        reg_state[CTX_PDP3_LDW] = GEN8_RING_PDP_LDW(ring, 3);
+        reg_state[CTX_PDP2_UDW] = GEN8_RING_PDP_UDW(ring, 2);
+        reg_state[CTX_PDP2_LDW] = GEN8_RING_PDP_LDW(ring, 2);
+        reg_state[CTX_PDP1_UDW] = GEN8_RING_PDP_UDW(ring, 1);
+        reg_state[CTX_PDP1_LDW] = GEN8_RING_PDP_LDW(ring, 1);
+        reg_state[CTX_PDP0_UDW] = GEN8_RING_PDP_UDW(ring, 0);
+        reg_state[CTX_PDP0_LDW] = GEN8_RING_PDP_LDW(ring, 0);
+        reg_state[CTX_PDP3_UDW+1] = upper_32_bits(ppgtt->pd_dma_addr[3]);
+        reg_state[CTX_PDP3_LDW+1] = lower_32_bits(ppgtt->pd_dma_addr[3]);
+        reg_state[CTX_PDP2_UDW+1] = upper_32_bits(ppgtt->pd_dma_addr[2]);
+        reg_state[CTX_PDP2_LDW+1] = lower_32_bits(ppgtt->pd_dma_addr[2]);
+        reg_state[CTX_PDP1_UDW+1] = upper_32_bits(ppgtt->pd_dma_addr[1]);
+        reg_state[CTX_PDP1_LDW+1] = lower_32_bits(ppgtt->pd_dma_addr[1]);
+        reg_state[CTX_PDP0_UDW+1] = upper_32_bits(ppgtt->pd_dma_addr[0]);
+        reg_state[CTX_PDP0_LDW+1] = lower_32_bits(ppgtt->pd_dma_addr[0]);
+        if (ring->id == RCS) {
+                reg_state[CTX_LRI_HEADER_2] = MI_LOAD_REGISTER_IMM(1);
+                reg_state[CTX_R_PWR_CLK_STATE] = 0x20c8;
+                reg_state[CTX_R_PWR_CLK_STATE+1] = 0;
+        }
+
+        kunmap_atomic(reg_state);
+
+        ctx_obj->dirty = 1;
+        set_page_dirty(page);
+        i915_gem_object_unpin_pages(ctx_obj);
+
+        return 0;
+}
+
+/**
+ * intel_lr_context_free() - free the LRC specific bits of a context
+ * @ctx: the LR context to free.
+ *
+ * The real context freeing is done in i915_gem_context_free: this only
+ * takes care of the bits that are LRC related: the per-engine backing
+ * objects and the logical ringbuffer.
+ */
+void intel_lr_context_free(struct intel_context *ctx)
+{
+        int i;
+
+        for (i = 0; i < I915_NUM_RINGS; i++) {
+                struct drm_i915_gem_object *ctx_obj = ctx->engine[i].state;
+                struct intel_ringbuffer *ringbuf = ctx->engine[i].ringbuf;
+
+                if (ctx_obj) {
+                        intel_destroy_ringbuffer_obj(ringbuf);
+                        kfree(ringbuf);
+                        i915_gem_object_ggtt_unpin(ctx_obj);
+                        drm_gem_object_unreference(&ctx_obj->base);
+                }
+        }
+}
+
+static uint32_t get_lr_context_size(struct intel_engine_cs *ring)
+{
+        int ret = 0;
+
+        WARN_ON(INTEL_INFO(ring->dev)->gen != 8);
+
+        switch (ring->id) {
+        case RCS:
+                ret = GEN8_LR_CONTEXT_RENDER_SIZE;
+                break;
+        case VCS:
+        case BCS:
+        case VECS:
+        case VCS2:
+                ret = GEN8_LR_CONTEXT_OTHER_SIZE;
+                break;
+        }
+
+        return ret;
+}
+
+/**
+ * intel_lr_context_deferred_create() - create the LRC specific bits of a context
+ * @ctx: LR context to create.
+ * @ring: engine to be used with the context.
+ *
+ * This function can be called more than once, with different engines, if we plan
+ * to use the context with them. The context backing objects and the ringbuffers
+ * (specially the ringbuffer backing objects) suck a lot of memory up, and that's why
+ * the creation is a deferred call: it's better to make sure first that we need to use
+ * a given ring with the context.
+ *
+ * Return: non-zero on eror.
+ */
+int intel_lr_context_deferred_create(struct intel_context *ctx,
+                                     struct intel_engine_cs *ring)
+{
+        struct drm_device *dev = ring->dev;
+        struct drm_i915_gem_object *ctx_obj;
+        uint32_t context_size;
+        struct intel_ringbuffer *ringbuf;
+        int ret;
+
+        WARN_ON(ctx->legacy_hw_ctx.rcs_state != NULL);
+        if (ctx->engine[ring->id].state)
+                return 0;
+
+        context_size = round_up(get_lr_context_size(ring), 4096);
+
+        ctx_obj = i915_gem_alloc_context_obj(dev, context_size);
+        if (IS_ERR(ctx_obj)) {
+                ret = PTR_ERR(ctx_obj);
+                DRM_DEBUG_DRIVER("Alloc LRC backing obj failed: %d\n", ret);
+                return ret;
+        }
+
+        ret = i915_gem_obj_ggtt_pin(ctx_obj, GEN8_LR_CONTEXT_ALIGN, 0);
+        if (ret) {
+                DRM_DEBUG_DRIVER("Pin LRC backing obj failed: %d\n", ret);
+                drm_gem_object_unreference(&ctx_obj->base);
+                return ret;
+        }
+
+        ringbuf = kzalloc(sizeof(*ringbuf), GFP_KERNEL);
+        if (!ringbuf) {
+                DRM_DEBUG_DRIVER("Failed to allocate ringbuffer %s\n",
+                                ring->name);
+                i915_gem_object_ggtt_unpin(ctx_obj);
+                drm_gem_object_unreference(&ctx_obj->base);
+                ret = -ENOMEM;
+                return ret;
+        }
+
+        ringbuf->ring = ring;
+        ringbuf->FIXME_lrc_ctx = ctx;
+
+        ringbuf->size = 32 * PAGE_SIZE;
+        ringbuf->effective_size = ringbuf->size;
+        ringbuf->head = 0;
+        ringbuf->tail = 0;
+        ringbuf->space = ringbuf->size;
+        ringbuf->last_retired_head = -1;
+
+        /* TODO: For now we put this in the mappable region so that we can reuse
+         * the existing ringbuffer code which ioremaps it. When we start
+         * creating many contexts, this will no longer work and we must switch
+         * to a kmapish interface.
+         */
+        ret = intel_alloc_ringbuffer_obj(dev, ringbuf);
+        if (ret) {
+                DRM_DEBUG_DRIVER("Failed to allocate ringbuffer obj %s: %d\n",
+                                ring->name, ret);
+                goto error;
+        }
+
+        ret = populate_lr_context(ctx, ctx_obj, ring, ringbuf);
+        if (ret) {
+                DRM_DEBUG_DRIVER("Failed to populate LRC: %d\n", ret);
+                intel_destroy_ringbuffer_obj(ringbuf);
+                goto error;
+        }
+
+        ctx->engine[ring->id].ringbuf = ringbuf;
+        ctx->engine[ring->id].state = ctx_obj;
+
+        if (ctx == ring->default_context) {
+                /* The status page is offset 0 from the default context object
+                 * in LRC mode. */
+                ring->status_page.gfx_addr = i915_gem_obj_ggtt_offset(ctx_obj);
+                ring->status_page.page_addr =
+                                kmap(sg_page(ctx_obj->pages->sgl));
+                if (ring->status_page.page_addr == NULL)
+                        return -ENOMEM;
+                ring->status_page.obj = ctx_obj;
+        }
+
+        if (ring->id == RCS && !ctx->rcs_initialized) {
+                ret = intel_lr_context_render_state_init(ring, ctx);
+                if (ret) {
+                        DRM_ERROR("Init render state failed: %d\n", ret);
+                        ctx->engine[ring->id].ringbuf = NULL;
+                        ctx->engine[ring->id].state = NULL;
+                        intel_destroy_ringbuffer_obj(ringbuf);
+                        goto error;
+                }
+                ctx->rcs_initialized = true;
+        }
+
+        return 0;
+
+error:
+        kfree(ringbuf);
+        i915_gem_object_ggtt_unpin(ctx_obj);
+        drm_gem_object_unreference(&ctx_obj->base);
+        return ret;
+}
diff --git a/drivers/gpu/drm/i915/intel_lrc.h b/drivers/gpu/drm/i915/intel_lrc.h
new file mode 100644
index 000000000000..33c3b4bf28c5
--- /dev/null
+++ b/drivers/gpu/drm/i915/intel_lrc.h
@@ -0,0 +1,114 @@
+/*
+ * Copyright © 2014 Intel Corporation
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the next
+ * paragraph) shall be included in all copies or substantial portions of the
+ * Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+ * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+#ifndef _INTEL_LRC_H_
+#define _INTEL_LRC_H_
+
+/* Execlists regs */
+#define RING_ELSP(ring)                 ((ring)->mmio_base+0x230)
+#define RING_EXECLIST_STATUS(ring)      ((ring)->mmio_base+0x234)
+#define RING_CONTEXT_CONTROL(ring)      ((ring)->mmio_base+0x244)
+#define RING_CONTEXT_STATUS_BUF(ring)   ((ring)->mmio_base+0x370)
+#define RING_CONTEXT_STATUS_PTR(ring)   ((ring)->mmio_base+0x3a0)
+
+/* Logical Rings */
+void intel_logical_ring_stop(struct intel_engine_cs *ring);
+void intel_logical_ring_cleanup(struct intel_engine_cs *ring);
+int intel_logical_rings_init(struct drm_device *dev);
+
+int logical_ring_flush_all_caches(struct intel_ringbuffer *ringbuf);
+void intel_logical_ring_advance_and_submit(struct intel_ringbuffer *ringbuf);
+/**
+ * intel_logical_ring_advance() - advance the ringbuffer tail
+ * @ringbuf: Ringbuffer to advance.
+ *
+ * The tail is only updated in our logical ringbuffer struct.
+ */
+static inline void intel_logical_ring_advance(struct intel_ringbuffer *ringbuf)
+{
+        ringbuf->tail &= ringbuf->size - 1;
+}
+/**
+ * intel_logical_ring_emit() - write a DWORD to the ringbuffer.
+ * @ringbuf: Ringbuffer to write to.
+ * @data: DWORD to write.
+ */
+static inline void intel_logical_ring_emit(struct intel_ringbuffer *ringbuf,
+                                           u32 data)
+{
+        iowrite32(data, ringbuf->virtual_start + ringbuf->tail);
+        ringbuf->tail += 4;
+}
+int intel_logical_ring_begin(struct intel_ringbuffer *ringbuf, int num_dwords);
+
+/* Logical Ring Contexts */
+int intel_lr_context_render_state_init(struct intel_engine_cs *ring,
+                                       struct intel_context *ctx);
+void intel_lr_context_free(struct intel_context *ctx);
+int intel_lr_context_deferred_create(struct intel_context *ctx,
+                                     struct intel_engine_cs *ring);
+
+/* Execlists */
+int intel_sanitize_enable_execlists(struct drm_device *dev, int enable_execlists);
+int intel_execlists_submission(struct drm_device *dev, struct drm_file *file,
+                               struct intel_engine_cs *ring,
+                               struct intel_context *ctx,
+                               struct drm_i915_gem_execbuffer2 *args,
+                               struct list_head *vmas,
+                               struct drm_i915_gem_object *batch_obj,
+                               u64 exec_start, u32 flags);
+u32 intel_execlists_ctx_id(struct drm_i915_gem_object *ctx_obj);
+
+/**
+ * struct intel_ctx_submit_request - queued context submission request
+ * @ctx: Context to submit to the ELSP.
+ * @ring: Engine to submit it to.
+ * @tail: how far in the context's ringbuffer this request goes to.
+ * @execlist_link: link in the submission queue.
+ * @work: workqueue for processing this request in a bottom half.
+ * @elsp_submitted: no. of times this request has been sent to the ELSP.
+ *
+ * The ELSP only accepts two elements at a time, so we queue context/tail
+ * pairs on a given queue (ring->execlist_queue) until the hardware is
+ * available. The queue serves a double purpose: we also use it to keep track
+ * of the up to 2 contexts currently in the hardware (usually one in execution
+ * and the other queued up by the GPU): We only remove elements from the head
+ * of the queue when the hardware informs us that an element has been
+ * completed.
+ *
+ * All accesses to the queue are mediated by a spinlock (ring->execlist_lock).
+ */
+struct intel_ctx_submit_request {
+        struct intel_context *ctx;
+        struct intel_engine_cs *ring;
+        u32 tail;
+
+        struct list_head execlist_link;
+        struct work_struct work;
+
+        int elsp_submitted;
+};
+
+void intel_execlists_handle_ctx_events(struct intel_engine_cs *ring);
+
+#endif /* _INTEL_LRC_H_ */
diff --git a/drivers/gpu/drm/i915/intel_lvds.c b/drivers/gpu/drm/i915/intel_lvds.c
index fdf40267249c..a6bd1422e38f 100644
--- a/drivers/gpu/drm/i915/intel_lvds.c
+++ b/drivers/gpu/drm/i915/intel_lvds.c
@@ -823,8 +823,7 @@ bool intel_is_dual_link_lvds(struct drm_device *dev)
         struct intel_encoder *encoder;
         struct intel_lvds_encoder *lvds_encoder;
 
-        list_for_each_entry(encoder, &dev->mode_config.encoder_list,
-                            base.head) {
+        for_each_intel_encoder(dev, encoder) {
                 if (encoder->type == INTEL_OUTPUT_LVDS) {
                         lvds_encoder = to_lvds_encoder(&encoder->base);
 
diff --git a/drivers/gpu/drm/i915/intel_panel.c b/drivers/gpu/drm/i915/intel_panel.c
index 8e374449c6b5..18784470a760 100644
--- a/drivers/gpu/drm/i915/intel_panel.c
+++ b/drivers/gpu/drm/i915/intel_panel.c
@@ -751,6 +751,8 @@ void intel_panel_disable_backlight(struct intel_connector *connector)
 
         spin_lock_irqsave(&dev_priv->backlight_lock, flags);
 
+        if (panel->backlight.device)
+                panel->backlight.device->props.power = FB_BLANK_POWERDOWN;
         panel->backlight.enabled = false;
         dev_priv->display.disable_backlight(connector);
 
@@ -957,6 +959,8 @@ void intel_panel_enable_backlight(struct intel_connector *connector)
 
         dev_priv->display.enable_backlight(connector);
         panel->backlight.enabled = true;
+        if (panel->backlight.device)
+                panel->backlight.device->props.power = FB_BLANK_UNBLANK;
 
         spin_unlock_irqrestore(&dev_priv->backlight_lock, flags);
 }
@@ -965,6 +969,7 @@ void intel_panel_enable_backlight(struct intel_connector *connector)
 static int intel_backlight_device_update_status(struct backlight_device *bd)
 {
         struct intel_connector *connector = bl_get_data(bd);
+        struct intel_panel *panel = &connector->panel;
         struct drm_device *dev = connector->base.dev;
 
         drm_modeset_lock(&dev->mode_config.connection_mutex, NULL);
@@ -972,6 +977,23 @@ static int intel_backlight_device_update_status(struct backlight_device *bd)
                       bd->props.brightness, bd->props.max_brightness);
         intel_panel_set_backlight(connector, bd->props.brightness,
                                   bd->props.max_brightness);
+
+        /*
+         * Allow flipping bl_power as a sub-state of enabled. Sadly the
+         * backlight class device does not make it easy to to differentiate
+         * between callbacks for brightness and bl_power, so our backlight_power
+         * callback needs to take this into account.
+         */
+        if (panel->backlight.enabled) {
+                if (panel->backlight_power) {
+                        bool enable = bd->props.power == FB_BLANK_UNBLANK &&
+                                bd->props.brightness != 0;
+                        panel->backlight_power(connector, enable);
+                }
+        } else {
+                bd->props.power = FB_BLANK_POWERDOWN;
+        }
+
         drm_modeset_unlock(&dev->mode_config.connection_mutex);
         return 0;
 }
@@ -1023,6 +1045,11 @@ static int intel_backlight_device_register(struct intel_connector *connector)
                                             panel->backlight.level,
                                             props.max_brightness);
 
+        if (panel->backlight.enabled)
+                props.power = FB_BLANK_UNBLANK;
+        else
+                props.power = FB_BLANK_POWERDOWN;
+
         /*
          * Note: using the same name independent of the connector prevents
          * registration of multiple backlight devices in the driver.
@@ -1203,7 +1230,7 @@ static int vlv_setup_backlight(struct intel_connector *connector)
         enum pipe pipe;
         u32 ctl, ctl2, val;
 
-        for_each_pipe(pipe) {
+        for_each_pipe(dev_priv, pipe) {
                 u32 cur_val = I915_READ(VLV_BLC_PWM_CTL(pipe));
 
                 /* Skip if the modulation freq is already set */
diff --git a/drivers/gpu/drm/i915/intel_pm.c b/drivers/gpu/drm/i915/intel_pm.c
index 40c12295c0bd..c27b6140bfd1 100644
--- a/drivers/gpu/drm/i915/intel_pm.c
+++ b/drivers/gpu/drm/i915/intel_pm.c
@@ -309,6 +309,9 @@ static void gen7_enable_fbc(struct drm_crtc *crtc)
 
         dpfc_ctl |= IVB_DPFC_CTL_FENCE_EN;
 
+        if (dev_priv->fbc.false_color)
+                dpfc_ctl |= FBC_CTL_FALSE_COLOR;
+
         I915_WRITE(ILK_DPFC_CONTROL, dpfc_ctl | DPFC_CTL_EN);
 
         if (IS_IVYBRIDGE(dev)) {
@@ -342,6 +345,16 @@ bool intel_fbc_enabled(struct drm_device *dev)
         return dev_priv->display.fbc_enabled(dev);
 }
 
+void gen8_fbc_sw_flush(struct drm_device *dev, u32 value)
+{
+        struct drm_i915_private *dev_priv = dev->dev_private;
+
+        if (!IS_GEN8(dev))
+                return;
+
+        I915_WRITE(MSG_FBC_REND_STATE, value);
+}
+
 static void intel_fbc_work_fn(struct work_struct *__work)
 {
         struct intel_fbc_work *work =
@@ -578,6 +591,12 @@ void intel_update_fbc(struct drm_device *dev)
                         DRM_DEBUG_KMS("framebuffer not tiled or fenced, disabling compression\n");
                 goto out_disable;
         }
+        if (INTEL_INFO(dev)->gen <= 4 && !IS_G4X(dev) &&
+            to_intel_plane(crtc->primary)->rotation != BIT(DRM_ROTATE_0)) {
+                if (set_no_fbc_reason(dev_priv, FBC_UNSUPPORTED_MODE))
+                        DRM_DEBUG_KMS("Rotation unsupported, disabling\n");
+                goto out_disable;
+        }
 
         /* If the kernel debugger is active, always disable compression */
         if (in_dbg_master())
@@ -853,7 +872,7 @@ void intel_set_memory_cxsr(struct drm_i915_private *dev_priv, bool enable)
  * A value of 5us seems to be a good balance; safe for very low end
  * platforms but not overly aggressive on lower latency configs.
  */
-static const int latency_ns = 5000;
+static const int pessimal_latency_ns = 5000;
 
 static int i9xx_get_fifo_size(struct drm_device *dev, int plane)
 {
@@ -982,13 +1001,20 @@ static const struct intel_watermark_params i915_wm_info = {
         .guard_size = 2,
         .cacheline_size = I915_FIFO_LINE_SIZE,
 };
-static const struct intel_watermark_params i830_wm_info = {
+static const struct intel_watermark_params i830_a_wm_info = {
         .fifo_size = I855GM_FIFO_SIZE,
         .max_wm = I915_MAX_WM,
         .default_wm = 1,
         .guard_size = 2,
         .cacheline_size = I830_FIFO_LINE_SIZE,
 };
+static const struct intel_watermark_params i830_bc_wm_info = {
+        .fifo_size = I855GM_FIFO_SIZE,
+        .max_wm = I915_MAX_WM/2,
+        .default_wm = 1,
+        .guard_size = 2,
+        .cacheline_size = I830_FIFO_LINE_SIZE,
+};
 static const struct intel_watermark_params i845_wm_info = {
         .fifo_size = I830_FIFO_SIZE,
         .max_wm = I915_MAX_WM,
@@ -1044,6 +1070,17 @@ static unsigned long intel_calculate_wm(unsigned long clock_in_khz,
                 wm_size = wm->max_wm;
         if (wm_size <= 0)
                 wm_size = wm->default_wm;
+
+        /*
+         * Bspec seems to indicate that the value shouldn't be lower than
+         * 'burst size + 1'. Certainly 830 is quite unhappy with low values.
+         * Lets go for 8 which is the burst size since certain platforms
+         * already use a hardcoded 8 (which is what the spec says should be
+         * done).
+         */
+        if (wm_size <= 8)
+                wm_size = 8;
+
         return wm_size;
 }
 
@@ -1268,33 +1305,27 @@ static bool g4x_compute_srwm(struct drm_device *dev,
                               display, cursor);
 }
 
-static bool vlv_compute_drain_latency(struct drm_device *dev,
-                                     int plane,
-                                     int *plane_prec_mult,
-                                     int *plane_dl,
-                                     int *cursor_prec_mult,
-                                     int *cursor_dl)
+static bool vlv_compute_drain_latency(struct drm_crtc *crtc,
+                                      int pixel_size,
+                                      int *prec_mult,
+                                      int *drain_latency)
 {
-        struct drm_crtc *crtc;
-        int clock, pixel_size;
         int entries;
+        int clock = to_intel_crtc(crtc)->config.adjusted_mode.crtc_clock;
 
-        crtc = intel_get_crtc_for_plane(dev, plane);
-        if (!intel_crtc_active(crtc))
+        if (WARN(clock == 0, "Pixel clock is zero!\n"))
                 return false;
 
-        clock = to_intel_crtc(crtc)->config.adjusted_mode.crtc_clock;
-        pixel_size = crtc->primary->fb->bits_per_pixel / 8;     /* BPP */
+        if (WARN(pixel_size == 0, "Pixel size is zero!\n"))
+                return false;
 
-        entries = (clock / 1000) * pixel_size;
-        *plane_prec_mult = (entries > 128) ?
-                DRAIN_LATENCY_PRECISION_64 : DRAIN_LATENCY_PRECISION_32;
-        *plane_dl = (64 * (*plane_prec_mult) * 4) / entries;
+        entries = DIV_ROUND_UP(clock, 1000) * pixel_size;
+        *prec_mult = (entries > 128) ? DRAIN_LATENCY_PRECISION_64 :
+                                       DRAIN_LATENCY_PRECISION_32;
+        *drain_latency = (64 * (*prec_mult) * 4) / entries;
 
-        entries = (clock / 1000) * 4;   /* BPP is always 4 for cursor */
-        *cursor_prec_mult = (entries > 128) ?
-                DRAIN_LATENCY_PRECISION_64 : DRAIN_LATENCY_PRECISION_32;
-        *cursor_dl = (64 * (*cursor_prec_mult) * 4) / entries;
+        if (*drain_latency > DRAIN_LATENCY_MASK)
+                *drain_latency = DRAIN_LATENCY_MASK;
 
         return true;
 }
@@ -1307,39 +1338,48 @@ static bool vlv_compute_drain_latency(struct drm_device *dev,
  * latency value.
  */
 
-static void vlv_update_drain_latency(struct drm_device *dev)
+static void vlv_update_drain_latency(struct drm_crtc *crtc)
 {
-        struct drm_i915_private *dev_priv = dev->dev_private;
-        int planea_prec, planea_dl, planeb_prec, planeb_dl;
-        int cursora_prec, cursora_dl, cursorb_prec, cursorb_dl;
-        int plane_prec_mult, cursor_prec_mult; /* Precision multiplier is
-                                                        either 16 or 32 */
+        struct drm_i915_private *dev_priv = crtc->dev->dev_private;
+        struct intel_crtc *intel_crtc = to_intel_crtc(crtc);
+        int pixel_size;
+        int drain_latency;
+        enum pipe pipe = intel_crtc->pipe;
+        int plane_prec, prec_mult, plane_dl;
+
+        plane_dl = I915_READ(VLV_DDL(pipe)) & ~(DDL_PLANE_PRECISION_64 |
+                   DRAIN_LATENCY_MASK | DDL_CURSOR_PRECISION_64 |
+                   (DRAIN_LATENCY_MASK << DDL_CURSOR_SHIFT));
 
-        /* For plane A, Cursor A */
-        if (vlv_compute_drain_latency(dev, 0, &plane_prec_mult, &planea_dl,
-                                      &cursor_prec_mult, &cursora_dl)) {
-                cursora_prec = (cursor_prec_mult == DRAIN_LATENCY_PRECISION_32) ?
-                        DDL_CURSORA_PRECISION_32 : DDL_CURSORA_PRECISION_64;
-                planea_prec = (plane_prec_mult == DRAIN_LATENCY_PRECISION_32) ?
-                        DDL_PLANEA_PRECISION_32 : DDL_PLANEA_PRECISION_64;
+        if (!intel_crtc_active(crtc)) {
+                I915_WRITE(VLV_DDL(pipe), plane_dl);
+                return;
+        }
 
-                I915_WRITE(VLV_DDL1, cursora_prec |
-                                (cursora_dl << DDL_CURSORA_SHIFT) |
-                                planea_prec | planea_dl);
+        /* Primary plane Drain Latency */
+        pixel_size = crtc->primary->fb->bits_per_pixel / 8;     /* BPP */
+        if (vlv_compute_drain_latency(crtc, pixel_size, &prec_mult, &drain_latency)) {
+                plane_prec = (prec_mult == DRAIN_LATENCY_PRECISION_64) ?
+                                           DDL_PLANE_PRECISION_64 :
+                                           DDL_PLANE_PRECISION_32;
+                plane_dl |= plane_prec | drain_latency;
         }
 
-        /* For plane B, Cursor B */
-        if (vlv_compute_drain_latency(dev, 1, &plane_prec_mult, &planeb_dl,
-                                      &cursor_prec_mult, &cursorb_dl)) {
-                cursorb_prec = (cursor_prec_mult == DRAIN_LATENCY_PRECISION_32) ?
-                        DDL_CURSORB_PRECISION_32 : DDL_CURSORB_PRECISION_64;
-                planeb_prec = (plane_prec_mult == DRAIN_LATENCY_PRECISION_32) ?
-                        DDL_PLANEB_PRECISION_32 : DDL_PLANEB_PRECISION_64;
+        /* Cursor Drain Latency
+         * BPP is always 4 for cursor
+         */
+        pixel_size = 4;
 
-                I915_WRITE(VLV_DDL2, cursorb_prec |
-                                (cursorb_dl << DDL_CURSORB_SHIFT) |
-                                planeb_prec | planeb_dl);
+        /* Program cursor DL only if it is enabled */
+        if (intel_crtc->cursor_base &&
+            vlv_compute_drain_latency(crtc, pixel_size, &prec_mult, &drain_latency)) {
+                plane_prec = (prec_mult == DRAIN_LATENCY_PRECISION_64) ?
+                                           DDL_CURSOR_PRECISION_64 :
+                                           DDL_CURSOR_PRECISION_32;
+                plane_dl |= plane_prec | (drain_latency << DDL_CURSOR_SHIFT);
         }
+
+        I915_WRITE(VLV_DDL(pipe), plane_dl);
 }
 
 #define single_plane_enabled(mask) is_power_of_2(mask)
@@ -1355,20 +1395,92 @@ static void valleyview_update_wm(struct drm_crtc *crtc)
         unsigned int enabled = 0;
         bool cxsr_enabled;
 
-        vlv_update_drain_latency(dev);
+        vlv_update_drain_latency(crtc);
+
+        if (g4x_compute_wm0(dev, PIPE_A,
+                            &valleyview_wm_info, pessimal_latency_ns,
+                            &valleyview_cursor_wm_info, pessimal_latency_ns,
+                            &planea_wm, &cursora_wm))
+                enabled |= 1 << PIPE_A;
+
+        if (g4x_compute_wm0(dev, PIPE_B,
+                            &valleyview_wm_info, pessimal_latency_ns,
+                            &valleyview_cursor_wm_info, pessimal_latency_ns,
+                            &planeb_wm, &cursorb_wm))
+                enabled |= 1 << PIPE_B;
+
+        if (single_plane_enabled(enabled) &&
+            g4x_compute_srwm(dev, ffs(enabled) - 1,
+                             sr_latency_ns,
+                             &valleyview_wm_info,
+                             &valleyview_cursor_wm_info,
+                             &plane_sr, &ignore_cursor_sr) &&
+            g4x_compute_srwm(dev, ffs(enabled) - 1,
+                             2*sr_latency_ns,
+                             &valleyview_wm_info,
+                             &valleyview_cursor_wm_info,
+                             &ignore_plane_sr, &cursor_sr)) {
+                cxsr_enabled = true;
+        } else {
+                cxsr_enabled = false;
+                intel_set_memory_cxsr(dev_priv, false);
+                plane_sr = cursor_sr = 0;
+        }
+
+        DRM_DEBUG_KMS("Setting FIFO watermarks - A: plane=%d, cursor=%d, "
+                      "B: plane=%d, cursor=%d, SR: plane=%d, cursor=%d\n",
+                      planea_wm, cursora_wm,
+                      planeb_wm, cursorb_wm,
+                      plane_sr, cursor_sr);
+
+        I915_WRITE(DSPFW1,
+                   (plane_sr << DSPFW_SR_SHIFT) |
+                   (cursorb_wm << DSPFW_CURSORB_SHIFT) |
+                   (planeb_wm << DSPFW_PLANEB_SHIFT) |
+                   (planea_wm << DSPFW_PLANEA_SHIFT));
+        I915_WRITE(DSPFW2,
+                   (I915_READ(DSPFW2) & ~DSPFW_CURSORA_MASK) |
+                   (cursora_wm << DSPFW_CURSORA_SHIFT));
+        I915_WRITE(DSPFW3,
+                   (I915_READ(DSPFW3) & ~DSPFW_CURSOR_SR_MASK) |
+                   (cursor_sr << DSPFW_CURSOR_SR_SHIFT));
+
+        if (cxsr_enabled)
+                intel_set_memory_cxsr(dev_priv, true);
+}
+
+static void cherryview_update_wm(struct drm_crtc *crtc)
+{
+        struct drm_device *dev = crtc->dev;
+        static const int sr_latency_ns = 12000;
+        struct drm_i915_private *dev_priv = dev->dev_private;
+        int planea_wm, planeb_wm, planec_wm;
+        int cursora_wm, cursorb_wm, cursorc_wm;
+        int plane_sr, cursor_sr;
+        int ignore_plane_sr, ignore_cursor_sr;
+        unsigned int enabled = 0;
+        bool cxsr_enabled;
+
+        vlv_update_drain_latency(crtc);
 
         if (g4x_compute_wm0(dev, PIPE_A,
-                            &valleyview_wm_info, latency_ns,
-                            &valleyview_cursor_wm_info, latency_ns,
+                            &valleyview_wm_info, pessimal_latency_ns,
+                            &valleyview_cursor_wm_info, pessimal_latency_ns,
                             &planea_wm, &cursora_wm))
                 enabled |= 1 << PIPE_A;
 
         if (g4x_compute_wm0(dev, PIPE_B,
-                            &valleyview_wm_info, latency_ns,
-                            &valleyview_cursor_wm_info, latency_ns,
+                            &valleyview_wm_info, pessimal_latency_ns,
+                            &valleyview_cursor_wm_info, pessimal_latency_ns,
                             &planeb_wm, &cursorb_wm))
                 enabled |= 1 << PIPE_B;
 
+        if (g4x_compute_wm0(dev, PIPE_C,
+                            &valleyview_wm_info, pessimal_latency_ns,
+                            &valleyview_cursor_wm_info, pessimal_latency_ns,
+                            &planec_wm, &cursorc_wm))
+                enabled |= 1 << PIPE_C;
+
         if (single_plane_enabled(enabled) &&
             g4x_compute_srwm(dev, ffs(enabled) - 1,
                              sr_latency_ns,
@@ -1387,27 +1499,66 @@ static void valleyview_update_wm(struct drm_crtc *crtc)
                 plane_sr = cursor_sr = 0;
         }
 
-        DRM_DEBUG_KMS("Setting FIFO watermarks - A: plane=%d, cursor=%d, B: plane=%d, cursor=%d, SR: plane=%d, cursor=%d\n",
+        DRM_DEBUG_KMS("Setting FIFO watermarks - A: plane=%d, cursor=%d, "
+                      "B: plane=%d, cursor=%d, C: plane=%d, cursor=%d, "
+                      "SR: plane=%d, cursor=%d\n",
                       planea_wm, cursora_wm,
                       planeb_wm, cursorb_wm,
+                      planec_wm, cursorc_wm,
                       plane_sr, cursor_sr);
 
         I915_WRITE(DSPFW1,
                    (plane_sr << DSPFW_SR_SHIFT) |
                    (cursorb_wm << DSPFW_CURSORB_SHIFT) |
                    (planeb_wm << DSPFW_PLANEB_SHIFT) |
-                   planea_wm);
+                   (planea_wm << DSPFW_PLANEA_SHIFT));
         I915_WRITE(DSPFW2,
                    (I915_READ(DSPFW2) & ~DSPFW_CURSORA_MASK) |
                    (cursora_wm << DSPFW_CURSORA_SHIFT));
         I915_WRITE(DSPFW3,
                    (I915_READ(DSPFW3) & ~DSPFW_CURSOR_SR_MASK) |
                    (cursor_sr << DSPFW_CURSOR_SR_SHIFT));
+        I915_WRITE(DSPFW9_CHV,
+                   (I915_READ(DSPFW9_CHV) & ~(DSPFW_PLANEC_MASK |
+                                              DSPFW_CURSORC_MASK)) |
+                   (planec_wm << DSPFW_PLANEC_SHIFT) |
+                   (cursorc_wm << DSPFW_CURSORC_SHIFT));
 
         if (cxsr_enabled)
                 intel_set_memory_cxsr(dev_priv, true);
 }
 
+static void valleyview_update_sprite_wm(struct drm_plane *plane,
+                                        struct drm_crtc *crtc,
+                                        uint32_t sprite_width,
+                                        uint32_t sprite_height,
+                                        int pixel_size,
+                                        bool enabled, bool scaled)
+{
+        struct drm_device *dev = crtc->dev;
+        struct drm_i915_private *dev_priv = dev->dev_private;
+        int pipe = to_intel_plane(plane)->pipe;
+        int sprite = to_intel_plane(plane)->plane;
+        int drain_latency;
+        int plane_prec;
+        int sprite_dl;
+        int prec_mult;
+
+        sprite_dl = I915_READ(VLV_DDL(pipe)) & ~(DDL_SPRITE_PRECISION_64(sprite) |
+                    (DRAIN_LATENCY_MASK << DDL_SPRITE_SHIFT(sprite)));
+
+        if (enabled && vlv_compute_drain_latency(crtc, pixel_size, &prec_mult,
+                                                 &drain_latency)) {
+                plane_prec = (prec_mult == DRAIN_LATENCY_PRECISION_64) ?
+                                           DDL_SPRITE_PRECISION_64(sprite) :
+                                           DDL_SPRITE_PRECISION_32(sprite);
+                sprite_dl |= plane_prec |
+                             (drain_latency << DDL_SPRITE_SHIFT(sprite));
+        }
+
+        I915_WRITE(VLV_DDL(pipe), sprite_dl);
+}
+
 static void g4x_update_wm(struct drm_crtc *crtc)
 {
         struct drm_device *dev = crtc->dev;
@@ -1419,14 +1570,14 @@ static void g4x_update_wm(struct drm_crtc *crtc)
         bool cxsr_enabled;
 
         if (g4x_compute_wm0(dev, PIPE_A,
-                            &g4x_wm_info, latency_ns,
-                            &g4x_cursor_wm_info, latency_ns,
+                            &g4x_wm_info, pessimal_latency_ns,
+                            &g4x_cursor_wm_info, pessimal_latency_ns,
                             &planea_wm, &cursora_wm))
                 enabled |= 1 << PIPE_A;
 
         if (g4x_compute_wm0(dev, PIPE_B,
-                            &g4x_wm_info, latency_ns,
-                            &g4x_cursor_wm_info, latency_ns,
+                            &g4x_wm_info, pessimal_latency_ns,
+                            &g4x_cursor_wm_info, pessimal_latency_ns,
                             &planeb_wm, &cursorb_wm))
                 enabled |= 1 << PIPE_B;
 
@@ -1443,7 +1594,8 @@ static void g4x_update_wm(struct drm_crtc *crtc)
                 plane_sr = cursor_sr = 0;
         }
 
-        DRM_DEBUG_KMS("Setting FIFO watermarks - A: plane=%d, cursor=%d, B: plane=%d, cursor=%d, SR: plane=%d, cursor=%d\n",
+        DRM_DEBUG_KMS("Setting FIFO watermarks - A: plane=%d, cursor=%d, "
+                      "B: plane=%d, cursor=%d, SR: plane=%d, cursor=%d\n",
                       planea_wm, cursora_wm,
                       planeb_wm, cursorb_wm,
                       plane_sr, cursor_sr);
@@ -1452,7 +1604,7 @@ static void g4x_update_wm(struct drm_crtc *crtc)
                    (plane_sr << DSPFW_SR_SHIFT) |
                    (cursorb_wm << DSPFW_CURSORB_SHIFT) |
                    (planeb_wm << DSPFW_PLANEB_SHIFT) |
-                   planea_wm);
+                   (planea_wm << DSPFW_PLANEA_SHIFT));
         I915_WRITE(DSPFW2,
                    (I915_READ(DSPFW2) & ~DSPFW_CURSORA_MASK) |
                    (cursora_wm << DSPFW_CURSORA_SHIFT));
@@ -1526,8 +1678,11 @@ static void i965_update_wm(struct drm_crtc *unused_crtc)
 
         /* 965 has limitations... */
         I915_WRITE(DSPFW1, (srwm << DSPFW_SR_SHIFT) |
-                   (8 << 16) | (8 << 8) | (8 << 0));
-        I915_WRITE(DSPFW2, (8 << 8) | (8 << 0));
+                   (8 << DSPFW_CURSORB_SHIFT) |
+                   (8 << DSPFW_PLANEB_SHIFT) |
+                   (8 << DSPFW_PLANEA_SHIFT));
+        I915_WRITE(DSPFW2, (8 << DSPFW_CURSORA_SHIFT) |
+                   (8 << DSPFW_PLANEC_SHIFT_OLD));
         /* update cursor SR watermark */
         I915_WRITE(DSPFW3, (cursor_sr << DSPFW_CURSOR_SR_SHIFT));
 
@@ -1552,7 +1707,7 @@ static void i9xx_update_wm(struct drm_crtc *unused_crtc)
         else if (!IS_GEN2(dev))
                 wm_info = &i915_wm_info;
         else
-                wm_info = &i830_wm_info;
+                wm_info = &i830_a_wm_info;
 
         fifo_size = dev_priv->display.get_fifo_size(dev, 0);
         crtc = intel_get_crtc_for_plane(dev, 0);
@@ -1565,10 +1720,16 @@ static void i9xx_update_wm(struct drm_crtc *unused_crtc)
                 adjusted_mode = &to_intel_crtc(crtc)->config.adjusted_mode;
                 planea_wm = intel_calculate_wm(adjusted_mode->crtc_clock,
                                                wm_info, fifo_size, cpp,
-                                               latency_ns);
+                                               pessimal_latency_ns);
                 enabled = crtc;
-        } else
+        } else {
                 planea_wm = fifo_size - wm_info->guard_size;
+                if (planea_wm > (long)wm_info->max_wm)
+                        planea_wm = wm_info->max_wm;
+        }
+
+        if (IS_GEN2(dev))
+                wm_info = &i830_bc_wm_info;
 
         fifo_size = dev_priv->display.get_fifo_size(dev, 1);
         crtc = intel_get_crtc_for_plane(dev, 1);
@@ -1581,13 +1742,16 @@ static void i9xx_update_wm(struct drm_crtc *unused_crtc)
                 adjusted_mode = &to_intel_crtc(crtc)->config.adjusted_mode;
                 planeb_wm = intel_calculate_wm(adjusted_mode->crtc_clock,
                                                wm_info, fifo_size, cpp,
-                                               latency_ns);
+                                               pessimal_latency_ns);
                 if (enabled == NULL)
                         enabled = crtc;
                 else
                         enabled = NULL;
-        } else
+        } else {
                 planeb_wm = fifo_size - wm_info->guard_size;
+                if (planeb_wm > (long)wm_info->max_wm)
+                        planeb_wm = wm_info->max_wm;
+        }
 
         DRM_DEBUG_KMS("FIFO watermarks - A: %d, B: %d\n", planea_wm, planeb_wm);
 
@@ -1674,7 +1838,7 @@ static void i845_update_wm(struct drm_crtc *unused_crtc)
         planea_wm = intel_calculate_wm(adjusted_mode->crtc_clock,
                                        &i845_wm_info,
                                        dev_priv->display.get_fifo_size(dev, 0),
-                                       4, latency_ns);
+                                       4, pessimal_latency_ns);
         fwater_lo = I915_READ(FW_BLC) & ~0xfff;
         fwater_lo |= (3<<8) | planea_wm;
 
@@ -2527,7 +2691,7 @@ static struct intel_pipe_wm *ilk_find_best_result(struct drm_device *dev,
 #define WM_DIRTY_FBC (1 << 24)
 #define WM_DIRTY_DDB (1 << 25)
 
-static unsigned int ilk_compute_wm_dirty(struct drm_device *dev,
+static unsigned int ilk_compute_wm_dirty(struct drm_i915_private *dev_priv,
                                          const struct ilk_wm_values *old,
                                          const struct ilk_wm_values *new)
 {
@@ -2535,7 +2699,7 @@ static unsigned int ilk_compute_wm_dirty(struct drm_device *dev,
         enum pipe pipe;
         int wm_lp;
 
-        for_each_pipe(pipe) {
+        for_each_pipe(dev_priv, pipe) {
                 if (old->wm_linetime[pipe] != new->wm_linetime[pipe]) {
                         dirty |= WM_DIRTY_LINETIME(pipe);
                         /* Must disable LP1+ watermarks too */
@@ -2621,7 +2785,7 @@ static void ilk_write_wm_values(struct drm_i915_private *dev_priv,
         unsigned int dirty;
         uint32_t val;
 
-        dirty = ilk_compute_wm_dirty(dev, previous, results);
+        dirty = ilk_compute_wm_dirty(dev_priv, previous, results);
         if (!dirty)
                 return;
 
@@ -3327,13 +3491,18 @@ void valleyview_set_rps(struct drm_device *dev, u8 val)
         WARN_ON(val > dev_priv->rps.max_freq_softlimit);
         WARN_ON(val < dev_priv->rps.min_freq_softlimit);
 
-        DRM_DEBUG_DRIVER("GPU freq request from %d MHz (%u) to %d MHz (%u)\n",
-                         vlv_gpu_freq(dev_priv, dev_priv->rps.cur_freq),
-                         dev_priv->rps.cur_freq,
-                         vlv_gpu_freq(dev_priv, val), val);
+        if (WARN_ONCE(IS_CHERRYVIEW(dev) && (val & 1),
+                      "Odd GPU freq value\n"))
+                val &= ~1;
+
+        if (val != dev_priv->rps.cur_freq) {
+                DRM_DEBUG_DRIVER("GPU freq request from %d MHz (%u) to %d MHz (%u)\n",
+                                 vlv_gpu_freq(dev_priv, dev_priv->rps.cur_freq),
+                                 dev_priv->rps.cur_freq,
+                                 vlv_gpu_freq(dev_priv, val), val);
 
-        if (val != dev_priv->rps.cur_freq)
                 vlv_punit_write(dev_priv, PUNIT_REG_GPU_FREQ_REQ, val);
+        }
 
         I915_WRITE(GEN6_PMINTRMSK, gen6_rps_pm_mask(dev_priv, val));
 
@@ -3406,8 +3575,14 @@ static void valleyview_disable_rps(struct drm_device *dev)
 {
         struct drm_i915_private *dev_priv = dev->dev_private;
 
+        /* we're doing forcewake before Disabling RC6,
+         * This what the BIOS expects when going into suspend */
+        gen6_gt_force_wake_get(dev_priv, FORCEWAKE_ALL);
+
         I915_WRITE(GEN6_RC_CONTROL, 0);
 
+        gen6_gt_force_wake_put(dev_priv, FORCEWAKE_ALL);
+
         gen6_disable_rps_interrupts(dev);
 }
 
@@ -3598,7 +3773,6 @@ static void gen6_enable_rps(struct drm_device *dev)
         struct drm_i915_private *dev_priv = dev->dev_private;
         struct intel_engine_cs *ring;
         u32 rp_state_cap;
-        u32 gt_perf_status;
         u32 rc6vids, pcu_mbox = 0, rc6_mask = 0;
         u32 gtfifodbg;
         int rc6_mode;
@@ -3623,7 +3797,6 @@ static void gen6_enable_rps(struct drm_device *dev)
         gen6_gt_force_wake_get(dev_priv, FORCEWAKE_ALL);
 
         rp_state_cap = I915_READ(GEN6_RP_STATE_CAP);
-        gt_perf_status = I915_READ(GEN6_GT_PERF_STATUS);
 
         parse_rp_state_cap(dev_priv, rp_state_cap);
 
@@ -3965,11 +4138,27 @@ static void valleyview_cleanup_pctx(struct drm_device *dev)
 static void valleyview_init_gt_powersave(struct drm_device *dev)
 {
         struct drm_i915_private *dev_priv = dev->dev_private;
+        u32 val;
 
         valleyview_setup_pctx(dev);
 
         mutex_lock(&dev_priv->rps.hw_lock);
 
+        val = vlv_punit_read(dev_priv, PUNIT_REG_GPU_FREQ_STS);
+        switch ((val >> 6) & 3) {
+        case 0:
+        case 1:
+                dev_priv->mem_freq = 800;
+                break;
+        case 2:
+                dev_priv->mem_freq = 1066;
+                break;
+        case 3:
+                dev_priv->mem_freq = 1333;
+                break;
+        }
+        DRM_DEBUG_DRIVER("DDR speed: %d MHz", dev_priv->mem_freq);
+
         dev_priv->rps.max_freq = valleyview_rps_max_freq(dev_priv);
         dev_priv->rps.rp0_freq = dev_priv->rps.max_freq;
         DRM_DEBUG_DRIVER("max GPU freq: %d MHz (%u)\n",
@@ -4004,11 +4193,38 @@ static void valleyview_init_gt_powersave(struct drm_device *dev)
 static void cherryview_init_gt_powersave(struct drm_device *dev)
 {
         struct drm_i915_private *dev_priv = dev->dev_private;
+        u32 val;
 
         cherryview_setup_pctx(dev);
 
         mutex_lock(&dev_priv->rps.hw_lock);
 
+        val = vlv_punit_read(dev_priv, CCK_FUSE_REG);
+        switch ((val >> 2) & 0x7) {
+        case 0:
+        case 1:
+                dev_priv->rps.cz_freq = 200;
+                dev_priv->mem_freq = 1600;
+                break;
+        case 2:
+                dev_priv->rps.cz_freq = 267;
+                dev_priv->mem_freq = 1600;
+                break;
+        case 3:
+                dev_priv->rps.cz_freq = 333;
+                dev_priv->mem_freq = 2000;
+                break;
+        case 4:
+                dev_priv->rps.cz_freq = 320;
+                dev_priv->mem_freq = 1600;
+                break;
+        case 5:
+                dev_priv->rps.cz_freq = 400;
+                dev_priv->mem_freq = 1600;
+                break;
+        }
+        DRM_DEBUG_DRIVER("DDR speed: %d MHz", dev_priv->mem_freq);
+
         dev_priv->rps.max_freq = cherryview_rps_max_freq(dev_priv);
         dev_priv->rps.rp0_freq = dev_priv->rps.max_freq;
         DRM_DEBUG_DRIVER("max GPU freq: %d MHz (%u)\n",
@@ -4030,6 +4246,12 @@ static void cherryview_init_gt_powersave(struct drm_device *dev)
                          vlv_gpu_freq(dev_priv, dev_priv->rps.min_freq),
                          dev_priv->rps.min_freq);
 
+        WARN_ONCE((dev_priv->rps.max_freq |
+                   dev_priv->rps.efficient_freq |
+                   dev_priv->rps.rp1_freq |
+                   dev_priv->rps.min_freq) & 1,
+                  "Odd GPU freq values\n");
+
         /* Preserve min/max settings in case of re-init */
         if (dev_priv->rps.max_freq_softlimit == 0)
                 dev_priv->rps.max_freq_softlimit = dev_priv->rps.max_freq;
@@ -5088,7 +5310,7 @@ static void g4x_disable_trickle_feed(struct drm_device *dev)
         struct drm_i915_private *dev_priv = dev->dev_private;
         int pipe;
 
-        for_each_pipe(pipe) {
+        for_each_pipe(dev_priv, pipe) {
                 I915_WRITE(DSPCNTR(pipe),
                            I915_READ(DSPCNTR(pipe)) |
                            DISPPLANE_TRICKLE_FEED_DISABLE);
@@ -5203,7 +5425,7 @@ static void cpt_init_clock_gating(struct drm_device *dev)
         /* The below fixes the weird display corruption, a few pixels shifted
          * downward, on (only) LVDS of some HP laptops with IVY.
          */
-        for_each_pipe(pipe) {
+        for_each_pipe(dev_priv, pipe) {
                 val = I915_READ(TRANS_CHICKEN2(pipe));
                 val |= TRANS_CHICKEN2_TIMING_OVERRIDE;
                 val &= ~TRANS_CHICKEN2_FDI_POLARITY_REVERSED;
@@ -5215,7 +5437,7 @@ static void cpt_init_clock_gating(struct drm_device *dev)
                 I915_WRITE(TRANS_CHICKEN2(pipe), val);
         }
         /* WADP0ClockGatingDisable */
-        for_each_pipe(pipe) {
+        for_each_pipe(dev_priv, pipe) {
                 I915_WRITE(TRANS_CHICKEN1(pipe),
                            TRANS_CHICKEN1_DP0UNIT_GC_DISABLE);
         }
@@ -5383,7 +5605,7 @@ static void lpt_suspend_hw(struct drm_device *dev)
         }
 }
 
-static void gen8_init_clock_gating(struct drm_device *dev)
+static void broadwell_init_clock_gating(struct drm_device *dev)
 {
         struct drm_i915_private *dev_priv = dev->dev_private;
         enum pipe pipe;
@@ -5395,37 +5617,12 @@ static void gen8_init_clock_gating(struct drm_device *dev)
         /* FIXME(BDW): Check all the w/a, some might only apply to
          * pre-production hw. */
 
-        /* WaDisablePartialInstShootdown:bdw */
-        I915_WRITE(GEN8_ROW_CHICKEN,
-                   _MASKED_BIT_ENABLE(PARTIAL_INSTRUCTION_SHOOTDOWN_DISABLE));
 
-        /* WaDisableThreadStallDopClockGating:bdw */
-        /* FIXME: Unclear whether we really need this on production bdw. */
-        I915_WRITE(GEN8_ROW_CHICKEN,
-                   _MASKED_BIT_ENABLE(STALL_DOP_GATING_DISABLE));
-
-        /*
-         * This GEN8_CENTROID_PIXEL_OPT_DIS W/A is only needed for
-         * pre-production hardware
-         */
-        I915_WRITE(HALF_SLICE_CHICKEN3,
-                   _MASKED_BIT_ENABLE(GEN8_CENTROID_PIXEL_OPT_DIS));
-        I915_WRITE(HALF_SLICE_CHICKEN3,
-                   _MASKED_BIT_ENABLE(GEN8_SAMPLER_POWER_BYPASS_DIS));
         I915_WRITE(GAMTARBMODE, _MASKED_BIT_ENABLE(ARB_MODE_BWGTLB_DISABLE));
 
         I915_WRITE(_3D_CHICKEN3,
                    _MASKED_BIT_ENABLE(_3D_CHICKEN_SDE_LIMIT_FIFO_POLY_DEPTH(2)));
 
-        I915_WRITE(COMMON_SLICE_CHICKEN2,
-                   _MASKED_BIT_ENABLE(GEN8_CSC2_SBE_VUE_CACHE_CONSERVATIVE));
-
-        I915_WRITE(GEN7_HALF_SLICE_CHICKEN1,
-                   _MASKED_BIT_ENABLE(GEN7_SINGLE_SUBSCAN_DISPATCH_ENABLE));
-
-        /* WaDisableDopClockGating:bdw May not be needed for production */
-        I915_WRITE(GEN7_ROW_CHICKEN2,
-                   _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
 
         /* WaSwitchSolVfFArbitrationPriority:bdw */
         I915_WRITE(GAM_ECOCHK, I915_READ(GAM_ECOCHK) | HSW_ECOCHK_ARB_PRIO_SOL);
@@ -5435,37 +5632,18 @@ static void gen8_init_clock_gating(struct drm_device *dev)
                    I915_READ(CHICKEN_PAR1_1) | DPA_MASK_VBLANK_SRD);
 
         /* WaPsrDPRSUnmaskVBlankInSRD:bdw */
-        for_each_pipe(pipe) {
+        for_each_pipe(dev_priv, pipe) {
                 I915_WRITE(CHICKEN_PIPESL_1(pipe),
                            I915_READ(CHICKEN_PIPESL_1(pipe)) |
                            BDW_DPRS_MASK_VBLANK_SRD);
         }
 
-        /* Use Force Non-Coherent whenever executing a 3D context. This is a
-         * workaround for for a possible hang in the unlikely event a TLB
-         * invalidation occurs during a PSD flush.
-         */
-        I915_WRITE(HDC_CHICKEN0,
-                   I915_READ(HDC_CHICKEN0) |
-                   _MASKED_BIT_ENABLE(HDC_FORCE_NON_COHERENT));
-
         /* WaVSRefCountFullforceMissDisable:bdw */
         /* WaDSRefCountFullforceMissDisable:bdw */
         I915_WRITE(GEN7_FF_THREAD_MODE,
                    I915_READ(GEN7_FF_THREAD_MODE) &
                    ~(GEN8_FF_DS_REF_CNT_FFME | GEN7_FF_VS_REF_CNT_FFME));
 
-        /*
-         * BSpec recommends 8x4 when MSAA is used,
-         * however in practice 16x4 seems fastest.
-         *
-         * Note that PS/WM thread counts depend on the WIZ hashing
-         * disable bit, which we don't touch here, but it's good
-         * to keep in mind (see 3DSTATE_PS and 3DSTATE_WM).
-         */
-        I915_WRITE(GEN7_GT_MODE,
-                   GEN6_WIZ_HASHING_MASK | GEN6_WIZ_HASHING_16x4);
-
         I915_WRITE(GEN6_RC_SLEEP_PSMI_CONTROL,
                    _MASKED_BIT_ENABLE(GEN8_RC_SEMA_IDLE_MSG_DISABLE));
 
@@ -5473,9 +5651,7 @@ static void gen8_init_clock_gating(struct drm_device *dev)
         I915_WRITE(GEN8_UCGCTL6, I915_READ(GEN8_UCGCTL6) |
                    GEN8_SDEUNIT_CLOCK_GATE_DISABLE);
 
-        /* Wa4x4STCOptimizationDisable:bdw */
-        I915_WRITE(CACHE_MODE_1,
-                   _MASKED_BIT_ENABLE(GEN8_4x4_STC_OPTIMIZATION_DISABLE));
+        lpt_init_clock_gating(dev);
 }
 
 static void haswell_init_clock_gating(struct drm_device *dev)
@@ -5631,24 +5807,6 @@ static void ivybridge_init_clock_gating(struct drm_device *dev)
 static void valleyview_init_clock_gating(struct drm_device *dev)
 {
         struct drm_i915_private *dev_priv = dev->dev_private;
-        u32 val;
-
-        mutex_lock(&dev_priv->rps.hw_lock);
-        val = vlv_punit_read(dev_priv, PUNIT_REG_GPU_FREQ_STS);
-        mutex_unlock(&dev_priv->rps.hw_lock);
-        switch ((val >> 6) & 3) {
-        case 0:
-        case 1:
-                dev_priv->mem_freq = 800;
-                break;
-        case 2:
-                dev_priv->mem_freq = 1066;
-                break;
-        case 3:
-                dev_priv->mem_freq = 1333;
-                break;
-        }
-        DRM_DEBUG_DRIVER("DDR speed: %d MHz", dev_priv->mem_freq);
 
         I915_WRITE(DSPCLK_GATE_D, VRHUNIT_CLOCK_GATE_DISABLE);
 
@@ -5724,48 +5882,11 @@ static void valleyview_init_clock_gating(struct drm_device *dev)
 static void cherryview_init_clock_gating(struct drm_device *dev)
 {
         struct drm_i915_private *dev_priv = dev->dev_private;
-        u32 val;
-
-        mutex_lock(&dev_priv->rps.hw_lock);
-        val = vlv_punit_read(dev_priv, CCK_FUSE_REG);
-        mutex_unlock(&dev_priv->rps.hw_lock);
-        switch ((val >> 2) & 0x7) {
-        case 0:
-        case 1:
-                        dev_priv->rps.cz_freq = CHV_CZ_CLOCK_FREQ_MODE_200;
-                        dev_priv->mem_freq = 1600;
-                        break;
-        case 2:
-                        dev_priv->rps.cz_freq = CHV_CZ_CLOCK_FREQ_MODE_267;
-                        dev_priv->mem_freq = 1600;
-                        break;
-        case 3:
-                        dev_priv->rps.cz_freq = CHV_CZ_CLOCK_FREQ_MODE_333;
-                        dev_priv->mem_freq = 2000;
-                        break;
-        case 4:
-                        dev_priv->rps.cz_freq = CHV_CZ_CLOCK_FREQ_MODE_320;
-                        dev_priv->mem_freq = 1600;
-                        break;
-        case 5:
-                        dev_priv->rps.cz_freq = CHV_CZ_CLOCK_FREQ_MODE_400;
-                        dev_priv->mem_freq = 1600;
-                        break;
-        }
-        DRM_DEBUG_DRIVER("DDR speed: %d MHz", dev_priv->mem_freq);
 
         I915_WRITE(DSPCLK_GATE_D, VRHUNIT_CLOCK_GATE_DISABLE);
 
         I915_WRITE(MI_ARB_VLV, MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE);
 
-        /* WaDisablePartialInstShootdown:chv */
-        I915_WRITE(GEN8_ROW_CHICKEN,
-                   _MASKED_BIT_ENABLE(PARTIAL_INSTRUCTION_SHOOTDOWN_DISABLE));
-
-        /* WaDisableThreadStallDopClockGating:chv */
-        I915_WRITE(GEN8_ROW_CHICKEN,
-                   _MASKED_BIT_ENABLE(STALL_DOP_GATING_DISABLE));
-
         /* WaVSRefCountFullforceMissDisable:chv */
         /* WaDSRefCountFullforceMissDisable:chv */
         I915_WRITE(GEN7_FF_THREAD_MODE,
@@ -5784,10 +5905,6 @@ static void cherryview_init_clock_gating(struct drm_device *dev)
         I915_WRITE(GEN8_UCGCTL6, I915_READ(GEN8_UCGCTL6) |
                    GEN8_SDEUNIT_CLOCK_GATE_DISABLE);
 
-        /* WaDisableSamplerPowerBypass:chv (pre-production hw) */
-        I915_WRITE(HALF_SLICE_CHICKEN3,
-                   _MASKED_BIT_ENABLE(GEN8_SAMPLER_POWER_BYPASS_DIS));
-
         /* WaDisableGunitClockGating:chv (pre-production hw) */
         I915_WRITE(VLV_GUNIT_CLOCK_GATE, I915_READ(VLV_GUNIT_CLOCK_GATE) |
                    GINT_DIS);
@@ -5797,8 +5914,6 @@ static void cherryview_init_clock_gating(struct drm_device *dev)
                    _MASKED_BIT_ENABLE(GEN8_FF_DOP_CLOCK_GATE_DISABLE));
 
         /* WaDisableDopClockGating:chv (pre-production hw) */
-        I915_WRITE(GEN7_ROW_CHICKEN2,
-                   _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
         I915_WRITE(GEN6_UCGCTL1, I915_READ(GEN6_UCGCTL1) |
                    GEN6_EU_TCUNIT_CLOCK_GATE_DISABLE);
 }
@@ -5883,6 +5998,9 @@ static void gen3_init_clock_gating(struct drm_device *dev)
 
         /* On GEN3 we really need to make sure the ARB C3 LP bit is set */
         I915_WRITE(MI_ARB_STATE, _MASKED_BIT_ENABLE(MI_ARB_C3_LP_WRITE_ENABLE));
+
+        I915_WRITE(MI_ARB_STATE,
+                   _MASKED_BIT_ENABLE(MI_ARB_DISPLAY_TRICKLE_FEED_DISABLE));
 }
 
 static void i85x_init_clock_gating(struct drm_device *dev)
@@ -5894,6 +6012,9 @@ static void i85x_init_clock_gating(struct drm_device *dev)
         /* interrupts should cause a wake up from C3 */
         I915_WRITE(MI_STATE, _MASKED_BIT_ENABLE(MI_AGPBUSY_INT_EN) |
                    _MASKED_BIT_DISABLE(MI_AGPBUSY_830_MODE));
+
+        I915_WRITE(MEM_MODE,
+                   _MASKED_BIT_ENABLE(MEM_DISPLAY_TRICKLE_FEED_DISABLE));
 }
 
 static void i830_init_clock_gating(struct drm_device *dev)
@@ -5901,6 +6022,10 @@ static void i830_init_clock_gating(struct drm_device *dev)
         struct drm_i915_private *dev_priv = dev->dev_private;
 
         I915_WRITE(DSPCLK_GATE_D, OVRUNIT_CLOCK_GATE_DISABLE);
+
+        I915_WRITE(MEM_MODE,
+                   _MASKED_BIT_ENABLE(MEM_DISPLAY_A_TRICKLE_FEED_DISABLE) |
+                   _MASKED_BIT_ENABLE(MEM_DISPLAY_B_TRICKLE_FEED_DISABLE));
 }
 
 void intel_init_clock_gating(struct drm_device *dev)
@@ -6203,6 +6328,8 @@ static void vlv_display_power_well_disable(struct drm_i915_private *dev_priv,
         spin_unlock_irq(&dev_priv->irq_lock);
 
         vlv_set_power_well(dev_priv, power_well, false);
+
+        vlv_power_sequencer_reset(dev_priv);
 }
 
 static void vlv_dpio_cmn_power_well_enable(struct drm_i915_private *dev_priv,
@@ -6238,12 +6365,11 @@ static void vlv_dpio_cmn_power_well_enable(struct drm_i915_private *dev_priv,
 static void vlv_dpio_cmn_power_well_disable(struct drm_i915_private *dev_priv,
                                             struct i915_power_well *power_well)
 {
-        struct drm_device *dev = dev_priv->dev;
         enum pipe pipe;
 
         WARN_ON_ONCE(power_well->data != PUNIT_POWER_WELL_DPIO_CMN_BC);
 
-        for_each_pipe(pipe)
+        for_each_pipe(dev_priv, pipe)
                 assert_pll_disabled(dev_priv, pipe);
 
         /* Assert common reset */
@@ -6252,6 +6378,153 @@ static void vlv_dpio_cmn_power_well_disable(struct drm_i915_private *dev_priv,
         vlv_set_power_well(dev_priv, power_well, false);
 }
 
+static void chv_dpio_cmn_power_well_enable(struct drm_i915_private *dev_priv,
+                                           struct i915_power_well *power_well)
+{
+        enum dpio_phy phy;
+
+        WARN_ON_ONCE(power_well->data != PUNIT_POWER_WELL_DPIO_CMN_BC &&
+                     power_well->data != PUNIT_POWER_WELL_DPIO_CMN_D);
+
+        /*
+         * Enable the CRI clock source so we can get at the
+         * display and the reference clock for VGA
+         * hotplug / manual detection.
+         */
+        if (power_well->data == PUNIT_POWER_WELL_DPIO_CMN_BC) {
+                phy = DPIO_PHY0;
+                I915_WRITE(DPLL(PIPE_B), I915_READ(DPLL(PIPE_B)) |
+                           DPLL_REFA_CLK_ENABLE_VLV);
+                I915_WRITE(DPLL(PIPE_B), I915_READ(DPLL(PIPE_B)) |
+                           DPLL_REFA_CLK_ENABLE_VLV | DPLL_INTEGRATED_CRI_CLK_VLV);
+        } else {
+                phy = DPIO_PHY1;
+                I915_WRITE(DPLL(PIPE_C), I915_READ(DPLL(PIPE_C)) |
+                           DPLL_REFA_CLK_ENABLE_VLV | DPLL_INTEGRATED_CRI_CLK_VLV);
+        }
+        udelay(1); /* >10ns for cmnreset, >0ns for sidereset */
+        vlv_set_power_well(dev_priv, power_well, true);
+
+        /* Poll for phypwrgood signal */
+        if (wait_for(I915_READ(DISPLAY_PHY_STATUS) & PHY_POWERGOOD(phy), 1))
+                DRM_ERROR("Display PHY %d is not power up\n", phy);
+
+        I915_WRITE(DISPLAY_PHY_CONTROL, I915_READ(DISPLAY_PHY_CONTROL) |
+                   PHY_COM_LANE_RESET_DEASSERT(phy));
+}
+
+static void chv_dpio_cmn_power_well_disable(struct drm_i915_private *dev_priv,
+                                            struct i915_power_well *power_well)
+{
+        enum dpio_phy phy;
+
+        WARN_ON_ONCE(power_well->data != PUNIT_POWER_WELL_DPIO_CMN_BC &&
+                     power_well->data != PUNIT_POWER_WELL_DPIO_CMN_D);
+
+        if (power_well->data == PUNIT_POWER_WELL_DPIO_CMN_BC) {
+                phy = DPIO_PHY0;
+                assert_pll_disabled(dev_priv, PIPE_A);
+                assert_pll_disabled(dev_priv, PIPE_B);
+        } else {
+                phy = DPIO_PHY1;
+                assert_pll_disabled(dev_priv, PIPE_C);
+        }
+
+        I915_WRITE(DISPLAY_PHY_CONTROL, I915_READ(DISPLAY_PHY_CONTROL) &
+                   ~PHY_COM_LANE_RESET_DEASSERT(phy));
+
+        vlv_set_power_well(dev_priv, power_well, false);
+}
+
+static bool chv_pipe_power_well_enabled(struct drm_i915_private *dev_priv,
+                                        struct i915_power_well *power_well)
+{
+        enum pipe pipe = power_well->data;
+        bool enabled;
+        u32 state, ctrl;
+
+        mutex_lock(&dev_priv->rps.hw_lock);
+
+        state = vlv_punit_read(dev_priv, PUNIT_REG_DSPFREQ) & DP_SSS_MASK(pipe);
+        /*
+         * We only ever set the power-on and power-gate states, anything
+         * else is unexpected.
+         */
+        WARN_ON(state != DP_SSS_PWR_ON(pipe) && state != DP_SSS_PWR_GATE(pipe));
+        enabled = state == DP_SSS_PWR_ON(pipe);
+
+        /*
+         * A transient state at this point would mean some unexpected party
+         * is poking at the power controls too.
+         */
+        ctrl = vlv_punit_read(dev_priv, PUNIT_REG_DSPFREQ) & DP_SSC_MASK(pipe);
+        WARN_ON(ctrl << 16 != state);
+
+        mutex_unlock(&dev_priv->rps.hw_lock);
+
+        return enabled;
+}
+
+static void chv_set_pipe_power_well(struct drm_i915_private *dev_priv,
+                                    struct i915_power_well *power_well,
+                                    bool enable)
+{
+        enum pipe pipe = power_well->data;
+        u32 state;
+        u32 ctrl;
+
+        state = enable ? DP_SSS_PWR_ON(pipe) : DP_SSS_PWR_GATE(pipe);
+
+        mutex_lock(&dev_priv->rps.hw_lock);
+
+#define COND \
+        ((vlv_punit_read(dev_priv, PUNIT_REG_DSPFREQ) & DP_SSS_MASK(pipe)) == state)
+
+        if (COND)
+                goto out;
+
+        ctrl = vlv_punit_read(dev_priv, PUNIT_REG_DSPFREQ);
+        ctrl &= ~DP_SSC_MASK(pipe);
+        ctrl |= enable ? DP_SSC_PWR_ON(pipe) : DP_SSC_PWR_GATE(pipe);
+        vlv_punit_write(dev_priv, PUNIT_REG_DSPFREQ, ctrl);
+
+        if (wait_for(COND, 100))
+                DRM_ERROR("timout setting power well state %08x (%08x)\n",
+                          state,
+                          vlv_punit_read(dev_priv, PUNIT_REG_DSPFREQ));
+
+#undef COND
+
+out:
+        mutex_unlock(&dev_priv->rps.hw_lock);
+}
+
+static void chv_pipe_power_well_sync_hw(struct drm_i915_private *dev_priv,
+                                        struct i915_power_well *power_well)
+{
+        chv_set_pipe_power_well(dev_priv, power_well, power_well->count > 0);
+}
+
+static void chv_pipe_power_well_enable(struct drm_i915_private *dev_priv,
+                                       struct i915_power_well *power_well)
+{
+        WARN_ON_ONCE(power_well->data != PIPE_A &&
+                     power_well->data != PIPE_B &&
+                     power_well->data != PIPE_C);
+
+        chv_set_pipe_power_well(dev_priv, power_well, true);
+}
+
+static void chv_pipe_power_well_disable(struct drm_i915_private *dev_priv,
+                                        struct i915_power_well *power_well)
+{
+        WARN_ON_ONCE(power_well->data != PIPE_A &&
+                     power_well->data != PIPE_B &&
+                     power_well->data != PIPE_C);
+
+        chv_set_pipe_power_well(dev_priv, power_well, false);
+}
+
 static void check_power_well_state(struct drm_i915_private *dev_priv,
                                    struct i915_power_well *power_well)
 {
@@ -6443,6 +6716,39 @@ EXPORT_SYMBOL_GPL(i915_get_cdclk_freq);
         BIT(POWER_DOMAIN_PORT_DDI_C_4_LANES) |  \
         BIT(POWER_DOMAIN_INIT))
 
+#define CHV_PIPE_A_POWER_DOMAINS (      \
+        BIT(POWER_DOMAIN_PIPE_A) |      \
+        BIT(POWER_DOMAIN_INIT))
+
+#define CHV_PIPE_B_POWER_DOMAINS (      \
+        BIT(POWER_DOMAIN_PIPE_B) |      \
+        BIT(POWER_DOMAIN_INIT))
+
+#define CHV_PIPE_C_POWER_DOMAINS (      \
+        BIT(POWER_DOMAIN_PIPE_C) |      \
+        BIT(POWER_DOMAIN_INIT))
+
+#define CHV_DPIO_CMN_BC_POWER_DOMAINS (         \
+        BIT(POWER_DOMAIN_PORT_DDI_B_2_LANES) |  \
+        BIT(POWER_DOMAIN_PORT_DDI_B_4_LANES) |  \
+        BIT(POWER_DOMAIN_PORT_DDI_C_2_LANES) |  \
+        BIT(POWER_DOMAIN_PORT_DDI_C_4_LANES) |  \
+        BIT(POWER_DOMAIN_INIT))
+
+#define CHV_DPIO_CMN_D_POWER_DOMAINS (          \
+        BIT(POWER_DOMAIN_PORT_DDI_D_2_LANES) |  \
+        BIT(POWER_DOMAIN_PORT_DDI_D_4_LANES) |  \
+        BIT(POWER_DOMAIN_INIT))
+
+#define CHV_DPIO_TX_D_LANES_01_POWER_DOMAINS (  \
+        BIT(POWER_DOMAIN_PORT_DDI_D_2_LANES) |  \
+        BIT(POWER_DOMAIN_PORT_DDI_D_4_LANES) |  \
+        BIT(POWER_DOMAIN_INIT))
+
+#define CHV_DPIO_TX_D_LANES_23_POWER_DOMAINS (  \
+        BIT(POWER_DOMAIN_PORT_DDI_D_4_LANES) |  \
+        BIT(POWER_DOMAIN_INIT))
+
 static const struct i915_power_well_ops i9xx_always_on_power_well_ops = {
         .sync_hw = i9xx_always_on_power_well_noop,
         .enable = i9xx_always_on_power_well_noop,
@@ -6450,6 +6756,20 @@ static const struct i915_power_well_ops i9xx_always_on_power_well_ops = {
         .is_enabled = i9xx_always_on_power_well_enabled,
 };
 
+static const struct i915_power_well_ops chv_pipe_power_well_ops = {
+        .sync_hw = chv_pipe_power_well_sync_hw,
+        .enable = chv_pipe_power_well_enable,
+        .disable = chv_pipe_power_well_disable,
+        .is_enabled = chv_pipe_power_well_enabled,
+};
+
+static const struct i915_power_well_ops chv_dpio_cmn_power_well_ops = {
+        .sync_hw = vlv_power_well_sync_hw,
+        .enable = chv_dpio_cmn_power_well_enable,
+        .disable = chv_dpio_cmn_power_well_disable,
+        .is_enabled = vlv_power_well_enabled,
+};
+
 static struct i915_power_well i9xx_always_on_power_well[] = {
         {
                 .name = "always-on",
@@ -6572,6 +6892,107 @@ static struct i915_power_well vlv_power_wells[] = {
         },
 };
 
+static struct i915_power_well chv_power_wells[] = {
+        {
+                .name = "always-on",
+                .always_on = 1,
+                .domains = VLV_ALWAYS_ON_POWER_DOMAINS,
+                .ops = &i9xx_always_on_power_well_ops,
+        },
+#if 0
+        {
+                .name = "display",
+                .domains = VLV_DISPLAY_POWER_DOMAINS,
+                .data = PUNIT_POWER_WELL_DISP2D,
+                .ops = &vlv_display_power_well_ops,
+        },
+        {
+                .name = "pipe-a",
+                .domains = CHV_PIPE_A_POWER_DOMAINS,
+                .data = PIPE_A,
+                .ops = &chv_pipe_power_well_ops,
+        },
+        {
+                .name = "pipe-b",
+                .domains = CHV_PIPE_B_POWER_DOMAINS,
+                .data = PIPE_B,
+                .ops = &chv_pipe_power_well_ops,
+        },
+        {
+                .name = "pipe-c",
+                .domains = CHV_PIPE_C_POWER_DOMAINS,
+                .data = PIPE_C,
+                .ops = &chv_pipe_power_well_ops,
+        },
+#endif
+        {
+                .name = "dpio-common-bc",
+                /*
+                 * XXX: cmnreset for one PHY seems to disturb the other.
+                 * As a workaround keep both powered on at the same
+                 * time for now.
+                 */
+                .domains = CHV_DPIO_CMN_BC_POWER_DOMAINS | CHV_DPIO_CMN_D_POWER_DOMAINS,
+                .data = PUNIT_POWER_WELL_DPIO_CMN_BC,
+                .ops = &chv_dpio_cmn_power_well_ops,
+        },
+        {
+                .name = "dpio-common-d",
+                /*
+                 * XXX: cmnreset for one PHY seems to disturb the other.
+                 * As a workaround keep both powered on at the same
+                 * time for now.
+                 */
+                .domains = CHV_DPIO_CMN_BC_POWER_DOMAINS | CHV_DPIO_CMN_D_POWER_DOMAINS,
+                .data = PUNIT_POWER_WELL_DPIO_CMN_D,
+                .ops = &chv_dpio_cmn_power_well_ops,
+        },
+#if 0
+        {
+                .name = "dpio-tx-b-01",
+                .domains = VLV_DPIO_TX_B_LANES_01_POWER_DOMAINS |
+                           VLV_DPIO_TX_B_LANES_23_POWER_DOMAINS,
+                .ops = &vlv_dpio_power_well_ops,
+                .data = PUNIT_POWER_WELL_DPIO_TX_B_LANES_01,
+        },
+        {
+                .name = "dpio-tx-b-23",
+                .domains = VLV_DPIO_TX_B_LANES_01_POWER_DOMAINS |
+                           VLV_DPIO_TX_B_LANES_23_POWER_DOMAINS,
+                .ops = &vlv_dpio_power_well_ops,
+                .data = PUNIT_POWER_WELL_DPIO_TX_B_LANES_23,
+        },
+        {
+                .name = "dpio-tx-c-01",
+                .domains = VLV_DPIO_TX_C_LANES_01_POWER_DOMAINS |
+                           VLV_DPIO_TX_C_LANES_23_POWER_DOMAINS,
+                .ops = &vlv_dpio_power_well_ops,
+                .data = PUNIT_POWER_WELL_DPIO_TX_C_LANES_01,
+        },
+        {
+                .name = "dpio-tx-c-23",
+                .domains = VLV_DPIO_TX_C_LANES_01_POWER_DOMAINS |
+                           VLV_DPIO_TX_C_LANES_23_POWER_DOMAINS,
+                .ops = &vlv_dpio_power_well_ops,
+                .data = PUNIT_POWER_WELL_DPIO_TX_C_LANES_23,
+        },
+        {
+                .name = "dpio-tx-d-01",
+                .domains = CHV_DPIO_TX_D_LANES_01_POWER_DOMAINS |
+                           CHV_DPIO_TX_D_LANES_23_POWER_DOMAINS,
+                .ops = &vlv_dpio_power_well_ops,
+                .data = PUNIT_POWER_WELL_DPIO_TX_D_LANES_01,
+        },
+        {
+                .name = "dpio-tx-d-23",
+                .domains = CHV_DPIO_TX_D_LANES_01_POWER_DOMAINS |
+                           CHV_DPIO_TX_D_LANES_23_POWER_DOMAINS,
+                .ops = &vlv_dpio_power_well_ops,
+                .data = PUNIT_POWER_WELL_DPIO_TX_D_LANES_23,
+        },
+#endif
+};
+
 static struct i915_power_well *lookup_power_well(struct drm_i915_private *dev_priv,
                                                  enum punit_power_well power_well_id)
 {
@@ -6608,6 +7029,8 @@ int intel_power_domains_init(struct drm_i915_private *dev_priv)
         } else if (IS_BROADWELL(dev_priv->dev)) {
                 set_power_wells(power_domains, bdw_power_wells);
                 hsw_pwr = power_domains;
+        } else if (IS_CHERRYVIEW(dev_priv->dev)) {
+                set_power_wells(power_domains, chv_power_wells);
         } else if (IS_VALLEYVIEW(dev_priv->dev)) {
                 set_power_wells(power_domains, vlv_power_wells);
         } else {
@@ -6833,13 +7256,15 @@ void intel_init_pm(struct drm_device *dev)
                 else if (IS_HASWELL(dev))
                         dev_priv->display.init_clock_gating = haswell_init_clock_gating;
                 else if (INTEL_INFO(dev)->gen == 8)
-                        dev_priv->display.init_clock_gating = gen8_init_clock_gating;
+                        dev_priv->display.init_clock_gating = broadwell_init_clock_gating;
         } else if (IS_CHERRYVIEW(dev)) {
-                dev_priv->display.update_wm = valleyview_update_wm;
+                dev_priv->display.update_wm = cherryview_update_wm;
+                dev_priv->display.update_sprite_wm = valleyview_update_sprite_wm;
                 dev_priv->display.init_clock_gating =
                         cherryview_init_clock_gating;
         } else if (IS_VALLEYVIEW(dev)) {
                 dev_priv->display.update_wm = valleyview_update_wm;
+                dev_priv->display.update_sprite_wm = valleyview_update_sprite_wm;
                 dev_priv->display.init_clock_gating =
                         valleyview_init_clock_gating;
         } else if (IS_PINEVIEW(dev)) {
@@ -7025,6 +7450,7 @@ static int chv_freq_opcode(struct drm_i915_private *dev_priv, int val)
                 return -1;
         }
 
+        /* CHV needs even values */
         opcode = (DIV_ROUND_CLOSEST((val * 2 * mul), dev_priv->rps.cz_freq) * 2);
 
         return opcode;
diff --git a/drivers/gpu/drm/i915/intel_renderstate.h b/drivers/gpu/drm/i915/intel_renderstate.h
index fd4f66231d30..6c792d3a9c9c 100644
--- a/drivers/gpu/drm/i915/intel_renderstate.h
+++ b/drivers/gpu/drm/i915/intel_renderstate.h
@@ -24,13 +24,7 @@
 #ifndef _INTEL_RENDERSTATE_H
 #define _INTEL_RENDERSTATE_H
 
-#include <linux/types.h>
-
-struct intel_renderstate_rodata {
-        const u32 *reloc;
-        const u32 *batch;
-        const u32 batch_items;
-};
+#include "i915_drv.h"
 
 extern const struct intel_renderstate_rodata gen6_null_state;
 extern const struct intel_renderstate_rodata gen7_null_state;
diff --git a/drivers/gpu/drm/i915/intel_ringbuffer.c b/drivers/gpu/drm/i915/intel_ringbuffer.c
index 47a126a0493f..0a80e419b589 100644
--- a/drivers/gpu/drm/i915/intel_ringbuffer.c
+++ b/drivers/gpu/drm/i915/intel_ringbuffer.c
@@ -33,14 +33,24 @@
 #include "i915_trace.h"
 #include "intel_drv.h"
 
-/* Early gen2 devices have a cacheline of just 32 bytes, using 64 is overkill,
- * but keeps the logic simple. Indeed, the whole purpose of this macro is just
- * to give some inclination as to some of the magic values used in the various
- * workarounds!
- */
-#define CACHELINE_BYTES 64
+bool
+intel_ring_initialized(struct intel_engine_cs *ring)
+{
+        struct drm_device *dev = ring->dev;
+
+        if (!dev)
+                return false;
+
+        if (i915.enable_execlists) {
+                struct intel_context *dctx = ring->default_context;
+                struct intel_ringbuffer *ringbuf = dctx->engine[ring->id].ringbuf;
+
+                return ringbuf->obj;
+        } else
+                return ring->buffer && ring->buffer->obj;
+}
 
-static inline int __ring_space(int head, int tail, int size)
+int __intel_ring_space(int head, int tail, int size)
 {
         int space = head - (tail + I915_RING_FREE_SPACE);
         if (space < 0)
@@ -48,12 +58,13 @@ static inline int __ring_space(int head, int tail, int size)
         return space;
 }
 
-static inline int ring_space(struct intel_ringbuffer *ringbuf)
+int intel_ring_space(struct intel_ringbuffer *ringbuf)
 {
-        return __ring_space(ringbuf->head & HEAD_ADDR, ringbuf->tail, ringbuf->size);
+        return __intel_ring_space(ringbuf->head & HEAD_ADDR,
+                                  ringbuf->tail, ringbuf->size);
 }
 
-static bool intel_ring_stopped(struct intel_engine_cs *ring)
+bool intel_ring_stopped(struct intel_engine_cs *ring)
 {
         struct drm_i915_private *dev_priv = ring->dev->dev_private;
         return dev_priv->gpu_error.stop_rings & intel_ring_flag(ring);
@@ -433,7 +444,14 @@ gen8_render_ring_flush(struct intel_engine_cs *ring,
                         return ret;
         }
 
-        return gen8_emit_pipe_control(ring, flags, scratch_addr);
+        ret = gen8_emit_pipe_control(ring, flags, scratch_addr);
+        if (ret)
+                return ret;
+
+        if (!invalidate_domains && flush_domains)
+                return gen7_ring_fbc_flush(ring, FBC_REND_NUKE);
+
+        return 0;
 }
 
 static void ring_write_tail(struct intel_engine_cs *ring,
@@ -476,9 +494,14 @@ static bool stop_ring(struct intel_engine_cs *ring)
 
         if (!IS_GEN2(ring->dev)) {
                 I915_WRITE_MODE(ring, _MASKED_BIT_ENABLE(STOP_RING));
-                if (wait_for_atomic((I915_READ_MODE(ring) & MODE_IDLE) != 0, 1000)) {
-                        DRM_ERROR("%s :timed out trying to stop ring\n", ring->name);
-                        return false;
+                if (wait_for((I915_READ_MODE(ring) & MODE_IDLE) != 0, 1000)) {
+                        DRM_ERROR("%s : timed out trying to stop ring\n", ring->name);
+                        /* Sometimes we observe that the idle flag is not
+                         * set even though the ring is empty. So double
+                         * check before giving up.
+                         */
+                        if (I915_READ_HEAD(ring) != I915_READ_TAIL(ring))
+                                return false;
                 }
         }
 
@@ -540,6 +563,14 @@ static int init_ring_common(struct intel_engine_cs *ring)
          * also enforces ordering), otherwise the hw might lose the new ring
          * register values. */
         I915_WRITE_START(ring, i915_gem_obj_ggtt_offset(obj));
+
+        /* WaClearRingBufHeadRegAtInit:ctg,elk */
+        if (I915_READ_HEAD(ring))
+                DRM_DEBUG("%s initialization failed [head=%08x], fudging\n",
+                          ring->name, I915_READ_HEAD(ring));
+        I915_WRITE_HEAD(ring, 0);
+        (void)I915_READ_HEAD(ring);
+
         I915_WRITE_CTL(ring,
                         ((ringbuf->size - PAGE_SIZE) & RING_NR_PAGES)
                         | RING_VALID);
@@ -563,7 +594,7 @@ static int init_ring_common(struct intel_engine_cs *ring)
         else {
                 ringbuf->head = I915_READ_HEAD(ring);
                 ringbuf->tail = I915_READ_TAIL(ring) & TAIL_ADDR;
-                ringbuf->space = ring_space(ringbuf);
+                ringbuf->space = intel_ring_space(ringbuf);
                 ringbuf->last_retired_head = -1;
         }
 
@@ -575,8 +606,25 @@ out:
         return ret;
 }
 
-static int
-init_pipe_control(struct intel_engine_cs *ring)
+void
+intel_fini_pipe_control(struct intel_engine_cs *ring)
+{
+        struct drm_device *dev = ring->dev;
+
+        if (ring->scratch.obj == NULL)
+                return;
+
+        if (INTEL_INFO(dev)->gen >= 5) {
+                kunmap(sg_page(ring->scratch.obj->pages->sgl));
+                i915_gem_object_ggtt_unpin(ring->scratch.obj);
+        }
+
+        drm_gem_object_unreference(&ring->scratch.obj->base);
+        ring->scratch.obj = NULL;
+}
+
+int
+intel_init_pipe_control(struct intel_engine_cs *ring)
 {
         int ret;
 
@@ -617,6 +665,135 @@ err:
         return ret;
 }
 
+static inline void intel_ring_emit_wa(struct intel_engine_cs *ring,
+                                       u32 addr, u32 value)
+{
+        struct drm_device *dev = ring->dev;
+        struct drm_i915_private *dev_priv = dev->dev_private;
+
+        if (WARN_ON(dev_priv->num_wa_regs >= I915_MAX_WA_REGS))
+                return;
+
+        intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(1));
+        intel_ring_emit(ring, addr);
+        intel_ring_emit(ring, value);
+
+        dev_priv->intel_wa_regs[dev_priv->num_wa_regs].addr = addr;
+        dev_priv->intel_wa_regs[dev_priv->num_wa_regs].mask = value & 0xFFFF;
+        /* value is updated with the status of remaining bits of this
+         * register when it is read from debugfs file
+         */
+        dev_priv->intel_wa_regs[dev_priv->num_wa_regs].value = value;
+        dev_priv->num_wa_regs++;
+
+        return;
+}
+
+static int bdw_init_workarounds(struct intel_engine_cs *ring)
+{
+        int ret;
+        struct drm_device *dev = ring->dev;
+        struct drm_i915_private *dev_priv = dev->dev_private;
+
+        /*
+         * workarounds applied in this fn are part of register state context,
+         * they need to be re-initialized followed by gpu reset, suspend/resume,
+         * module reload.
+         */
+        dev_priv->num_wa_regs = 0;
+        memset(dev_priv->intel_wa_regs, 0, sizeof(dev_priv->intel_wa_regs));
+
+        /*
+         * update the number of dwords required based on the
+         * actual number of workarounds applied
+         */
+        ret = intel_ring_begin(ring, 18);
+        if (ret)
+                return ret;
+
+        /* WaDisablePartialInstShootdown:bdw */
+        /* WaDisableThreadStallDopClockGating:bdw */
+        /* FIXME: Unclear whether we really need this on production bdw. */
+        intel_ring_emit_wa(ring, GEN8_ROW_CHICKEN,
+                           _MASKED_BIT_ENABLE(PARTIAL_INSTRUCTION_SHOOTDOWN_DISABLE
+                                             | STALL_DOP_GATING_DISABLE));
+
+        /* WaDisableDopClockGating:bdw May not be needed for production */
+        intel_ring_emit_wa(ring, GEN7_ROW_CHICKEN2,
+                           _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
+
+        intel_ring_emit_wa(ring, HALF_SLICE_CHICKEN3,
+                           _MASKED_BIT_ENABLE(GEN8_SAMPLER_POWER_BYPASS_DIS));
+
+        /* Use Force Non-Coherent whenever executing a 3D context. This is a
+         * workaround for for a possible hang in the unlikely event a TLB
+         * invalidation occurs during a PSD flush.
+         */
+        intel_ring_emit_wa(ring, HDC_CHICKEN0,
+                           _MASKED_BIT_ENABLE(HDC_FORCE_NON_COHERENT));
+
+        /* Wa4x4STCOptimizationDisable:bdw */
+        intel_ring_emit_wa(ring, CACHE_MODE_1,
+                           _MASKED_BIT_ENABLE(GEN8_4x4_STC_OPTIMIZATION_DISABLE));
+
+        /*
+         * BSpec recommends 8x4 when MSAA is used,
+         * however in practice 16x4 seems fastest.
+         *
+         * Note that PS/WM thread counts depend on the WIZ hashing
+         * disable bit, which we don't touch here, but it's good
+         * to keep in mind (see 3DSTATE_PS and 3DSTATE_WM).
+         */
+        intel_ring_emit_wa(ring, GEN7_GT_MODE,
+                           GEN6_WIZ_HASHING_MASK | GEN6_WIZ_HASHING_16x4);
+
+        intel_ring_advance(ring);
+
+        DRM_DEBUG_DRIVER("Number of Workarounds applied: %d\n",
+                         dev_priv->num_wa_regs);
+
+        return 0;
+}
+
+static int chv_init_workarounds(struct intel_engine_cs *ring)
+{
+        int ret;
+        struct drm_device *dev = ring->dev;
+        struct drm_i915_private *dev_priv = dev->dev_private;
+
+        /*
+         * workarounds applied in this fn are part of register state context,
+         * they need to be re-initialized followed by gpu reset, suspend/resume,
+         * module reload.
+         */
+        dev_priv->num_wa_regs = 0;
+        memset(dev_priv->intel_wa_regs, 0, sizeof(dev_priv->intel_wa_regs));
+
+        ret = intel_ring_begin(ring, 12);
+        if (ret)
+                return ret;
+
+        /* WaDisablePartialInstShootdown:chv */
+        intel_ring_emit_wa(ring, GEN8_ROW_CHICKEN,
+                           _MASKED_BIT_ENABLE(PARTIAL_INSTRUCTION_SHOOTDOWN_DISABLE));
+
+        /* WaDisableThreadStallDopClockGating:chv */
+        intel_ring_emit_wa(ring, GEN8_ROW_CHICKEN,
+                           _MASKED_BIT_ENABLE(STALL_DOP_GATING_DISABLE));
+
+        /* WaDisableDopClockGating:chv (pre-production hw) */
+        intel_ring_emit_wa(ring, GEN7_ROW_CHICKEN2,
+                           _MASKED_BIT_ENABLE(DOP_CLOCK_GATING_DISABLE));
+
+        /* WaDisableSamplerPowerBypass:chv (pre-production hw) */
+        intel_ring_emit_wa(ring, HALF_SLICE_CHICKEN3,
+                           _MASKED_BIT_ENABLE(GEN8_SAMPLER_POWER_BYPASS_DIS));
+
+        intel_ring_advance(ring);
+
+        return 0;
+}
+
 static int init_render_ring(struct intel_engine_cs *ring)
 {
         struct drm_device *dev = ring->dev;
@@ -651,7 +828,7 @@ static int init_render_ring(struct intel_engine_cs *ring)
                            _MASKED_BIT_ENABLE(GFX_REPLAY_MODE));
 
         if (INTEL_INFO(dev)->gen >= 5) {
-                ret = init_pipe_control(ring);
+                ret = intel_init_pipe_control(ring);
                 if (ret)
                         return ret;
         }
@@ -686,16 +863,7 @@ static void render_ring_cleanup(struct intel_engine_cs *ring)
                 dev_priv->semaphore_obj = NULL;
         }
 
-        if (ring->scratch.obj == NULL)
-                return;
-
-        if (INTEL_INFO(dev)->gen >= 5) {
-                kunmap(sg_page(ring->scratch.obj->pages->sgl));
-                i915_gem_object_ggtt_unpin(ring->scratch.obj);
-        }
-
-        drm_gem_object_unreference(&ring->scratch.obj->base);
-        ring->scratch.obj = NULL;
+        intel_fini_pipe_control(ring);
 }
 
 static int gen8_rcs_signal(struct intel_engine_cs *signaller,
@@ -1526,7 +1694,7 @@ static int init_phys_status_page(struct intel_engine_cs *ring)
         return 0;
 }
 
-static void intel_destroy_ringbuffer_obj(struct intel_ringbuffer *ringbuf)
+void intel_destroy_ringbuffer_obj(struct intel_ringbuffer *ringbuf)
 {
         if (!ringbuf->obj)
                 return;
@@ -1537,8 +1705,8 @@ static void intel_destroy_ringbuffer_obj(struct intel_ringbuffer *ringbuf)
         ringbuf->obj = NULL;
 }
 
-static int intel_alloc_ringbuffer_obj(struct drm_device *dev,
-                                      struct intel_ringbuffer *ringbuf)
+int intel_alloc_ringbuffer_obj(struct drm_device *dev,
+                               struct intel_ringbuffer *ringbuf)
 {
         struct drm_i915_private *dev_priv = to_i915(dev);
         struct drm_i915_gem_object *obj;
@@ -1600,7 +1768,9 @@ static int intel_init_ring_buffer(struct drm_device *dev,
         ring->dev = dev;
         INIT_LIST_HEAD(&ring->active_list);
         INIT_LIST_HEAD(&ring->request_list);
+        INIT_LIST_HEAD(&ring->execlist_queue);
         ringbuf->size = 32 * PAGE_SIZE;
+        ringbuf->ring = ring;
         memset(ring->semaphore.sync_seqno, 0, sizeof(ring->semaphore.sync_seqno));
 
         init_waitqueue_head(&ring->irq_queue);
@@ -1683,13 +1853,14 @@ static int intel_ring_wait_request(struct intel_engine_cs *ring, int n)
                 ringbuf->head = ringbuf->last_retired_head;
                 ringbuf->last_retired_head = -1;
 
-                ringbuf->space = ring_space(ringbuf);
+                ringbuf->space = intel_ring_space(ringbuf);
                 if (ringbuf->space >= n)
                         return 0;
         }
 
         list_for_each_entry(request, &ring->request_list, list) {
-                if (__ring_space(request->tail, ringbuf->tail, ringbuf->size) >= n) {
+                if (__intel_ring_space(request->tail, ringbuf->tail,
+                                       ringbuf->size) >= n) {
                         seqno = request->seqno;
                         break;
                 }
@@ -1706,7 +1877,7 @@ static int intel_ring_wait_request(struct intel_engine_cs *ring, int n)
         ringbuf->head = ringbuf->last_retired_head;
         ringbuf->last_retired_head = -1;
 
-        ringbuf->space = ring_space(ringbuf);
+        ringbuf->space = intel_ring_space(ringbuf);
         return 0;
 }
 
@@ -1735,7 +1906,7 @@ static int ring_wait_for_space(struct intel_engine_cs *ring, int n)
         trace_i915_ring_wait_begin(ring);
         do {
                 ringbuf->head = I915_READ_HEAD(ring);
-                ringbuf->space = ring_space(ringbuf);
+                ringbuf->space = intel_ring_space(ringbuf);
                 if (ringbuf->space >= n) {
                         ret = 0;
                         break;
@@ -1787,7 +1958,7 @@ static int intel_wrap_ring_buffer(struct intel_engine_cs *ring)
                 iowrite32(MI_NOOP, virt++);
 
         ringbuf->tail = 0;
-        ringbuf->space = ring_space(ringbuf);
+        ringbuf->space = intel_ring_space(ringbuf);
 
         return 0;
 }
@@ -1992,9 +2163,7 @@ gen8_ring_dispatch_execbuffer(struct intel_engine_cs *ring,
                               u64 offset, u32 len,
                               unsigned flags)
 {
-        struct drm_i915_private *dev_priv = ring->dev->dev_private;
-        bool ppgtt = dev_priv->mm.aliasing_ppgtt != NULL &&
-                !(flags & I915_DISPATCH_SECURE);
+        bool ppgtt = USES_PPGTT(ring->dev) && !(flags & I915_DISPATCH_SECURE);
         int ret;
 
         ret = intel_ring_begin(ring, 4);
@@ -2023,8 +2192,9 @@ hsw_ring_dispatch_execbuffer(struct intel_engine_cs *ring,
                 return ret;
 
         intel_ring_emit(ring,
-                        MI_BATCH_BUFFER_START | MI_BATCH_PPGTT_HSW |
-                        (flags & I915_DISPATCH_SECURE ? 0 : MI_BATCH_NON_SECURE_HSW));
+                        MI_BATCH_BUFFER_START |
+                        (flags & I915_DISPATCH_SECURE ?
+                         0 : MI_BATCH_PPGTT_HSW | MI_BATCH_NON_SECURE_HSW));
         /* bit0-7 is the length on GEN6+ */
         intel_ring_emit(ring, offset);
         intel_ring_advance(ring);
@@ -2123,6 +2293,10 @@ int intel_init_render_ring_buffer(struct drm_device *dev)
                                         dev_priv->semaphore_obj = obj;
                         }
                 }
+                if (IS_CHERRYVIEW(dev))
+                        ring->init_context = chv_init_workarounds;
+                else
+                        ring->init_context = bdw_init_workarounds;
                 ring->add_request = gen6_add_request;
                 ring->flush = gen8_render_ring_flush;
                 ring->irq_get = gen8_ring_get_irq;
diff --git a/drivers/gpu/drm/i915/intel_ringbuffer.h b/drivers/gpu/drm/i915/intel_ringbuffer.h
index 70525d0c2c74..96479c89f4bd 100644
--- a/drivers/gpu/drm/i915/intel_ringbuffer.h
+++ b/drivers/gpu/drm/i915/intel_ringbuffer.h
@@ -5,6 +5,13 @@
 
 #define I915_CMD_HASH_ORDER 9
 
+/* Early gen2 devices have a cacheline of just 32 bytes, using 64 is overkill,
+ * but keeps the logic simple. Indeed, the whole purpose of this macro is just
+ * to give some inclination as to some of the magic values used in the various
+ * workarounds!
+ */
+#define CACHELINE_BYTES 64
+
 /*
  * Gen2 BSpec "1. Programming Environment" / 1.4.4.6 "Ring Buffer Use"
  * Gen3 BSpec "vol1c Memory Interface Functions" / 2.3.4.5 "Ring Buffer Use"
@@ -90,6 +97,15 @@ struct intel_ringbuffer {
         struct drm_i915_gem_object *obj;
         void __iomem *virtual_start;
 
+        struct intel_engine_cs *ring;
+
+        /*
+         * FIXME: This backpointer is an artifact of the history of how the
+         * execlist patches came into being. It will get removed once the basic
+         * code has landed.
+         */
+        struct intel_context *FIXME_lrc_ctx;
+
         u32 head;
         u32 tail;
         int space;
@@ -132,6 +148,8 @@ struct  intel_engine_cs {
 
         int             (*init)(struct intel_engine_cs *ring);
 
+        int             (*init_context)(struct intel_engine_cs *ring);
+
         void            (*write_tail)(struct intel_engine_cs *ring,
                                       u32 value);
         int __must_check (*flush)(struct intel_engine_cs *ring,
@@ -214,6 +232,18 @@ struct  intel_engine_cs {
                                   unsigned int num_dwords);
         } semaphore;
 
+        /* Execlists */
+        spinlock_t execlist_lock;
+        struct list_head execlist_queue;
+        u8 next_context_status_buffer;
+        u32             irq_keep_mask; /* bitmask for interrupts that should not be masked */
+        int             (*emit_request)(struct intel_ringbuffer *ringbuf);
+        int             (*emit_flush)(struct intel_ringbuffer *ringbuf,
+                                      u32 invalidate_domains,
+                                      u32 flush_domains);
+        int             (*emit_bb_start)(struct intel_ringbuffer *ringbuf,
+                                         u64 offset, unsigned flags);
+
         /**
          * List of objects currently involved in rendering from the
          * ringbuffer.
@@ -287,11 +317,7 @@ struct  intel_engine_cs {
         u32 (*get_cmd_length_mask)(u32 cmd_header);
 };
 
-static inline bool
-intel_ring_initialized(struct intel_engine_cs *ring)
-{
-        return ring->buffer && ring->buffer->obj;
-}
+bool intel_ring_initialized(struct intel_engine_cs *ring);
 
 static inline unsigned
 intel_ring_flag(struct intel_engine_cs *ring)
@@ -355,6 +381,10 @@ intel_write_status_page(struct intel_engine_cs *ring,
 #define I915_GEM_HWS_SCRATCH_INDEX      0x30
 #define I915_GEM_HWS_SCRATCH_ADDR (I915_GEM_HWS_SCRATCH_INDEX << MI_STORE_DWORD_INDEX_SHIFT)
 
+void intel_destroy_ringbuffer_obj(struct intel_ringbuffer *ringbuf);
+int intel_alloc_ringbuffer_obj(struct drm_device *dev,
+                               struct intel_ringbuffer *ringbuf);
+
 void intel_stop_ring_buffer(struct intel_engine_cs *ring);
 void intel_cleanup_ring_buffer(struct intel_engine_cs *ring);
 
@@ -372,6 +402,9 @@ static inline void intel_ring_advance(struct intel_engine_cs *ring)
         struct intel_ringbuffer *ringbuf = ring->buffer;
         ringbuf->tail &= ringbuf->size - 1;
 }
+int __intel_ring_space(int head, int tail, int size);
+int intel_ring_space(struct intel_ringbuffer *ringbuf);
+bool intel_ring_stopped(struct intel_engine_cs *ring);
 void __intel_ring_advance(struct intel_engine_cs *ring);
 
 int __must_check intel_ring_idle(struct intel_engine_cs *ring);
@@ -379,6 +412,9 @@ void intel_ring_init_seqno(struct intel_engine_cs *ring, u32 seqno);
 int intel_ring_flush_all_caches(struct intel_engine_cs *ring);
 int intel_ring_invalidate_all_caches(struct intel_engine_cs *ring);
 
+void intel_fini_pipe_control(struct intel_engine_cs *ring);
+int intel_init_pipe_control(struct intel_engine_cs *ring);
+
 int intel_init_render_ring_buffer(struct drm_device *dev);
 int intel_init_bsd_ring_buffer(struct drm_device *dev);
 int intel_init_bsd2_ring_buffer(struct drm_device *dev);
diff --git a/drivers/gpu/drm/i915/intel_sprite.c b/drivers/gpu/drm/i915/intel_sprite.c
index 168c6652cda1..07a74ef589bd 100644
--- a/drivers/gpu/drm/i915/intel_sprite.c
+++ b/drivers/gpu/drm/i915/intel_sprite.c
@@ -53,6 +53,7 @@ static bool intel_pipe_update_start(struct intel_crtc *crtc, uint32_t *start_vbl
         enum pipe pipe = crtc->pipe;
         long timeout = msecs_to_jiffies_timeout(1);
         int scanline, min, max, vblank_start;
+        wait_queue_head_t *wq = drm_crtc_vblank_waitqueue(&crtc->base);
         DEFINE_WAIT(wait);
 
         WARN_ON(!drm_modeset_is_locked(&crtc->base.mutex));
@@ -81,7 +82,7 @@ static bool intel_pipe_update_start(struct intel_crtc *crtc, uint32_t *start_vbl
                  * other CPUs can see the task state update by the time we
                  * read the scanline.
                  */
-                prepare_to_wait(&crtc->vbl_wait, &wait, TASK_UNINTERRUPTIBLE);
+                prepare_to_wait(wq, &wait, TASK_UNINTERRUPTIBLE);
 
                 scanline = intel_get_crtc_scanline(crtc);
                 if (scanline < min || scanline > max)
@@ -100,7 +101,7 @@ static bool intel_pipe_update_start(struct intel_crtc *crtc, uint32_t *start_vbl
                 local_irq_disable();
         }
 
-        finish_wait(&crtc->vbl_wait, &wait);
+        finish_wait(wq, &wait);
 
         drm_vblank_put(dev, pipe);
 
@@ -163,6 +164,7 @@ vlv_update_plane(struct drm_plane *dplane, struct drm_crtc *crtc,
         sprctl &= ~SP_PIXFORMAT_MASK;
         sprctl &= ~SP_YUV_BYTE_ORDER_MASK;
         sprctl &= ~SP_TILED;
+        sprctl &= ~SP_ROTATE_180;
 
         switch (fb->pixel_format) {
         case DRM_FORMAT_YUYV:
@@ -235,6 +237,14 @@ vlv_update_plane(struct drm_plane *dplane, struct drm_crtc *crtc,
                                                         fb->pitches[0]);
         linear_offset -= sprsurf_offset;
 
+        if (intel_plane->rotation == BIT(DRM_ROTATE_180)) {
+                sprctl |= SP_ROTATE_180;
+
+                x += src_w;
+                y += src_h;
+                linear_offset += src_h * fb->pitches[0] + src_w * pixel_size;
+        }
+
         atomic_update = intel_pipe_update_start(intel_crtc, &start_vbl_count);
 
         intel_update_primary_plane(intel_crtc);
@@ -364,6 +374,7 @@ ivb_update_plane(struct drm_plane *plane, struct drm_crtc *crtc,
         sprctl &= ~SPRITE_RGB_ORDER_RGBX;
         sprctl &= ~SPRITE_YUV_BYTE_ORDER_MASK;
         sprctl &= ~SPRITE_TILED;
+        sprctl &= ~SPRITE_ROTATE_180;
 
         switch (fb->pixel_format) {
         case DRM_FORMAT_XBGR8888:
@@ -426,6 +437,18 @@ ivb_update_plane(struct drm_plane *plane, struct drm_crtc *crtc,
                                                pixel_size, fb->pitches[0]);
         linear_offset -= sprsurf_offset;
 
+        if (intel_plane->rotation == BIT(DRM_ROTATE_180)) {
+                sprctl |= SPRITE_ROTATE_180;
+
+                /* HSW and BDW does this automagically in hardware */
+                if (!IS_HASWELL(dev) && !IS_BROADWELL(dev)) {
+                        x += src_w;
+                        y += src_h;
+                        linear_offset += src_h * fb->pitches[0] +
+                                src_w * pixel_size;
+                }
+        }
+
         atomic_update = intel_pipe_update_start(intel_crtc, &start_vbl_count);
 
         intel_update_primary_plane(intel_crtc);
@@ -571,6 +594,7 @@ ilk_update_plane(struct drm_plane *plane, struct drm_crtc *crtc,
         dvscntr &= ~DVS_RGB_ORDER_XBGR;
         dvscntr &= ~DVS_YUV_BYTE_ORDER_MASK;
         dvscntr &= ~DVS_TILED;
+        dvscntr &= ~DVS_ROTATE_180;
 
         switch (fb->pixel_format) {
         case DRM_FORMAT_XBGR8888:
@@ -628,6 +652,14 @@ ilk_update_plane(struct drm_plane *plane, struct drm_crtc *crtc,
                                                pixel_size, fb->pitches[0]);
         linear_offset -= dvssurf_offset;
 
+        if (intel_plane->rotation == BIT(DRM_ROTATE_180)) {
+                dvscntr |= DVS_ROTATE_180;
+
+                x += src_w;
+                y += src_h;
+                linear_offset += src_h * fb->pitches[0] + src_w * pixel_size;
+        }
+
         atomic_update = intel_pipe_update_start(intel_crtc, &start_vbl_count);
 
         intel_update_primary_plane(intel_crtc);
@@ -895,6 +927,9 @@ intel_update_plane(struct drm_plane *plane, struct drm_crtc *crtc,
         max_scale = intel_plane->max_downscale << 16;
         min_scale = intel_plane->can_scale ? 1 : (1 << 16);
 
+        drm_rect_rotate(&src, fb->width << 16, fb->height << 16,
+                        intel_plane->rotation);
+
         hscale = drm_rect_calc_hscale_relaxed(&src, &dst, min_scale, max_scale);
         BUG_ON(hscale < 0);
 
@@ -933,6 +968,9 @@ intel_update_plane(struct drm_plane *plane, struct drm_crtc *crtc,
                                      drm_rect_width(&dst) * hscale - drm_rect_width(&src),
                                      drm_rect_height(&dst) * vscale - drm_rect_height(&src));
 
+                drm_rect_rotate_inv(&src, fb->width << 16, fb->height << 16,
+                                    intel_plane->rotation);
+
                 /* sanity check to make sure the src viewport wasn't enlarged */
                 WARN_ON(src.x1 < (int) src_x ||
                         src.y1 < (int) src_y ||
@@ -1180,18 +1218,45 @@ out_unlock:
         return ret;
 }
 
-void intel_plane_restore(struct drm_plane *plane)
+int intel_plane_set_property(struct drm_plane *plane,
+                             struct drm_property *prop,
+                             uint64_t val)
+{
+        struct drm_device *dev = plane->dev;
+        struct intel_plane *intel_plane = to_intel_plane(plane);
+        uint64_t old_val;
+        int ret = -ENOENT;
+
+        if (prop == dev->mode_config.rotation_property) {
+                /* exactly one rotation angle please */
+                if (hweight32(val & 0xf) != 1)
+                        return -EINVAL;
+
+                if (intel_plane->rotation == val)
+                        return 0;
+
+                old_val = intel_plane->rotation;
+                intel_plane->rotation = val;
+                ret = intel_plane_restore(plane);
+                if (ret)
+                        intel_plane->rotation = old_val;
+        }
+
+        return ret;
+}
+
+int intel_plane_restore(struct drm_plane *plane)
 {
         struct intel_plane *intel_plane = to_intel_plane(plane);
 
         if (!plane->crtc || !plane->fb)
-                return;
+                return 0;
 
-        intel_update_plane(plane, plane->crtc, plane->fb,
-                           intel_plane->crtc_x, intel_plane->crtc_y,
-                           intel_plane->crtc_w, intel_plane->crtc_h,
-                           intel_plane->src_x, intel_plane->src_y,
-                           intel_plane->src_w, intel_plane->src_h);
+        return plane->funcs->update_plane(plane, plane->crtc, plane->fb,
+                                  intel_plane->crtc_x, intel_plane->crtc_y,
+                                  intel_plane->crtc_w, intel_plane->crtc_h,
+                                  intel_plane->src_x, intel_plane->src_y,
+                                  intel_plane->src_w, intel_plane->src_h);
 }
 
 void intel_plane_disable(struct drm_plane *plane)
@@ -1206,6 +1271,7 @@ static const struct drm_plane_funcs intel_plane_funcs = {
         .update_plane = intel_update_plane,
         .disable_plane = intel_disable_plane,
         .destroy = intel_destroy_plane,
+        .set_property = intel_plane_set_property,
 };
 
 static uint32_t ilk_plane_formats[] = {
@@ -1310,13 +1376,28 @@ intel_plane_init(struct drm_device *dev, enum pipe pipe, int plane)
 
         intel_plane->pipe = pipe;
         intel_plane->plane = plane;
+        intel_plane->rotation = BIT(DRM_ROTATE_0);
         possible_crtcs = (1 << pipe);
-        ret = drm_plane_init(dev, &intel_plane->base, possible_crtcs,
-                             &intel_plane_funcs,
-                             plane_formats, num_plane_formats,
-                             false);
-        if (ret)
+        ret = drm_universal_plane_init(dev, &intel_plane->base, possible_crtcs,
+                                       &intel_plane_funcs,
+                                       plane_formats, num_plane_formats,
+                                       DRM_PLANE_TYPE_OVERLAY);
+        if (ret) {
                 kfree(intel_plane);
+                goto out;
+        }
+
+        if (!dev->mode_config.rotation_property)
+                dev->mode_config.rotation_property =
+                        drm_mode_create_rotation_property(dev,
+                                                          BIT(DRM_ROTATE_0) |
+                                                          BIT(DRM_ROTATE_180));
+
+        if (dev->mode_config.rotation_property)
+                drm_object_attach_property(&intel_plane->base.base,
+                                           dev->mode_config.rotation_property,
+                                           intel_plane->rotation);
 
+ out:
         return ret;
 }
diff --git a/drivers/gpu/drm/i915/intel_uncore.c b/drivers/gpu/drm/i915/intel_uncore.c
index e81bc3bdc533..918b76163965 100644
--- a/drivers/gpu/drm/i915/intel_uncore.c
+++ b/drivers/gpu/drm/i915/intel_uncore.c
@@ -101,7 +101,7 @@ static void __gen7_gt_force_wake_mt_get(struct drm_i915_private *dev_priv,
 {
         u32 forcewake_ack;
 
-        if (IS_HASWELL(dev_priv->dev) || IS_GEN8(dev_priv->dev))
+        if (IS_HASWELL(dev_priv->dev) || IS_BROADWELL(dev_priv->dev))
                 forcewake_ack = FORCEWAKE_ACK_HSW;
         else
                 forcewake_ack = FORCEWAKE_MT_ACK;
@@ -334,7 +334,7 @@ void intel_uncore_forcewake_reset(struct drm_device *dev, bool restore)
         else if (IS_GEN6(dev) || IS_GEN7(dev))
                 __gen6_gt_force_wake_reset(dev_priv);
 
-        if (IS_IVYBRIDGE(dev) || IS_HASWELL(dev) || IS_GEN8(dev))
+        if (IS_IVYBRIDGE(dev) || IS_HASWELL(dev) || IS_BROADWELL(dev))
                 __gen7_gt_force_wake_mt_reset(dev_priv);
 
         if (restore) { /* If reset with a user forcewake, try to restore */
@@ -838,7 +838,7 @@ void intel_uncore_init(struct drm_device *dev)
         if (IS_VALLEYVIEW(dev)) {
                 dev_priv->uncore.funcs.force_wake_get = __vlv_force_wake_get;
                 dev_priv->uncore.funcs.force_wake_put = __vlv_force_wake_put;
-        } else if (IS_HASWELL(dev) || IS_GEN8(dev)) {
+        } else if (IS_HASWELL(dev) || IS_BROADWELL(dev)) {
                 dev_priv->uncore.funcs.force_wake_get = __gen7_gt_force_wake_mt_get;
                 dev_priv->uncore.funcs.force_wake_put = __gen7_gt_force_wake_mt_put;
         } else if (IS_IVYBRIDGE(dev)) {